diff --git a/.agents/skills/verify-mcode/references/features/README.md b/.agents/skills/verify-mcode/references/features/README.md index ae99cca14..9f635abb5 100644 --- a/.agents/skills/verify-mcode/references/features/README.md +++ b/.agents/skills/verify-mcode/references/features/README.md @@ -26,6 +26,7 @@ | Last turn shows native agent changes, settles durably, and identifies Git fallback | [Last turn native diff](turn-diff-review.md) | Codex composer and Review Electron proof, plus focused Git/RPC and migration checks | | A thread starts, runs, stops, clears active state, and retains a cancelled reconnect snapshot | [Thread lifecycle](thread-lifecycle.md) | `runtime live --scenario stop` | | Provider events become durable assistant conversation data | [Provider events and durability](provider-events-and-durability.md) | `runtime live --scenario completion` | +| Consecutive replies settle while switching away and back during streaming | [Provider events and durability](provider-events-and-durability.md#consecutive-turns-and-navigation) | Real-provider Electron follow-ups, mid-turn navigation, idle observation, and reload | | Seven active tasks keep event order while model and terminal controls settle | [Multi-surface journeys](multi-surface-journeys.md#seven-active-tasks-and-controls) | Fixture-owned seven-task public RPC run, then Electron controls and reload | | Codex reroutes, warnings, diagnostics, and authentication recovery remain bounded, durable, and thread-scoped | [Provider events and durability](provider-events-and-durability.md) | Composer notice journey and `runtime health` | | Approval review offers only valid access modes, waits for a real permission request, and keeps Full Access review-free | [Provider events and durability](provider-events-and-durability.md) | Approval review journey and focused policy and mapper tests | diff --git a/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md b/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md index fdcb8c8d6..82ac18326 100644 --- a/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md +++ b/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md @@ -19,6 +19,14 @@ If cleanup is incomplete, use the receipt's exact path with `--cleanup-receipt --confirm-run` before another run. Do not delete tasks by title or age. +For the worker-owned turn change in issue #1760, collect at least ten matched +receipts. Compare server-loop delay, public event throughput, memory, and +`metrics.workerQueue` peaks with the recorded baseline. Require model-picker +and Stop acknowledgement within 2 seconds, terminal readiness within 5 seconds, +and ordered durable events in every run. Run `--stop-one` as a separate case; +require the six peers to finish and the stopped task to remain cancelled after +reload. Trace and fix a missed budget before calling the run complete. + In the owned Electron app, create separate direct tasks in `.dev/fixture-repo`. Open and close the model picker, open a Terminal, switch tasks, and reload. Capture the settled controls and terminal output in a screenshot, record the @@ -28,6 +36,10 @@ final conversation check and [Electron live testing](../../../electorn-live-test for the desktop control and capture. Report any unavailable control or missing capture as a gap. A public RPC receipt alone does not prove the desktop path. +Also run the [consecutive-turn navigation journey](provider-events-and-durability.md#consecutive-turns-and-navigation) +with real Codex. The seven-task fixture starts one turn per task and cannot prove +successful follow-ups or renderer state while switching during streaming. + ## Provider completeness Use this journey when a change affects turn diffs, Review, provider events, automatic review, or workspace invalidation. Resolve the upstream Codex source with the OpenSrc command in the repository instructions before a Codex run. Record the resolved upstream commit in the receipt. The cache is read-only. diff --git a/.agents/skills/verify-mcode/references/features/provider-events-and-durability.md b/.agents/skills/verify-mcode/references/features/provider-events-and-durability.md index 9626b837c..ab61c4f88 100644 --- a/.agents/skills/verify-mcode/references/features/provider-events-and-durability.md +++ b/.agents/skills/verify-mcode/references/features/provider-events-and-durability.md @@ -17,7 +17,8 @@ 1. Open the project for this worktree. 2. Start a short thread with a selected provider and model. 3. Wait for the final reply. -4. Reload or reopen the conversation and confirm that the reply remains. +4. Send a follow-up in the same conversation. Require its reply, cleared activity indicator, and restored Send control. +5. Run the consecutive-turn journey below, then reload and confirm that replies remain. ## Driving it with verify-mcode @@ -32,6 +33,28 @@ This fixture checks the production Codex event boundary, not a real model run or ## Runtime checks +### Consecutive turns and navigation + +In the owned Electron app, select a real provider and model and create a direct +task under `.dev/fixture-repo`. Record its ID before proceeding. + +1. Send a short prompt, wait for completion, then send a second prompt in the + same task. Require both replies and a restored Send control after each turn. +2. Send a longer follow-up. After assistant text starts streaming, switch to + another fixture task and back before completion. Capture active runtime state + before leaving, while away, and after returning. The response must continue + and finish once, without cancellation or text appearing in the other task. +3. Send another follow-up after navigation. Require a completed public runtime, + no Stop control or thinking indicator, and no reactivation for 40 seconds. +4. Reload, reopen the same task, verify the durable replies through + `conversation.page`, and send one more follow-up to completion. +5. Save a video of the mid-turn switch, settled screenshots, terminal outcomes, + provider/model, and exact owned IDs. Delete only the tasks created for proof. + +A first-turn server receipt does not cover this journey. Compare desktop state +with the public runtime from that desktop's server; its isolated database can +differ from the worktree server. Report which providers were actually exercised. + Run `runtime health`, then run: ```sh diff --git a/.agents/skills/verify-mcode/scripts/runtime.mjs b/.agents/skills/verify-mcode/scripts/runtime.mjs index 00d5351f0..eb4979761 100644 --- a/.agents/skills/verify-mcode/scripts/runtime.mjs +++ b/.agents/skills/verify-mcode/scripts/runtime.mjs @@ -67,7 +67,7 @@ const FOCUSED_TEST_FILES = [ "src/features/agents/turns/__tests__/turn-event-sink.test.ts", "src/features/agents/turns/__tests__/turn-finalizer.test.ts", "src/features/agents/turns/__tests__/turn-runtime.test.ts", - "src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts", + "src/features/agents/canonical/__tests__/canonical-agent-boundary.test.ts", "src/features/agents/collaboration/adapters/__tests__/codex-collaboration-event-adapter.test.ts", "src/features/providers/composition/__tests__/provider-event-ingress.test.ts", "src/features/projects/git/__tests__/git-repository-fetch.test.ts", diff --git a/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs b/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs index 1e030af0b..fb319bc21 100644 --- a/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs +++ b/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it } from "vitest"; +import * as NodeChildProcess from "node:child_process"; import * as NodeFSPromises from "node:fs/promises"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; @@ -24,6 +25,8 @@ describe("buildServerRuntimeBundles", () => { const serverOutFile = NodePath.join(outputRoot, "server.cjs"); const ptyHostOutFile = NodePath.join(outputRoot, "pty-host.cjs"); const providerEventWorkerOutFile = NodePath.join(outputRoot, "provider-event.worker.cjs"); + const canonicalWriterWorkerOutFile = NodePath.join(outputRoot, "canonical-agent-writer.worker.cjs"); + const executionWorkerOutFile = NodePath.join(outputRoot, "execution.worker.cjs"); try { compileServerWithSwc(serverRoot); @@ -36,7 +39,11 @@ describe("buildServerRuntimeBundles", () => { expect(() => new NodeVM.Script(bundledEntry, { filename: "server.cjs" })).not.toThrow(); await NodeFSPromises.access(providerEventWorkerOutFile); + await NodeFSPromises.access(canonicalWriterWorkerOutFile); + await NodeFSPromises.access(executionWorkerOutFile); + expect(await NodeFSPromises.readFile(executionWorkerOutFile, "utf8")).toContain("writer-request"); expect(await NodeFSPromises.readFile(serverOutFile, "utf8")).toContain("provider-event.worker.cjs"); + expect(startAndCloseBundledWorker(executionWorkerOutFile)).toBe("ready,closed"); } finally { await NodeFSPromises.rm(outputRoot, { recursive: true, force: true }); await NodeFSPromises.rm(distTsc, { recursive: true, force: true }); @@ -63,17 +70,39 @@ describe("buildServerRuntimeBundles", () => { "composition", "provider-event.worker.js", ); + const canonicalWriterWorkerEntry = NodePath.join( + serverRoot, + "dist-tsc", + "features", + "agents", + "canonical", + "canonical-agent-writer.worker.js", + ); + const executionWorkerEntry = NodePath.join( + serverRoot, + "dist-tsc", + "features", + "agents", + "execution", + "execution.worker.js", + ); const serverOutFile = NodePath.join(fixtureRoot, "dist", "server.cjs"); const ptyHostOutFile = NodePath.join(fixtureRoot, "dist", "pty-host.cjs"); const providerEventWorkerOutFile = NodePath.join(fixtureRoot, "dist", "provider-event.worker.cjs"); + const canonicalWriterWorkerOutFile = NodePath.join(fixtureRoot, "dist", "canonical-agent-writer.worker.cjs"); + const executionWorkerOutFile = NodePath.join(fixtureRoot, "dist", "execution.worker.cjs"); await Promise.all([ NodeFSPromises.mkdir(NodePath.dirname(ptyHostEntry), { recursive: true }), NodeFSPromises.mkdir(NodePath.dirname(providerEventWorkerEntry), { recursive: true }), + NodeFSPromises.mkdir(NodePath.dirname(canonicalWriterWorkerEntry), { recursive: true }), + NodeFSPromises.mkdir(NodePath.dirname(executionWorkerEntry), { recursive: true }), ]); await NodeFSPromises.writeFile(serverEntry, 'console.log("server-entry");\n'); await NodeFSPromises.writeFile(ptyHostEntry, 'console.log("pty-host-entry");\n'); await NodeFSPromises.writeFile(providerEventWorkerEntry, 'console.log("provider-event-worker");\n'); + await NodeFSPromises.writeFile(canonicalWriterWorkerEntry, 'console.log("canonical-writer-worker");\n'); + await NodeFSPromises.writeFile(executionWorkerEntry, 'console.log("execution-worker");\n'); await buildServerRuntimeBundles({ serverRoot, @@ -85,8 +114,36 @@ describe("buildServerRuntimeBundles", () => { await NodeFSPromises.access(serverOutFile); await NodeFSPromises.access(ptyHostOutFile); await NodeFSPromises.access(providerEventWorkerOutFile); + await NodeFSPromises.access(canonicalWriterWorkerOutFile); + await NodeFSPromises.access(executionWorkerOutFile); expect(await NodeFSPromises.readFile(serverOutFile, "utf8")).toContain("server-entry"); expect(await NodeFSPromises.readFile(ptyHostOutFile, "utf8")).toContain("pty-host-entry"); expect(await NodeFSPromises.readFile(providerEventWorkerOutFile, "utf8")).toContain("provider-event-worker"); + expect(await NodeFSPromises.readFile(canonicalWriterWorkerOutFile, "utf8")).toContain("canonical-writer-worker"); + expect(await NodeFSPromises.readFile(executionWorkerOutFile, "utf8")).toContain("execution-worker"); }, 30_000); }); + +function startAndCloseBundledWorker(workerPath) { + const script = ` + import { pathToFileURL } from "node:url"; + const worker = new Worker(pathToFileURL(process.env.MCODE_TEST_EXECUTION_WORKER), { type: "module", ref: true }); + const received = []; + const timer = setTimeout(() => { worker.terminate(); process.exit(1); }, 3000); + worker.onmessage = (event) => { + received.push(event.data.kind); + if (event.data.kind === "ready") worker.postMessage({ kind: "close" }); + if (event.data.kind === "closed") { + clearTimeout(timer); + worker.terminate(); + process.stdout.write(received.join(",")); + } + }; + worker.onerror = () => { clearTimeout(timer); process.exit(1); }; + `; + return NodeChildProcess.execFileSync("bun", ["-e", script], { + encoding: "utf8", + timeout: 5_000, + env: { ...process.env, MCODE_TEST_EXECUTION_WORKER: workerPath }, + }); +} diff --git a/apps/desktop/scripts/build-main.mjs b/apps/desktop/scripts/build-main.mjs index 174c6e132..07e29409d 100644 --- a/apps/desktop/scripts/build-main.mjs +++ b/apps/desktop/scripts/build-main.mjs @@ -90,7 +90,7 @@ await buildServerRuntimeBundles({ production: true, }); -console.log("Server bundles complete: dist/server/server.cjs, dist/server/pty-host.cjs"); +console.log("Server bundles complete: dist/server/server.cjs, dist/server/pty-host.cjs, dist/server/execution.worker.cjs"); const drizzleSrc = NodePath.resolve(serverRoot, "drizzle"); const drizzleDst = NodePath.resolve(desktopRoot, "dist/server/drizzle"); diff --git a/apps/desktop/scripts/dev-electron.mjs b/apps/desktop/scripts/dev-electron.mjs index ed807079e..834834b69 100644 --- a/apps/desktop/scripts/dev-electron.mjs +++ b/apps/desktop/scripts/dev-electron.mjs @@ -160,7 +160,7 @@ const serverEsbuildCfg = { ...shared, entryPoints: [NodePath.resolve(serverRoot, "dist-tsc/index.js")], outfile: serverOutFile, - external: ["bun:sqlite", "node-pty", "electron", "koffi"], + external: ["bun", "bun:sqlite", "node-pty", "electron", "koffi"], banner: { js: 'var __importMetaUrl = require("url").pathToFileURL(__filename).href;', }, diff --git a/apps/server/drizzle/0065_ambitious_blindfold.sql b/apps/server/drizzle/0065_ambitious_blindfold.sql new file mode 100644 index 000000000..7b69d5cc8 --- /dev/null +++ b/apps/server/drizzle/0065_ambitious_blindfold.sql @@ -0,0 +1,10 @@ +CREATE TABLE `canonical_writer_operation_receipts` ( + `execution_id` text NOT NULL, + `operation_id` text NOT NULL, + `kind` text NOT NULL, + `input_hash` text NOT NULL, + `receipt_json` text NOT NULL, + `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, + PRIMARY KEY(`execution_id`, `operation_id`), + FOREIGN KEY (`execution_id`) REFERENCES `canonical_agent_turns`(`execution_id`) ON UPDATE no action ON DELETE cascade +); diff --git a/apps/server/drizzle/0066_fat_amazoness.sql b/apps/server/drizzle/0066_fat_amazoness.sql new file mode 100644 index 000000000..c22cb0107 --- /dev/null +++ b/apps/server/drizzle/0066_fat_amazoness.sql @@ -0,0 +1,5 @@ +CREATE TABLE `canonical_writer_live_publication_heads` ( + `thread_id` text PRIMARY KEY NOT NULL, + `last_sequence` integer NOT NULL, + FOREIGN KEY (`thread_id`) REFERENCES `threads`(`id`) ON UPDATE no action ON DELETE cascade +); diff --git a/apps/server/drizzle/meta/0065_snapshot.json b/apps/server/drizzle/meta/0065_snapshot.json new file mode 100644 index 000000000..c9c8fbfdf --- /dev/null +++ b/apps/server/drizzle/meta/0065_snapshot.json @@ -0,0 +1,4989 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d6ce0589-8bee-4f7f-b188-033550ae9cf0", + "prevId": "991b4c44-b789-4554-99a3-77caea9d753d", + "tables": { + "canonical_agent_events": { + "name": "canonical_agent_events", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_sequence": { + "name": "accepted_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "durable_revision": { + "name": "durable_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "roster_revision": { + "name": "roster_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envelope_json": { + "name": "envelope_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "persisted_at": { + "name": "persisted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_events_execution_sequence": { + "name": "idx_canonical_agent_events_execution_sequence", + "columns": [ + "execution_id", + "accepted_sequence" + ], + "isUnique": true + }, + "idx_canonical_agent_events_thread_revision": { + "name": "idx_canonical_agent_events_thread_revision", + "columns": [ + "thread_id", + "durable_revision" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_events_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_events_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_events", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_ingest_checkpoints": { + "name": "canonical_agent_ingest_checkpoints", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_accepted_sequence": { + "name": "last_accepted_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_durable_sequence": { + "name": "last_durable_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "native_cursor_json": { + "name": "native_cursor_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "terminal_outcome": { + "name": "terminal_outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recovery_incident_id": { + "name": "recovery_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_checkpoints_thread": { + "name": "idx_canonical_agent_checkpoints_thread", + "columns": [ + "thread_id", + "updated_at" + ], + "isUnique": false + }, + "idx_canonical_agent_checkpoints_recovery_incident": { + "name": "idx_canonical_agent_checkpoints_recovery_incident", + "columns": [ + "recovery_incident_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_ingest_checkpoints_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_ingest_checkpoints_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_ingest_checkpoints", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_ingest_checkpoints_turn_id_canonical_agent_turns_id_fk": { + "name": "canonical_agent_ingest_checkpoints_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "canonical_agent_ingest_checkpoints", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_items": { + "name": "canonical_agent_items", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_item_id": { + "name": "parent_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_items_turn": { + "name": "idx_canonical_agent_items_turn", + "columns": [ + "turn_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_agent_items_thread": { + "name": "idx_canonical_agent_items_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_agent_items_created_id": { + "name": "idx_canonical_agent_items_created_id", + "columns": [ + "created_at", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_items_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_items_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_items", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_items_turn_id_canonical_agent_turns_id_fk": { + "name": "canonical_agent_items_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "canonical_agent_items", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_threads": { + "name": "canonical_agent_threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "root_thread_id": { + "name": "root_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owning_parent_thread_id": { + "name": "owning_parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "activity_state": { + "name": "activity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversation_revision": { + "name": "conversation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "roster_revision": { + "name": "roster_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_threads_workspace": { + "name": "idx_canonical_agent_threads_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_canonical_agent_threads_root": { + "name": "idx_canonical_agent_threads_root", + "columns": [ + "root_thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_threads_id_threads_id_fk": { + "name": "canonical_agent_threads_id_threads_id_fk", + "tableFrom": "canonical_agent_threads", + "tableTo": "threads", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_threads_workspace_id_workspaces_id_fk": { + "name": "canonical_agent_threads_workspace_id_workspaces_id_fk", + "tableFrom": "canonical_agent_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_turns": { + "name": "canonical_agent_turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trigger_json": { + "name": "trigger_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approval_review_mode": { + "name": "approval_review_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "approval_review_reason": { + "name": "approval_review_reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual-requested'" + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_turns_execution": { + "name": "idx_canonical_agent_turns_execution", + "columns": [ + "execution_id" + ], + "isUnique": true + }, + "idx_canonical_agent_turns_thread": { + "name": "idx_canonical_agent_turns_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_turns_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_turns_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_turns", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_collaboration_actions": { + "name": "canonical_collaboration_actions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_item_id": { + "name": "source_item_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_thread_id": { + "name": "target_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_turn_id": { + "name": "target_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_unknown": { + "name": "delivery_unknown", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_collaboration_source": { + "name": "idx_canonical_collaboration_source", + "columns": [ + "source_thread_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_collaboration_target": { + "name": "idx_canonical_collaboration_target", + "columns": [ + "target_thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_conversation_display_mappings": { + "name": "canonical_conversation_display_mappings", + "columns": { + "source_item_id": { + "name": "source_item_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "target_kind": { + "name": "target_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_canonical_conversation_display_mappings_target": { + "name": "idx_canonical_conversation_display_mappings_target", + "columns": [ + "target_kind", + "target_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_conversation_display_mappings_source_item_id_canonical_agent_items_id_fk": { + "name": "canonical_conversation_display_mappings_source_item_id_canonical_agent_items_id_fk", + "tableFrom": "canonical_conversation_display_mappings", + "tableTo": "canonical_agent_items", + "columnsFrom": [ + "source_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_legacy_message_provenance": { + "name": "canonical_legacy_message_provenance", + "columns": { + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "migration_version": { + "name": "migration_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mapping_status": { + "name": "mapping_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_thread_id": { + "name": "canonical_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "canonical_turn_id": { + "name": "canonical_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "canonical_item_id": { + "name": "canonical_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_canonical_legacy_message_mapping": { + "name": "idx_canonical_legacy_message_mapping", + "columns": [ + "mapping_status", + "message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_legacy_message_provenance_message_id_messages_id_fk": { + "name": "canonical_legacy_message_provenance_message_id_messages_id_fk", + "tableFrom": "canonical_legacy_message_provenance", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_legacy_migration_checkpoints": { + "name": "canonical_legacy_migration_checkpoints", + "columns": { + "version": { + "name": "version", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "migrated_messages": { + "name": "migrated_messages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ambiguous_messages": { + "name": "ambiguous_messages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_writer_operation_receipts": { + "name": "canonical_writer_operation_receipts", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_hash": { + "name": "input_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "receipt_json": { + "name": "receipt_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "canonical_writer_operation_receipts_execution_id_canonical_agent_turns_execution_id_fk": { + "name": "canonical_writer_operation_receipts_execution_id_canonical_agent_turns_execution_id_fk", + "tableFrom": "canonical_writer_operation_receipts", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "execution_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "canonical_writer_operation_receipts_execution_id_operation_id_pk": { + "columns": [ + "execution_id", + "operation_id" + ], + "name": "canonical_writer_operation_receipts_execution_id_operation_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_jobs": { + "name": "cleanup_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_path": { + "name": "workspace_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'explicit'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cleanup_jobs_thread_id_unique": { + "name": "cleanup_jobs_thread_id_unique", + "columns": [ + "thread_id" + ], + "isUnique": true + }, + "idx_cleanup_jobs_retry": { + "name": "idx_cleanup_jobs_retry", + "columns": [ + "next_retry_at", + "attempts", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "conversation_display_materialization_state": { + "name": "conversation_display_materialization_state", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "last_source_created_at": { + "name": "last_source_created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_source_id": { + "name": "last_source_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed": { + "name": "completed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "diff_summaries": { + "name": "diff_summaries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_count": { + "name": "turn_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_turn_id": { + "name": "last_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_diff_summaries_thread": { + "name": "idx_diff_summaries_thread", + "columns": [ + "thread_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "diff_summaries_thread_id_threads_id_fk": { + "name": "diff_summaries_thread_id_threads_id_fk", + "tableFrom": "diff_summaries", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "external_thread_control_deliveries": { + "name": "external_thread_control_deliveries", + "columns": { + "pairing_id": { + "name": "pairing_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "authority_epoch": { + "name": "authority_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'in_flight'" + }, + "result_json": { + "name": "result_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_external_thread_control_delivery_identity": { + "name": "idx_external_thread_control_delivery_identity", + "columns": [ + "pairing_id", + "authority_epoch", + "delivery_id" + ], + "isUnique": true + }, + "idx_external_thread_control_delivery_retention": { + "name": "idx_external_thread_control_delivery_retention", + "columns": [ + "pairing_id", + "status", + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "external_thread_control_pairings": { + "name": "external_thread_control_pairings", + "columns": { + "pairing_id": { + "name": "pairing_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_hash": { + "name": "credential_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_ids_json": { + "name": "workspace_ids_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "scopes_json": { + "name": "scopes_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "calls_per_minute": { + "name": "calls_per_minute", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_active_threads": { + "name": "max_active_threads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "authority_epoch": { + "name": "authority_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "replaced_by_pairing_id": { + "name": "replaced_by_pairing_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "replaces_pairing_id": { + "name": "replaces_pairing_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "external_thread_control_pairings_credential_hash_unique": { + "name": "external_thread_control_pairings_credential_hash_unique", + "columns": [ + "credential_hash" + ], + "isUnique": true + }, + "idx_external_thread_control_pairings_integration": { + "name": "idx_external_thread_control_pairings_integration", + "columns": [ + "integration_id", + "status" + ], + "isUnique": false + }, + "idx_external_thread_control_active_integration": { + "name": "idx_external_thread_control_active_integration", + "columns": [ + "integration_id" + ], + "isUnique": true, + "where": "status = 'active'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "hook_executions": { + "name": "hook_executions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hook_name": { + "name": "hook_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "did_block": { + "name": "did_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_hook_executions_message_sort_order_id": { + "name": "idx_hook_executions_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "hook_executions_message_id_messages_id_fk": { + "name": "hook_executions_message_id_messages_id_fk", + "tableFrom": "hook_executions", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "messages": { + "name": "messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "files_changed": { + "name": "files_changed", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tokens_used": { + "name": "tokens_used", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachments": { + "name": "attachments", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preview_annotations": { + "name": "preview_annotations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mentions": { + "name": "mentions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_text_comments": { + "name": "selected_text_comments", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reply_to_message_id": { + "name": "reply_to_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "quoted_text": { + "name": "quoted_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_type": { + "name": "origin_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'legacy'" + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_provider_id": { + "name": "source_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "legacy_provenance": { + "name": "legacy_provenance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_agent_provenance": { + "name": "parent_agent_provenance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_internal": { + "name": "is_internal", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "outcome_execution_id": { + "name": "outcome_execution_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "system_notice": { + "name": "system_notice", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_thread": { + "name": "idx_messages_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + }, + "idx_messages_sequence": { + "name": "idx_messages_sequence", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false + }, + "idx_messages_thread_sequence_id": { + "name": "idx_messages_thread_sequence_id", + "columns": [ + "thread_id", + "sequence", + "id" + ], + "isUnique": false + }, + "idx_messages_notice_session_sequence": { + "name": "idx_messages_notice_session_sequence", + "columns": [ + "thread_id", + "json_extract(\"system_notice\", '$.sessionId')", + "\"sequence\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messages_thread_id_threads_id_fk": { + "name": "messages_thread_id_threads_id_fk", + "tableFrom": "messages", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "messages_reply_to_message_id_messages_id_fk": { + "name": "messages_reply_to_message_id_messages_id_fk", + "tableFrom": "messages", + "tableTo": "messages", + "columnsFrom": [ + "reply_to_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "parent_assistant_text_checkpoint_chunks": { + "name": "parent_assistant_text_checkpoint_chunks", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_sequence": { + "name": "first_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "byte_length": { + "name": "byte_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_parent_assistant_text_checkpoint_chunks_sequence": { + "name": "idx_parent_assistant_text_checkpoint_chunks_sequence", + "columns": [ + "execution_id", + "first_sequence" + ], + "isUnique": true + } + }, + "foreignKeys": { + "parent_assistant_text_checkpoint_chunks_execution_id_parent_assistant_text_checkpoints_execution_id_fk": { + "name": "parent_assistant_text_checkpoint_chunks_execution_id_parent_assistant_text_checkpoints_execution_id_fk", + "tableFrom": "parent_assistant_text_checkpoint_chunks", + "tableTo": "parent_assistant_text_checkpoints", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "execution_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "parent_assistant_text_checkpoints": { + "name": "parent_assistant_text_checkpoints", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_bytes": { + "name": "retained_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_chunks": { + "name": "retained_chunks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_parent_assistant_text_checkpoints_thread": { + "name": "idx_parent_assistant_text_checkpoints_thread", + "columns": [ + "thread_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "parent_assistant_text_checkpoints_thread_id_canonical_agent_threads_id_fk": { + "name": "parent_assistant_text_checkpoints_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "parent_assistant_text_checkpoints", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parent_assistant_text_checkpoints_turn_id_canonical_agent_turns_id_fk": { + "name": "parent_assistant_text_checkpoints_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "parent_assistant_text_checkpoints", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plan_question_answers": { + "name": "plan_question_answers", + "columns": { + "assistant_message_id": { + "name": "assistant_message_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answered_at": { + "name": "answered_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_plan_question_answers_thread_answered_at": { + "name": "idx_plan_question_answers_thread_answered_at", + "columns": [ + "thread_id", + "answered_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "plan_question_answers_assistant_message_id_messages_id_fk": { + "name": "plan_question_answers_assistant_message_id_messages_id_fk", + "tableFrom": "plan_question_answers", + "tableTo": "messages", + "columnsFrom": [ + "assistant_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "plan_question_answers_thread_id_threads_id_fk": { + "name": "plan_question_answers_thread_id_threads_id_fk", + "tableFrom": "plan_question_answers", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plans": { + "name": "plans", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_md": { + "name": "content_md", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sections_json": { + "name": "sections_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_plans_thread": { + "name": "idx_plans_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + }, + "idx_plans_thread_version": { + "name": "idx_plans_thread_version", + "columns": [ + "thread_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "plans_thread_id_threads_id_fk": { + "name": "plans_thread_id_threads_id_fk", + "tableFrom": "plans", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "plans_message_id_messages_id_fk": { + "name": "plans_message_id_messages_id_fk", + "tableFrom": "plans", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_action_runs": { + "name": "project_action_runs", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action_id": { + "name": "action_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "terminal_session_id": { + "name": "terminal_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "action_name": { + "name": "action_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcript": { + "name": "transcript", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "transcript_truncated": { + "name": "transcript_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "idx_project_action_runs_slot": { + "name": "idx_project_action_runs_slot", + "columns": [ + "thread_id", + "action_id" + ], + "isUnique": true + }, + "idx_project_action_runs_thread": { + "name": "idx_project_action_runs_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "project_action_runs_thread_id_threads_id_fk": { + "name": "project_action_runs_thread_id_threads_id_fk", + "tableFrom": "project_action_runs", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_action_runs_workspace_id_workspaces_id_fk": { + "name": "project_action_runs_workspace_id_workspaces_id_fk", + "tableFrom": "project_action_runs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_catalog_snapshots": { + "name": "provider_catalog_snapshots", + "columns": { + "context_key": { + "name": "context_key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cwd": { + "name": "cwd", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_provider_catalog_snapshots_workspace": { + "name": "idx_provider_catalog_snapshots_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_provider_catalog_snapshots_provider": { + "name": "idx_provider_catalog_snapshots_provider", + "columns": [ + "provider_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "provider_catalog_snapshots_workspace_id_workspaces_id_fk": { + "name": "provider_catalog_snapshots_workspace_id_workspaces_id_fk", + "tableFrom": "provider_catalog_snapshots", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_model_cache": { + "name": "provider_model_cache", + "columns": { + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "models_json": { + "name": "models_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pull_request_review_links": { + "name": "pull_request_review_links", + "columns": { + "worktree_id": { + "name": "worktree_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_node_id": { + "name": "repository_node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pull_request_number": { + "name": "pull_request_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pr_url": { + "name": "pr_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pr_state": { + "name": "pr_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_managed": { + "name": "worktree_managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "head_repository_node_id": { + "name": "head_repository_node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_repository_owner": { + "name": "head_repository_owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_repository_name": { + "name": "head_repository_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_ref": { + "name": "head_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_oid": { + "name": "head_oid", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "local_branch": { + "name": "local_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "push_remote": { + "name": "push_remote", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "push_ref": { + "name": "push_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "managed_remote_name": { + "name": "managed_remote_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "primary_thread_id": { + "name": "primary_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_pull_request_review_links_identity": { + "name": "idx_pull_request_review_links_identity", + "columns": [ + "provider", + "repository_node_id", + "pull_request_number" + ], + "isUnique": true + }, + "idx_pull_request_review_links_primary_thread": { + "name": "idx_pull_request_review_links_primary_thread", + "columns": [ + "primary_thread_id" + ], + "isUnique": true + }, + "idx_pull_request_review_links_workspace": { + "name": "idx_pull_request_review_links_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_pull_request_review_links_worktree_path": { + "name": "idx_pull_request_review_links_worktree_path", + "columns": [ + "worktree_path" + ], + "isUnique": false + } + }, + "foreignKeys": { + "pull_request_review_links_workspace_id_workspaces_id_fk": { + "name": "pull_request_review_links_workspace_id_workspaces_id_fk", + "tableFrom": "pull_request_review_links", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pull_request_review_links_primary_thread_id_threads_id_fk": { + "name": "pull_request_review_links_primary_thread_id_threads_id_fk", + "tableFrom": "pull_request_review_links", + "tableTo": "threads", + "columnsFrom": [ + "primary_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "terminal_cleanup_ledger": { + "name": "terminal_cleanup_ledger", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "host_generation": { + "name": "host_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "root_pid": { + "name": "root_pid", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "process_group_id": { + "name": "process_group_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "containment": { + "name": "containment", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thought_segments": { + "name": "thought_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "is_final_response": { + "name": "is_final_response", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_thought_segments_message_final_sort_order_id": { + "name": "idx_thought_segments_message_final_sort_order_id", + "columns": [ + "message_id", + "is_final_response", + "sort_order", + "id" + ], + "isUnique": false + }, + "idx_thought_segments_final_message_sort_order_id": { + "name": "idx_thought_segments_final_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false, + "where": "\"thought_segments\".\"is_final_response\" <> 0" + } + }, + "foreignKeys": { + "thought_segments_message_id_messages_id_fk": { + "name": "thought_segments_message_id_messages_id_fk", + "tableFrom": "thought_segments", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_control_approvals": { + "name": "thread_control_approvals", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "execution_json": { + "name": "execution_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "placement_json": { + "name": "placement_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "caller_id": { + "name": "caller_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_provider_id": { + "name": "source_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'thread_create_batch'" + }, + "operation_phase": { + "name": "operation_phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pre_provision'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "resolved_at": { + "name": "resolved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_thread_control_approvals_thread": { + "name": "idx_thread_control_approvals_thread", + "columns": [ + "thread_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_control_approvals_thread_id_threads_id_fk": { + "name": "thread_control_approvals_thread_id_threads_id_fk", + "tableFrom": "thread_control_approvals", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thread_control_approvals_workspace_id_workspaces_id_fk": { + "name": "thread_control_approvals_workspace_id_workspaces_id_fk", + "tableFrom": "thread_control_approvals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_control_audit": { + "name": "thread_control_audit", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "caller_id": { + "name": "caller_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_thread_control_audit_thread": { + "name": "idx_thread_control_audit_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_startups": { + "name": "thread_startups", + "columns": { + "startup_id": { + "name": "startup_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "steps_json": { + "name": "steps_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "transcript_json": { + "name": "transcript_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cancellation": { + "name": "cancellation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_json": { + "name": "error_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "block_json": { + "name": "block_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_thread_startups_workspace_updated": { + "name": "idx_thread_startups_workspace_updated", + "columns": [ + "workspace_id", + "\"updated_at\" desc" + ], + "isUnique": false + }, + "idx_thread_startups_state": { + "name": "idx_thread_startups_state", + "columns": [ + "state" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_startups_workspace_id_workspaces_id_fk": { + "name": "thread_startups_workspace_id_workspaces_id_fk", + "tableFrom": "thread_startups", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thread_startups_thread_id_threads_id_fk": { + "name": "thread_startups_thread_id_threads_id_fk", + "tableFrom": "thread_startups", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_tasks": { + "name": "thread_tasks", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tasks_json": { + "name": "tasks_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "thread_tasks_thread_id_threads_id_fk": { + "name": "thread_tasks_thread_id_threads_id_fk", + "tableFrom": "thread_tasks", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "threads": { + "name": "threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'direct'" + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkout_state": { + "name": "checkout_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'named'" + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "issue_number": { + "name": "issue_number", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_status": { + "name": "pr_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_completed_at": { + "name": "user_completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scheduled_deletion_at": { + "name": "scheduled_deletion_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleanup_state": { + "name": "cleanup_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleanup_reason": { + "name": "cleanup_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "worktree_managed": { + "name": "worktree_managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "sdk_session_id": { + "name": "sdk_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_context_tokens": { + "name": "last_context_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window": { + "name": "context_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'claude'" + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interaction_mode": { + "name": "interaction_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "orchestration_mode": { + "name": "orchestration_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "forked_from_message_id": { + "name": "forked_from_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_coordinator_thread_id": { + "name": "delegation_coordinator_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creator_turn_id": { + "name": "delegation_creator_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creator_tool_call_id": { + "name": "delegation_creator_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creation_kind": { + "name": "delegation_creation_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by_integration_id": { + "name": "created_by_integration_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_compact_summary": { + "name": "last_compact_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "copilot_agent": { + "name": "copilot_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "devin_mode": { + "name": "devin_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window_mode": { + "name": "context_window_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thinking": { + "name": "thinking", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "codex_fast_mode": { + "name": "codex_fast_mode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_open_in_app": { + "name": "default_open_in_app", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "has_file_changes": { + "name": "has_file_changes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "current_notice_session_id": { + "name": "current_notice_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_threads_workspace": { + "name": "idx_threads_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_threads_workspace_deleted": { + "name": "idx_threads_workspace_deleted", + "columns": [ + "workspace_id", + "deleted_at" + ], + "isUnique": false + }, + "idx_threads_workspace_completed": { + "name": "idx_threads_workspace_completed", + "columns": [ + "workspace_id", + "user_completed_at" + ], + "isUnique": false + }, + "idx_threads_cleanup_due": { + "name": "idx_threads_cleanup_due", + "columns": [ + "cleanup_state", + "scheduled_deletion_at" + ], + "isUnique": false + }, + "idx_threads_workspace_recency": { + "name": "idx_threads_workspace_recency", + "columns": [ + "workspace_id", + "\"updated_at\" desc" + ], + "isUnique": false + }, + "idx_threads_status": { + "name": "idx_threads_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_threads_parent_thread_id": { + "name": "idx_threads_parent_thread_id", + "columns": [ + "parent_thread_id" + ], + "isUnique": false + }, + "idx_threads_forked_from_message_id": { + "name": "idx_threads_forked_from_message_id", + "columns": [ + "forked_from_message_id" + ], + "isUnique": false + }, + "idx_threads_delegation_coordinator": { + "name": "idx_threads_delegation_coordinator", + "columns": [ + "delegation_coordinator_thread_id" + ], + "isUnique": false + }, + "idx_threads_created_by_integration": { + "name": "idx_threads_created_by_integration", + "columns": [ + "created_by_integration_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "threads_workspace_id_workspaces_id_fk": { + "name": "threads_workspace_id_workspaces_id_fk", + "tableFrom": "threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "threads_delegation_coordinator_thread_id_threads_id_fk": { + "name": "threads_delegation_coordinator_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "delegation_coordinator_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tool_call_records": { + "name": "tool_call_records", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_tool_call_id": { + "name": "parent_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_agent_key": { + "name": "provider_agent_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_identity_key": { + "name": "subagent_identity_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_provider_name": { + "name": "subagent_provider_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_prompt": { + "name": "subagent_prompt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_type": { + "name": "subagent_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_agent_id": { + "name": "subagent_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_duration_ms": { + "name": "subagent_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_summary": { + "name": "input_summary", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_summary": { + "name": "output_summary", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_truncated": { + "name": "output_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_total_bytes": { + "name": "output_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_artifact_path": { + "name": "output_artifact_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_tool_call_records_message_sort_order_id": { + "name": "idx_tool_call_records_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false + }, + "idx_tool_call_records_parent": { + "name": "idx_tool_call_records_parent", + "columns": [ + "parent_tool_call_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tool_call_records_message_id_messages_id_fk": { + "name": "tool_call_records_message_id_messages_id_fk", + "tableFrom": "tool_call_records", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turn_diff_snapshots": { + "name": "turn_diff_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fidelity": { + "name": "fidelity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_turn_diff_snapshots_message": { + "name": "idx_turn_diff_snapshots_message", + "columns": [ + "message_id" + ], + "isUnique": true + }, + "idx_turn_diff_snapshots_thread": { + "name": "idx_turn_diff_snapshots_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turn_diff_snapshots_message_id_messages_id_fk": { + "name": "turn_diff_snapshots_message_id_messages_id_fk", + "tableFrom": "turn_diff_snapshots", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_diff_snapshots_thread_id_threads_id_fk": { + "name": "turn_diff_snapshots_thread_id_threads_id_fk", + "tableFrom": "turn_diff_snapshots", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turn_snapshots": { + "name": "turn_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_before": { + "name": "ref_before", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_after": { + "name": "ref_after", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "files_changed": { + "name": "files_changed", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "file_effects": { + "name": "file_effects", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{\"revision\":0,\"fileCount\":0,\"additions\":0,\"deletions\":0,\"effects\":[]}'" + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_turn_snapshots_message": { + "name": "idx_turn_snapshots_message", + "columns": [ + "message_id" + ], + "isUnique": false + }, + "idx_turn_snapshots_thread": { + "name": "idx_turn_snapshots_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turn_snapshots_message_id_messages_id_fk": { + "name": "turn_snapshots_message_id_messages_id_fk", + "tableFrom": "turn_snapshots", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_snapshots_thread_id_threads_id_fk": { + "name": "turn_snapshots_thread_id_threads_id_fk", + "tableFrom": "turn_snapshots", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_automatic_setup_attempts": { + "name": "workspace_environment_automatic_setup_attempts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "launch_snapshot_json": { + "name": "launch_snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_truncated": { + "name": "output_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "idx_workspace_environment_automatic_setup_attempts_thread": { + "name": "idx_workspace_environment_automatic_setup_attempts_thread", + "columns": [ + "thread_id", + "\"created_at\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_environment_automatic_setup_attempts_thread_id_threads_id_fk": { + "name": "workspace_environment_automatic_setup_attempts_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_automatic_setup_attempts", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_command_approvals": { + "name": "workspace_environment_command_approvals", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "command_id": { + "name": "command_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_workspace_environment_command_approvals_command": { + "name": "idx_workspace_environment_command_approvals_command", + "columns": [ + "workspace_id", + "command_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "workspace_environment_command_approvals_workspace_id_workspaces_id_fk": { + "name": "workspace_environment_command_approvals_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_environment_command_approvals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_queued_turns": { + "name": "workspace_environment_queued_turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "queue_position": { + "name": "queue_position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submission_json": { + "name": "submission_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "released_at": { + "name": "released_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dispatching_at": { + "name": "dispatching_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_workspace_environment_queued_turns_state": { + "name": "idx_workspace_environment_queued_turns_state", + "columns": [ + "state", + "created_at" + ], + "isUnique": false + }, + "idx_workspace_environment_queued_turns_thread_state_position": { + "name": "idx_workspace_environment_queued_turns_thread_state_position", + "columns": [ + "thread_id", + "state", + "queue_position" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_environment_queued_turns_thread_id_threads_id_fk": { + "name": "workspace_environment_queued_turns_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_queued_turns", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_setup_gates": { + "name": "workspace_environment_setup_gates", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt_id": { + "name": "attempt_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_environment_setup_gates_thread_id_threads_id_fk": { + "name": "workspace_environment_setup_gates_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_setup_gates", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_storage_settings": { + "name": "workspace_environment_storage_settings", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "storage_mode": { + "name": "storage_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_environment_storage_settings_workspace_id_workspaces_id_fk": { + "name": "workspace_environment_storage_settings_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_environment_storage_settings", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_terminal_preferences": { + "name": "workspace_terminal_preferences", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "default_profile_id": { + "name": "default_profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_terminal_preferences_workspace_id_workspaces_id_fk": { + "name": "workspace_terminal_preferences_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_terminal_preferences", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_worktrees": { + "name": "workspace_worktrees", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_path": { + "name": "canonical_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_ref": { + "name": "base_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "managed": { + "name": "managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "stale": { + "name": "stale", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_workspace_worktrees_path": { + "name": "idx_workspace_worktrees_path", + "columns": [ + "workspace_id", + "canonical_path" + ], + "isUnique": true + }, + "idx_workspace_worktrees_workspace": { + "name": "idx_workspace_worktrees_workspace", + "columns": [ + "workspace_id", + "stale" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_worktrees_workspace_id_workspaces_id_fk": { + "name": "workspace_worktrees_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_worktrees", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspaces": { + "name": "workspaces", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_config": { + "name": "provider_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "pinned": { + "name": "pinned", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_opened_at": { + "name": "last_opened_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "is_git_repo": { + "name": "is_git_repo", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "workspaces_path_unique": { + "name": "workspaces_path_unique", + "columns": [ + "path" + ], + "isUnique": true + }, + "idx_workspaces_sort_order": { + "name": "idx_workspaces_sort_order", + "columns": [ + "\"sort_order\" asc" + ], + "isUnique": false + }, + "idx_workspaces_pinned_last_opened": { + "name": "idx_workspaces_pinned_last_opened", + "columns": [ + "\"pinned\" desc", + "\"last_opened_at\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_messages_notice_session_sequence": { + "columns": { + "json_extract(\"system_notice\", '$.sessionId')": { + "isExpression": true + }, + "\"sequence\" desc": { + "isExpression": true + } + } + }, + "idx_thread_startups_workspace_updated": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_threads_workspace_recency": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_workspace_environment_automatic_setup_attempts_thread": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_workspaces_sort_order": { + "columns": { + "\"sort_order\" asc": { + "isExpression": true + } + } + }, + "idx_workspaces_pinned_last_opened": { + "columns": { + "\"pinned\" desc": { + "isExpression": true + }, + "\"last_opened_at\" desc": { + "isExpression": true + } + } + } + } + } +} \ No newline at end of file diff --git a/apps/server/drizzle/meta/0066_snapshot.json b/apps/server/drizzle/meta/0066_snapshot.json new file mode 100644 index 000000000..192a5b933 --- /dev/null +++ b/apps/server/drizzle/meta/0066_snapshot.json @@ -0,0 +1,5027 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "171945ab-d900-46c6-9acd-e89704d225f9", + "prevId": "d6ce0589-8bee-4f7f-b188-033550ae9cf0", + "tables": { + "canonical_agent_events": { + "name": "canonical_agent_events", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_sequence": { + "name": "accepted_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "durable_revision": { + "name": "durable_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "roster_revision": { + "name": "roster_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envelope_json": { + "name": "envelope_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "persisted_at": { + "name": "persisted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_events_execution_sequence": { + "name": "idx_canonical_agent_events_execution_sequence", + "columns": [ + "execution_id", + "accepted_sequence" + ], + "isUnique": true + }, + "idx_canonical_agent_events_thread_revision": { + "name": "idx_canonical_agent_events_thread_revision", + "columns": [ + "thread_id", + "durable_revision" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_events_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_events_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_events", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_ingest_checkpoints": { + "name": "canonical_agent_ingest_checkpoints", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_accepted_sequence": { + "name": "last_accepted_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_durable_sequence": { + "name": "last_durable_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "native_cursor_json": { + "name": "native_cursor_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "terminal_outcome": { + "name": "terminal_outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recovery_incident_id": { + "name": "recovery_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_checkpoints_thread": { + "name": "idx_canonical_agent_checkpoints_thread", + "columns": [ + "thread_id", + "updated_at" + ], + "isUnique": false + }, + "idx_canonical_agent_checkpoints_recovery_incident": { + "name": "idx_canonical_agent_checkpoints_recovery_incident", + "columns": [ + "recovery_incident_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_ingest_checkpoints_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_ingest_checkpoints_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_ingest_checkpoints", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_ingest_checkpoints_turn_id_canonical_agent_turns_id_fk": { + "name": "canonical_agent_ingest_checkpoints_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "canonical_agent_ingest_checkpoints", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_items": { + "name": "canonical_agent_items", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_item_id": { + "name": "parent_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_items_turn": { + "name": "idx_canonical_agent_items_turn", + "columns": [ + "turn_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_agent_items_thread": { + "name": "idx_canonical_agent_items_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_agent_items_created_id": { + "name": "idx_canonical_agent_items_created_id", + "columns": [ + "created_at", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_items_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_items_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_items", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_items_turn_id_canonical_agent_turns_id_fk": { + "name": "canonical_agent_items_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "canonical_agent_items", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_threads": { + "name": "canonical_agent_threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "root_thread_id": { + "name": "root_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owning_parent_thread_id": { + "name": "owning_parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "activity_state": { + "name": "activity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversation_revision": { + "name": "conversation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "roster_revision": { + "name": "roster_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_threads_workspace": { + "name": "idx_canonical_agent_threads_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_canonical_agent_threads_root": { + "name": "idx_canonical_agent_threads_root", + "columns": [ + "root_thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_threads_id_threads_id_fk": { + "name": "canonical_agent_threads_id_threads_id_fk", + "tableFrom": "canonical_agent_threads", + "tableTo": "threads", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_threads_workspace_id_workspaces_id_fk": { + "name": "canonical_agent_threads_workspace_id_workspaces_id_fk", + "tableFrom": "canonical_agent_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_turns": { + "name": "canonical_agent_turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trigger_json": { + "name": "trigger_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approval_review_mode": { + "name": "approval_review_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "approval_review_reason": { + "name": "approval_review_reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual-requested'" + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_turns_execution": { + "name": "idx_canonical_agent_turns_execution", + "columns": [ + "execution_id" + ], + "isUnique": true + }, + "idx_canonical_agent_turns_thread": { + "name": "idx_canonical_agent_turns_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_turns_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_turns_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_turns", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_collaboration_actions": { + "name": "canonical_collaboration_actions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_item_id": { + "name": "source_item_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_thread_id": { + "name": "target_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_turn_id": { + "name": "target_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_unknown": { + "name": "delivery_unknown", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_collaboration_source": { + "name": "idx_canonical_collaboration_source", + "columns": [ + "source_thread_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_collaboration_target": { + "name": "idx_canonical_collaboration_target", + "columns": [ + "target_thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_conversation_display_mappings": { + "name": "canonical_conversation_display_mappings", + "columns": { + "source_item_id": { + "name": "source_item_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "target_kind": { + "name": "target_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_canonical_conversation_display_mappings_target": { + "name": "idx_canonical_conversation_display_mappings_target", + "columns": [ + "target_kind", + "target_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_conversation_display_mappings_source_item_id_canonical_agent_items_id_fk": { + "name": "canonical_conversation_display_mappings_source_item_id_canonical_agent_items_id_fk", + "tableFrom": "canonical_conversation_display_mappings", + "tableTo": "canonical_agent_items", + "columnsFrom": [ + "source_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_legacy_message_provenance": { + "name": "canonical_legacy_message_provenance", + "columns": { + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "migration_version": { + "name": "migration_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mapping_status": { + "name": "mapping_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_thread_id": { + "name": "canonical_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "canonical_turn_id": { + "name": "canonical_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "canonical_item_id": { + "name": "canonical_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_canonical_legacy_message_mapping": { + "name": "idx_canonical_legacy_message_mapping", + "columns": [ + "mapping_status", + "message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_legacy_message_provenance_message_id_messages_id_fk": { + "name": "canonical_legacy_message_provenance_message_id_messages_id_fk", + "tableFrom": "canonical_legacy_message_provenance", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_legacy_migration_checkpoints": { + "name": "canonical_legacy_migration_checkpoints", + "columns": { + "version": { + "name": "version", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "migrated_messages": { + "name": "migrated_messages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ambiguous_messages": { + "name": "ambiguous_messages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_writer_live_publication_heads": { + "name": "canonical_writer_live_publication_heads", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "canonical_writer_live_publication_heads_thread_id_threads_id_fk": { + "name": "canonical_writer_live_publication_heads_thread_id_threads_id_fk", + "tableFrom": "canonical_writer_live_publication_heads", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_writer_operation_receipts": { + "name": "canonical_writer_operation_receipts", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_hash": { + "name": "input_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "receipt_json": { + "name": "receipt_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "canonical_writer_operation_receipts_execution_id_canonical_agent_turns_execution_id_fk": { + "name": "canonical_writer_operation_receipts_execution_id_canonical_agent_turns_execution_id_fk", + "tableFrom": "canonical_writer_operation_receipts", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "execution_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "canonical_writer_operation_receipts_execution_id_operation_id_pk": { + "columns": [ + "execution_id", + "operation_id" + ], + "name": "canonical_writer_operation_receipts_execution_id_operation_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_jobs": { + "name": "cleanup_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_path": { + "name": "workspace_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'explicit'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cleanup_jobs_thread_id_unique": { + "name": "cleanup_jobs_thread_id_unique", + "columns": [ + "thread_id" + ], + "isUnique": true + }, + "idx_cleanup_jobs_retry": { + "name": "idx_cleanup_jobs_retry", + "columns": [ + "next_retry_at", + "attempts", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "conversation_display_materialization_state": { + "name": "conversation_display_materialization_state", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "last_source_created_at": { + "name": "last_source_created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_source_id": { + "name": "last_source_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed": { + "name": "completed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "diff_summaries": { + "name": "diff_summaries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_count": { + "name": "turn_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_turn_id": { + "name": "last_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_diff_summaries_thread": { + "name": "idx_diff_summaries_thread", + "columns": [ + "thread_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "diff_summaries_thread_id_threads_id_fk": { + "name": "diff_summaries_thread_id_threads_id_fk", + "tableFrom": "diff_summaries", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "external_thread_control_deliveries": { + "name": "external_thread_control_deliveries", + "columns": { + "pairing_id": { + "name": "pairing_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "authority_epoch": { + "name": "authority_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'in_flight'" + }, + "result_json": { + "name": "result_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_external_thread_control_delivery_identity": { + "name": "idx_external_thread_control_delivery_identity", + "columns": [ + "pairing_id", + "authority_epoch", + "delivery_id" + ], + "isUnique": true + }, + "idx_external_thread_control_delivery_retention": { + "name": "idx_external_thread_control_delivery_retention", + "columns": [ + "pairing_id", + "status", + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "external_thread_control_pairings": { + "name": "external_thread_control_pairings", + "columns": { + "pairing_id": { + "name": "pairing_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_hash": { + "name": "credential_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_ids_json": { + "name": "workspace_ids_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "scopes_json": { + "name": "scopes_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "calls_per_minute": { + "name": "calls_per_minute", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_active_threads": { + "name": "max_active_threads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "authority_epoch": { + "name": "authority_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "replaced_by_pairing_id": { + "name": "replaced_by_pairing_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "replaces_pairing_id": { + "name": "replaces_pairing_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "external_thread_control_pairings_credential_hash_unique": { + "name": "external_thread_control_pairings_credential_hash_unique", + "columns": [ + "credential_hash" + ], + "isUnique": true + }, + "idx_external_thread_control_pairings_integration": { + "name": "idx_external_thread_control_pairings_integration", + "columns": [ + "integration_id", + "status" + ], + "isUnique": false + }, + "idx_external_thread_control_active_integration": { + "name": "idx_external_thread_control_active_integration", + "columns": [ + "integration_id" + ], + "isUnique": true, + "where": "status = 'active'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "hook_executions": { + "name": "hook_executions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hook_name": { + "name": "hook_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "did_block": { + "name": "did_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_hook_executions_message_sort_order_id": { + "name": "idx_hook_executions_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "hook_executions_message_id_messages_id_fk": { + "name": "hook_executions_message_id_messages_id_fk", + "tableFrom": "hook_executions", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "messages": { + "name": "messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "files_changed": { + "name": "files_changed", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tokens_used": { + "name": "tokens_used", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachments": { + "name": "attachments", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preview_annotations": { + "name": "preview_annotations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mentions": { + "name": "mentions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_text_comments": { + "name": "selected_text_comments", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reply_to_message_id": { + "name": "reply_to_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "quoted_text": { + "name": "quoted_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_type": { + "name": "origin_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'legacy'" + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_provider_id": { + "name": "source_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "legacy_provenance": { + "name": "legacy_provenance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_agent_provenance": { + "name": "parent_agent_provenance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_internal": { + "name": "is_internal", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "outcome_execution_id": { + "name": "outcome_execution_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "system_notice": { + "name": "system_notice", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_thread": { + "name": "idx_messages_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + }, + "idx_messages_sequence": { + "name": "idx_messages_sequence", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false + }, + "idx_messages_thread_sequence_id": { + "name": "idx_messages_thread_sequence_id", + "columns": [ + "thread_id", + "sequence", + "id" + ], + "isUnique": false + }, + "idx_messages_notice_session_sequence": { + "name": "idx_messages_notice_session_sequence", + "columns": [ + "thread_id", + "json_extract(\"system_notice\", '$.sessionId')", + "\"sequence\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messages_thread_id_threads_id_fk": { + "name": "messages_thread_id_threads_id_fk", + "tableFrom": "messages", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "messages_reply_to_message_id_messages_id_fk": { + "name": "messages_reply_to_message_id_messages_id_fk", + "tableFrom": "messages", + "tableTo": "messages", + "columnsFrom": [ + "reply_to_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "parent_assistant_text_checkpoint_chunks": { + "name": "parent_assistant_text_checkpoint_chunks", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_sequence": { + "name": "first_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "byte_length": { + "name": "byte_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_parent_assistant_text_checkpoint_chunks_sequence": { + "name": "idx_parent_assistant_text_checkpoint_chunks_sequence", + "columns": [ + "execution_id", + "first_sequence" + ], + "isUnique": true + } + }, + "foreignKeys": { + "parent_assistant_text_checkpoint_chunks_execution_id_parent_assistant_text_checkpoints_execution_id_fk": { + "name": "parent_assistant_text_checkpoint_chunks_execution_id_parent_assistant_text_checkpoints_execution_id_fk", + "tableFrom": "parent_assistant_text_checkpoint_chunks", + "tableTo": "parent_assistant_text_checkpoints", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "execution_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "parent_assistant_text_checkpoints": { + "name": "parent_assistant_text_checkpoints", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_bytes": { + "name": "retained_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_chunks": { + "name": "retained_chunks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_parent_assistant_text_checkpoints_thread": { + "name": "idx_parent_assistant_text_checkpoints_thread", + "columns": [ + "thread_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "parent_assistant_text_checkpoints_thread_id_canonical_agent_threads_id_fk": { + "name": "parent_assistant_text_checkpoints_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "parent_assistant_text_checkpoints", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parent_assistant_text_checkpoints_turn_id_canonical_agent_turns_id_fk": { + "name": "parent_assistant_text_checkpoints_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "parent_assistant_text_checkpoints", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plan_question_answers": { + "name": "plan_question_answers", + "columns": { + "assistant_message_id": { + "name": "assistant_message_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answered_at": { + "name": "answered_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_plan_question_answers_thread_answered_at": { + "name": "idx_plan_question_answers_thread_answered_at", + "columns": [ + "thread_id", + "answered_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "plan_question_answers_assistant_message_id_messages_id_fk": { + "name": "plan_question_answers_assistant_message_id_messages_id_fk", + "tableFrom": "plan_question_answers", + "tableTo": "messages", + "columnsFrom": [ + "assistant_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "plan_question_answers_thread_id_threads_id_fk": { + "name": "plan_question_answers_thread_id_threads_id_fk", + "tableFrom": "plan_question_answers", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plans": { + "name": "plans", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_md": { + "name": "content_md", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sections_json": { + "name": "sections_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_plans_thread": { + "name": "idx_plans_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + }, + "idx_plans_thread_version": { + "name": "idx_plans_thread_version", + "columns": [ + "thread_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "plans_thread_id_threads_id_fk": { + "name": "plans_thread_id_threads_id_fk", + "tableFrom": "plans", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "plans_message_id_messages_id_fk": { + "name": "plans_message_id_messages_id_fk", + "tableFrom": "plans", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_action_runs": { + "name": "project_action_runs", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action_id": { + "name": "action_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "terminal_session_id": { + "name": "terminal_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "action_name": { + "name": "action_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcript": { + "name": "transcript", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "transcript_truncated": { + "name": "transcript_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "idx_project_action_runs_slot": { + "name": "idx_project_action_runs_slot", + "columns": [ + "thread_id", + "action_id" + ], + "isUnique": true + }, + "idx_project_action_runs_thread": { + "name": "idx_project_action_runs_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "project_action_runs_thread_id_threads_id_fk": { + "name": "project_action_runs_thread_id_threads_id_fk", + "tableFrom": "project_action_runs", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_action_runs_workspace_id_workspaces_id_fk": { + "name": "project_action_runs_workspace_id_workspaces_id_fk", + "tableFrom": "project_action_runs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_catalog_snapshots": { + "name": "provider_catalog_snapshots", + "columns": { + "context_key": { + "name": "context_key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cwd": { + "name": "cwd", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_provider_catalog_snapshots_workspace": { + "name": "idx_provider_catalog_snapshots_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_provider_catalog_snapshots_provider": { + "name": "idx_provider_catalog_snapshots_provider", + "columns": [ + "provider_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "provider_catalog_snapshots_workspace_id_workspaces_id_fk": { + "name": "provider_catalog_snapshots_workspace_id_workspaces_id_fk", + "tableFrom": "provider_catalog_snapshots", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_model_cache": { + "name": "provider_model_cache", + "columns": { + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "models_json": { + "name": "models_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pull_request_review_links": { + "name": "pull_request_review_links", + "columns": { + "worktree_id": { + "name": "worktree_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_node_id": { + "name": "repository_node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pull_request_number": { + "name": "pull_request_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pr_url": { + "name": "pr_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pr_state": { + "name": "pr_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_managed": { + "name": "worktree_managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "head_repository_node_id": { + "name": "head_repository_node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_repository_owner": { + "name": "head_repository_owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_repository_name": { + "name": "head_repository_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_ref": { + "name": "head_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_oid": { + "name": "head_oid", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "local_branch": { + "name": "local_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "push_remote": { + "name": "push_remote", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "push_ref": { + "name": "push_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "managed_remote_name": { + "name": "managed_remote_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "primary_thread_id": { + "name": "primary_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_pull_request_review_links_identity": { + "name": "idx_pull_request_review_links_identity", + "columns": [ + "provider", + "repository_node_id", + "pull_request_number" + ], + "isUnique": true + }, + "idx_pull_request_review_links_primary_thread": { + "name": "idx_pull_request_review_links_primary_thread", + "columns": [ + "primary_thread_id" + ], + "isUnique": true + }, + "idx_pull_request_review_links_workspace": { + "name": "idx_pull_request_review_links_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_pull_request_review_links_worktree_path": { + "name": "idx_pull_request_review_links_worktree_path", + "columns": [ + "worktree_path" + ], + "isUnique": false + } + }, + "foreignKeys": { + "pull_request_review_links_workspace_id_workspaces_id_fk": { + "name": "pull_request_review_links_workspace_id_workspaces_id_fk", + "tableFrom": "pull_request_review_links", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pull_request_review_links_primary_thread_id_threads_id_fk": { + "name": "pull_request_review_links_primary_thread_id_threads_id_fk", + "tableFrom": "pull_request_review_links", + "tableTo": "threads", + "columnsFrom": [ + "primary_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "terminal_cleanup_ledger": { + "name": "terminal_cleanup_ledger", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "host_generation": { + "name": "host_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "root_pid": { + "name": "root_pid", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "process_group_id": { + "name": "process_group_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "containment": { + "name": "containment", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thought_segments": { + "name": "thought_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "is_final_response": { + "name": "is_final_response", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_thought_segments_message_final_sort_order_id": { + "name": "idx_thought_segments_message_final_sort_order_id", + "columns": [ + "message_id", + "is_final_response", + "sort_order", + "id" + ], + "isUnique": false + }, + "idx_thought_segments_final_message_sort_order_id": { + "name": "idx_thought_segments_final_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false, + "where": "\"thought_segments\".\"is_final_response\" <> 0" + } + }, + "foreignKeys": { + "thought_segments_message_id_messages_id_fk": { + "name": "thought_segments_message_id_messages_id_fk", + "tableFrom": "thought_segments", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_control_approvals": { + "name": "thread_control_approvals", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "execution_json": { + "name": "execution_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "placement_json": { + "name": "placement_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "caller_id": { + "name": "caller_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_provider_id": { + "name": "source_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'thread_create_batch'" + }, + "operation_phase": { + "name": "operation_phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pre_provision'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "resolved_at": { + "name": "resolved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_thread_control_approvals_thread": { + "name": "idx_thread_control_approvals_thread", + "columns": [ + "thread_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_control_approvals_thread_id_threads_id_fk": { + "name": "thread_control_approvals_thread_id_threads_id_fk", + "tableFrom": "thread_control_approvals", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thread_control_approvals_workspace_id_workspaces_id_fk": { + "name": "thread_control_approvals_workspace_id_workspaces_id_fk", + "tableFrom": "thread_control_approvals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_control_audit": { + "name": "thread_control_audit", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "caller_id": { + "name": "caller_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_thread_control_audit_thread": { + "name": "idx_thread_control_audit_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_startups": { + "name": "thread_startups", + "columns": { + "startup_id": { + "name": "startup_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "steps_json": { + "name": "steps_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "transcript_json": { + "name": "transcript_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cancellation": { + "name": "cancellation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_json": { + "name": "error_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "block_json": { + "name": "block_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_thread_startups_workspace_updated": { + "name": "idx_thread_startups_workspace_updated", + "columns": [ + "workspace_id", + "\"updated_at\" desc" + ], + "isUnique": false + }, + "idx_thread_startups_state": { + "name": "idx_thread_startups_state", + "columns": [ + "state" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_startups_workspace_id_workspaces_id_fk": { + "name": "thread_startups_workspace_id_workspaces_id_fk", + "tableFrom": "thread_startups", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thread_startups_thread_id_threads_id_fk": { + "name": "thread_startups_thread_id_threads_id_fk", + "tableFrom": "thread_startups", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_tasks": { + "name": "thread_tasks", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tasks_json": { + "name": "tasks_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "thread_tasks_thread_id_threads_id_fk": { + "name": "thread_tasks_thread_id_threads_id_fk", + "tableFrom": "thread_tasks", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "threads": { + "name": "threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'direct'" + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkout_state": { + "name": "checkout_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'named'" + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "issue_number": { + "name": "issue_number", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_status": { + "name": "pr_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_completed_at": { + "name": "user_completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scheduled_deletion_at": { + "name": "scheduled_deletion_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleanup_state": { + "name": "cleanup_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleanup_reason": { + "name": "cleanup_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "worktree_managed": { + "name": "worktree_managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "sdk_session_id": { + "name": "sdk_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_context_tokens": { + "name": "last_context_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window": { + "name": "context_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'claude'" + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interaction_mode": { + "name": "interaction_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "orchestration_mode": { + "name": "orchestration_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "forked_from_message_id": { + "name": "forked_from_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_coordinator_thread_id": { + "name": "delegation_coordinator_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creator_turn_id": { + "name": "delegation_creator_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creator_tool_call_id": { + "name": "delegation_creator_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creation_kind": { + "name": "delegation_creation_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by_integration_id": { + "name": "created_by_integration_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_compact_summary": { + "name": "last_compact_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "copilot_agent": { + "name": "copilot_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "devin_mode": { + "name": "devin_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window_mode": { + "name": "context_window_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thinking": { + "name": "thinking", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "codex_fast_mode": { + "name": "codex_fast_mode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_open_in_app": { + "name": "default_open_in_app", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "has_file_changes": { + "name": "has_file_changes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "current_notice_session_id": { + "name": "current_notice_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_threads_workspace": { + "name": "idx_threads_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_threads_workspace_deleted": { + "name": "idx_threads_workspace_deleted", + "columns": [ + "workspace_id", + "deleted_at" + ], + "isUnique": false + }, + "idx_threads_workspace_completed": { + "name": "idx_threads_workspace_completed", + "columns": [ + "workspace_id", + "user_completed_at" + ], + "isUnique": false + }, + "idx_threads_cleanup_due": { + "name": "idx_threads_cleanup_due", + "columns": [ + "cleanup_state", + "scheduled_deletion_at" + ], + "isUnique": false + }, + "idx_threads_workspace_recency": { + "name": "idx_threads_workspace_recency", + "columns": [ + "workspace_id", + "\"updated_at\" desc" + ], + "isUnique": false + }, + "idx_threads_status": { + "name": "idx_threads_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_threads_parent_thread_id": { + "name": "idx_threads_parent_thread_id", + "columns": [ + "parent_thread_id" + ], + "isUnique": false + }, + "idx_threads_forked_from_message_id": { + "name": "idx_threads_forked_from_message_id", + "columns": [ + "forked_from_message_id" + ], + "isUnique": false + }, + "idx_threads_delegation_coordinator": { + "name": "idx_threads_delegation_coordinator", + "columns": [ + "delegation_coordinator_thread_id" + ], + "isUnique": false + }, + "idx_threads_created_by_integration": { + "name": "idx_threads_created_by_integration", + "columns": [ + "created_by_integration_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "threads_workspace_id_workspaces_id_fk": { + "name": "threads_workspace_id_workspaces_id_fk", + "tableFrom": "threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "threads_delegation_coordinator_thread_id_threads_id_fk": { + "name": "threads_delegation_coordinator_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "delegation_coordinator_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tool_call_records": { + "name": "tool_call_records", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_tool_call_id": { + "name": "parent_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_agent_key": { + "name": "provider_agent_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_identity_key": { + "name": "subagent_identity_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_provider_name": { + "name": "subagent_provider_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_prompt": { + "name": "subagent_prompt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_type": { + "name": "subagent_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_agent_id": { + "name": "subagent_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_duration_ms": { + "name": "subagent_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_summary": { + "name": "input_summary", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_summary": { + "name": "output_summary", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_truncated": { + "name": "output_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_total_bytes": { + "name": "output_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_artifact_path": { + "name": "output_artifact_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_tool_call_records_message_sort_order_id": { + "name": "idx_tool_call_records_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false + }, + "idx_tool_call_records_parent": { + "name": "idx_tool_call_records_parent", + "columns": [ + "parent_tool_call_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tool_call_records_message_id_messages_id_fk": { + "name": "tool_call_records_message_id_messages_id_fk", + "tableFrom": "tool_call_records", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turn_diff_snapshots": { + "name": "turn_diff_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fidelity": { + "name": "fidelity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_turn_diff_snapshots_message": { + "name": "idx_turn_diff_snapshots_message", + "columns": [ + "message_id" + ], + "isUnique": true + }, + "idx_turn_diff_snapshots_thread": { + "name": "idx_turn_diff_snapshots_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turn_diff_snapshots_message_id_messages_id_fk": { + "name": "turn_diff_snapshots_message_id_messages_id_fk", + "tableFrom": "turn_diff_snapshots", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_diff_snapshots_thread_id_threads_id_fk": { + "name": "turn_diff_snapshots_thread_id_threads_id_fk", + "tableFrom": "turn_diff_snapshots", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turn_snapshots": { + "name": "turn_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_before": { + "name": "ref_before", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_after": { + "name": "ref_after", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "files_changed": { + "name": "files_changed", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "file_effects": { + "name": "file_effects", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{\"revision\":0,\"fileCount\":0,\"additions\":0,\"deletions\":0,\"effects\":[]}'" + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_turn_snapshots_message": { + "name": "idx_turn_snapshots_message", + "columns": [ + "message_id" + ], + "isUnique": false + }, + "idx_turn_snapshots_thread": { + "name": "idx_turn_snapshots_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turn_snapshots_message_id_messages_id_fk": { + "name": "turn_snapshots_message_id_messages_id_fk", + "tableFrom": "turn_snapshots", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_snapshots_thread_id_threads_id_fk": { + "name": "turn_snapshots_thread_id_threads_id_fk", + "tableFrom": "turn_snapshots", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_automatic_setup_attempts": { + "name": "workspace_environment_automatic_setup_attempts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "launch_snapshot_json": { + "name": "launch_snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_truncated": { + "name": "output_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "idx_workspace_environment_automatic_setup_attempts_thread": { + "name": "idx_workspace_environment_automatic_setup_attempts_thread", + "columns": [ + "thread_id", + "\"created_at\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_environment_automatic_setup_attempts_thread_id_threads_id_fk": { + "name": "workspace_environment_automatic_setup_attempts_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_automatic_setup_attempts", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_command_approvals": { + "name": "workspace_environment_command_approvals", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "command_id": { + "name": "command_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_workspace_environment_command_approvals_command": { + "name": "idx_workspace_environment_command_approvals_command", + "columns": [ + "workspace_id", + "command_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "workspace_environment_command_approvals_workspace_id_workspaces_id_fk": { + "name": "workspace_environment_command_approvals_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_environment_command_approvals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_queued_turns": { + "name": "workspace_environment_queued_turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "queue_position": { + "name": "queue_position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submission_json": { + "name": "submission_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "released_at": { + "name": "released_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dispatching_at": { + "name": "dispatching_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_workspace_environment_queued_turns_state": { + "name": "idx_workspace_environment_queued_turns_state", + "columns": [ + "state", + "created_at" + ], + "isUnique": false + }, + "idx_workspace_environment_queued_turns_thread_state_position": { + "name": "idx_workspace_environment_queued_turns_thread_state_position", + "columns": [ + "thread_id", + "state", + "queue_position" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_environment_queued_turns_thread_id_threads_id_fk": { + "name": "workspace_environment_queued_turns_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_queued_turns", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_setup_gates": { + "name": "workspace_environment_setup_gates", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt_id": { + "name": "attempt_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_environment_setup_gates_thread_id_threads_id_fk": { + "name": "workspace_environment_setup_gates_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_setup_gates", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_storage_settings": { + "name": "workspace_environment_storage_settings", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "storage_mode": { + "name": "storage_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_environment_storage_settings_workspace_id_workspaces_id_fk": { + "name": "workspace_environment_storage_settings_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_environment_storage_settings", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_terminal_preferences": { + "name": "workspace_terminal_preferences", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "default_profile_id": { + "name": "default_profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_terminal_preferences_workspace_id_workspaces_id_fk": { + "name": "workspace_terminal_preferences_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_terminal_preferences", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_worktrees": { + "name": "workspace_worktrees", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_path": { + "name": "canonical_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_ref": { + "name": "base_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "managed": { + "name": "managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "stale": { + "name": "stale", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_workspace_worktrees_path": { + "name": "idx_workspace_worktrees_path", + "columns": [ + "workspace_id", + "canonical_path" + ], + "isUnique": true + }, + "idx_workspace_worktrees_workspace": { + "name": "idx_workspace_worktrees_workspace", + "columns": [ + "workspace_id", + "stale" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_worktrees_workspace_id_workspaces_id_fk": { + "name": "workspace_worktrees_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_worktrees", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspaces": { + "name": "workspaces", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_config": { + "name": "provider_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "pinned": { + "name": "pinned", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_opened_at": { + "name": "last_opened_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "is_git_repo": { + "name": "is_git_repo", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "workspaces_path_unique": { + "name": "workspaces_path_unique", + "columns": [ + "path" + ], + "isUnique": true + }, + "idx_workspaces_sort_order": { + "name": "idx_workspaces_sort_order", + "columns": [ + "\"sort_order\" asc" + ], + "isUnique": false + }, + "idx_workspaces_pinned_last_opened": { + "name": "idx_workspaces_pinned_last_opened", + "columns": [ + "\"pinned\" desc", + "\"last_opened_at\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_messages_notice_session_sequence": { + "columns": { + "json_extract(\"system_notice\", '$.sessionId')": { + "isExpression": true + }, + "\"sequence\" desc": { + "isExpression": true + } + } + }, + "idx_thread_startups_workspace_updated": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_threads_workspace_recency": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_workspace_environment_automatic_setup_attempts_thread": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_workspaces_sort_order": { + "columns": { + "\"sort_order\" asc": { + "isExpression": true + } + } + }, + "idx_workspaces_pinned_last_opened": { + "columns": { + "\"pinned\" desc": { + "isExpression": true + }, + "\"last_opened_at\" desc": { + "isExpression": true + } + } + } + } + } +} \ No newline at end of file diff --git a/apps/server/drizzle/meta/_journal.json b/apps/server/drizzle/meta/_journal.json index af57bc677..ee9e5a450 100644 --- a/apps/server/drizzle/meta/_journal.json +++ b/apps/server/drizzle/meta/_journal.json @@ -463,6 +463,20 @@ "when": 1790262706352, "tag": "0064_fuzzy_jimmy_woo", "breakpoints": true + }, + { + "idx": 65, + "version": "6", + "when": 1790275595252, + "tag": "0065_ambitious_blindfold", + "breakpoints": true + }, + { + "idx": 66, + "version": "6", + "when": 1790286163932, + "tag": "0066_fat_amazoness", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/server/src/application/bootstrap/server-bootstrap.ts b/apps/server/src/application/bootstrap/server-bootstrap.ts index 5975addb5..f9855b707 100644 --- a/apps/server/src/application/bootstrap/server-bootstrap.ts +++ b/apps/server/src/application/bootstrap/server-bootstrap.ts @@ -70,6 +70,7 @@ import { startAgentOrchestration, } from "../../features/agents"; import { AgentEventPublicationRegistry } from "../../features/agents/orchestration/agent-event-publication-registry.js"; +import { WorkerOwnedTurnRuntime } from "../../features/agents/execution/worker-owned-turn-runtime.js"; import { AgentEventPublicationRuntimePort, AgentReliabilityPort, @@ -342,6 +343,7 @@ const messageRepo = container.resolve(MessageRepo); const threadRepo = container.resolve(ThreadRepo); const providerRegistry = container.resolve(ProviderRegistry); const providerEventIngress = container.resolve(ProviderEventIngress); +const workerOwnedTurnRuntime = container.resolve(WorkerOwnedTurnRuntime); const cursorProvider = container.resolve("CursorProvider"); const providerAvailability = container.resolve(ProviderAvailabilityService); const toolCallRecordRepo = container.resolve(ToolCallRecordRepo); @@ -664,6 +666,7 @@ codexCatalogService.onSkillsChanged((cwd) => { // Create and start HTTP + WS server const { httpServer, wss } = createWsServer({ + workerQueueDepth: () => workerOwnedTurnRuntime.scheduler.depth(), runtime: hostRuntime, workspaceService, workspaceEnvironmentService, @@ -849,9 +852,14 @@ async function bootstrapServer(): Promise { interruptThreadStartupsAtStartup(); projectActionService.recoverStaleRuns(); - cleanupWorker.start(); + cleanupWorker.start(process.env.MCODE_AGENT_RUNTIME === "1" + ? process.env.MCODE_AGENT_FIXTURE_REPO?.trim() + : undefined); recordStartupCheckpoint("stale startup work recovery completed"); + await workerOwnedTurnRuntime.whenReady(); + recordStartupCheckpoint("execution workers ready"); + // Provider work must start only after every client-visible history route has // one durable display representation. startAgentOrchestration({ @@ -951,6 +959,8 @@ async function shutdown(): Promise { await captureCleanupFailure(() => providerRegistry.shutdown()); shutdownCoordinator.setPhase("shutdown provider event workers"); providerEventIngress.shutdown(); + shutdownCoordinator.setPhase("shutdown execution workers and writer"); + await captureCleanupFailure(() => workerOwnedTurnRuntime.close()); browserAutomationBroker.shutdown(); browserAutomationSessionLease.shutdown(); diff --git a/apps/server/src/application/transport/__tests__/push.test.ts b/apps/server/src/application/transport/__tests__/push.test.ts index 871ad7851..ee37655eb 100644 --- a/apps/server/src/application/transport/__tests__/push.test.ts +++ b/apps/server/src/application/transport/__tests__/push.test.ts @@ -11,9 +11,10 @@ import { setClientThreadSubscriptions, subscribeClientToThread, unsubscribeClientFromThread, + MAX_AGENT_EVENT_JOURNAL_THREADS, _resetForTest, } from "../push.js"; -import { decodeTerminalDataFrame } from "@mcode/contracts"; +import { AgentEventSchema, decodeTerminalDataFrame } from "@mcode/contracts"; import { createPassThroughTransportPayloadValidator, createValidatingTransportPayloadValidator, @@ -127,6 +128,22 @@ describe("broadcast", () => { resetTransportPayloadValidatorForTest(); }); + it("keeps the durable publication identity while assigning a process sequence", () => { + const received: Array<{ buf: Buffer; binary: boolean }> = []; + const socket = fakeOpenSocket(received); + addClient(socket); + subscribeClientToThread(socket, "thread-a"); + const publicationId = "3"; + const first = broadcast("agent.event", { type: "turnStarted", threadId: "thread-a", + turnExecutionId: "00000000-0000-4000-8000-000000000001", publicationId }); + const replay = broadcast("agent.event", { type: "turnStarted", threadId: "thread-a", + turnExecutionId: "00000000-0000-4000-8000-000000000001", publicationId }); + expect(first).toMatchObject({ publicationId, sequence: 1 }); + expect(replay).toMatchObject({ publicationId, sequence: 2 }); + expect(received.map(({ buf }) => JSON.parse(buf.toString("utf-8")).data.publicationId)) + .toEqual([publicationId, publicationId]); + }); + it("routes thread-scoped events only to clients subscribed to that thread", () => { const a: Array<{ buf: Buffer; binary: boolean }> = []; const b: Array<{ buf: Buffer; binary: boolean }> = []; @@ -256,6 +273,51 @@ describe("broadcast", () => { expect(received.map((entry) => JSON.parse(entry.buf.toString("utf-8")).data.sequence)).toEqual([1, 2, 3]); }); + it("keeps an evicted thread's next event distinguishable from its old cursor", () => { + const first = AgentEventSchema().parse(broadcast("agent.event", { type: "turnStarted", threadId: "evicted-thread" })); + if (!first.epoch) throw new Error("Agent event has no journal epoch"); + for (let index = 0; index < MAX_AGENT_EVENT_JOURNAL_THREADS; index++) { + broadcast("agent.event", { type: "turnStarted", threadId: `other-${index}` }); + } + const next = broadcast("agent.event", { type: "turnStarted", threadId: "evicted-thread" }); + expect(next).not.toEqual(first); + const received: Array<{ buf: Buffer; binary: boolean }> = []; + const ws = fakeOpenSocket(received); + addClient(ws); + const result = setClientThreadSubscriptions(ws, ["evicted-thread"], { + "evicted-thread": { epoch: first.epoch, sequence: 1 }, + }); + expect(result.hydrationRequiredThreadIds).toEqual(["evicted-thread"]); + expect(received).toHaveLength(0); + }); + + it("requires hydration for an ambiguous legacy cursor or a cursor ahead of its journal", () => { + const first = AgentEventSchema().parse(broadcast("agent.event", { type: "turnStarted", threadId: "cursor-thread" })); + if (!first.epoch) throw new Error("Agent event has no journal epoch"); + const received: Array<{ buf: Buffer; binary: boolean }> = []; + const ws = fakeOpenSocket(received); + addClient(ws); + for (const cursor of [1, { epoch: first.epoch, sequence: 2 }]) { + const result = setClientThreadSubscriptions(ws, ["cursor-thread"], { "cursor-thread": cursor }); + expect(result.hydrationRequiredThreadIds).toEqual(["cursor-thread"]); + } + expect(received).toHaveLength(0); + }); + + it("replays only newer events for a cursor in the current journal generation", () => { + const first = AgentEventSchema().parse(broadcast("agent.event", { type: "turnStarted", threadId: "current-thread" })); + if (!first.epoch) throw new Error("Agent event has no journal epoch"); + broadcast("agent.event", { type: "textDelta", threadId: "current-thread", delta: "new text" }); + const received: Array<{ buf: Buffer; binary: boolean }> = []; + const ws = fakeOpenSocket(received); + addClient(ws); + const result = setClientThreadSubscriptions(ws, ["current-thread"], { + "current-thread": { epoch: first.epoch, sequence: 1 }, + }); + expect(result).toEqual({ hydrationRequiredThreadIds: [], replayedThrough: { "current-thread": 2 } }); + expect(received).toHaveLength(1); + }); + it("does not replay history for legacy subscription calls without cursors", () => { const received: Array<{ buf: Buffer; binary: boolean }> = []; broadcast("agent.event", { type: "textDelta", threadId: "thread-legacy", delta: "old" }); diff --git a/apps/server/src/application/transport/__tests__/ws-server-health.test.ts b/apps/server/src/application/transport/__tests__/ws-server-health.test.ts index 2d4dad2ea..627c8a98f 100644 --- a/apps/server/src/application/transport/__tests__/ws-server-health.test.ts +++ b/apps/server/src/application/transport/__tests__/ws-server-health.test.ts @@ -8,18 +8,12 @@ import "reflect-metadata"; import { describe, it, expect, afterEach, vi } from "vitest"; import * as NodeHTTP from "node:http"; import { WebSocket } from "ws"; -import { createWsServer } from "../ws-server.js"; +import { createWsServer, type WsServerDeps } from "../ws-server.js"; import type { RouterDeps } from "../ws-router.js"; /** Minimal RouterDeps stub — only agentService is called by the health handler. */ function makeMinimalDeps( - overrides: Partial void; - }> = {}, + overrides: Partial = {}, ): RouterDeps & { authToken: string; singleInstance: boolean; @@ -219,6 +213,28 @@ describe("/health endpoint", () => { expect((body as Record).activeAgents).toBe(2); }); + it("exposes bounded execution queue counts without command content", async () => { + const deps = makeMinimalDeps({ + workerQueueDepth: () => ({ + pending: 3, + pendingBytes: 512, + activeExecutions: 7, + workers: [{ index: 0, queued: 2, inFlight: true }], + }), + }); + ({ httpServer: server } = createWsServer(deps)); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + const { body } = await getHealth(server); + expect((body as Record).workerQueue).toEqual({ + pending: 3, + pendingBytes: 512, + activeExecutions: 7, + workers: [{ index: 0, queued: 2, inFlight: true }], + }); + expect(JSON.stringify((body as Record).workerQueue)).not.toMatch(/thread|command|prompt|token/i); + }); + it("exposes content-free browser automation reliability diagnostics", async () => { const deps = makeMinimalDeps({ browserAutomationBroker: { diff --git a/apps/server/src/application/transport/push.ts b/apps/server/src/application/transport/push.ts index 57203ef1c..a3b187c25 100644 --- a/apps/server/src/application/transport/push.ts +++ b/apps/server/src/application/transport/push.ts @@ -19,9 +19,13 @@ export const MAX_QUEUED_PUSH_BYTES = 16 * 1_024 * 1_024; const clients = new Set(); const queuedPushBytes = new Map(); const threadSubscriptions = new Map>(); -const threadJournals = new Map(); -const nextSequenceByThread = new Map(); -const eventEpoch = NodeCrypto.randomUUID(); +interface AgentEventJournal { + epoch: string; + sequence: number; + events: unknown[]; +} + +const threadJournals = new Map(); const SUBSCRIPTION_SCOPED_CHANNELS = new Set([ "agent.event", "agent.canonical", @@ -134,10 +138,12 @@ function replayThreadSubscription( function requiresThreadHydration( rawCursor: NonNullable[string], cursor: number, - journal: { events: unknown[] } | undefined, + journal: AgentEventJournal | undefined, ): boolean { - if (typeof rawCursor !== "number" && rawCursor.epoch !== eventEpoch) return true; + if (typeof rawCursor === "number" && cursor > 0) return true; if (!journal) return cursor > 0; + if (typeof rawCursor !== "number" && rawCursor.epoch !== journal.epoch) return true; + if (cursor > journal.sequence) return true; return journal.events.length > 0 && cursor < (journal.events[0] as { sequence: number }).sequence - 1; } @@ -206,21 +212,22 @@ export function broadcast( function decorateAgentEvent(channel: WsChannelName, data: unknown, threadId: string | undefined): unknown { if (channel !== "agent.event" || !threadId || !data || typeof data !== "object") return data; - const sequence = (nextSequenceByThread.get(threadId) ?? 0) + 1; - return { ...(data as Record), sequence, epoch: eventEpoch }; + const journal = threadJournals.get(threadId); + const sequence = (journal?.sequence ?? 0) + 1; + const epoch = journal?.epoch ?? NodeCrypto.randomUUID(); + return { ...(data as Record), sequence, epoch }; } function retainAgentEvent(channel: WsChannelName, threadId: string | undefined, event: unknown): void { if (channel !== "agent.event" || !threadId) return; - nextSequenceByThread.delete(threadId); - nextSequenceByThread.set(threadId, (event as { sequence: number }).sequence); + if (!event || typeof event !== "object" || !("sequence" in event) || !("epoch" in event)) return; + if (typeof event.sequence !== "number" || typeof event.epoch !== "string") return; const existing = threadJournals.get(threadId); const events = existing ? [...existing.events, event] : [event]; events.splice(0, Math.max(0, events.length - MAX_AGENT_EVENT_JOURNAL_EVENTS_PER_THREAD)); threadJournals.delete(threadId); - threadJournals.set(threadId, { events }); + threadJournals.set(threadId, { epoch: event.epoch, sequence: event.sequence, events }); trimJournalMap(threadJournals); - trimJournalMap(nextSequenceByThread); } function trimJournalMap(map: Map): void { @@ -336,5 +343,4 @@ export function _resetForTest(): void { queuedPushBytes.clear(); threadSubscriptions.clear(); threadJournals.clear(); - nextSequenceByThread.clear(); } diff --git a/apps/server/src/application/transport/ws-server.ts b/apps/server/src/application/transport/ws-server.ts index dfb60f805..1931db1f4 100644 --- a/apps/server/src/application/transport/ws-server.ts +++ b/apps/server/src/application/transport/ws-server.ts @@ -34,6 +34,7 @@ import type { } from "../../features/browser-automation/index.js"; import { EXTERNAL_THREAD_CONTROL_MCP_PATH } from "../../features/thread-control/index.js"; import type { ReliabilityHarnessAdapter } from "../../runtime/reliability-harness/control.js"; +import type { ExecutionMailboxDepth } from "../../features/agents/execution/execution-mailbox-scheduler.js"; /** Constant-time string comparison to prevent timing attacks on token validation. */ function safeTokenEqual(a: string, b: string): boolean { @@ -74,6 +75,8 @@ export type WsServerDeps = RouterDeps & { browserAutomationMcpHandler?: BrowserAutomationMcpHandler; /** Optional opt-in packaged reliability controls. */ reliabilityHarness?: ReliabilityHarnessAdapter; + /** Content-free counts for active execution worker queues. */ + workerQueueDepth?: () => ExecutionMailboxDepth; }; /** Refreshes mutable workspace authorization while preserving one connection's desktop identity. */ @@ -358,6 +361,7 @@ function createHealthBody(deps: WsServerDeps): Record { nightlyEvidence: deps.browserAutomationBroker.nightlyEvidenceStatus(), }; } + if (deps.workerQueueDepth) body.workerQueue = deps.workerQueueDepth(); if (!deps.singleInstance) body.authToken = deps.authToken; return body; } diff --git a/apps/server/src/features/agents/__tests__/index.test.ts b/apps/server/src/features/agents/__tests__/index.test.ts index 57ac2dc9e..3619de974 100644 --- a/apps/server/src/features/agents/__tests__/index.test.ts +++ b/apps/server/src/features/agents/__tests__/index.test.ts @@ -8,7 +8,6 @@ describe("agents feature boundary", () => { "AgentPermissionService", "AgentService", "CanonicalAgentBoundary", - "CanonicalAgentEventSink", "DelegationTargetResolver", "GoalLifecycleService", "ParentAssistantTextCheckpointService", diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink-stub.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary-stub.ts similarity index 75% rename from apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink-stub.ts rename to apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary-stub.ts index 853b84038..b2a7526f8 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink-stub.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary-stub.ts @@ -1,13 +1,13 @@ -import type { CanonicalAgentEventSink } from "../../index.js"; +import type { CanonicalAgentBoundary } from "../../index.js"; import type { Database } from "bun:sqlite"; /** Creates an AgentService test seam that runs compatibility writes without canonical persistence. */ -export function createCanonicalAgentEventSinkStub( +export function createCanonicalAgentBoundaryStub( db: Pick, -): CanonicalAgentEventSink { +): CanonicalAgentBoundary { return { startParentTurn: ( - input: Parameters[0], + input: Parameters[0], ) => { db.transaction(input.projectUserMessage)(); return { @@ -26,5 +26,5 @@ export function createCanonicalAgentEventSinkStub( finishCanonicalChildTurn: () => null, recordProviderDiagnostic: () => undefined, recordCodexChildRoutingDiagnostic: () => false, - } as unknown as CanonicalAgentEventSink; + } as unknown as CanonicalAgentBoundary; } diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary.test.ts similarity index 93% rename from apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts rename to apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary.test.ts index 2d86a68a6..54620ab03 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary.test.ts @@ -1,5 +1,8 @@ import "reflect-metadata"; import type { Database } from "bun:sqlite"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { CANONICAL_AGENT_EVENT_BATCH_MAX, @@ -19,16 +22,16 @@ import { type ProviderIdentity, type ProviderRuntimeExtension, } from "@mcode/contracts"; -import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { openDatabase, openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { CANONICAL_AGENT_CONTROL_EVENT_RESERVE, - CanonicalAgentEventSink, + CanonicalAgentBoundary, type CanonicalAgentEventDraft, -} from "../canonical-agent-event-sink.js"; +} from "../canonical-agent-boundary.js"; import { CodexCollaborationEventAdapter } from "../../collaboration/adapters/codex-collaboration-event-adapter.js"; import type { CodexCollaborationDurability } from "../../collaboration/codex-collaboration-durability.js"; @@ -114,7 +117,7 @@ function initialDrafts(): CanonicalAgentEventDraft[] { function seedUnfinishedCheckpointCount( db: Database, - sink: CanonicalAgentEventSink, + sink: CanonicalAgentBoundary, count: number, ): void { const messageRepo = new MessageRepo(db); @@ -168,7 +171,7 @@ function terminalDraft( } function startCanonicalParent( - sink: CanonicalAgentEventSink, + sink: CanonicalAgentBoundary, db: Database, approvalReview: { mode: "manual" | "automatic"; reason: string } = { mode: "manual", reason: "manual-requested" }, ): void { @@ -271,7 +274,7 @@ function appendItemDraft(eventId: string, itemId: string): CanonicalAgentEventDr }; } -function parentNarrativeToolCall(index: number): ParentNarrativeRecoveryItem { +function parentNarrativeToolCall(index: number): Extract { return { kind: "toolCall", record: { @@ -309,21 +312,42 @@ function executionIdForTurn(db: Database, turnId: string): string { return row.execution_id; } -describe("CanonicalAgentEventSink", () => { +function parentTerminalInput(db: Database, toolCount = 0): Parameters[0] { + const messageRepo = new MessageRepo(db); + const message = messageRepo.create(THREAD_ID, "assistant", "answer", 2, undefined, undefined, undefined, undefined, true); + return { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed", + projectTurn: () => ({ + message: { ...message, is_internal: false }, + narrative: Array.from({ length: toolCount }, (_, index) => { + const item = parentNarrativeToolCall(index); + return { ...item, record: { ...item.record, message_id: message.id }, sequence: 2, sortOrder: index }; + }), + }), + finalizeCompatibility: () => messageRepo.publishAssistant(message.id), + }; +} + +describe("CanonicalAgentBoundary", () => { let db: Database; let published: ReturnType void>>; - let sink: CanonicalAgentEventSink; + let sink: CanonicalAgentBoundary; beforeEach(() => { db = openMemoryDatabase(); seedThread(db); published = vi.fn(); - sink = new CanonicalAgentEventSink(db, published); + sink = new CanonicalAgentBoundary(db, published); }); it("retains the resolved approval-review decision when a turn is read after reopening", () => { startCanonicalParent(sink, db, { mode: "automatic", reason: "automatic-review-available" }); - const reloaded = new CanonicalAgentEventSink(db, published); + const reloaded = new CanonicalAgentBoundary(db, published); expect(reloaded.loadTurnByExecution(EXECUTION_ID)).toMatchObject({ approvalReviewMode: "automatic", @@ -413,7 +437,7 @@ describe("CanonicalAgentEventSink", () => { it("reports deferred canonical delivery without undoing the durable commit", () => { const failingPublisher = vi.fn(() => { throw new Error("canonical delivery failed"); }); - const deferredSink = new CanonicalAgentEventSink(db, failingPublisher); + const deferredSink = new CanonicalAgentBoundary(db, failingPublisher); const result = deferredSink.commit({ threadId: THREAD_ID, @@ -533,7 +557,7 @@ describe("CanonicalAgentEventSink", () => { preparedSql.push(sql); return originalPrepare(sql); }) as Database["prepare"]; - const instrumentedSink = new CanonicalAgentEventSink(db, vi.fn()); + const instrumentedSink = new CanonicalAgentBoundary(db, vi.fn()); instrumentedSink.commit({ threadId: THREAD_ID, @@ -784,7 +808,7 @@ describe("CanonicalAgentEventSink", () => { userMessageId = message.id; return message; }, - } satisfies Parameters[0]; + } satisfies Parameters[0]; sink.startParentTurn(startInput); sink.startParentTurn(startInput); const finishInput = { @@ -831,7 +855,7 @@ describe("CanonicalAgentEventSink", () => { }], }; }, - } satisfies Parameters[0]; + } satisfies Parameters[0]; sink.finishParentTurn(finishInput); sink.finishParentTurn(finishInput); @@ -858,12 +882,12 @@ describe("CanonicalAgentEventSink", () => { expect(messageRepo.listByThread(THREAD_ID, 10).messages).toHaveLength(2); }); - it("resumes terminal batches when a recovered item changes", async () => { + it("resumes terminal batches over large history when a recovered item changes", async () => { const messageRepo = new MessageRepo(db); let interruptPublication = false; let interrupted = false; const batchRows: number[] = []; - sink = new CanonicalAgentEventSink(db, (events) => { + sink = new CanonicalAgentBoundary(db, (events) => { batchRows.push(3 + events.reduce( (rows, event) => rows + (event.payload.type === "item.recorded" ? 2 : 1), 0, @@ -888,6 +912,8 @@ describe("CanonicalAgentEventSink", () => { providerIdentities: [], projectUserMessage: () => messageRepo.create(THREAD_ID, "user", "question", 1), }); + const historyCount = 2_048; + seedCanonicalItemHistory(db, historyCount); interruptPublication = true; const message = messageRepo.create(THREAD_ID, "assistant", "answer", 2, undefined, undefined, undefined, undefined, true); const narrative = Array.from({ length: 100 }, (_, index) => ({ @@ -944,10 +970,10 @@ describe("CanonicalAgentEventSink", () => { expect(result.outcome).toBe("committed"); expect(sink.loadCheckpoint(EXECUTION_ID)).toMatchObject({ terminalOutcome: "completed", - lastAcceptedSequence: 107, - lastDurableSequence: 107, + lastAcceptedSequence: historyCount + 107, + lastDurableSequence: historyCount + 107, }); - expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 107 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: historyCount + 107 }); expect(messageRepo.listByThread(THREAD_ID, 10).messages).toHaveLength(2); expect(sink.loadConversationProjection(THREAD_ID, 10).messages).toHaveLength(2); expect(sink.loadConversationProjection(THREAD_ID, 10).narrativeByMessage[message.id]?.tools) @@ -965,11 +991,110 @@ describe("CanonicalAgentEventSink", () => { expect(db.prepare(` SELECT DISTINCT durable_revision FROM canonical_agent_events - WHERE accepted_sequence > 4 - `).all()).toEqual([{ durable_revision: 2 }]); + WHERE accepted_sequence > ? + `).all(historyCount + 4)).toEqual([{ durable_revision: 2 }]); expect(sink.loadThread(THREAD_ID)?.conversationRevision).toBe(2); }); + it("rolls back a failed terminal batch and resumes without publishing its writes", async () => { + startCanonicalParent(sink, db); + published.mockClear(); + const input = parentTerminalInput(db, ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows + 1); + let batches = 0; + await expect(sink.finishParentTurnBatched(input, () => { + batches += 1; + if (batches === 2) throw new Error("terminal batch write failed"); + })).rejects.toThrow("terminal batch write failed"); + + const committed = published.mock.calls.flatMap(([events]) => events); + expect(committed.length).toBeGreaterThan(0); + expect(sink.loadCheckpoint(EXECUTION_ID)).toMatchObject({ + lastAcceptedSequence: committed.length + 4, + lastDurableSequence: committed.length + 4, + terminalOutcome: null, + }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()) + .toEqual({ count: committed.length + 4 }); + expect(sink.loadTurn(TURN_ID)?.status).toBe("Running"); + expect(new MessageRepo(db).listByThread(THREAD_ID, 10).messages).toHaveLength(1); + + await sink.finishParentTurnBatched(input); + const events = published.mock.calls.flatMap(([batch]) => batch); + expect(events.map((event) => event.acceptedSequence)) + .toEqual(Array.from({ length: 67 }, (_, index) => index + 5)); + expect(new Set(events.map((event) => event.eventId)).size).toBe(67); + expect(sink.loadCheckpoint(EXECUTION_ID)?.terminalOutcome).toBe("completed"); + expect(new MessageRepo(db).listByThread(THREAD_ID, 10).messages).toHaveLength(2); + }); + + it("rejects terminal sequence collisions from a stale checkpoint before publication", async () => { + startCanonicalParent(sink, db); + seedCanonicalItemHistory(db, 1); + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET last_accepted_sequence = 4, last_durable_sequence = 4").run(); + published.mockClear(); + + await expect(sink.finishParentTurnBatched(parentTerminalInput(db))).rejects.toThrow("sequence-conflict"); + + expect(published).not.toHaveBeenCalled(); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 5 }); + expect(sink.loadTurn(TURN_ID)?.status).toBe("Running"); + expect(sink.loadCheckpoint(EXECUTION_ID)?.lastAcceptedSequence).toBe(4); + expect(new MessageRepo(db).listByThread(THREAD_ID, 10).messages).toHaveLength(1); + }); + + it("continues terminal sequences from durable events when the checkpoint is absent", async () => { + startCanonicalParent(sink, db); + seedCanonicalItemHistory(db, 256); + db.prepare("DELETE FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").run(EXECUTION_ID); + published.mockClear(); + + const result = await sink.finishParentTurnBatched(parentTerminalInput(db)); + + expect(result.events.map((event) => event.acceptedSequence)).toEqual([261, 262]); + expect(sink.loadCheckpoint(EXECUTION_ID)).toMatchObject({ + lastAcceptedSequence: 262, + lastDurableSequence: 262, + terminalOutcome: "completed", + }); + }); + + it("reloads thread and turn fields changed by another connection between terminal batches", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-terminal-freshness-")); + const path = NodePath.join(directory, "app.sqlite"); + const writerDb = openDatabase({ dbPath: path }); + const otherDb = openDatabase({ dbPath: path }); + try { + seedThread(writerDb); + let finishing = false; + let mutated = false; + const writer = new CanonicalAgentBoundary(writerDb, () => { + if (!finishing || mutated) return; + mutated = true; + otherDb.prepare("UPDATE canonical_agent_threads SET roster_revision = 17 WHERE id = ?").run(THREAD_ID); + otherDb.prepare("UPDATE canonical_agent_turns SET approval_review_reason = 'changed-between-batches' WHERE id = ?").run(TURN_ID); + otherDb.prepare("UPDATE canonical_agent_ingest_checkpoints SET native_cursor_json = ? WHERE execution_id = ?") + .run(JSON.stringify({ position: "changed-between-batches" }), EXECUTION_ID); + }); + startCanonicalParent(writer, writerDb); + finishing = true; + + const result = await writer.finishParentTurnBatched(parentTerminalInput(writerDb, ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows + 1)); + + expect(mutated).toBe(true); + expect(result.writeBatches.batches).toBeGreaterThan(1); + expect(writer.loadThread(THREAD_ID)?.rosterRevision).toBe(17); + expect(writer.loadTurn(TURN_ID)).toMatchObject({ + status: "Completed", + approvalReviewReason: "changed-between-batches", + }); + expect(writer.loadCheckpoint(EXECUTION_ID)?.nativeCursor).toEqual({ position: "changed-between-batches" }); + } finally { + otherDb.close(true); + writerDb.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); + it("commits terminal projections above the semantic event limit in bounded batches", async () => { const messageRepo = new MessageRepo(db); sink.startParentTurn({ @@ -1153,7 +1278,7 @@ describe("CanonicalAgentEventSink", () => { }, }], }); - const reloaded = new CanonicalAgentEventSink(db, vi.fn()); + const reloaded = new CanonicalAgentBoundary(db, vi.fn()); expect(committed.outcome).toBe("committed"); expect(duplicate.outcome).toBe("duplicate"); @@ -1731,7 +1856,7 @@ describe("CanonicalAgentEventSink", () => { }, }), "toolCall:spawn-generic-roster-source"); - const restoredSink = new CanonicalAgentEventSink(db, vi.fn()); + const restoredSink = new CanonicalAgentBoundary(db, vi.fn()); const row = restoredSink.loadSubagentRoster({ owningParentThreadId: THREAD_ID, limit: CANONICAL_SUBAGENT_ROSTER_MAX_CHILDREN, @@ -2414,7 +2539,7 @@ describe("CanonicalAgentEventSink", () => { }], payload: { projection: "codexCollaboration" }, }); - const restoredSink = new CanonicalAgentEventSink(db, vi.fn()); + const restoredSink = new CanonicalAgentBoundary(db, vi.fn()); expect(restoredSink.loadCollaborationActionBySourceProviderIdentity( delegation.childThread.id, childTurn.id, @@ -2796,7 +2921,7 @@ describe("CanonicalAgentEventSink", () => { outcome: "errored", })).toMatchObject({ status: "Completed" }); - const restored = new CanonicalAgentEventSink(db, vi.fn()); + const restored = new CanonicalAgentBoundary(db, vi.fn()); const recovered = restored.loadCodexChildDelegation(THREAD_ID, input.parentItemId); expect(recovered).toMatchObject({ childThread: { diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts new file mode 100644 index 000000000..7956e6a07 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -0,0 +1,718 @@ +import "reflect-metadata"; +import type { Database } from "bun:sqlite"; +import * as NodeFSPromises from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { AgentEventType, type ParentNarrativeRecoveryItem } from "@mcode/contracts"; +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import type { CanonicalAgentEventDraft } from "../canonical-agent-boundary.js"; +import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; +import { ExecutionLivePublicationRelease } from "../execution-live-publication-release.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; +import type { CanonicalWriterResponse } from "../canonical-agent-writer-protocol.js"; +import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; +import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; + +const THREAD_ID = "writer-thread"; +const TURN_ID = "writer-turn"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000176"; +const NOW = "2026-09-24T12:00:00.000Z"; + +function workerDroppingReply(kind: CanonicalWriterResponse["kind"]): Worker { + const worker = new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + return new Proxy(worker, { + set(target, property, value) { + if (property !== "onmessage" || typeof value !== "function") return Reflect.set(target, property, value); + target.onmessage = (message) => { + if (message.data.kind === kind) { + target.terminate(); + return; + } + value(message); + }; + return true; + }, + get(target, property) { + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + +function events(): CanonicalAgentEventDraft[] { + const sourceIdentities = [{ providerId: "codex" as const, scope: "thread" as const, value: "native-writer-thread", provenance: "native" as const }]; + return [ + { + eventId: `${EXECUTION_ID}:thread`, + routing: { threadId: THREAD_ID, executionId: EXECUTION_ID }, + sourceProviderId: "codex", + sourceIdentities, + payload: { + type: "thread.recorded", + thread: { + id: THREAD_ID, + workspaceId: "writer-workspace", + rootThreadId: THREAD_ID, + providerId: "codex", + providerIdentities: sourceIdentities, + activityState: "Active", + conversationRevision: 0, + rosterRevision: 0, + createdAt: NOW, + updatedAt: NOW, + }, + }, + }, + { + eventId: `${EXECUTION_ID}:turn`, + routing: { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }, + sourceProviderId: "codex", + sourceIdentities, + payload: { + type: "turn.created", + turn: { + id: TURN_ID, + threadId: THREAD_ID, + status: "Pending", + trigger: { kind: "user" }, + permissionMode: "supervised", + approvalReviewMode: "manual", + approvalReviewReason: "manual-requested", + providerIdentities: sourceIdentities, + startedAt: null, + endedAt: null, + createdAt: NOW, + updatedAt: NOW, + }, + }, + }, + { + eventId: `${EXECUTION_ID}:started`, + routing: { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }, + sourceProviderId: "codex", + sourceIdentities, + payload: { type: "turn.started", startedAt: NOW }, + }, + ]; +} + +function recoveryToolCall(id = "writer-recovery-tool"): ParentNarrativeRecoveryItem { + return { + kind: "toolCall", + record: { + id, + message_id: "", + parent_tool_call_id: null, + tool_name: "Read", + display_name: null, + provider_agent_key: null, + subagent_identity_key: null, + subagent_provider_name: null, + subagent_prompt: null, + subagent_type: null, + subagent_agent_id: null, + subagent_duration_ms: null, + model: null, + reasoning_effort: null, + input_summary: "a file", + output_summary: "read", + output_total_bytes: null, + output_artifact_path: null, + exit_code: null, + status: "completed", + started_at: NOW, + completed_at: NOW, + sort_order: 0, + }, + }; +} + +describe("canonical SQLite writer", () => { + let tempDir: string; + let dbPath: string; + let db: Database; + let writer: CanonicalAgentWriterClient | undefined; + + beforeEach(async () => { + tempDir = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-canonical-writer-")); + dbPath = NodePath.join(tempDir, "app.sqlite"); + db = openDatabase({ dbPath }); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("writer-workspace", "Writer test", tempDir, NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "writer-workspace", "Writer thread", "main", "codex", NOW, NOW); + }); + + afterEach(async () => { + await writer?.close(); + db.close(true); + await NodeFSPromises.rm(tempDir, { recursive: true, force: true }); + }); + + it("replays the original committed receipt and full envelopes", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + const first = await writer.commit("writer-operation-1", batch); + expect(first).toMatchObject({ outcome: "committed", acceptedThrough: 3, durableThrough: 3 }); + expect(first.events.map((event) => event.eventId)).toEqual(batch.events.map((event) => event.eventId)); + expect(first).not.toHaveProperty("canonicalDelivery"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + + const replay = await writer.commit("writer-operation-1", batch); + expect(replay).toEqual(first); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()).toEqual({ count: 1 }); + }); + + it("recovers one durable live publication identity after a file-backed writer loses its reply", async () => { + const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "publication-lease" }; + const operation: ExecutionSemanticOperation = { + operationId: "publication-lease:1", execution, lease, ordinal: 1, + mutation: { kind: "begin", providerId: "codex", input: { + thread: { id: THREAD_ID, workspaceId: "writer-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "publication-user", content: "Question", sequence: 1 }, + } }, + livePublication: [{ after: "writer", event: { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + } }], + }; + let created = 0; + const registry = new AgentEventPublicationRegistry(); + const published: string[] = []; + registry.bind((event) => published.push(event.type)); + const release = new ExecutionLivePublicationRelease(registry); + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("semantic-transacted") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + let port = new CanonicalExecutionWriterPort(writer, () => {}, release); + + const receipt = await port.transact(operation); + expect(created).toBe(2); + expect(published).toEqual([AgentEventType.TurnStarted]); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ + publicationId: "1", after: "writer", + event: { type: AgentEventType.TurnStarted, turnExecutionId: EXECUTION_ID }, + }] }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("publication-user")) + .toEqual({ count: 1 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, operation.operationId)).toEqual({ count: 1 }); + + await writer.close(); + writer = new CanonicalAgentWriterClient(dbPath); + port = new CanonicalExecutionWriterPort(writer, () => {}, release); + expect(await port.transact(operation)).toEqual(receipt); + expect(published).toEqual([AgentEventType.TurnStarted]); + }); + + it("replays durable semantic assistant text after a worker reply is lost", async () => { + const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "text-lease" }; + const start: ExecutionSemanticOperation = { + operationId: "text-lease:1", execution, lease, ordinal: 1, + mutation: { + kind: "begin", providerId: "codex", + input: { + thread: { id: THREAD_ID, workspaceId: "writer-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, + permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "text-user", content: "Question", sequence: 1 }, + }, + }, + }; + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(start, () => {})).toMatchObject({ kind: "committed" }); + await writer.close(); + + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("semantic-transacted") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + const textOperation: ExecutionSemanticOperation = { + operationId: "text-lease:2", execution, lease, ordinal: 2, + mutation: { kind: "append-assistant-text", inputs: [{ ...execution, sequence: 1, text: "Worker durable text" }] }, + }; + const receipt = await writer.transactSemantic(textOperation, () => {}); + expect(receipt).toMatchObject({ kind: "committed", operationId: "text-lease:2", + assistantTextCheckpoint: { outcome: "committed", durableThrough: 1 } }); + expect(created).toBe(2); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Worker durable text"); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoint_chunks WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 1 }); + await expect(NodeFSPromises.stat(NodePath.join(tempDir, "app.sqlite.recovery", "parent-assistant-text"))) + .rejects.toMatchObject({ code: "ENOENT" }); + await writer.close(); + + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(textOperation, () => {})).toEqual(receipt); + expect(await writer.interruptWorkerLoss({ execution, lease, reason: "worker exited", recoveryIncidentId: "text-loss" }, + () => {})).toMatchObject({ kind: "committed", operationId: "text-lease:worker-lost" }); + expect(db.prepare("SELECT content FROM messages WHERE role = 'assistant' AND is_internal = 0").get()) + .toEqual({ content: "Worker durable text" }); + }); + + it("replays a narrative delta after the worker commits but loses its reply", async () => { + const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "narrative-lease" }; + const begin: ExecutionSemanticOperation = { + operationId: "narrative-lease:1", execution, lease, ordinal: 1, + mutation: { kind: "begin", providerId: "codex", input: { + thread: { id: THREAD_ID, workspaceId: "writer-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "narrative-user", content: "Question", sequence: 1 }, + } }, + }; + writer = new CanonicalAgentWriterClient(dbPath); + expect((await writer.transactSemantic(begin, () => {})).kind).toBe("committed"); + await writer.close(); + + let created = 0; + const published: string[] = []; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("semantic-transacted") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + const delta: ExecutionSemanticOperation = { + operationId: "narrative-lease:2", execution, lease, ordinal: 2, + mutation: { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: [recoveryToolCall()], discardedItemIds: [], + } }, + }; + const receipt = await writer.transactSemantic(delta, (events) => published.push(...events.map((event) => event.eventId))); + expect(receipt).toMatchObject({ kind: "committed", operationId: "narrative-lease:2" }); + expect(created).toBe(2); + expect(published).toEqual([]); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + await writer.close(); + + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(delta, () => {})).toEqual(receipt); + expect(await writer.interruptWorkerLoss({ execution, lease, reason: "worker exited", recoveryIncidentId: "narrative-loss" }, + () => {})).toMatchObject({ kind: "committed", operationId: "narrative-lease:worker-lost" }); + expect(db.prepare("SELECT id, status FROM tool_call_records WHERE id = ?") + .get("writer-recovery-tool")).toEqual({ id: "writer-recovery-tool", status: "completed" }); + }); + + it("replays one live reclassification receipt after the worker loses its reply", async () => { + const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "live-text-lease" }; + const begin: ExecutionSemanticOperation = { + operationId: "live-text-lease:1", execution, lease, ordinal: 1, + mutation: { kind: "begin", providerId: "codex", input: { + thread: { id: THREAD_ID, workspaceId: "writer-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "live-text-user", content: "Question", sequence: 1 }, + } }, + }; + writer = new CanonicalAgentWriterClient(dbPath); + expect((await writer.transactSemantic(begin, () => {})).kind).toBe("committed"); + const provisional: ExecutionSemanticOperation = { + operationId: "live-text-lease:2", execution, lease, ordinal: 2, + mutation: { kind: "live-event", text: { kind: "append", inputs: [{ + ...execution, sequence: 1, text: "provisional thought", + }] } }, + livePublication: [{ after: "writer", event: { + type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "provisional thought", + } }], + }; + expect((await writer.transactSemantic(provisional, () => {})).kind).toBe("committed"); + await writer.close(); + + const thought = { kind: "narrationSegment" as const, record: { + id: "live-thought", message_id: "", text: "provisional thought", + started_at: NOW, ended_at: NOW, sort_order: 0, + } }; + const reclassified: ExecutionSemanticOperation = { + operationId: "live-text-lease:3", execution, lease, ordinal: 3, + mutation: { kind: "live-event", text: { kind: "reclassify", expectedText: "provisional thought" }, + narrative: { executionId: EXECUTION_ID, items: [thought], discardedItemIds: [] } }, + livePublication: [{ after: "writer", event: { + type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: false, + } }], + }; + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("semantic-transacted") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + const receipt = await writer.transactSemantic(reclassified, () => {}); + expect(receipt).toMatchObject({ kind: "committed", operationId: "live-text-lease:3", + livePublication: [{ publicationId: "2", after: "writer" }] }); + expect(created).toBe(2); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("narrationSegment:live-thought")).toEqual({ count: 1 }); + await writer.close(); + + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(reclassified, () => {})).toEqual(receipt); + expect((await writer.interruptWorkerLoss({ execution, lease, reason: "worker exited", + recoveryIncidentId: "live-thought-loss" }, () => {})).kind).toBe("committed"); + expect(db.prepare("SELECT id, text FROM thought_segments WHERE id = ?").get("live-thought")) + .toEqual({ id: "live-thought", text: "provisional thought" }); + }); + + it("rejects a database failure without reporting a durable receipt", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.whenReady(); + db.run("DROP TABLE canonical_agent_events"); + await expect(writer.commit("writer-operation-2", { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + phase: "running", + events: events(), + })).rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_ingest_checkpoints").get()).toEqual({ count: 0 }); + }); + + it("acknowledges structured recovery after persistence and converges on replay", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("narrative-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + const recovery = { executionId: EXECUTION_ID, items: [recoveryToolCall()] }; + expect(await writer.recordParentNarrativeRecovery("narrative-record", recovery)).toEqual({ recorded: true }); + const row = db.prepare("SELECT payload_json FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool") as { payload_json: string }; + expect(JSON.parse(row.payload_json)).toMatchObject({ + projection: "narrativeRecovery", + narrative: recoveryToolCall(), + }); + expect(await writer.recordParentNarrativeRecovery("narrative-record", recovery)).toEqual({ recorded: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + + expect(await writer.recordParentNarrativeRecovery("narrative-discard", { + executionId: EXECUTION_ID, + items: [], + discardedItemIds: ["toolCall:writer-recovery-tool"], + })).toEqual({ recorded: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 0 }); + expect(await writer.recordParentNarrativeRecovery("narrative-discard", { + executionId: EXECUTION_ID, + items: [], + discardedItemIds: ["toolCall:writer-recovery-tool"], + })).toEqual({ recorded: true }); + }); + + it("rejects a failed recovery write without a durability acknowledgement", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("narrative-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + db.run("DROP TABLE canonical_agent_items"); + await expect(writer.recordParentNarrativeRecovery("narrative-failed", { + executionId: EXECUTION_ID, + items: [recoveryToolCall()], + })).rejects.toThrow("Canonical writer write-failed"); + }); + + it("converges after a failure between bounded recovery batches", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("batched-recovery-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + const input = { + executionId: EXECUTION_ID, + items: Array.from({ length: 65 }, (_, index) => recoveryToolCall(`batched-${index}`)), + }; + db.run(`CREATE TRIGGER reject_last_recovery BEFORE INSERT ON canonical_agent_items + WHEN NEW.id = 'toolCall:batched-64' BEGIN SELECT RAISE(FAIL, 'injected recovery failure'); END`); + await expect(writer.recordParentNarrativeRecovery("batched-recovery", input)) + .rejects.toThrow("Canonical writer write-failed"); + const partial = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id LIKE 'toolCall:batched-%'") + .get() as { count: number }; + expect(partial.count).toBeGreaterThan(0); + expect(partial.count).toBeLessThan(65); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get("batched-recovery")).toEqual({ count: 0 }); + db.run("DROP TRIGGER reject_last_recovery"); + expect(await writer.recordParentNarrativeRecovery("batched-recovery", input)).toEqual({ recorded: true }); + expect(await writer.recordParentNarrativeRecovery("batched-recovery", input)).toEqual({ recorded: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id LIKE 'toolCall:batched-%'") + .get()).toEqual({ count: 65 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get("batched-recovery")).toEqual({ count: 1 }); + }, 30_000); + + it("lets a second SQLite writer proceed while recovery continues across batches", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("slow-recovery-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + db.run("CREATE TABLE writer_lock_probe (id TEXT PRIMARY KEY)"); + db.run(`CREATE TRIGGER slow_recovery BEFORE INSERT ON canonical_agent_items + WHEN NEW.id LIKE 'toolCall:slow-%' BEGIN SELECT randomblob(8000000); END`); + const input = { + executionId: EXECUTION_ID, + items: Array.from({ length: 80 }, (_, index) => recoveryToolCall(`slow-${index}`)), + }; + let settled = false; + const recovery = writer.recordParentNarrativeRecovery("slow-recovery", input) + .finally(() => { settled = true; }); + const count = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id LIKE 'toolCall:slow-%'"); + for (let attempts = 0; attempts < 500 && (count.get() as { count: number }).count === 0; attempts++) { + await new Promise((resolve) => setTimeout(resolve, 2)); + } + expect((count.get() as { count: number }).count).toBeGreaterThan(0); + expect(settled).toBe(false); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get("slow-recovery")).toEqual({ count: 0 }); + const started = performance.now(); + db.prepare("INSERT INTO writer_lock_probe (id) VALUES (?)").run("main-write"); + const mainWriteMs = performance.now() - started; + expect(mainWriteMs).toBeLessThan(250); + await recovery; + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get("slow-recovery")).toEqual({ count: 1 }); + }, 30_000); + + it("reports a missing execution without claiming recovery was recorded", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.recordParentNarrativeRecovery("missing-recovery", { + executionId: "missing-execution", + items: [recoveryToolCall()], + })).toEqual({ recorded: false }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items").get()).toEqual({ count: 0 }); + }); + + it("commits recovery and assistant-text reset together", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("classification-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + const checkpoints = new ParentAssistantTextCheckpointService(db); + checkpoints.appendChunk([{ + executionId: EXECUTION_ID, threadId: THREAD_ID, turnId: TURN_ID, sequence: 1, text: "provisional text", + }]); + const input = { executionId: EXECUTION_ID, items: [recoveryToolCall()] }; + expect(await writer.classifyParentNarrativeRecovery("classification-1", input)) + .toEqual({ recorded: true, reset: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + expect(await writer.classifyParentNarrativeRecovery("classification-1", input)) + .toEqual({ recorded: true, reset: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + }); + + it("rolls back recovery when the assistant-text checkpoint is missing", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("classification-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + await expect(writer.classifyParentNarrativeRecovery("classification-missing", { + executionId: EXECUTION_ID, + items: [recoveryToolCall()], + })).rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 0 }); + }); + + it("rejects an unmigrated path without creating a second database", async () => { + const missingPath = NodePath.join(tempDir, "missing.sqlite"); + writer = new CanonicalAgentWriterClient(missingPath); + await expect(writer.whenReady()).rejects.toThrow("Canonical writer open-failed"); + await expect(NodeFSPromises.stat(missingPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("bounds write admission while SQLite is busy", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const activeWriter = writer; + await activeWriter.whenReady(); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + db.run("BEGIN IMMEDIATE"); + let locked = true; + try { + const accepted = Array.from({ length: 64 }, (_, index) => activeWriter.commit(`queued-${index}`, batch)); + await expect(activeWriter.commit("overflow", batch)).rejects.toThrow("Canonical writer admission is full"); + db.run("ROLLBACK"); + locked = false; + const results = await Promise.all(accepted); + expect(results[0]?.outcome).toBe("committed"); + expect(results.slice(1).every((result) => result.outcome === "duplicate")).toBe(true); + } finally { + if (locked) db.run("ROLLBACK"); + } + }); + + it("restarts a closed worker and accepts later writes", async () => { + let worker: Worker | undefined; + writer = new CanonicalAgentWriterClient(dbPath, () => { + worker = new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + return worker; + }); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await writer.commit("writer-operation-3", batch); + const closed = new Promise((resolve) => worker?.addEventListener("close", resolve, { once: true })); + worker?.terminate(); + await closed; + expect(await writer.commit("writer-operation-4", batch)).toMatchObject({ outcome: "duplicate" }); + }); + + it("replays the committed envelope after a response is lost with the worker", async () => { + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => { + return created++ === 0 ? workerDroppingReply("committed") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + }); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + const receipt = await writer.commit("lost-committed-response", batch); + expect(receipt).toMatchObject({ outcome: "committed", acceptedThrough: 3 }); + expect(receipt.events.map((event) => event.eventId)).toEqual(batch.events.map((event) => event.eventId)); + expect(created).toBe(2); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + }); + + it("replays a classification after its reset committed but the reply was lost", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("classification-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + new ParentAssistantTextCheckpointService(db).appendChunk([{ + executionId: EXECUTION_ID, threadId: THREAD_ID, turnId: TURN_ID, sequence: 1, text: "provisional", + }]); + await writer.close(); + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => { + return created++ === 0 ? workerDroppingReply("parent-narrative-recovery-classified") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + }); + expect(await writer.classifyParentNarrativeRecovery("lost-classification", { + executionId: EXECUTION_ID, items: [recoveryToolCall()], + })).toEqual({ recorded: true, reset: true }); + expect(created).toBe(2); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + }); + + it("replays a recovery discard after its reply was lost", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("recovery-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + await writer.recordParentNarrativeRecovery("recovery-record", { + executionId: EXECUTION_ID, items: [recoveryToolCall()], + }); + await writer.close(); + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("parent-narrative-recovery-recorded") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + expect(await writer.recordParentNarrativeRecovery("lost-recovery-discard", { + executionId: EXECUTION_ID, items: [], discardedItemIds: ["toolCall:writer-recovery-tool"], + })).toEqual({ recorded: true }); + expect(created).toBe(2); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 0 }); + }); + + it("stops after three lost replies and permits explicit replay later", async () => { + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => { + created++; + return workerDroppingReply("committed"); + }); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await expect(writer.commit("lost-three-times", batch)).rejects.toThrow("Canonical writer worker closed"); + expect(created).toBe(3); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + await writer.close(); + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.commit("lost-three-times", batch)).toMatchObject({ outcome: "committed" }); + }); + + it("rejects reused operation IDs with changed phase or native cursor", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await writer.commit("same-id", batch); + await expect(writer.commit("same-id", { ...batch, phase: "finalizing" })) + .rejects.toThrow("Canonical writer operation-conflict"); + await expect(writer.commit("same-id", { ...batch, nativeCursor: "later" })) + .rejects.toThrow("Canonical writer operation-conflict"); + }); + + it("caps outstanding receipts and resumes after acknowledgement", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await writer.commit("capacity-start", batch); + const insert = db.prepare(`INSERT INTO canonical_writer_operation_receipts + (execution_id, operation_id, kind, input_hash, receipt_json, created_at) + VALUES (?, ?, 'commit', ?, '{}', ?)`); + db.transaction(() => { + for (let index = 1; index < 16_384; index++) { + insert.run(EXECUTION_ID, `seed-${index}`, "seed", NOW); + } + })(); + const before = db.prepare("SELECT COUNT(*) AS count, SUM(LENGTH(receipt_json)) AS bytes FROM canonical_writer_operation_receipts").get(); + expect(before).toEqual({ count: 16_384, bytes: expect.any(Number) }); + await expect(writer.commit("capacity-over", batch)).rejects.toThrow("Canonical writer receipt-capacity"); + await writer.acknowledgeOperation(EXECUTION_ID, "capacity-start"); + await writer.acknowledgeOperation(EXECUTION_ID, "capacity-start"); + expect(await writer.commit("capacity-over", batch)).toMatchObject({ outcome: "duplicate" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()) + .toEqual({ count: 16_384 }); + db.prepare("DELETE FROM canonical_agent_turns WHERE execution_id = ?").run(EXECUTION_ID); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()) + .toEqual({ count: 0 }); + }, 30_000); + + it("retries a failed acknowledgement before the next write", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await writer.commit("ack-retry-start", batch); + db.run(`CREATE TRIGGER reject_receipt_delete BEFORE DELETE ON canonical_writer_operation_receipts + BEGIN SELECT RAISE(FAIL, 'ack unavailable'); END`); + await expect(writer.acknowledgeOperation(EXECUTION_ID, "ack-retry-start")) + .rejects.toThrow("Canonical writer write-failed"); + expect(writer.pendingAcknowledgementCount).toBe(1); + db.run("DROP TRIGGER reject_receipt_delete"); + expect(await writer.commit("ack-retry-next", batch)).toMatchObject({ outcome: "duplicate" }); + expect(writer.pendingAcknowledgementCount).toBe(0); + expect(db.prepare("SELECT operation_id FROM canonical_writer_operation_receipts").all()) + .toEqual([{ operation_id: "ack-retry-next" }]); + }); + + it("retries failed acknowledgements on close", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("ack-close-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + db.run(`CREATE TRIGGER reject_receipt_delete BEFORE DELETE ON canonical_writer_operation_receipts + BEGIN SELECT RAISE(FAIL, 'ack unavailable'); END`); + await expect(writer.acknowledgeOperation(EXECUTION_ID, "ack-close-start")) + .rejects.toThrow("Canonical writer write-failed"); + db.run("DROP TRIGGER reject_receipt_delete"); + await writer.close(); + expect(writer.pendingAcknowledgementCount).toBe(0); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()) + .toEqual({ count: 0 }); + }); + + it("bounds failed acknowledgement retention and reports unfinished close cleanup", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + for (let index = 0; index <= 64; index++) await writer.commit(`ack-full-${index}`, batch); + db.run(`CREATE TRIGGER reject_receipt_delete BEFORE DELETE ON canonical_writer_operation_receipts + BEGIN SELECT RAISE(FAIL, 'ack unavailable'); END`); + for (let index = 0; index < 64; index++) { + await expect(writer.acknowledgeOperation(EXECUTION_ID, `ack-full-${index}`)) + .rejects.toThrow("Canonical writer write-failed"); + } + await expect(writer.acknowledgeOperation(EXECUTION_ID, "ack-full-64")) + .rejects.toThrow("Canonical writer acknowledgement retry queue is full"); + expect(writer.pendingAcknowledgementCount).toBe(64); + await expect(writer.close()).rejects.toThrow("64 unacknowledged operations"); + }, 30_000); +}); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-append-group.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-append-group.test.ts new file mode 100644 index 000000000..0cc1e2a45 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-append-group.test.ts @@ -0,0 +1,234 @@ +import "reflect-metadata"; +import { Database } from "bun:sqlite"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; +import { APPEND_GROUP_LIMITS, selectAppendGroup, type QueuedCanonicalWrite } from "../canonical-append-group.js"; +import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; +import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; + +const NOW = "2026-09-25T12:00:00.000Z"; + +function append(index: number, ordinal = 2): ExecutionSemanticOperation { + const execution = { threadId: `thread-${index}`, turnId: `turn-${index}`, + executionId: `00000000-0000-4000-8000-${String(index).padStart(12, "0")}` }; + const itemId = `item-${index}-${ordinal}`; + return { + execution, lease: { ownerEpoch: 1, workerIndex: index, workerGeneration: 1, leaseId: `lease-${index}` }, + operationId: `lease-${index}:${ordinal}`, ordinal, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, events: [{ + eventId: `${execution.executionId}:item-${ordinal}`, routing: { ...execution, itemId }, + sourceProviderId: "codex", sourceIdentities: [], + payload: { type: "item.recorded", item: { id: itemId, threadId: execution.threadId, + turnId: execution.turnId, kind: "message", providerIdentities: [], + payload: { projection: "message", content: "Durable event" }, createdAt: NOW, updatedAt: NOW } }, + }] }, + }; +} + +function queued(operation: ExecutionSemanticOperation, bytes = 100): QueuedCanonicalWrite { + return { bytes, request: { kind: "semantic-transact", requestId: `request-${operation.operationId}`, + operationId: operation.operationId, executionId: operation.execution.executionId, operation } }; +} + +describe("bounded append groups", () => { + it("preserves FIFO across repeated identities, controls, mismatched routing, and reclassification", () => { + const first = queued(append(1)); + const second = queued(append(2)); + const repeated = queued(append(1, 3)); + const control = queued({ ...append(3), mutation: { kind: "checkpoint", phase: "running", nativeCursor: null } }); + expect(selectAppendGroup([first, second, repeated, queued(append(4))])).toEqual([first.request, second.request]); + expect(selectAppendGroup([first, control, second])).toEqual([first.request]); + const mismatched: QueuedCanonicalWrite = { ...second, request: { ...second.request, executionId: "wrong" } }; + expect(selectAppendGroup([first, mismatched, queued(append(4))])).toEqual([first.request]); + const source = append(3); + if (source.mutation.kind !== "append-events") throw new Error("Expected append"); + const reclassify = queued({ ...source, mutation: { ...source.mutation, + parentLive: { text: { kind: "reclassify", expectedText: "" } } } }); + expect(selectAppendGroup([first, reclassify, second])).toEqual([first.request]); + }); + + it("caps count and bytes without consuming the rest of the queue", () => { + const queue = Array.from({ length: 10 }, (_, index) => queued(append(index + 1))); + expect(selectAppendGroup(queue)).toHaveLength(APPEND_GROUP_LIMITS.operations); + expect(queue).toHaveLength(10); + expect(selectAppendGroup([queued(append(1), APPEND_GROUP_LIMITS.bytes), queued(append(2))])).toHaveLength(1); + }); +}); + +describe("file-backed grouped semantic commits", () => { + let directory: string; + let db: Database; + let observer: Database; + let writer: CanonicalExecutionSemanticWriter; + let published: string[]; + + beforeEach(async () => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-append-group-")); + const path = NodePath.join(directory, "app.sqlite"); + db = openDatabase({ dbPath: path }); + observer = new Database(path, { strict: true, readonly: true }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((event) => event.eventId))); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-1", "Fixture", "C:/fixture", NOW, NOW); + for (const index of [1, 2]) { + const operation = append(index); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(operation.execution.threadId, "workspace-1", "Fixture", "main", "codex", NOW, NOW); + expect(await writer.transact({ ...operation, ordinal: 1, operationId: `lease-${index}:1`, mutation: { + kind: "begin", providerId: "codex", input: { + thread: { id: operation.execution.threadId, workspaceId: "workspace-1", providerId: "codex", createdAt: NOW }, + turnId: operation.execution.turnId, executionId: operation.execution.executionId, + permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: `user-${index}`, content: "Question", sequence: 1 }, + }, + } })).toMatchObject({ kind: "committed" }); + } + published.length = 0; + // A fixed clock makes the transaction tests independent of host load; SQLite remains real. + vi.spyOn(performance, "now").mockReturnValue(0); + }); + + afterEach(() => { + vi.restoreAllMocks(); + observer.close(true); + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + function persistedAppends() { + return observer.query("SELECT operation_id FROM canonical_writer_operation_receipts WHERE kind = 'semantic:append-events' ORDER BY execution_id").all(); + } + + it("returns each request's publications only after the outer commit is visible to another connection", () => { + const results = writer.transactAppendGroup([append(1), append(2)]); + expect(results).toHaveLength(2); + expect(results.map((result) => result.receipt.kind)).toEqual(["committed", "committed"]); + expect(db.inTransaction).toBe(false); + expect(persistedAppends()).toEqual([{ operation_id: "lease-1:2" }, { operation_id: "lease-2:2" }]); + expect(published).toEqual([]); + for (const result of results) { + expect(result.publications.flat().every((event) => event.routing.executionId === result.operation.execution.executionId)).toBe(true); + expect(result.publications.flat().length).toBeGreaterThan(0); + } + }); + + it("rolls back a conflicting append's savepoint while committing its peer", () => { + const conflicting = append(1); + if (conflicting.mutation.kind !== "append-events") throw new Error("Expected append"); + const results = writer.transactAppendGroup([{ ...conflicting, mutation: { ...conflicting.mutation, + parentLive: { text: { kind: "unchanged" } } } }, append(2)]); + expect(results.map((result) => result.receipt.kind)).toEqual(["conflict", "committed"]); + expect(results[0]?.publications).toEqual([]); + expect(persistedAppends()).toEqual([{ operation_id: "lease-2:2" }]); + expect(observer.query("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(`${conflicting.execution.executionId}:item-2`)).toBeNull(); + }); + + it("discards all results and publications when the physical outer commit fails", () => { + db.exec(`CREATE TABLE group_commit_parent (id INTEGER PRIMARY KEY); + CREATE TABLE group_commit_child (parent_id INTEGER REFERENCES group_commit_parent(id) DEFERRABLE INITIALLY DEFERRED); + CREATE TRIGGER fail_group_commit AFTER INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:append-events' + BEGIN INSERT INTO group_commit_child(parent_id) VALUES (999); END;`); + expect(() => writer.transactAppendGroup([append(1), append(2)])).toThrow(); + expect(db.inTransaction).toBe(false); + expect(persistedAppends()).toEqual([]); + expect(published).toEqual([]); + db.exec("DROP TRIGGER fail_group_commit"); + expect(writer.transactAppendGroup([append(1), append(2)]).map((result) => result.receipt.kind)) + .toEqual(["committed", "committed"]); + }); + + it("replays a lost acknowledgement with the original receipts and no duplicate rows", () => { + const initial = writer.transactAppendGroup([append(1), append(2)]); + const replay = writer.transactAppendGroup([append(1), append(2)]); + expect(replay).toEqual(initial); + expect(persistedAppends()).toHaveLength(2); + expect(published).toEqual([]); + }); + + it("returns the consumed prefix when the elapsed limit ends the transaction", () => { + vi.spyOn(performance, "now").mockReturnValueOnce(0).mockReturnValue(APPEND_GROUP_LIMITS.elapsedMs); + const operations = [append(1), append(2)]; + const first = writer.transactAppendGroup(operations); + expect(first).toHaveLength(1); + expect(writer.transactAppendGroup(operations.slice(first.length))).toHaveLength(1); + expect(persistedAppends()).toHaveLength(2); + }); + + it("correlates concurrent worker requests and publishes only rows visible from a separate connection", async () => { + vi.restoreAllMocks(); + const client = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const observed: string[] = []; + try { + const operations = [append(1), append(2)]; + const receipts = await Promise.all(operations.map(async (operation) => { + const receipt = await client.transactSemantic(operation, (events) => { + for (const event of events) { + expect(event.routing.executionId).toBe(operation.execution.executionId); + expect(observer.query("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(event.eventId)).toEqual({ event_id: event.eventId }); + expect(observer.query("SELECT operation_id FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(operation.execution.executionId, operation.operationId)).toEqual({ operation_id: operation.operationId }); + } + observed.push(`publication:${operation.operationId}`); + }); + observed.push(`receipt:${operation.operationId}`); + return receipt; + })); + expect(receipts.map((receipt) => receipt.kind)).toEqual(["committed", "committed"]); + for (const operation of operations) { + expect(observed.indexOf(`publication:${operation.operationId}`)).toBeGreaterThanOrEqual(0); + expect(observed.indexOf(`publication:${operation.operationId}`)).toBeLessThan(observed.indexOf(`receipt:${operation.operationId}`)); + } + expect(persistedAppends()).toHaveLength(2); + expect(observer.query("SELECT id, thread_id FROM canonical_agent_items WHERE id IN ('item-1-2', 'item-2-2') ORDER BY id").all()) + .toEqual([{ id: "item-1-2", thread_id: "thread-1" }, { id: "item-2-2", thread_id: "thread-2" }]); + } finally { + await client.close(); + } + }); + + it("keeps a concurrent worker peer usable when one execution's database write fails", async () => { + vi.restoreAllMocks(); + db.exec(`CREATE TRIGGER reject_first_append BEFORE INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:append-events' AND NEW.execution_id = '${append(1).execution.executionId}' + BEGIN SELECT RAISE(ABORT, 'execution write failed'); END;`); + const client = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const seen: string[] = []; + try { + const results = await Promise.allSettled([1, 2].map((index) => client.transactSemantic(append(index), + (events) => seen.push(...events.map((event) => event.routing.executionId))))); + expect(results[0]).toMatchObject({ status: "rejected" }); + expect(results[1]).toMatchObject({ status: "fulfilled", value: { kind: "committed" } }); + expect(seen).not.toContain(append(1).execution.executionId); + expect(seen).toContain(append(2).execution.executionId); + expect(persistedAppends()).toEqual([{ operation_id: "lease-2:2" }]); + } finally { + await client.close(); + } + }); + + it("rejects an unserializable queued append without losing a healthy worker peer", async () => { + vi.restoreAllMocks(); + const invalid = append(1); + if (invalid.mutation.kind !== "append-events") throw new Error("Expected append"); + const operation = { ...invalid, mutation: { ...invalid.mutation, nativeCursor: 1n } }; + const client = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + try { + const results = await Promise.allSettled([ + client.transactSemantic(operation, () => {}), client.transactSemantic(append(2), () => {}), + ]); + expect(results[0]).toMatchObject({ status: "rejected" }); + expect(results[1]).toMatchObject({ status: "fulfilled", value: { kind: "committed" } }); + expect(persistedAppends()).toEqual([{ operation_id: "lease-2:2" }]); + } finally { + await client.close(); + } + }); +}); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts new file mode 100644 index 000000000..81e6ba19c --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts @@ -0,0 +1,167 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import type { AgentEvent, ProviderId } from "@mcode/contracts"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { CodexLiveEventReducer } from "../../execution/codex-live-event-reducer.js"; +import { CanonicalAgentBoundary } from "../canonical-agent-boundary.js"; +import { CanonicalCodexSystemErrorProjection } from "../canonical-codex-system-error-projection.js"; +import { CanonicalParentTurnWrite } from "../canonical-parent-turn-write.js"; + +const THREAD_ID = "thread-system-projection"; +const TURN_ID = "turn-system-projection"; +const EXECUTION_ID = "11111111-1111-4111-8111-111111111111"; +const NOW = "2026-09-24T10:00:00.000Z"; +const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + +function seedThread(db: Database, providerId: ProviderId = "codex"): void { + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-system-projection", "Workspace", "C:/fixture", NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "workspace-system-projection", "Thread", "main", providerId, NOW, NOW); + const turns = new CanonicalParentTurnWrite(db, () => {}); + expect(turns.start({ + thread: { id: THREAD_ID, workspaceId: "workspace-system-projection", providerId, createdAt: NOW }, + turnId: TURN_ID, + executionId: EXECUTION_ID, + permissionMode: "supervised", + providerIdentities: [], + userMessage: { kind: "create", messageId: "user-system-projection", content: "Question", sequence: 1 }, + }).outcome).toBe("committed"); +} + +function boundSystem(subtype: string, extra: Partial> = {}): Extract { + return { type: "system", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, subtype, ...extra }; +} + +describe("CanonicalCodexSystemErrorProjection", () => { + let directory: string; + let path: string; + let db: Database; + let projection: CanonicalCodexSystemErrorProjection; + let reducer: CodexLiveEventReducer; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-codex-system-")); + path = NodePath.join(directory, "app.sqlite"); + db = openDatabase({ dbPath: path }); + seedThread(db); + projection = new CanonicalCodexSystemErrorProjection(db); + reducer = new CodexLiveEventReducer(execution); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + it("persists turn-scoped notices with publication identity and rejects an unbound session event", () => { + const session = boundSystem("provider.session.started", { + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "session", sessionId: "session-1" }, + }); + const unbound = reducer.reduce({ ...session, turnExecutionId: undefined }); + expect(unbound).toMatchObject({ kind: "unsupported", eventType: "system" }); + expect(() => projection.project(unbound)).toThrow("Unsupported Codex event"); + expect(db.prepare("SELECT current_notice_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ current_notice_session_id: null }); + + const sessionResult = projection.project(reducer.reduce(session)); + expect(sessionResult).toMatchObject({ kind: "system", event: { subtype: "provider.session.started" } }); + expect(db.prepare("SELECT current_notice_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ current_notice_session_id: "session-1" }); + expect(reducer.reduce({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }).kind).toBe("reduced"); + + const notice = boundSystem("provider.notice.unknown-event", { + message: "Codex reported an update.", + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "session", + sessionId: "session-1", noticeKey: "notice-1" }, + }); + const reduction = reducer.reduce(notice); + const first = projection.project(reduction); + expect(first.kind).toBe("system"); + if (first.kind !== "system") return; + expect(first.event.messageId).toBeDefined(); + if (!first.event.messageId) return; + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, first.event.messageId)) + .toMatchObject({ role: "system", content: "Codex reported an update." }); + expect(projection.project(reduction)).toEqual(first); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND content = ?") + .get(THREAD_ID, notice.message)).toEqual({ count: 1 }); + expect(structuredClone(first)).toEqual(first); + + db.close(true); + db = openDatabase({ dbPath: path }); + expect(db.prepare("SELECT current_notice_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ current_notice_session_id: "session-1" }); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, first.event.messageId)).not.toBeNull(); + }); + + it("persists a Codex cursor with canonical provenance and clears an invalidated cursor", () => { + const unbound = reducer.reduce({ ...boundSystem("sdk_session_id:session-owned"), turnExecutionId: undefined }); + expect(unbound).toMatchObject({ kind: "unsupported", eventType: "system" }); + expect(() => projection.project(unbound)).toThrow("Unsupported Codex event"); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + + expect(reducer.reduce({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }).kind).toBe("reduced"); + const saved = projection.project(reducer.reduce(boundSystem("sdk_session_id:native-thread"))); + expect(saved).toMatchObject({ kind: "system", event: { subtype: "sdk_session_id:native-thread" } }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: "native-thread" }); + expect(new CanonicalAgentBoundary(db, () => {}).loadCheckpoint(EXECUTION_ID)) + .toMatchObject({ nativeCursor: { providerId: "codex", scope: "thread", value: "native-thread", provenance: "native" } }); + expect(projection.project(reducer.reduce(boundSystem("sdk_session_invalidated")))).toMatchObject({ kind: "system" }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET terminal_outcome = 'completed' WHERE execution_id = ?") + .run(EXECUTION_ID); + expect(() => projection.project(reducer.reduce(boundSystem("sdk_session_id:late-thread")))) + .toThrow("could not be persisted"); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + }); + + it.each(["claude", "cursor"] as const)("persists a %s session cursor with its provider identity", (providerId) => { + const providerDb = openDatabase({ dbPath: NodePath.join(directory, `${providerId}.sqlite`) }); + try { + seedThread(providerDb, providerId); + const providerProjection = new CanonicalCodexSystemErrorProjection(providerDb); + const cursor = boundSystem("sdk_session_id:native-session"); + expect(providerProjection.projectBoundSystem(execution, providerId, cursor, + [{ kind: "session-cursor", event: cursor }], "writer")) + .toMatchObject({ kind: "system", event: { subtype: "sdk_session_id:native-session" } }); + expect(providerDb.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: "native-session" }); + expect(new CanonicalAgentBoundary(providerDb, () => {}).loadCheckpoint(EXECUTION_ID)) + .toMatchObject({ nativeCursor: { providerId, scope: "session", value: "native-session", provenance: "native" } }); + } finally { + providerDb.close(true); + } + }); + + it("returns cloneable error terminal input without publishing or terminalizing early", () => { + expect(reducer.reduce({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }).kind).toBe("reduced"); + expect(reducer.reduce({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + delta: "Partial answer", isFinalResponse: true }).kind).toBe("reduced"); + const reduction = reducer.reduce({ type: "error", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, error: "Provider failed" }); + expect(() => projection.project(reduction)).toThrow("end time"); + const result = projection.project(reduction, NOW); + expect(result).toMatchObject({ kind: "error-terminal", event: { error: "Provider failed" }, after: "terminal", + input: { threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "errored", assistant: { content: "Partial answer" } } }); + expect(structuredClone(result)).toEqual(result); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + if (result.kind !== "error-terminal") return; + const staged = new CanonicalParentTurnWrite(db, () => {}).stageTerminalProjection(result.input); + expect(staged.messageId).toBeDefined(); + if (!staged.messageId) return; + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, staged.messageId)) + .toMatchObject({ content: "Partial answer", is_internal: true }); + }); +}); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts new file mode 100644 index 000000000..7d007df48 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -0,0 +1,1645 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { AgentEventType, type AgentEvent, type ProviderRuntimeExtension } from "@mcode/contracts"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { TaskRepo } from "../../orchestration/persistence/task-repo.js"; +import { CodexLiveEventReducer } from "../../execution/codex-live-event-reducer.js"; +import type { ExecutionSemanticOperation, ExecutionWorkCommand } from "../../execution/execution-worker-handler.js"; +import { ExecutionWorkerHandler } from "../../execution/execution-worker-handler.js"; +import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; +import { CodexParentMessageProjection } from "../../turns/codex-parent-message-projection.js"; +import { NarrativeRecoveryDelta } from "../../turns/narrative-recovery-delta.js"; +import { CanonicalAgentBoundary } from "../canonical-agent-boundary.js"; +import type { CodexSystemWriterIntent } from "../canonical-codex-system-error-projection.js"; +import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; +import { ExecutionLivePublicationRelease } from "../execution-live-publication-release.js"; +import type { DataOnlyParentTurnStartInput } from "../canonical-parent-turn-write.js"; + +const THREAD_ID = "thread-1"; +const TURN_ID = "turn-1"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; +const NOW = "2026-09-24T10:00:00.000Z"; +const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID } as const; +const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "lease-1" } as const; + +function seedThread(db: Database): void { + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-1", "Workspace", "C:/fixture", NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "workspace-1", "Thread", "main", "codex", NOW, NOW); +} + +function startInput(): DataOnlyParentTurnStartInput { + return { + thread: { id: THREAD_ID, workspaceId: "workspace-1", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, + executionId: EXECUTION_ID, + permissionMode: "supervised", + providerIdentities: [], + userMessage: { kind: "create", messageId: "user-1", content: "Question", sequence: 1 }, + }; +} + +function event() { + return { + eventId: `${EXECUTION_ID}:item-1`, + routing: { ...execution, itemId: "item-1" }, + sourceProviderId: "codex", + sourceIdentities: [], + payload: { + type: "item.recorded" as const, + item: { + id: "item-1", + threadId: THREAD_ID, + turnId: TURN_ID, + kind: "message" as const, + providerIdentities: [], + payload: { projection: "message", content: "Event" }, + createdAt: NOW, + updatedAt: NOW, + }, + }, + }; +} + +function runtimeEvent(sequence: number, agentEvent: AgentEvent, extension?: ProviderRuntimeExtension): Extract< + ExecutionSemanticOperation["mutation"], { kind: "append-events" } +>["events"][number] { + const itemId = `runtime-${sequence}`; + return { + eventId: `${EXECUTION_ID}:${itemId}`, + routing: { ...execution, itemId }, + sourceProviderId: "codex", + sourceIdentities: [], + sourceSequence: sequence, + payload: { type: "item.recorded", item: { + id: itemId, threadId: THREAD_ID, turnId: TURN_ID, kind: "system", + providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent: { + event: agentEvent, ...(extension ? { extension } : {}), + } }, + createdAt: NOW, updatedAt: NOW, + } }, + }; +} + +function toolNarrative(messageId: string, count: number) { + return Array.from({ length: count }, (_, index) => ({ + kind: "toolCall" as const, + sequence: 2, + sortOrder: index, + record: { + id: `tool-${index}`, + message_id: messageId, + parent_tool_call_id: null, + tool_name: "Read", + input_summary: `file-${index}`, + output_summary: "ok", + status: "completed" as const, + started_at: NOW, + completed_at: NOW, + sort_order: index, + }, + })); +} + +function operation(ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation { + return { operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation }; +} + +function systemLiveOperation(ordinal: number, systemEvent: Extract): ExecutionSemanticOperation { + const reduction = new CodexLiveEventReducer(execution).reduce(systemEvent); + if (reduction.kind !== "reduced" || reduction.publication.after !== "writer") throw new Error("Expected a bound live system event"); + const systemIntents = reduction.writer.filter((intent): intent is CodexSystemWriterIntent => + intent.kind === "notice-session" || intent.kind === "system-notice" || intent.kind === "session-cursor"); + if (systemIntents.length !== reduction.writer.length) throw new Error("Unexpected system reducer intent"); + return { + ...operation(ordinal, { kind: "live-event", text: { kind: "unchanged" }, systemIntents }), + livePublication: [reduction.publication], + }; +} + +function parentTextOperation(text = "Durable parent text"): ExecutionSemanticOperation { + const agentEvent: AgentEvent = { + type: AgentEventType.TextDelta, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, delta: text, + }; + return { + ...operation(2, { + kind: "append-events", phase: "running", nativeCursor: null, + events: [runtimeEvent(2, agentEvent)], + parentLive: { text: { kind: "append", inputs: [{ ...execution, sequence: 1, text }] } }, + }), + livePublication: [{ after: "writer", event: agentEvent }], + }; +} + +function finishLiveCommand(outcome: "completed" | "errored" | "cancelled" = "completed"): + Extract { + const identity = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const terminal: AgentEvent = outcome === "completed" + ? { ...identity, type: "turnComplete", reason: "end_turn", costUsd: null, tokensIn: 20, tokensOut: 5 } + : outcome === "errored" ? { ...identity, type: "error", error: "Provider failed" } + : { ...identity, type: "ended", outcome: "interrupted" }; + return { + kind: "finish-live-event", outcome, + projection: { threadId: THREAD_ID, executionId: EXECUTION_ID, outcome, endedAt: NOW, + assistant: { content: "Final answer", model: null, attachments: [] }, narrative: [] }, + input: { ...execution, providerId: "codex", providerIdentities: [], outcome, + ...(outcome === "errored" ? { error: "Provider failed" } : {}), projection: { kind: "writer-staged" } }, + ...(outcome === "cancelled" ? {} : { providerEvent: { + phase: "running", nativeCursor: null, events: [runtimeEvent(2, terminal)], + } }), + livePublication: [{ after: "terminal", event: terminal }], + }; +} + +describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () => { + let directory: string; + let path: string; + let db: Database; + let published: string[]; + let writer: CanonicalExecutionSemanticWriter; + let handler: ExecutionWorkerHandler; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-semantic-write-")); + path = NodePath.join(directory, "mcode.db"); + db = openDatabase({ dbPath: path }); + seedThread(db); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + async function send(ordinal: number, command: Parameters[0]["command"]) { + return (await handler.handle({ requestId: ordinal, execution, lease, ordinal, command })).result; + } + + const loss = { + execution, lease, + reason: "The execution worker exited before its provider turn could be proved live.", + recoveryIncidentId: "incident-1", + }; + + it("persists raw late hooks after terminal and releases their committed message identity", async () => { + const operations: ExecutionSemanticOperation[] = []; + handler = new ExecutionWorkerHandler({ transact: async (input) => { + operations.push(input); + return writer.transact(input); + } }); + expect(await send(1, { kind: "start", providerId: "codex", input: startInput(), + parentLive: { planFeature: "none", precedingMessageId: "user-1" } })).toMatchObject({ kind: "committed" }); + const raw = (ordinal: number, event: AgentEvent) => send(ordinal, { kind: "event", phase: "running", + nativeCursor: null, events: [runtimeEvent(ordinal, event)] }); + const bound = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const started = await raw(2, { ...bound, type: "turnStarted" }); + expect(started.kind, JSON.stringify(started)).toBe("committed"); + expect((await raw(3, { ...bound, type: "textDelta", delta: "Final answer", isFinalResponse: true })).kind).toBe("committed"); + const finish = finishLiveCommand(); + const terminal = await send(4, { kind: "event", phase: "running", nativeCursor: null, + events: [runtimeEvent(4, { ...bound, type: "turnComplete", reason: "end_turn", costUsd: null, tokensIn: 20, tokensOut: 5 })], + terminalInput: finish.input }); + expect(terminal.kind, JSON.stringify(terminal)).toBe("committed"); + expect((await raw(5, { ...bound, type: "hookStarted", hookName: "Save", hookType: "stop" })).kind).toBe("committed"); + const completed = await raw(6, { ...bound, type: "hookCompleted", hookName: "Save", exitCode: 0, durationMs: 12, didBlock: false }); + expect(completed).toMatchObject({ kind: "committed", providerCommit: { eventCount: 1 }, providerEvents: [], + livePublication: [{ after: "terminal", event: { type: "hookCompleted", persistedMessageId: expect.any(String), persistedHookId: expect.any(String) } }] }); + const completedOperation = operations[5]; + if (!completedOperation) throw new Error("Expected hook operation"); + const receipt = await writer.transact(completedOperation); + const released: AgentEvent[] = []; + new ExecutionLivePublicationRelease({ isBound: () => true, publish: (event) => released.push(event) }).release(completedOperation, receipt); + expect(released).toEqual([expect.objectContaining({ type: "hookCompleted", persistedHookId: expect.any(String), persistedMessageId: expect.any(String) })]); + const canonical = new CanonicalAgentBoundary(db); + const message = canonical.loadTerminalProjection(TURN_ID).message; + expect(db.prepare("SELECT message_id, hook_name, duration_ms FROM hook_executions").all()) + .toEqual([{ message_id: message?.id, hook_name: "Save", duration_ms: 12 }]); + expect(JSON.stringify(canonical.loadConversationProjection(THREAD_ID, 10))).toContain('"duration_ms":12'); + expect(canonical.loadCheckpoint(EXECUTION_ID)?.terminalOutcome).toBe("completed"); + expect((await raw(7, { ...bound, type: "ended", outcome: "completed" })).kind).toBe("committed"); + expect(await raw(8, { ...bound, type: "hookProgress", hookName: "Save", output: "too late" })) + .toEqual({ kind: "rejected", reason: "invalid-event-routing" }); + }); + + it("atomically records and releases a late system notice without changing terminal outcome", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + await send(2, finishLiveCommand()); + const notice: Extract = { type: "system", threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, subtype: "provider.notice.unknown-event", message: "Late provider notice", + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "turn", noticeKey: "late-notice" } }; + const input: ExecutionSemanticOperation = { ...operation(3, { kind: "post-terminal-event", + systemIntents: [{ kind: "system-notice", event: notice }], + providerEvent: { phase: "running", nativeCursor: null, events: [runtimeEvent(3, notice)] } }), + livePublication: [{ after: "terminal", event: notice }] }; + db.run("CREATE TRIGGER fail_late_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:post-terminal-event' BEGIN SELECT RAISE(ABORT, 'late receipt unavailable'); END"); + await expect(writer.transact(input)).rejects.toThrow("late receipt unavailable"); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.content === notice.message)).toBe(false); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?").get(`${EXECUTION_ID}:runtime-3`)).toBeNull(); + db.run("DROP TRIGGER fail_late_receipt"); + const receipt = await writer.transact(input); + if (receipt.kind !== "committed") throw new Error("Expected late receipt"); + const released: AgentEvent[] = []; + new ExecutionLivePublicationRelease({ isBound: () => true, publish: (event) => released.push(event) }).release(input, receipt); + expect(released).toEqual([expect.objectContaining({ ...notice, messageId: expect.any(String) })]); + expect(await writer.transact(input)).toEqual(receipt); + expect(new CanonicalAgentBoundary(db).loadCheckpoint(EXECUTION_ID)?.terminalOutcome).toBe("completed"); + expect(db.prepare("SELECT status FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ status: "completed" }); + }); + + it("rejects late content, cursor changes, wrong outcome and stale leases without publishing", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + await send(2, finishLiveCommand()); + published.length = 0; + const bound = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const events: AgentEvent[] = [ + { ...bound, type: "textDelta", delta: "Must not append" }, + { ...bound, type: "toolUse", toolCallId: "late-tool", toolName: "Read", toolInput: {} }, + { ...bound, type: "system", subtype: "sdk_session_id:late-session" }, + { ...bound, type: "system", subtype: "sdk_session_invalidated" }, + { ...bound, type: "ended", outcome: "errored" }, + ]; + for (const event of events) { + expect(await writer.transact({ ...operation(3, { kind: "post-terminal-event" }), + livePublication: [{ after: "terminal", event }] })).toEqual({ kind: "conflict", operationId: "lease-1:3" }); + } + const input: ExecutionSemanticOperation = { ...operation(3, { kind: "post-terminal-event" }), + livePublication: [{ after: "terminal", event: { ...bound, type: "ended", outcome: "completed" } }] }; + expect((await writer.transact({ ...input, lease: { ...lease, workerGeneration: 2 } })).kind).toBe("conflict"); + expect((await writer.transact({ ...input, ordinal: 4 })).kind).toBe("conflict"); + expect(published).toEqual([]); + expect(new CanonicalAgentBoundary(db).loadCheckpoint(EXECUTION_ID)?.terminalOutcome).toBe("completed"); + expect((await writer.transact(input)).kind).toBe("committed"); + }); + + it.each(["completed", "errored"] as const)("commits a %s provider terminal event with its final projection in one command", async (outcome) => { + const sequences: number[] = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + for (const item of events) { + published.push(item.eventId); + if (item.eventId === `${EXECUTION_ID}:runtime-2`) { + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: outcome }); + } + sequences.push(item.acceptedSequence); + } + }); + handler = new ExecutionWorkerHandler(writer); + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + sequences.length = 0; + const command = finishLiveCommand(outcome); + const receipt = await send(2, command); + expect(receipt).toMatchObject({ kind: "committed", providerCommit: { outcome: "committed", eventCount: 1 }, + providerEvents: [], livePublication: [{ publicationId: "1", after: "terminal" }] }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).filter((message) => !message.is_internal)).toEqual(expect.arrayContaining([ + expect.objectContaining({ role: "assistant", content: "Final answer", outcome }), + ])); + expect(sequences).toEqual([...sequences].sort((left, right) => left - right)); + expect(published).toContain(`${EXECUTION_ID}:runtime-2`); + expect(await send(3, { kind: "release" })).toEqual({ kind: "released" }); + const { livePublication, ...mutation } = command; + if (receipt.kind !== "committed") throw new Error("Expected a terminal receipt"); + const released: AgentEvent[] = []; + const release = new ExecutionLivePublicationRelease({ isBound: () => true, publish: (event) => released.push(event) }); + release.release({ ...operation(2, mutation), livePublication }, receipt); + expect(released).toEqual([expect.objectContaining({ type: outcome === "completed" ? "turnComplete" : "error", + publicationId: "1" })]); + published.length = 0; + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((event) => event.eventId))); + expect(await writer.transact({ ...operation(2, mutation), livePublication })).toEqual(receipt); + expect(published).toEqual([]); + }); + + it("settles Stop through one synthetic terminal command without a provider draft", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const stop = await handler.handle({ requestId: 2, execution, lease, ordinal: 2, stopWatermark: 1, + command: { kind: "stop", requestId: "stop-1" } }); + expect(stop.result.kind).toBe("committed"); + expect(await send(3, finishLiveCommand("cancelled"))).toMatchObject({ kind: "committed", + livePublication: [{ event: { type: "ended", outcome: "interrupted" } }] }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "cancelled" }); + expect(db.prepare("SELECT status FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ status: "interrupted" }); + }); + + it.each(["codex", "claude"] as const)("keeps the latest %s session cursor when terminal input has stale identities", async (providerId) => { + db.prepare("UPDATE threads SET provider = ? WHERE id = ?").run(providerId, THREAD_ID); + const start = startInput(); + await send(1, { kind: "start", providerId, input: { ...start, thread: { ...start.thread, providerId } } }); + const cursor: Extract = { type: "system", threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, subtype: "sdk_session_id:latest-session" }; + expect(await send(2, { kind: "live-event", text: { kind: "unchanged" }, + systemIntents: [{ kind: "session-cursor", event: cursor }], publication: { after: "writer", event: cursor }, + })).toMatchObject({ kind: "committed" }); + const command = finishLiveCommand(); + if (!command.providerEvent) throw new Error("Expected terminal provider evidence"); + const scope = providerId === "codex" ? "thread" : "session"; + expect(await send(3, { ...command, + input: { ...command.input, providerId, + providerIdentities: [{ providerId, scope, value: "stale-session", provenance: "native" }] }, + providerEvent: { ...command.providerEvent, + events: command.providerEvent.events.map((event) => ({ ...event, sourceProviderId: providerId })) }, + })).toMatchObject({ kind: "committed" }); + const expected = { providerId, scope, value: "latest-session", provenance: "native" }; + const canonical = new CanonicalAgentBoundary(db); + expect(canonical.loadThread(THREAD_ID)?.providerIdentities).toEqual([expected]); + expect(canonical.loadCheckpoint(EXECUTION_ID)?.nativeCursor).toEqual(expected); + }); + + it("rolls back terminal provider evidence and staging when preparation cannot reserve its operation", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + db.run("CREATE TRIGGER fail_terminal_reservation BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish-pending' BEGIN SELECT RAISE(ABORT, 'reservation unavailable'); END"); + await expect(send(2, finishLiveCommand())).rejects.toThrow("reservation unavailable"); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(`${EXECUTION_ID}:runtime-2`)).toBeNull(); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.role === "assistant")).toBe(false); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + db.run("DROP TRIGGER fail_terminal_reservation"); + expect((await send(2, finishLiveCommand())).kind).toBe("committed"); + }); + + it("rejects compound terminal identity, outcome, and provider publication mismatches before finalization", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const { livePublication, ...mutation } = finishLiveCommand(); + const input = { ...operation(2, mutation), livePublication }; + const invalid = [ + { ...input, mutation: { ...mutation, projection: { ...mutation.projection, executionId: "other-execution" } } }, + { ...input, mutation: { ...mutation, input: { ...mutation.input, outcome: "errored" as const } } }, + { ...input, mutation: { ...mutation, providerEvent: { phase: "running", nativeCursor: null, events: [event()] } } }, + ]; + for (const candidate of invalid) { + expect(await writer.transact(candidate)).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + } + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toBeNull(); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.role === "assistant")).toBe(false); + expect(await writer.transact(input)).toMatchObject({ kind: "committed" }); + }); + + it("resumes a compound terminal operation after final receipt failure and database reopen", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + db.run("CREATE TRIGGER fail_compound_terminal_receipt BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish-live-event' BEGIN SELECT RAISE(ABORT, 'terminal receipt unavailable'); END"); + const command = finishLiveCommand(); + await expect(send(2, command)).rejects.toThrow("terminal receipt unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ kind: "semantic:finish-pending" }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.role === "assistant" && !message.is_internal)).toBe(false); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + db.run("DROP TRIGGER fail_compound_terminal_receipt"); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((event) => event.eventId))); + const { livePublication, ...mutation } = command; + const input = { ...operation(2, mutation), livePublication }; + expect(await writer.transact({ ...input, mutation: { ...mutation, + projection: { ...mutation.projection, assistant: { ...mutation.projection.assistant, content: "Changed retry" } }, + } })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(input)).toMatchObject({ kind: "committed", livePublication: [{ publicationId: "1" }] }); + expect(published.filter((id) => id === `${EXECUTION_ID}:runtime-2`)).toHaveLength(1); + }); + + it("retries only unacknowledged compound terminal publication chunks", async () => { + let failTerminalPublication = false; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + if (failTerminalPublication && events.some((event) => event.eventId !== `${EXECUTION_ID}:runtime-2`)) { + throw new Error("terminal publisher unavailable"); + } + published.push(...events.map((event) => event.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + failTerminalPublication = true; + const command = finishLiveCommand(); + await expect(send(2, command)).rejects.toThrow("terminal publisher unavailable"); + expect(published.filter((id) => id === `${EXECUTION_ID}:runtime-2`)).toHaveLength(1); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((event) => event.eventId))); + const { livePublication, ...mutation } = command; + expect(await writer.transact({ ...operation(2, mutation), livePublication })).toMatchObject({ kind: "committed" }); + expect(published.filter((id) => id === `${EXECUTION_ID}:runtime-2`)).toHaveLength(1); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + }); + + it("recovers a prepared compound terminal projection when its worker is lost", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + db.run("CREATE TRIGGER fail_terminal_before_loss BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish-live-event' BEGIN SELECT RAISE(ABORT, 'terminal receipt unavailable'); END"); + await expect(send(2, finishLiveCommand())).rejects.toThrow("terminal receipt unavailable"); + expect(writer.interruptWorkerLoss(loss)).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "interrupted" }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toEqual(expect.arrayContaining([ + expect.objectContaining({ role: "assistant", content: "Final answer", outcome: "interrupted", is_internal: false }), + ])); + db.run("DROP TRIGGER fail_terminal_before_loss"); + const { livePublication, ...mutation } = finishLiveCommand(); + expect(await writer.transact({ ...operation(2, mutation), livePublication })) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + }); + + it("commits a provider event and its parent effects with one replayable publication", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const input = parentTextOperation(); + const receipt = await writer.transact(input); + expect(receipt).toMatchObject({ kind: "committed", providerCommit: { eventCount: 1 }, providerEvents: [], + assistantTextCheckpoint: { outcome: "committed" }, + livePublication: [{ publicationId: "1", event: { delta: "Durable parent text" } }] }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Durable parent text"); + expect(published).toContain(`${EXECUTION_ID}:runtime-2`); + expect(await writer.transact(input)).toEqual(receipt); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Durable parent text"); + }); + + it("rolls back canonical append, parent effects, and publication when their receipt fails", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const input = parentTextOperation(); + db.run("CREATE TRIGGER fail_parent_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-events' BEGIN SELECT RAISE(ABORT, 'parent receipt unavailable'); END"); + await expect(writer.transact(input)).rejects.toThrow("parent receipt unavailable"); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(`${EXECUTION_ID}:runtime-2`)).toBeNull(); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + db.run("DROP TRIGGER fail_parent_receipt"); + expect(await writer.transact(input)).toMatchObject({ kind: "committed", providerEvents: [], + livePublication: [{ publicationId: "1" }] }); + }); + + it("rejects parent effects without the matching provider event or exact execution", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const input = parentTextOperation(); + if (input.mutation.kind !== "append-events") throw new Error("Expected provider append"); + const unrelated: AgentEvent = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "A different event" }; + expect(await writer.transact({ ...input, mutation: { ...input.mutation, events: [runtimeEvent(2, unrelated)] } })) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact({ ...input, mutation: { ...input.mutation, parentLive: { + text: { kind: "append", inputs: [{ ...execution, executionId: "wrong-execution", sequence: 1, text: "Durable parent text" }] }, + } } })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(`${EXECUTION_ID}:runtime-2`)).toBeNull(); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + }); + + it("replays a compound system notice with its writer-assigned message identity", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const notice: Extract = { + type: "system", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + subtype: "provider.notice.unknown-event", message: "Codex reported an update.", + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "turn", noticeKey: "compound-notice" }, + }; + const live = systemLiveOperation(2, notice); + if (live.mutation.kind !== "live-event") throw new Error("Expected live system effects"); + const input = { ...live, mutation: { + kind: "append-events" as const, phase: "running", nativeCursor: null, + events: [runtimeEvent(2, notice)], parentLive: { text: live.mutation.text, systemIntents: live.mutation.systemIntents }, + } }; + const receipt = await writer.transact(input); + expect(receipt).toMatchObject({ kind: "committed", providerEvents: [], livePublication: [ + { event: { ...notice, messageId: expect.any(String) } }, + ] }); + const released: AgentEvent[] = []; + new ExecutionLivePublicationRelease({ + isBound: () => true, + publish: (event) => { released.push(event); }, + }).release(input, receipt); + expect(released).toEqual([expect.objectContaining({ + ...notice, messageId: expect.any(String), publicationId: "1", + })]); + expect(await writer.transact(input)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'system'").get(THREAD_ID)) + .toEqual({ count: 1 }); + }); + + it("commits context and compaction projections before publishing their canonical events", async () => { + const seenAtPublication: { eventId: string; state: unknown }[] = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + for (const item of events) { + const state = db.prepare("SELECT last_context_tokens, last_compact_summary FROM threads WHERE id = ?").get(THREAD_ID); + seenAtPublication.push({ eventId: item.eventId, state }); + } + }); + handler = new ExecutionWorkerHandler(writer); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + async function append(ordinal: number, agentEvent: AgentEvent) { + return send(ordinal, { kind: "event", phase: "running", nativeCursor: null, events: [runtimeEvent(ordinal, agentEvent)] }); + } + const identity = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + expect((await append(2, { ...identity, type: "contextEstimate", tokensIn: 100, totalProcessedTokens: 120, contextWindow: 200_000 })).kind).toBe("committed"); + expect((await append(3, { ...identity, type: "compacting", active: true })).kind).toBe("committed"); + expect((await append(4, { ...identity, type: "contextEstimate", tokensIn: 90, totalProcessedTokens: 140 })).kind).toBe("committed"); + expect((await append(5, { ...identity, type: "compactSummary", summary: "Short context" })).kind).toBe("committed"); + expect((await append(6, { ...identity, type: "contextEstimate", tokensIn: 30, totalProcessedTokens: 150 })).kind).toBe("committed"); + expect((await append(7, { ...identity, type: "compacting", active: true })).kind).toBe("committed"); + expect((await append(8, { ...identity, type: "compacting", active: false })).kind).toBe("committed"); + expect((await append(9, { ...identity, type: "turnComplete", reason: "end_turn", costUsd: null, tokensIn: 40, tokensOut: 5 })).kind).toBe("committed"); + + expect(seenAtPublication.filter(({ eventId }) => eventId.includes(":runtime-"))).toEqual([ + { eventId: `${EXECUTION_ID}:runtime-2`, state: { last_context_tokens: 100, last_compact_summary: null } }, + { eventId: `${EXECUTION_ID}:runtime-3`, state: { last_context_tokens: 100, last_compact_summary: null } }, + { eventId: `${EXECUTION_ID}:runtime-4`, state: { last_context_tokens: 100, last_compact_summary: null } }, + { eventId: `${EXECUTION_ID}:runtime-5`, state: { last_context_tokens: 100, last_compact_summary: "Short context" } }, + { eventId: `${EXECUTION_ID}:runtime-6`, state: { last_context_tokens: 30, last_compact_summary: "Short context" } }, + { eventId: `${EXECUTION_ID}:runtime-7`, state: { last_context_tokens: 30, last_compact_summary: "Short context" } }, + { eventId: `${EXECUTION_ID}:runtime-8`, state: { last_context_tokens: 30, last_compact_summary: "Short context" } }, + { eventId: `${EXECUTION_ID}:runtime-9`, state: { last_context_tokens: 40, last_compact_summary: "Short context" } }, + ]); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ role: "system", content: "Context compacted" })); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + const dividerCount = db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND content = 'Context compacted'") + .get(THREAD_ID); + expect(await writer.transact(operation(8, { kind: "append-events", phase: "running", nativeCursor: null, + events: [runtimeEvent(8, { ...identity, type: "compacting", active: false })] }))).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND content = 'Context compacted'") + .get(THREAD_ID)).toEqual(dividerCount); + }); + + it("rolls back context projections and rejects completion during compaction", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const identity = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + db.run("CREATE TRIGGER fail_context_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-events' BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END"); + await expect(send(2, { kind: "event", phase: "running", nativeCursor: null, events: [ + runtimeEvent(2, { ...identity, type: "contextEstimate", tokensIn: 80, totalProcessedTokens: 90 }), + ] })).rejects.toThrow("receipt unavailable"); + expect(db.prepare("SELECT last_context_tokens FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ last_context_tokens: null }); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + db.run("DROP TRIGGER fail_context_receipt"); + expect((await send(2, { kind: "event", phase: "running", nativeCursor: null, events: [ + runtimeEvent(2, { ...identity, type: "compacting", active: true }), + ] })).kind).toBe("committed"); + expect(await send(3, { kind: "event", phase: "running", nativeCursor: null, events: [ + runtimeEvent(3, { ...identity, type: "turnComplete", reason: "end_turn", costUsd: null, tokensIn: 50, tokensOut: 1 }), + ] })).toEqual({ kind: "rejected", reason: "writer-conflict" }); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?").get(`${EXECUTION_ID}:runtime-3`)).toBeNull(); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-3`); + expect((await send(3, { kind: "event", phase: "running", nativeCursor: null, events: [ + runtimeEvent(3, { ...identity, type: "compactSummary", summary: "Recovered" }), + ] })).kind).toBe("committed"); + }); + + it("fences a lost worker while retaining durable text and narrative", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const text = new ParentAssistantTextCheckpointService(db); + expect(text.appendChunk([{ ...execution, sequence: 1, text: "Partial answer" }]).outcome).toBe("committed"); + text.recoveryJournal.append([{ ...execution, sequence: 2, text: " from journal" }]); + const { CanonicalAgentBoundary } = await import("../canonical-agent-boundary.js"); + const canonical = new CanonicalAgentBoundary(db, () => {}); + expect(canonical.recordParentNarrativeRecovery({ + executionId: EXECUTION_ID, items: toolNarrative("", 1), + })).toBe(true); + + const receipt = writer.interruptWorkerLoss(loss); + expect(receipt).toMatchObject({ kind: "committed", operationId: "lease-1:worker-lost" }); + expect(canonical.loadTurn(TURN_ID)).toMatchObject({ status: "Interrupted" }); + expect(canonical.loadCheckpoint(EXECUTION_ID)).toMatchObject({ phase: "interrupted", terminalOutcome: "interrupted" }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Partial answer from journal", outcome: "interrupted", is_internal: false, + })); + expect(db.prepare("SELECT id, status, message_id FROM tool_call_records WHERE id = ?").get("tool-0")) + .toMatchObject({ id: "tool-0", status: "completed", + message_id: canonical.loadTerminalProjection(TURN_ID).message?.id }); + expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); + expect(await writer.transact(operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }))) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((item) => item.eventId))); + expect(writer.interruptWorkerLoss(loss)).toEqual(receipt); + expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); + }); + + it("keeps an assistant-text receipt and text across writer restart and worker loss", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const inputs = [{ ...execution, sequence: 1, text: "Durable partial answer" }]; + const textOperation = operation(2, { kind: "append-assistant-text", inputs }); + const appended = await send(2, { kind: "assistant-text", inputs }); + expect(appended).toMatchObject({ + kind: "committed", operationId: "lease-1:2", + assistantTextCheckpoint: { outcome: "committed", durableThrough: 1, committedItems: 1 }, + }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Durable partial answer"); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(textOperation)).toEqual(appended); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Durable partial answer"); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoint_chunks WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 1 }); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Durable partial answer", outcome: "interrupted", + })); + }); + + it("cancels after a text checkpoint and retires it only after terminal commit", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect((await send(2, { kind: "assistant-text", inputs: [ + { ...execution, sequence: 1, text: "Saved before cancellation" }, + ] })).kind).toBe("committed"); + expect((await handler.handle({ requestId: 3, execution, lease, ordinal: 3, stopWatermark: 2, + command: { kind: "stop", requestId: "cancel-1" } })).result.kind).toBe("committed"); + expect((await send(4, { kind: "provider-outcome", outcome: "cancelled" })).kind).toBe("committed"); + expect((await send(5, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "cancelled", endedAt: NOW, + assistant: { content: "Saved before cancellation", model: null, attachments: [] }, narrative: [], + } })).kind).toBe("committed"); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Saved before cancellation"); + const finish = operation(6, { kind: "finish", outcome: "cancelled", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex", providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" }, + } }); + db.run("CREATE TRIGGER fail_text_retirement BEFORE DELETE ON parent_assistant_text_checkpoints BEGIN SELECT RAISE(ABORT, 'retirement unavailable'); END"); + await expect(writer.transact(finish)).rejects.toThrow("retirement unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "cancelled" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 1 }); + db.run("DROP TRIGGER fail_text_retirement"); + expect((await writer.transact(finish)).kind).toBe("committed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Saved before cancellation", outcome: "cancelled", is_internal: false, + })); + expect((await writer.transact(finish)).kind).toBe("committed"); + }); + + it("publishes the assigned live assistant ID only when finish names the staged ID", async () => { + const assignedId = "a".repeat(64); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect((await send(2, { kind: "provider-outcome", outcome: "completed" })).kind).toBe("committed"); + expect((await send(3, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, + assistant: { content: "Live answer", model: null, attachments: [], messageId: assignedId }, narrative: [], + } })).kind).toBe("committed"); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + const finishInput = { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex" as const, providerIdentities: [], outcome: "completed" as const, + projection: { kind: "writer-staged" as const, messageId: assignedId }, + }; + expect(await writer.transact(operation(4, { kind: "finish", outcome: "completed", + input: { ...finishInput, projection: { kind: "writer-staged" } } }))) + .toEqual({ kind: "conflict", operationId: "lease-1:4" }); + expect(await writer.transact(operation(4, { kind: "finish", outcome: "completed", + input: { ...finishInput, projection: { kind: "writer-staged", messageId: "b".repeat(64) } } }))) + .toEqual({ kind: "conflict", operationId: "lease-1:4" }); + expect((await writer.transact(operation(4, { kind: "finish", outcome: "completed", input: finishInput }))).kind) + .toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, assignedId)) + .toMatchObject({ id: assignedId, content: "Live answer", is_internal: false }); + }); + + it("stages a projected Codex message with its receipt before live publication and reuses it at finish", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const projection = new CodexParentMessageProjection({ execution, turnKind: "ordinary" }); + const projected = projection.projectMessage({ + execution, content: "Live answer", model: null, precedingMessageId: "user-1", postTurnGoalReceipt: false, + }); + const message = { + precedingMessageId: "user-1", messageId: projected.messageId, + content: "Live answer", model: projected.model, attachments: projected.attachments ?? [], + }; + const event = { type: AgentEventType.Message, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + content: message.content, tokens: null, messageId: message.messageId, model: message.model }; + const liveOperation = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" }, message }), + livePublication: [{ after: "writer" as const, event }] }; + + expect(await writer.transact({ ...liveOperation, livePublication: [{ after: "writer", + event: { ...event, content: "Different public body" } }] })) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)).toBeNull(); + + db.run("CREATE TRIGGER fail_message_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' AND NEW.operation_id = 'lease-1:2' BEGIN SELECT RAISE(ABORT, 'message receipt unavailable'); END"); + await expect(writer.transact(liveOperation)).rejects.toThrow("message receipt unavailable"); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)).toBeNull(); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 0 }); + db.run("DROP TRIGGER fail_message_receipt"); + + const committedMessage = await send(2, { + kind: "live-event", text: { kind: "unchanged" }, message, + publication: { after: "writer", event }, + }); + expect(committedMessage).toMatchObject({ kind: "committed", livePublication: [{ + publicationId: "1", event, + }] }); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)) + .toMatchObject({ id: message.messageId, content: message.content, model: null, is_internal: true }); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(liveOperation)).toEqual(committedMessage); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get(message.messageId)) + .toEqual({ count: 1 }); + expect(await writer.transact({ ...liveOperation, mutation: { kind: "live-event", text: { kind: "unchanged" }, + message: { ...message, content: "Different body" } }, + livePublication: [{ after: "writer", event: { ...event, content: "Different body" } }] })) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact({ ...liveOperation, ordinal: 3, operationId: "lease-1:3", + mutation: { kind: "live-event", text: { kind: "unchanged" }, message: { ...message, content: "Different body" } }, + livePublication: [{ after: "writer", event: { ...event, content: "Different body" } }], + })).toEqual({ kind: "conflict", operationId: "lease-1:3" }); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)) + .toMatchObject({ content: "Live answer" }); + + expect((await writer.transact(operation(3, { kind: "provider-outcome", outcome: "completed" }))).kind) + .toBe("committed"); + const terminal = projection.projectTerminal({ + execution, outcome: "completed", endedAt: NOW, fallbackModel: null, narrative: [], + }); + expect(await writer.transact(operation(4, { kind: "stage-terminal", input: { + ...terminal, assistant: { ...terminal.assistant, messageId: "b".repeat(64) }, + } }))).toEqual({ kind: "conflict", operationId: "lease-1:4" }); + expect((await writer.transact(operation(4, { kind: "stage-terminal", input: terminal }))).kind) + .toBe("committed"); + expect((await writer.transact(operation(5, { kind: "finish", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex", providerIdentities: [], outcome: "completed", + projection: { kind: "writer-staged", messageId: message.messageId }, + } }))).kind).toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, message.messageId)) + .toMatchObject({ id: message.messageId, content: "Live answer", outcome: "completed", is_internal: false }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get(message.messageId)) + .toEqual({ count: 1 }); + }); + + it("uses a live message row with its assigned ID when the worker is lost", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const projection = new CodexParentMessageProjection({ execution, turnKind: "ordinary" }); + const projected = projection.projectMessage({ + execution, content: "Durable body", model: null, precedingMessageId: "user-1", postTurnGoalReceipt: false, + }); + const message = { + precedingMessageId: "user-1", messageId: projected.messageId, + content: "Durable body", model: projected.model, attachments: projected.attachments ?? [], + }; + expect((await send(2, { kind: "live-event", text: { kind: "unchanged" }, message, + publication: { after: "writer", event: { type: AgentEventType.Message, + threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, content: message.content, + tokens: null, messageId: message.messageId, model: message.model } }, + })).kind).toBe("committed"); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, message.messageId)) + .toMatchObject({ content: "Durable body", outcome: "interrupted", is_internal: false }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'assistant'") + .get(THREAD_ID)).toEqual({ count: 1 }); + }); + + it("fences assistant-text routing, lease, ordinal, input size, and conflicting replay", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const inputs = [{ ...execution, sequence: 1, text: "Saved text" }]; + const appendText = operation(2, { kind: "append-assistant-text", inputs }); + expect(await writer.transact({ ...appendText, lease: { ...lease, ownerEpoch: 2, leaseId: "lease-2" }, + operationId: "lease-2:2" })).toEqual({ kind: "conflict", operationId: "lease-2:2" }); + expect(await writer.transact({ ...appendText, ordinal: 3, operationId: "lease-1:3" })) + .toEqual({ kind: "conflict", operationId: "lease-1:3" }); + expect(await writer.transact(operation(2, { kind: "append-assistant-text", + inputs: [{ ...inputs[0]!, threadId: "wrong-thread" }], + }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(operation(2, { kind: "append-assistant-text", + inputs: [{ ...inputs[0]!, text: "x".repeat(16 * 1024 + 1) }], + }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(appendText)).toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(await writer.transact(operation(2, { kind: "append-assistant-text", + inputs: [{ ...inputs[0]!, text: "Changed text" }], + }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(operation(3, { kind: "append-assistant-text", inputs }))) + .toEqual({ kind: "conflict", operationId: "lease-1:3" }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Saved text"); + }); + + it("rolls back assistant text if its semantic receipt cannot commit", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + db.run("CREATE TRIGGER fail_text_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-assistant-text' BEGIN SELECT RAISE(ABORT, 'text receipt unavailable'); END"); + const input = operation(2, { kind: "append-assistant-text", + inputs: [{ ...execution, sequence: 1, text: "Keep me out until receipt" }], + }); + await expect(writer.transact(input)).rejects.toThrow("text receipt unavailable"); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoint_chunks").get()) + .toEqual({ count: 0 }); + expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "semantic:head")).toMatchObject({ receipt_json: expect.stringContaining('"ordinal":1') }); + db.run("DROP TRIGGER fail_text_receipt"); + expect(await writer.transact(input)).toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + }); + + it("replays narrative changes and discards, then recovers only the saved narrative after worker loss", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const canonical = new CanonicalAgentBoundary(db, () => {}); + const reducer = new NarrativeRecoveryDelta(); + const publicationCount = published.length; + const initialDelta = reducer.prepare(toolNarrative("", 1)); + if (!initialDelta) throw new Error("Expected an initial narrative delta"); + const first = operation(2, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: initialDelta.items, discardedItemIds: initialDelta.discardedItemIds, + } }); + expect((await writer.transact(first)).kind).toBe("committed"); + initialDelta.acknowledge(); + expect(canonical.loadParentNarrativeRecovery(TURN_ID)).toHaveLength(1); + expect(published).toHaveLength(publicationCount); + const discardedDelta = reducer.prepare([]); + if (!discardedDelta) throw new Error("Expected a discarded narrative delta"); + const discard = operation(3, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: discardedDelta.items, discardedItemIds: discardedDelta.discardedItemIds, + } }); + expect((await writer.transact(discard)).kind).toBe("committed"); + discardedDelta.acknowledge(); + expect(canonical.loadParentNarrativeRecovery(TURN_ID)).toEqual([]); + const latestDelta = reducer.prepare([toolNarrative("", 2)[1]!]); + if (!latestDelta) throw new Error("Expected a new narrative delta"); + const latest = operation(4, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: latestDelta.items, discardedItemIds: latestDelta.discardedItemIds, + } }); + const receipt = await writer.transact(latest); + expect(receipt).toMatchObject({ kind: "committed", operationId: "lease-1:4" }); + latestDelta.acknowledge(); + expect(published).toHaveLength(publicationCount); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((item) => item.eventId))); + expect(await writer.transact(first)).toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(await writer.transact(discard)).toMatchObject({ kind: "committed", operationId: "lease-1:3" }); + expect(await writer.transact(latest)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?").get("toolCall:tool-1")) + .toEqual({ count: 1 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?").get("toolCall:tool-0")) + .toEqual({ count: 0 }); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(db.prepare("SELECT id, status FROM tool_call_records WHERE id = ?").get("tool-1")) + .toEqual({ id: "tool-1", status: "completed" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM tool_call_records WHERE id = ?").get("tool-0")) + .toEqual({ count: 0 }); + }); + + it("rolls back narrative changes when the semantic receipt cannot commit", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const delta = operation(2, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: toolNarrative("", 1), discardedItemIds: [], + } }); + expect(await writer.transact(operation(2, { kind: "narrative-delta", input: { + executionId: "wrong-execution", items: toolNarrative("", 1), discardedItemIds: [], + } }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(operation(2, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: [{ ...toolNarrative("", 1)[0]!, record: { + ...toolNarrative("", 1)[0]!.record, input_summary: "x".repeat(256 * 1024 + 1), + } }], discardedItemIds: [], + } }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + db.run("CREATE TRIGGER fail_narrative_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:narrative-delta' BEGIN SELECT RAISE(ABORT, 'narrative receipt unavailable'); END"); + await expect(writer.transact(delta)).rejects.toThrow("narrative receipt unavailable"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?").get("toolCall:tool-0")) + .toEqual({ count: 0 }); + db.run("DROP TRIGGER fail_narrative_receipt"); + expect((await writer.transact(delta)).kind).toBe("committed"); + }); + + it("commits reclassification and promotion with their narrative changes before live publication", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const publicationCount = published.length; + const unknown = operation(2, { kind: "live-event", text: { kind: "append", inputs: [ + { ...execution, sequence: 1, text: "unknown draft" }, + ] } }); + const unknownEvent = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "unknown draft" }; + const appended = await writer.transact({ ...unknown, livePublication: [{ after: "writer", event: unknownEvent }] }); + expect(appended).toMatchObject({ kind: "committed", assistantTextCheckpoint: { durableThrough: 1 }, + livePublication: [{ publicationId: "1", event: unknownEvent }] }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("unknown draft"); + + const thought = { kind: "narrationSegment" as const, record: { + id: "thought-1", message_id: "", text: "unknown draft", started_at: NOW, + ended_at: NOW, sort_order: 0, + } }; + const boundary = { type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: false }; + const reclassified = { ...operation(3, { kind: "live-event", text: { + kind: "reclassify", expectedText: "unknown draft", + }, narrative: { executionId: EXECUTION_ID, items: [thought], discardedItemIds: [] } }), + livePublication: [{ after: "writer" as const, event: boundary }] }; + db.run("CREATE TRIGGER fail_compound_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' AND NEW.operation_id = 'lease-1:3' BEGIN SELECT RAISE(ABORT, 'compound receipt unavailable'); END"); + await expect(writer.transact(reclassified)).rejects.toThrow("compound receipt unavailable"); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("unknown draft"); + expect(db.prepare("SELECT last_sequence FROM canonical_writer_live_publication_heads WHERE thread_id = ?") + .get(THREAD_ID)).toEqual({ last_sequence: 1 }); + expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([]); + expect(published).toHaveLength(publicationCount); + db.run("DROP TRIGGER fail_compound_receipt"); + const reclassifiedReceipt = await writer.transact(reclassified); + expect(reclassifiedReceipt).toMatchObject({ kind: "committed", livePublication: [ + { publicationId: "2", event: boundary }, + ] }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([thought]); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(reclassified)).toEqual(reclassifiedReceipt); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + + const finalBoundary = { ...boundary, isFinalResponse: true }; + const promoted = { ...operation(4, { kind: "live-event", text: { kind: "promote", input: { + ...execution, sequence: 1, text: "final thought", + } }, narrative: { executionId: EXECUTION_ID, items: [], discardedItemIds: ["narrationSegment:thought-1"] } }), + livePublication: [{ after: "writer" as const, event: finalBoundary }] }; + expect(await writer.transact(promoted)).toMatchObject({ kind: "committed", + assistantTextCheckpoint: { durableThrough: 1 }, + livePublication: [{ publicationId: "3", event: finalBoundary }], + }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("final thought"); + expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([]); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "final thought", outcome: "interrupted", + })); + }); + + it("advances one thread publication identity across terminal and a new execution", async () => { + const firstStart = operation(1, { kind: "begin", providerId: "codex", input: startInput() }); + const firstEvent = { type: AgentEventType.TurnStarted, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID }; + expect(await writer.transact({ ...firstStart, livePublication: [{ after: "writer", event: firstEvent }] })) + .toMatchObject({ kind: "committed", livePublication: [{ publicationId: "1" }] }); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + + const nextExecution = { ...execution, turnId: "turn-next", + executionId: "00000000-0000-4000-8000-000000000002" }; + const nextLease = { ...lease, leaseId: "lease-next" }; + const nextStart: ExecutionSemanticOperation = { + operationId: "lease-next:1", execution: nextExecution, lease: nextLease, ordinal: 1, + mutation: { kind: "begin", providerId: "codex", input: { + ...startInput(), turnId: nextExecution.turnId, executionId: nextExecution.executionId, + userMessage: { kind: "create", messageId: "next-user", content: "Next question", sequence: 3 }, + } }, + livePublication: [{ after: "writer", event: { ...firstEvent, + turnExecutionId: nextExecution.executionId } }], + }; + expect(await writer.transact(nextStart)) + .toMatchObject({ kind: "committed", livePublication: [{ publicationId: "2" }] }); + expect(db.prepare("SELECT last_sequence FROM canonical_writer_live_publication_heads WHERE thread_id = ?") + .get(THREAD_ID)).toEqual({ last_sequence: 2 }); + }); + + it("durably acknowledges a boundary with no new text or narrative rows", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const boundary = { type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: true }; + const op = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" } }), + livePublication: [{ after: "writer" as const, event: boundary }] }; + const receipt = await writer.transact(op); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [ + { publicationId: "1", event: boundary }, + ] }); + expect(await writer.transact(op)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 1 }); + }); + + it("commits a system notice and its generated message ID in one replayable live receipt", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const notice: Extract = { + type: "system", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + subtype: "provider.notice.unknown-event", message: "Codex reported an update.", + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "turn", noticeKey: "notice-1" }, + }; + const op = systemLiveOperation(2, notice); + const publishedBefore = [...published]; + db.run("CREATE TRIGGER fail_system_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' BEGIN SELECT RAISE(ABORT, 'system receipt unavailable'); END"); + await expect(writer.transact(op)).rejects.toThrow("system receipt unavailable"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'system'").get(THREAD_ID)) + .toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, op.operationId)).toEqual({ count: 0 }); + expect(published).toEqual(publishedBefore); + db.run("DROP TRIGGER fail_system_receipt"); + + const receipt = await writer.transact(op); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ publicationId: "1", + after: "writer", event: { ...notice, messageId: expect.any(String) } }] }); + if (receipt.kind !== "committed") return; + const messageId = receipt.livePublication?.[0]?.event.type === "system" + ? receipt.livePublication[0].event.messageId : undefined; + expect(messageId).toBeDefined(); + if (!messageId) return; + const released: AgentEvent[] = []; + const release = new ExecutionLivePublicationRelease({ + isBound: () => true, + publish: (event) => { released.push(event); }, + }); + release.release(op, receipt); + expect(released).toEqual([{ ...notice, messageId, publicationId: "1" }]); + const actualPublication = receipt.livePublication?.[0]; + if (!actualPublication) throw new Error("Missing system live publication"); + expect(() => release.release(op, { ...receipt, livePublication: [{ ...actualPublication, + event: { ...notice, messageId, message: "Changed notice" }, + }] })).toThrow("Live AgentEvent publication receipt is invalid"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, messageId)) + .toMatchObject({ role: "system", content: notice.message }); + expect(await writer.transact(op)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'system'").get(THREAD_ID)) + .toEqual({ count: 1 }); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(op)).toEqual(receipt); + const stored = db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, op.operationId) as { receipt_json: string }; + const corrupted = JSON.parse(stored.receipt_json) as { livePublication: { event: { messageId: string } }[] }; + corrupted.livePublication[0]!.event.messageId = "not-a-message-id"; + db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") + .run(JSON.stringify(corrupted), EXECUTION_ID, op.operationId); + await expect(writer.transact(op)).rejects.toThrow("Invalid uuid"); + const altered = JSON.parse(stored.receipt_json) as { livePublication: { event: { message: string } }[] }; + altered.livePublication[0]!.event.message = "Different notice"; + db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") + .run(JSON.stringify(altered), EXECUTION_ID, op.operationId); + await expect(writer.transact(op)).rejects.toThrow("Live publication receipt does not match"); + }); + + it("fences session cursors to the selected execution in the compound live writer", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const cursor: Extract = { + type: "system", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, subtype: "sdk_session_id:native-cursor", + }; + const unbound = { ...cursor, turnExecutionId: undefined }; + expect(await send(2, { kind: "live-event", text: { kind: "unchanged" }, + systemIntents: [{ kind: "session-cursor", event: unbound }], publication: { event: unbound, after: "writer" }, + })).toEqual({ kind: "rejected", reason: "writer-conflict" }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + expect(await send(2, { kind: "live-event", text: { kind: "unchanged" }, + systemIntents: [{ kind: "session-cursor", event: { ...cursor, turnExecutionId: "other-execution" } }], + publication: { event: cursor, after: "writer" }, + })).toEqual({ kind: "rejected", reason: "writer-conflict" }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + const receipt = await send(2, { kind: "live-event", text: { kind: "unchanged" }, + systemIntents: [{ kind: "session-cursor", event: cursor }], publication: { event: cursor, after: "writer" }, + }); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ event: cursor }] }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: "native-cursor" }); + expect(new CanonicalAgentBoundary(db, () => {}).loadCheckpoint(EXECUTION_ID)) + .toMatchObject({ nativeCursor: { providerId: "codex", scope: "thread", value: "native-cursor" } }); + }); + + it("stores task-tool rows before the matching live tool publication", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const toolUse = { type: AgentEventType.ToolUse, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + toolCallId: "todo-1", toolName: "TodoWrite", toolInput: { todos: [{ content: "Ship task" }] } }; + const op = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" }, + taskIntents: [{ kind: "upsert-group", group: "Tasks", + tasks: [{ content: "Ship task", status: "pending", group: "Tasks" }] }], + }), livePublication: [{ after: "writer" as const, event: toolUse }] }; + db.run("CREATE TRIGGER fail_task_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' BEGIN SELECT RAISE(ABORT, 'task receipt unavailable'); END"); + await expect(writer.transact(op)).rejects.toThrow("task receipt unavailable"); + expect(new TaskRepo(db).get(THREAD_ID)).toBeNull(); + db.run("DROP TRIGGER fail_task_receipt"); + const receipt = await writer.transact(op); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [ + { publicationId: "1", event: toolUse }, + ] }); + expect(new TaskRepo(db).get(THREAD_ID)).toEqual([ + { content: "Ship task", status: "pending", group: "Tasks" }, + ]); + expect(await writer.transact(op)).toEqual(receipt); + }); + + it("acknowledges transient tool progress without inventing a durable tool row", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const event = { type: AgentEventType.ToolProgress, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + toolCallId: "tool-1", toolName: "command_execution", elapsedSeconds: 1 }; + const op = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" } }), + livePublication: [{ after: "writer" as const, event }] }; + const receipt = await writer.transact(op); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ event }] }); + expect(await writer.transact(op)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE turn_id = ? AND id LIKE 'toolCall:%'") + .get(TURN_ID)).toEqual({ count: 0 }); + }); + + it("rejects an error publication before terminal durability", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const error: AgentEvent = { type: AgentEventType.Error, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, error: "failed" }; + const op = { ...operation(2, { kind: "append-events", phase: "running", nativeCursor: null, + events: [runtimeEvent(2, error)] }), livePublication: [{ after: "writer" as const, event: error }] }; + expect(await writer.transact(op)).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 0 }); + }); + + it("rejects a compound live event that exceeds its shared row or byte budget", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "short text" }; + const text = { kind: "append" as const, inputs: [{ ...execution, sequence: 1, text: event.delta }] }; + const publication = [{ after: "writer" as const, event }]; + expect(await writer.transact({ ...operation(2, { kind: "live-event", text, + narrative: { executionId: EXECUTION_ID, items: toolNarrative("", 62), discardedItemIds: [] }, + }), livePublication: publication })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + const largeItem = { ...toolNarrative("", 1)[0]!, record: { + ...toolNarrative("", 1)[0]!.record, input_summary: "x".repeat(256 * 1024 - 900), + } }; + expect(await writer.transact({ ...operation(2, { kind: "live-event", text, + narrative: { executionId: EXECUTION_ID, items: [largeItem], discardedItemIds: [] }, + }), livePublication: publication })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact({ ...operation(2, { kind: "live-event", text }), + livePublication: [{ after: "writer", event: { ...event, delta: "different" } }], + })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE turn_id = ? AND id LIKE 'toolCall:%'") + .get(TURN_ID)).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 0 }); + }); + + it("rejects stale worker-loss leases without changing the unfinished turn", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect(writer.interruptWorkerLoss({ ...loss, lease: { ...lease, ownerEpoch: 2 } })) + .toEqual({ kind: "conflict", operationId: "lease-1:worker-lost" }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(published).not.toContain(`${EXECUTION_ID}:recovery-interrupted`); + }); + + it("distinguishes a lost worker before start from one after a durable terminal", async () => { + expect(writer.interruptWorkerLoss(loss)).toEqual({ + kind: "conflict", operationId: "lease-1:worker-lost", recoveryState: "not-started", + }); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect((await send(2, { kind: "provider-outcome", outcome: "completed" })).kind).toBe("committed"); + expect((await send(3, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, + assistant: { content: "Answer", model: null, attachments: [] }, narrative: [], + } })).kind).toBe("committed"); + expect((await send(4, { kind: "finalize", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex", providerIdentities: [], outcome: "completed", + projection: { kind: "writer-staged" }, + } })).kind).toBe("committed"); + expect(writer.interruptWorkerLoss(loss)).toEqual({ + kind: "conflict", operationId: "lease-1:worker-lost", recoveryState: "already-terminal", + }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "completed" }); + }); + + it("rolls back worker-loss interruption and publication when its receipt cannot commit", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const text = new ParentAssistantTextCheckpointService(db); + text.appendChunk([{ ...execution, sequence: 1, text: "Keep me" }]); + text.recoveryJournal.append([{ ...execution, sequence: 2, text: " through failure" }]); + db.run("CREATE TRIGGER fail_loss_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:worker-lost' BEGIN SELECT RAISE(ABORT, 'loss receipt unavailable'); END"); + expect(() => writer.interruptWorkerLoss(loss)).toThrow("loss receipt unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.role === "assistant")).toBe(false); + expect(text.restore(EXECUTION_ID)).toBe("Keep me through failure"); + expect(published).not.toContain(`${EXECUTION_ID}:recovery-interrupted`); + db.run("DROP TRIGGER fail_loss_receipt"); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(db.prepare("SELECT content FROM messages WHERE role = 'assistant'").get()) + .toEqual({ content: "Keep me through failure" }); + }); + + it("commits start, event, and finalization with durable receipts across a database reload", async () => { + const begin = operation(1, { kind: "begin", providerId: "codex", input: startInput() }); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect((await send(2, { kind: "event", phase: "running", nativeCursor: null, events: [event()] })).kind).toBe("committed"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed" as const, + projection: { message: staged, narrative: [] }, + }; + const terminal = await send(3, { kind: "finalize", outcome: "completed", input: finish }); + expect(terminal.kind).toBe("committed"); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(begin)).toMatchObject({ kind: "committed", operationId: "lease-1:1" }); + expect(await writer.transact(operation(1, { + kind: "begin", + providerId: "codex", + input: { ...startInput(), permissionMode: "full" }, + }))).toEqual({ kind: "conflict", operationId: "lease-1:1" }); + expect(await writer.transact(operation(3, { kind: "finish", outcome: "completed", input: finish }))) + .toEqual(terminal); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: "completed" }); + expect(db.prepare("SELECT kind, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "semantic:head")).toMatchObject({ kind: "semantic-head" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind != 'semantic-publication'").get(EXECUTION_ID)) + .toEqual({ count: 4 }); + }); + + it("retains terminal live publication behind a completed finalization and rejects invalid routing", async () => { + const ended = { type: AgentEventType.Ended, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, outcome: "completed" as const }; + const invalidStart: ExecutionSemanticOperation = { + ...operation(1, { kind: "begin", providerId: "codex", input: startInput() }), + livePublication: [{ after: "terminal", event: ended }], + }; + expect(await writer.transact(invalidStart)).toEqual({ kind: "conflict", operationId: "lease-1:1" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const input = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex", providerIdentities: [], outcome: "completed" as const, + projection: { message: staged, narrative: [] } }; + const finish: ExecutionSemanticOperation = { + ...operation(2, { kind: "finish", outcome: "completed", input }), + livePublication: [{ after: "terminal", event: ended }], + }; + db.run("CREATE TRIGGER fail_live_finish BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish' BEGIN SELECT RAISE(ABORT, 'finish unavailable'); END"); + await expect(writer.transact(finish)).rejects.toThrow("finish unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT status FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ status: "active" }); + db.run("DROP TRIGGER fail_live_finish"); + + const receipt = await writer.transact(finish); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ + publicationId: "1", after: "terminal", event: ended, + }] }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "completed" }); + expect(db.prepare("SELECT status FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ status: "completed" }); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(finish)).toEqual(receipt); + expect(await writer.transact({ ...finish, livePublication: [{ after: "terminal", event: { + ...ended, threadId: "another-thread", + } }] })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + }); + + it("rejects oversized live intents before writes and oversized receipts before replay publication", async () => { + const started = { type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const begin: ExecutionSemanticOperation = { + ...operation(1, { kind: "begin", providerId: "codex", input: startInput() }), + livePublication: [{ after: "writer", event: started }], + }; + expect(await writer.transact({ ...begin, livePublication: Array.from({ length: 65 }, () => ({ + after: "writer" as const, event: started, + })) })).toEqual({ kind: "conflict", operationId: "lease-1:1" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + + const beginReceipt = await writer.transact(begin); + expect(beginReceipt.kind).toBe("committed"); + const before = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?") + .get(EXECUTION_ID); + published.length = 0; + const largeDelta: ExecutionSemanticOperation = { + ...operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }), + livePublication: [{ after: "writer", event: { + type: AgentEventType.TextDelta, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + delta: "x".repeat(256 * 1024), + } }], + }; + expect(await writer.transact(largeDelta)).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual(before); + expect(published).toEqual([]); + + db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") + .run("x".repeat(512 * 1024 + 1), EXECUTION_ID, begin.operationId); + await expect(writer.transact(begin)).rejects.toThrow("Live publication receipt exceeds its size limit"); + expect(published).toEqual([]); + + db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") + .run(JSON.stringify({ ...beginReceipt, publicationVersion: 1, livePublication: [{ + publicationId: "1", after: "writer", + event: { ...started, threadId: "another-thread" }, + }] }), EXECUTION_ID, begin.operationId); + await expect(writer.transact(begin)).rejects.toThrow("Live publication receipt does not match its operation"); + expect(published).toEqual([]); + }); + + it("rejects stale leases and skipped ordinals while checkpointing without a new event", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const eventsBefore = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?").get(EXECUTION_ID); + const append = operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }); + expect(await writer.transact({ ...append, lease: { ...lease, ownerEpoch: 2, leaseId: "lease-2" }, operationId: "lease-2:2" })) + .toEqual({ kind: "conflict", operationId: "lease-2:2" }); + expect(await writer.transact({ ...append, ordinal: 3, operationId: "lease-1:3" })) + .toEqual({ kind: "conflict", operationId: "lease-1:3" }); + expect(await send(2, { kind: "checkpoint", phase: "running", nativeCursor: null })) + .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(await send(3, { kind: "effect-result", effectId: "unsupported", settled: true })) + .toEqual({ kind: "rejected", reason: "writer-conflict" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual(eventsBefore); + expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "semantic:head")).toMatchObject({ receipt_json: expect.stringContaining('"ordinal":2') }); + expect(db.prepare("SELECT phase, native_cursor_json FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ phase: "running", native_cursor_json: null }); + }); + + it("rolls back terminal checkpoint and semantic receipt together when receipt storage fails", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed" as const, + projection: { message: staged, narrative: [] }, + }; + db.run("CREATE TRIGGER fail_semantic_receipt BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish' BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END"); + await expect(send(2, { kind: "finalize", outcome: "completed", input: finish })).rejects.toThrow("receipt unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ kind: "semantic:finish-pending" }); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, staged.id)).toBeNull(); + expect(published).not.toContain(`${EXECUTION_ID}:turn.completed`); + + db.run("DROP TRIGGER fail_semantic_receipt"); + expect((await send(2, { kind: "finalize", outcome: "completed", input: finish })).kind).toBe("committed"); + }); + + it("rolls back an event and its checkpoint when its semantic receipt cannot be stored", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const before = db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID); + db.run("CREATE TRIGGER fail_semantic_event_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-events' BEGIN SELECT RAISE(ABORT, 'event receipt unavailable'); END"); + await expect(send(2, { kind: "event", phase: "running", nativeCursor: null, events: [event()] })).rejects.toThrow("event receipt unavailable"); + expect(db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual(before); + expect(db.prepare("SELECT id FROM canonical_agent_items WHERE id = ?").get("item-1")).toBeNull(); + expect(published).not.toContain(event().eventId); + + db.run("DROP TRIGGER fail_semantic_event_receipt"); + expect((await send(2, { kind: "event", phase: "running", nativeCursor: null, events: [event()] })).kind).toBe("committed"); + expect(published).toContain(event().eventId); + }); + + it("replays committed event publication after a failed publisher and database reopen", async () => { + let failPublication = false; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + if (failPublication) throw new Error("publisher unavailable"); + published.push(...events.map((item) => item.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + failPublication = true; + await expect(send(2, { kind: "event", phase: "running", nativeCursor: null, events: [event()] })).rejects.toThrow("publisher unavailable"); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?").get(event().eventId)) + .toEqual({ event_id: event().eventId }); + const receipt = db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2") as { receipt_json: string }; + expect(receipt.receipt_json.length).toBeLessThan(256); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + }); + expect(await writer.transact(operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }))) + .toMatchObject({ + kind: "committed", operationId: "lease-1:2", + providerCommit: { outcome: "committed", eventCount: 1 }, + }); + expect(published).toContain(event().eventId); + }); + + it("replays terminal publication after its checkpoint committed and the publisher failed", async () => { + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + if (events.some((item) => item.eventId === `${EXECUTION_ID}:turn.completed`)) { + throw new Error("terminal publisher unavailable"); + } + published.push(...events.map((item) => item.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed" as const, + projection: { message: staged, narrative: [] }, + }; + await expect(send(2, { kind: "finalize", outcome: "completed", input: finish })) + .rejects.toThrow("terminal publisher unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: "completed" }); + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ kind: "semantic:finish" }); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + }); + expect(await writer.transact(operation(2, { kind: "finish", outcome: "completed", input: finish }))) + .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + }); + + it("replays earlier terminal batches after publication failed before finalization", async () => { + let failTerminalBatch = false; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + if (failTerminalBatch) { + failTerminalBatch = false; + throw new Error("first terminal batch unavailable"); + } + published.push(...events.map((item) => item.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + const start = await send(1, { kind: "start", providerId: "codex", input: startInput() }); + expect(start.kind).toBe("committed"); + if (start.kind !== "committed") throw new Error("Canonical start did not commit"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed" as const, + projection: { message: staged, narrative: toolNarrative(staged.id, 100) }, + }; + failTerminalBatch = true; + await expect(send(2, { kind: "finalize", outcome: "completed", input: finish })) + .rejects.toThrow("first terminal batch unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ kind: "semantic:finish-pending" }); + const priorChunks = db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind = 'semantic-publication'") + .get(EXECUTION_ID) as { count: number }; + expect(priorChunks.count).toBeGreaterThan(0); + expect(await writer.transact(operation(2, { + kind: "finish", + outcome: "completed", + input: { ...finish, error: "different input" }, + }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + }); + expect(await writer.transact(operation(2, { kind: "finish", outcome: "completed", input: finish }))) + .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + const terminalEventCount = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence > ?") + .get(EXECUTION_ID, start.durableRevision) as { count: number }; + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + expect(published).toHaveLength(terminalEventCount.count); + }); + + it("rolls back the canonical start and user message when the begin receipt fails", async () => { + db.run("CREATE TRIGGER fail_semantic_begin_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:begin' BEGIN SELECT RAISE(ABORT, 'begin receipt unavailable'); END"); + await expect(send(1, { kind: "start", providerId: "codex", input: startInput() })) + .rejects.toThrow("begin receipt unavailable"); + expect(db.prepare("SELECT execution_id FROM canonical_agent_turns WHERE execution_id = ?").get(EXECUTION_ID)).toBeNull(); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, "user-1")).toBeNull(); + expect(db.prepare("SELECT operation_id FROM canonical_writer_operation_receipts WHERE execution_id = ?") + .all(EXECUTION_ID)).toEqual([]); + expect(published).toEqual([]); + + db.run("DROP TRIGGER fail_semantic_begin_receipt"); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + }); + + it("keeps publication separate while a batched finish starts another execution", async () => { + const otherExecution = { + threadId: "thread-2", + turnId: "turn-2", + executionId: "00000000-0000-4000-8000-000000000002", + }; + const otherLease = { ...lease, workerIndex: 1, leaseId: "lease-2" }; + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(otherExecution.threadId, "workspace-1", "Second", "main", "codex", NOW, NOW); + let finishing = false; + let secondStartRequested = false; + let secondStart: Promise | null = null; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + if (!finishing || secondStartRequested) return; + secondStartRequested = true; + secondStart = handler.handle({ + requestId: 1, + execution: otherExecution, + lease: otherLease, + ordinal: 1, + command: { + kind: "start", + providerId: "codex", + input: { + ...startInput(), + thread: { ...startInput().thread, id: otherExecution.threadId }, + turnId: otherExecution.turnId, + executionId: otherExecution.executionId, + userMessage: { kind: "create", messageId: "user-2", content: "Second question", sequence: 1 }, + }, + }, + }).then((reply) => reply.result); + }); + handler = new ExecutionWorkerHandler(writer); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const narrative = toolNarrative(staged.id, 100); + finishing = true; + expect((await send(2, { + kind: "finalize", + outcome: "completed", + input: { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed", + projection: { message: staged, narrative }, + }, + })).kind).toBe("committed"); + expect(await secondStart).toMatchObject({ kind: "committed", operationId: "lease-2:1" }); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + expect(published.some((eventId) => eventId.startsWith(otherExecution.executionId))).toBe(true); + const terminalReceipt = db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2") as { receipt_json: string }; + expect(terminalReceipt.receipt_json.length).toBeLessThan(256); + const chunks = db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind = 'semantic-publication'") + .get(EXECUTION_ID) as { count: number }; + expect(chunks.count).toBeGreaterThan(1); + const chunkSize = db.prepare("SELECT MAX(json_array_length(receipt_json)) AS size FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind = 'semantic-publication'") + .get(EXECUTION_ID) as { size: number }; + expect(chunkSize.size).toBeLessThanOrEqual(64); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE kind = 'semantic-head'").get()) + .toEqual({ count: 2 }); + }); + + it("replays terminal publication across multiple bounded receipt pages", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const staged = new MessageRepo(db).create( + THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true, + ); + const input = { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "completed" as const, + projection: { message: staged, narrative: toolNarrative(staged.id, 1_025) }, + }; + published = []; + expect((await send(2, { kind: "finalize", outcome: "completed", input })).kind).toBe("committed"); + const firstPublication = [...published]; + expect(firstPublication).toContain(`${EXECUTION_ID}:turn.completed`); + expect((db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind = 'semantic-publication'") + .get(EXECUTION_ID) as { count: number }).count).toBeGreaterThan(16); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((item) => item.eventId))); + expect(await writer.transact(operation(2, { kind: "finish", outcome: "completed", input }))) + .toMatchObject({ kind: "committed" }); + expect(published).toEqual(firstPublication); + }, 30_000); +}); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts new file mode 100644 index 000000000..2a691de47 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -0,0 +1,733 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { AgentEventType, type AgentEvent, type ProviderRuntimeExtension } from "@mcode/contracts"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; +import { ExecutionMailboxScheduler } from "../../execution/execution-mailbox-scheduler.js"; +import type { ExecutionLease } from "../../execution/execution-mailbox-protocol.js"; +import { ExecutionThreadWorkerPort } from "../../execution/execution-worker-port.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; +import type { ExecutionSemanticOperation, ExecutionWorkCommand, ExecutionWorkerResult } from "../../execution/execution-worker-handler.js"; +import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; +import { ExecutionLivePublicationRelease } from "../execution-live-publication-release.js"; + +const NOW = "2026-09-24T10:00:00.000Z"; +const THREAD_ID = "semantic-worker-thread"; +const TURN_ID = "semantic-worker-turn"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000177"; +const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID } as const; +const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "semantic-worker-lease" } as const; + +function beginOperation(): ExecutionSemanticOperation { + return { + operationId: `${lease.leaseId}:1`, execution, lease, ordinal: 1, + mutation: { + kind: "begin", providerId: "codex", + input: { + thread: { id: THREAD_ID, workspaceId: "semantic-worker-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "semantic-worker-user", content: "Question", sequence: 1 }, + }, + }, + }; +} + +function runtimeDraft( + sequence: number, + event: AgentEvent, + extension?: ProviderRuntimeExtension, +): Extract["events"][number] { + const itemId = `runtime-item-${sequence}`; + return { + eventId: `${EXECUTION_ID}:runtime-${sequence}`, + routing: { ...execution, itemId }, + sourceProviderId: "codex", sourceIdentities: [], sourceSequence: sequence, + payload: { type: "item.recorded", item: { + id: itemId, threadId: THREAD_ID, turnId: TURN_ID, kind: "system", + providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent: { + event, ...(extension ? { extension } : {}), + } }, + createdAt: NOW, updatedAt: NOW, + } }, + }; +} + +describe("execution semantic writer transport", () => { + let directory: string; + let db: Database; + let writer: CanonicalAgentWriterClient | undefined; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-execution-writer-")); + db = openDatabase({ dbPath: NodePath.join(directory, "app.sqlite") }); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("semantic-worker-workspace", "Workspace", directory, NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "semantic-worker-workspace", "Thread", "main", "codex", NOW, NOW); + }); + + afterEach(async () => { + await writer?.close(); + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + it("releases live receipts in order after their writer and terminal barriers", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + const release = new ExecutionLivePublicationRelease(registry); + const port = new CanonicalExecutionWriterPort(writer, () => {}, release); + const started = { type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const begin: ExecutionSemanticOperation = { + ...beginOperation(), livePublication: [{ after: "writer", event: started }], + }; + expect((await port.transact(begin)).kind).toBe("committed"); + expect(published.map((event) => event.type)).toEqual([AgentEventType.TurnStarted]); + expect(published[0]?.publicationId).toBe("1"); + + const delta: AgentEvent = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "answer", isFinalResponse: true }; + const system: AgentEvent = { type: AgentEventType.System, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, subtype: "test" }; + const append: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, + events: [runtimeDraft(1, delta), runtimeDraft(2, system)] }, + livePublication: [{ after: "writer", event: delta }, { after: "writer", event: system }], + }; + expect((await port.transact(append)).kind).toBe("committed"); + expect(published.map((event) => event.type)) + .toEqual([AgentEventType.TurnStarted, AgentEventType.TextDelta, AgentEventType.System]); + expect(published.slice(1).map((event) => event.publicationId)) + .toEqual(["2", "3"]); + expect((await port.transact(append)).kind).toBe("committed"); + expect(published).toHaveLength(3); + + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, + undefined, undefined, undefined, "model", true); + const ended: AgentEvent = { type: AgentEventType.Ended, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, outcome: "completed" }; + const finish: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:3`, execution, lease, ordinal: 3, + mutation: { kind: "finish", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "completed", projection: { message: staged, narrative: [] }, + } }, + livePublication: [{ after: "terminal", event: ended }], + }; + db.run("CREATE TRIGGER fail_live_release BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish' BEGIN SELECT RAISE(ABORT, 'finish unavailable'); END"); + await expect(port.transact(finish)).rejects.toThrow("Canonical writer write-failed"); + expect(published).toHaveLength(3); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: null }); + db.run("DROP TRIGGER fail_live_release"); + + expect((await port.transact(finish)).kind).toBe("committed"); + expect(published.map((event) => event.type)) + .toEqual([AgentEventType.TurnStarted, AgentEventType.TextDelta, AgentEventType.System, AgentEventType.Ended]); + expect((await port.transact(finish)).kind).toBe("committed"); + expect(published).toHaveLength(4); + }); + + it("carries a live publication from the execution worker to the bound host publisher", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + const release = new ExecutionLivePublicationRelease(registry); + const port = new CanonicalExecutionWriterPort(writer, () => {}, release); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 4, maxPendingBytes: 8_000, reservedControl: 2, reservedControlBytes: 4_000, + maxPerExecutionPending: 4, maxPerExecutionBytes: 8_000, + reservedPerExecutionControl: 2, reservedPerExecutionControlBytes: 4_000, + }, + createWorker: () => new ExecutionThreadWorkerPort(port), + onWorkerLost: () => { throw new Error("Execution worker was lost"); }, + }); + try { + const claim = scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const begin = beginOperation().mutation; + if (begin.kind !== "begin") throw new Error("Unexpected begin operation"); + const admission = scheduler.submit({ execution, lease: claim.lease, byteLength: 1_000, + command: { kind: "start", providerId: "codex", input: begin.input, + livePublication: [{ after: "writer", event: { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + } }], + }, + }); + if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); + await expect(admission.completion).resolves.toMatchObject({ kind: "reply", result: { + kind: "committed", livePublication: [{ publicationId: "1" }], + } }); + expect(published.map((event) => event.type)).toEqual([AgentEventType.TurnStarted]); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get(THREAD_ID)) + .toEqual({ count: 1 }); + } finally { + scheduler.shutdown(); + } + }); + + it("releases a compound live event after its single writer receipt", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + const port = new CanonicalExecutionWriterPort(writer, () => {}, new ExecutionLivePublicationRelease(registry)); + expect((await port.transact(beginOperation())).kind).toBe("committed"); + const event: AgentEvent = { type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: false }; + const operation: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "live-event", text: { kind: "unchanged" } }, + livePublication: [{ after: "writer", event }], + }; + expect((await port.transact(operation)).kind).toBe("committed"); + expect(published).toEqual([{ ...event, publicationId: "1" }]); + expect((await port.transact(operation)).kind).toBe("committed"); + expect(published).toEqual([{ ...event, publicationId: "1" }]); + }); + + it("replays a committed live receipt when the public publisher becomes available", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const release = new ExecutionLivePublicationRelease(registry); + const port = new CanonicalExecutionWriterPort(writer, () => {}, release); + const begin: ExecutionSemanticOperation = { ...beginOperation(), livePublication: [{ + after: "writer", event: { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + }, + }] }; + await expect(port.transact(begin)).rejects.toThrow("Live AgentEvent publisher is not bound"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get(THREAD_ID)) + .toEqual({ count: 1 }); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + expect((await port.transact(begin)).kind).toBe("committed"); + expect(published.map((event) => event.type)).toEqual([AgentEventType.TurnStarted]); + expect((await port.transact(begin)).kind).toBe("committed"); + expect(published).toHaveLength(1); + }); + + it("replays the same public identity when publication succeeds before the host loses its reply", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const published: AgentEvent[] = []; + let loseReply = true; + registry.bind((event) => { + published.push(event); + if (loseReply) { + loseReply = false; + throw new Error("host lost reply after publish"); + } + }); + const operation: ExecutionSemanticOperation = { ...beginOperation(), livePublication: [{ + after: "writer", event: { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + }, + }] }; + const firstHost = new CanonicalExecutionWriterPort(writer, () => {}, new ExecutionLivePublicationRelease(registry)); + await expect(firstHost.transact(operation)).rejects.toThrow("host lost reply after publish"); + const restartedHost = new CanonicalExecutionWriterPort(writer, () => {}, new ExecutionLivePublicationRelease(registry)); + expect((await restartedHost.transact(operation)).kind).toBe("committed"); + expect(published.map((event) => event.publicationId)) + .toEqual(["1", "1"]); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get(THREAD_ID)) + .toEqual({ count: 1 }); + }); + + it("keeps releasing after the recent host cache reaches an all-day event count", () => { + let published = 0; + const release = new ExecutionLivePublicationRelease({ + isBound: () => true, + publish: () => { published++; }, + }); + const started: AgentEvent = { type: AgentEventType.TurnStarted, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID }; + for (let ordinal = 1; ordinal <= 8_193; ordinal++) { + const operationId = `${lease.leaseId}:${ordinal}`; + const operation: ExecutionSemanticOperation = { ...beginOperation(), operationId, ordinal, + livePublication: [{ after: "writer", event: started }] }; + release.release(operation, { kind: "committed", operationId, durableRevision: ordinal, + livePublication: [{ after: "writer", event: started, + publicationId: String(ordinal) }] }); + } + expect(published).toBe(8_193); + }); + + it("releases a real Codex context estimate with optional undefined fields", () => { + const published: AgentEvent[] = []; + const release = new ExecutionLivePublicationRelease({ + isBound: () => true, + publish: (event) => { published.push(event); }, + }); + const event: AgentEvent = { + type: AgentEventType.ContextEstimate, + threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, + tokensIn: 30_787, + tokensOut: 25, + totalProcessedTokens: 30_812, + contextWindow: 258_400, + cacheReadTokens: undefined, + }; + const operation: ExecutionSemanticOperation = { + ...beginOperation(), + livePublication: [{ after: "writer", event }], + }; + release.release(operation, { + kind: "committed", + operationId: operation.operationId, + durableRevision: 1, + livePublication: [{ after: "writer", event: { ...event }, publicationId: "1" }], + }); + expect(published).toEqual([{ ...event, publicationId: "1" }]); + }); + + it("commits and replays a semantic start through the dedicated SQLite worker", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const operation = beginOperation(); + + const first = await port.transact(operation); + expect(first).toMatchObject({ kind: "committed", operationId: operation.operationId }); + expect(db.prepare("SELECT content FROM messages WHERE id = ?").get("semantic-worker-user")) + .toEqual({ content: "Question" }); + expect(published.length).toBeGreaterThan(0); + + published.length = 0; + expect(await port.transact(operation)).toEqual(first); + expect(published.length).toBeGreaterThan(0); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("semantic-worker-user")) + .toEqual({ count: 1 }); + }); + + it("delivers a large committed event batch in bounded publication pages", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const pageSizes: number[] = []; + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + pageSizes.push(events.length); + published.push(...events.map((event) => event.eventId)); + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + pageSizes.length = 0; + published.length = 0; + const events = Array.from({ length: 130 }, (_, index) => runtimeDraft(index + 1, { + type: AgentEventType.TextDelta, + threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, + delta: "x", + isFinalResponse: true, + })); + const operation: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, events }, + }; + expect(await port.transact(operation)).toMatchObject({ kind: "committed" }); + expect(pageSizes).toEqual([64, 64, 2]); + expect(published).toEqual(events.map((event) => event.eventId)); + pageSizes.length = 0; + published.length = 0; + expect(await port.transact(operation)).toMatchObject({ kind: "committed" }); + expect(pageSizes).toEqual([64, 64, 2]); + expect(published).toEqual(events.map((event) => event.eventId)); + }); + + it("does not acknowledge a failed database write and accepts a clean retry", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const port = new CanonicalExecutionWriterPort(writer, () => {}); + db.run(`CREATE TRIGGER reject_semantic_start BEFORE INSERT ON canonical_agent_events + BEGIN SELECT RAISE(FAIL, 'injected write failure'); END`); + + await expect(port.transact(beginOperation())).rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("semantic-worker-user")) + .toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + + db.run("DROP TRIGGER reject_semantic_start"); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + }); + + it("commits and replays worker-loss interruption through the SQLite writer worker", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const input = { execution, lease, reason: "Execution worker exited", recoveryIncidentId: "incident-1" }; + expect(await port.interruptWorkerLoss(input)).toEqual({ + kind: "conflict", operationId: `${lease.leaseId}:worker-lost`, recoveryState: "not-started", + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + expect(await port.interruptWorkerLoss({ ...input, lease: { ...lease, ownerEpoch: 2 } })) + .toEqual({ kind: "conflict", operationId: `${lease.leaseId}:worker-lost` }); + const first = await port.interruptWorkerLoss(input); + expect(first).toMatchObject({ kind: "committed", operationId: `${lease.leaseId}:worker-lost` }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "interrupted" }); + expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); + published.length = 0; + expect(await port.interruptWorkerLoss(input)).toEqual(first); + expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); + expect(await port.transact({ + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "checkpoint", phase: "running", nativeCursor: null }, + })).toEqual({ kind: "conflict", operationId: `${lease.leaseId}:2` }); + }); + + it("replays a durable semantic receipt after the writer loses its reply", async () => { + let dropReply = true; + const createWorker = (): Worker => { + const worker = new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + return new Proxy(worker, { + set(target, property, value) { + if (property !== "onmessage" || typeof value !== "function") return Reflect.set(target, property, value); + target.onmessage = (message) => { + if (dropReply && message.data.kind === "semantic-transacted") { + dropReply = false; + target.terminate(); + return; + } + value(message); + }; + return true; + }, + get(target, property) { + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }; + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite"), createWorker); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + expect(dropReply).toBe(false); + expect(published.length).toBeGreaterThan(0); + expect(new Set(published).size).toBe(published.length); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("semantic-worker-user")) + .toEqual({ count: 1 }); + }); + + it("stages and finalizes assistant rows without a main-connection projection write", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const op = (ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation => ({ + operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation, + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + const terminalInput = { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed" as const, endedAt: NOW, + assistant: { content: "Answer", model: "fixture", attachments: [] }, narrative: [], + }; + expect(await port.transact(op(2, { kind: "provider-outcome", outcome: "completed" }))) + .toMatchObject({ kind: "committed" }); + const staged = op(3, { kind: "stage-terminal", input: terminalInput }); + expect(await port.transact(staged)).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT content, is_internal FROM messages WHERE role = 'assistant'").get()) + .toEqual({ content: "Answer", is_internal: 1 }); + const stagedId = deriveTurnAssistantMessageId(THREAD_ID, `execution:${EXECUTION_ID}`); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, stagedId)) + .toMatchObject({ content: "Answer", is_internal: true }); + expect(published).not.toContain(`${EXECUTION_ID}:turn.completed`); + + const finish = op(4, { kind: "finish", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "completed", projection: { kind: "writer-staged" }, + } }); + expect(await port.transact(finish)).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT content, is_internal, outcome FROM messages WHERE role = 'assistant'").get()) + .toEqual({ content: "Answer", is_internal: 0, outcome: "completed" }); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + expect(await port.transact(staged)).toMatchObject({ kind: "committed" }); + expect(await port.transact(finish)).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant'").get()) + .toEqual({ count: 1 }); + }); + + it("rolls back a failed semantic terminal stage and retries at the same ordinal", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const port = new CanonicalExecutionWriterPort(writer, () => {}); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + expect(await port.transact({ + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "provider-outcome", outcome: "completed" }, + })).toMatchObject({ kind: "committed" }); + const stage: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:3`, execution, lease, ordinal: 3, + mutation: { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, + assistant: { content: "Answer", model: null, attachments: [] }, narrative: [], + } }, + }; + db.run(`CREATE TRIGGER reject_terminal_stage BEFORE INSERT ON messages + WHEN NEW.role = 'assistant' BEGIN SELECT RAISE(FAIL, 'injected terminal failure'); END`); + await expect(port.transact(stage)).rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant'").get()) + .toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get(stage.operationId)).toEqual({ count: 0 }); + + db.run("DROP TRIGGER reject_terminal_stage"); + expect(await port.transact(stage)).toMatchObject({ kind: "committed", operationId: stage.operationId }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant' AND is_internal = 1").get()) + .toEqual({ count: 1 }); + }); + + it("fences a stopped execution to a cancelled durable outcome", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const port = new CanonicalExecutionWriterPort(writer, () => {}); + const op = (ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation => ({ + operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation, + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + expect(await port.transact(op(2, { + kind: "stop-requested", requestId: "stop-1", lastAdmittedOrdinal: 1, + }))).toMatchObject({ kind: "committed" }); + const nativeCursor = { providerId: "codex", scope: "thread", value: "native-thread", provenance: "native" }; + expect(await port.transact(op(3, { kind: "checkpoint", phase: "stopping", nativeCursor }))) + .toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT phase, native_cursor_json FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ phase: "stopping", native_cursor_json: JSON.stringify(nativeCursor) }); + expect(await port.transact(op(4, { kind: "provider-outcome", outcome: "completed" }))) + .toEqual({ kind: "conflict", operationId: `${lease.leaseId}:4` }); + expect(await port.transact(op(4, { kind: "provider-outcome", outcome: "cancelled" }))) + .toMatchObject({ kind: "committed" }); + expect(await port.transact(op(5, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "cancelled", endedAt: NOW, + assistant: { content: "Partial answer", model: null, attachments: [] }, narrative: [], + } }))).toMatchObject({ kind: "committed" }); + expect(await port.transact(op(6, { kind: "finish", outcome: "cancelled", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" }, + } }))).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT outcome FROM messages WHERE role = 'assistant'").get()) + .toEqual({ outcome: "cancelled" }); + }); + + it("runs a complete assistant turn through an execution worker and the sole writer worker", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 16, maxPendingBytes: 64_000, reservedControl: 4, reservedControlBytes: 16_000, + maxPerExecutionPending: 12, maxPerExecutionBytes: 48_000, + reservedPerExecutionControl: 3, reservedPerExecutionControlBytes: 12_000, + }, + createWorker: () => new ExecutionThreadWorkerPort(port), + onWorkerLost: () => { throw new Error("Execution worker was lost"); }, + }); + try { + const claim = scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const send = (lease: ExecutionLease, command: Parameters[0]["command"]) => { + const admission = scheduler.submit({ execution, lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); + return admission.completion; + }; + const start = beginOperation().mutation; + if (start.kind !== "begin") throw new Error("Unexpected begin operation"); + await expect(send(claim.lease, { kind: "start", providerId: "codex", input: start.input })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + const nativeCursor = { providerId: "codex", scope: "thread", value: "native-worker-thread", provenance: "native" }; + await expect(send(claim.lease, { kind: "event", phase: "running", nativeCursor, events: [{ + eventId: `${EXECUTION_ID}:worker-item`, + routing: { ...execution, itemId: "worker-item" }, + sourceProviderId: "codex", sourceIdentities: [], sourceSequence: 1, + payload: { type: "item.recorded", item: { + id: "worker-item", threadId: THREAD_ID, turnId: TURN_ID, kind: "message", + providerIdentities: [], payload: { projection: "message", content: "Worker event" }, + createdAt: NOW, updatedAt: NOW, + } }, + }] })).resolves.toMatchObject({ + kind: "reply", + result: { kind: "committed", providerCommit: { outcome: "committed", eventCount: 1 } }, + }); + expect(db.prepare("SELECT native_cursor_json FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ native_cursor_json: JSON.stringify(nativeCursor) }); + await expect(send(claim.lease, { kind: "provider-outcome", outcome: "completed" })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + await expect(send(claim.lease, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, + assistant: { content: "Worker answer", model: null, attachments: [] }, narrative: [], + } })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + await expect(send(claim.lease, { kind: "finalize", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "completed", projection: { kind: "writer-staged" }, + } })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + await expect(send(claim.lease, { kind: "release" })) + .resolves.toEqual({ kind: "reply", result: { kind: "released" } }); + expect(scheduler.release(execution, claim.lease)).toBe(true); + expect(db.prepare("SELECT content, outcome FROM messages WHERE role = 'assistant' AND is_internal = 0").get()) + .toEqual({ content: "Worker answer", outcome: "completed" }); + expect(published).toContain(`${EXECUTION_ID}:worker-item`); + expect(published.indexOf(`${EXECUTION_ID}:worker-item`)) + .toBeLessThan(published.indexOf(`${EXECUTION_ID}:turn.completed`)); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + expect(await port.interruptWorkerLoss({ + execution, lease: claim.lease, reason: "Execution worker exited", recoveryIncidentId: "incident-complete", + })).toEqual({ + kind: "conflict", operationId: `${claim.lease.leaseId}:worker-lost`, recoveryState: "already-terminal", + }); + } finally { + scheduler.shutdown(); + } + }); + + it("stages a partial assistant after Stop before the cancelled terminal commit", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const port = new CanonicalExecutionWriterPort(writer, () => {}); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 16, maxPendingBytes: 64_000, reservedControl: 4, reservedControlBytes: 16_000, + maxPerExecutionPending: 12, maxPerExecutionBytes: 48_000, + reservedPerExecutionControl: 3, reservedPerExecutionControlBytes: 12_000, + }, + createWorker: () => new ExecutionThreadWorkerPort(port), + onWorkerLost: () => { throw new Error("Execution worker was lost"); }, + }); + try { + const claim = scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const send = (command: Parameters[0]["command"]) => { + const admission = scheduler.submit({ execution, lease: claim.lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); + return admission.completion; + }; + const start = beginOperation().mutation; + if (start.kind !== "begin") throw new Error("Unexpected begin operation"); + expect((await send({ kind: "start", providerId: "codex", input: start.input })).kind).toBe("reply"); + expect((await send({ kind: "stop", requestId: "stop-partial" })).kind).toBe("reply"); + expect((await send({ kind: "provider-outcome", outcome: "cancelled" })).kind).toBe("reply"); + expect((await send({ kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "cancelled", endedAt: NOW, + assistant: { content: "Partial answer", model: null, attachments: [] }, narrative: [], + } })).kind).toBe("reply"); + await expect(send({ kind: "finalize", outcome: "cancelled", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" }, + } })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + expect(db.prepare("SELECT content, outcome FROM messages WHERE role = 'assistant' AND is_internal = 0").get()) + .toEqual({ content: "Partial answer", outcome: "cancelled" }); + } finally { + scheduler.shutdown(); + } + }); + + it("projects Codex parent and child events on the writer and replays without repeating child writes", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 16, maxPendingBytes: 64_000, reservedControl: 4, reservedControlBytes: 16_000, + maxPerExecutionPending: 12, maxPerExecutionBytes: 48_000, + reservedPerExecutionControl: 3, reservedPerExecutionControlBytes: 12_000, + }, + createWorker: () => new ExecutionThreadWorkerPort(port), + onWorkerLost: () => { throw new Error("Execution worker was lost"); }, + }); + try { + const claim = scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const send = (command: Parameters[0]["command"]) => { + const admission = scheduler.submit({ execution, lease: claim.lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); + return admission.completion; + }; + const start = beginOperation().mutation; + if (start.kind !== "begin") throw new Error("Unexpected begin operation"); + await expect(send({ kind: "start", providerId: "codex", input: start.input })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + + const parent = runtimeDraft(1, { + type: AgentEventType.ToolUse, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + toolCallId: "spawn-1", toolName: "Agent", toolInput: {}, + }, { + providerId: "codex", kind: "codex-collaboration", + collaboration: { kind: "spawnAgent", receiverThreadIds: ["native-child"], prompt: "Inspect the task" }, + }); + const parentCommand = { kind: "event" as const, phase: "running", nativeCursor: null, events: [parent] }; + await expect(send(parentCommand)).resolves.toMatchObject({ + kind: "reply", result: { kind: "committed", providerEvents: [{ + event: { type: AgentEventType.ToolUse, toolInput: {}, subagentPresentation: { + detail: { kind: "canonical-child" }, + } }, + }] }, + }); + const delegation = db.prepare("SELECT target_thread_id FROM canonical_collaboration_actions WHERE source_item_id = ?") + .get("toolCall:spawn-1"); + expect(delegation).toMatchObject({ target_thread_id: expect.any(String) }); + const parentOperation: ExecutionSemanticOperation = { + operationId: `${claim.lease.leaseId}:2`, execution, lease: claim.lease, ordinal: 2, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, events: [parent] }, + }; + const replayedParent = await port.transact(parentOperation); + expect(replayedParent).toMatchObject({ kind: "committed", providerEvents: [{ + event: { toolInput: {}, subagentPresentation: { detail: { kind: "canonical-child" } } }, + }] }); + expect(replayedParent).toMatchObject(db.prepare("SELECT last_durable_sequence AS durableRevision FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_collaboration_actions WHERE source_item_id = ?") + .get("toolCall:spawn-1")).toEqual({ count: 1 }); + + const child = runtimeDraft(2, { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + }, { + providerId: "codex", kind: "codex-collaboration", + child: { nativeThreadId: "native-child", nativeTurnId: "native-turn", parentCollaborationItemId: "spawn-1" }, + }); + await expect(send({ kind: "event", phase: "running", nativeCursor: null, events: [child] })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", providerEvents: [] } }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_turns WHERE thread_id = (SELECT target_thread_id FROM canonical_collaboration_actions WHERE source_item_id = ?)") + .get("toolCall:spawn-1")).toEqual({ count: 1 }); + const childEventsBefore = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE thread_id = (SELECT target_thread_id FROM canonical_collaboration_actions WHERE source_item_id = ?)") + .get("toolCall:spawn-1"); + expect(childEventsBefore).toMatchObject({ count: expect.any(Number) }); + const publishedBeforeReplay = published.length; + expect(await port.transact({ + operationId: `${claim.lease.leaseId}:3`, execution, lease: claim.lease, ordinal: 3, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, events: [child] }, + })).toMatchObject({ kind: "committed", providerEvents: [] }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE thread_id = (SELECT target_thread_id FROM canonical_collaboration_actions WHERE source_item_id = ?)") + .get("toolCall:spawn-1")).toEqual(childEventsBefore); + expect(published.length).toBeGreaterThan(publishedBeforeReplay); + expect(published).toContain(parent.eventId); + expect(published).toContain(child.eventId); + } finally { + scheduler.shutdown(); + } + }); + +}); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts new file mode 100644 index 000000000..06c7f825d --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -0,0 +1,413 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { PlanQuestionAnswersRepo } from "../../planning/persistence/plan-question-answers-repo.js"; +import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; +import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; +import type { PreparedExecutionFileEvidence } from "../../turns/turn-execution-file-evidence.js"; +import { + CanonicalParentTurnWrite, + type DataOnlyParentTerminalProjectionInput, + type DataOnlyParentTurnFinishInput, + type DataOnlyParentTurnStartInput, +} from "../canonical-parent-turn-write.js"; + +const THREAD_ID = "thread-1"; +const TURN_ID = "turn-1"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; +const NOW = "2026-09-24T10:00:00.000Z"; + +function seedThread(db: Database): void { + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-1", "Workspace", "C:/fixture", NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "workspace-1", "Thread", "main", "codex", NOW, NOW); +} + +function startInput(): DataOnlyParentTurnStartInput { + return { + thread: { id: THREAD_ID, workspaceId: "workspace-1", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, + executionId: EXECUTION_ID, + permissionMode: "supervised", + providerIdentities: [], + userMessage: { kind: "create", messageId: "user-1", content: "Question", sequence: 1 }, + }; +} + +function finishInput(message: ReturnType): DataOnlyParentTurnFinishInput { + return { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed", + projection: { message, narrative: [] }, + }; +} + +function terminalProjectionInput(): DataOnlyParentTerminalProjectionInput { + const narrative: ParentNarrativeRecoveryItem[] = [ + { + kind: "toolCall", + record: { + id: "tool-1", message_id: "", parent_tool_call_id: null, tool_name: "Read", + display_name: null, provider_agent_key: null, subagent_identity_key: null, + subagent_provider_name: null, subagent_prompt: null, subagent_type: null, + subagent_agent_id: null, subagent_duration_ms: null, model: null, reasoning_effort: null, + input_summary: "file.txt", output_summary: "", output_total_bytes: null, + output_artifact_path: null, exit_code: null, status: "running", + started_at: NOW, completed_at: null, sort_order: 1, + }, + }, + { + kind: "narrationSegment", + record: { id: "thought-1", message_id: "", text: "Answer", started_at: NOW, ended_at: NOW, sort_order: 2 }, + }, + { + kind: "hook", + record: { + id: "hook-1", message_id: "", hook_name: "Stop", tool_name: null, phase: "stop", + payload: "{}", duration_ms: null, did_block: false, started_at: NOW, + ended_at: null, sort_order: 3, + }, + }, + ]; + return { + threadId: THREAD_ID, + executionId: EXECUTION_ID, + outcome: "completed", + endedAt: "2026-09-24T10:00:01.000Z", + assistant: { + content: "Answer", + model: "claude-sonnet-4-6", + attachments: [{ id: "attachment-1", name: "result.txt", mimeType: "text/plain", sizeBytes: 6 }], + }, + narrative, + }; +} + +describe("CanonicalParentTurnWrite", () => { + let directory: string; + let path: string; + let db: Database; + let published: string[][]; + let writer: CanonicalParentTurnWrite; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-parent-write-")); + path = NodePath.join(directory, "mcode.db"); + db = openDatabase({ dbPath: path }); + seedThread(db); + published = []; + writer = new CanonicalParentTurnWrite(db, (events) => { + published.push(events.map((event) => event.eventId)); + }); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + it("stages the assistant after provider system notices that follow its user prompt", () => { + writer.start(startInput()); + const messages = new MessageRepo(db); + messages.createSystemNotice(THREAD_ID, "Provider notice", 2, undefined); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + + expect(writer.stageLiveAssistant({ threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }, { + precedingMessageId: "user-1", messageId, content: "Answer", model: null, attachments: [], + })).toBe(true); + expect(messages.findByIdInThreadIncludingInternal(THREAD_ID, messageId)) + .toMatchObject({ sequence: 3, is_internal: true, content: "Answer" }); + }); + + it("does not stage the assistant after a later user prompt", () => { + writer.start(startInput()); + const messages = new MessageRepo(db); + messages.createSystemNotice(THREAD_ID, "Provider notice", 2, undefined); + messages.create(THREAD_ID, "user", "Later prompt", 3); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + + expect(writer.stageLiveAssistant({ threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }, { + precedingMessageId: "user-1", messageId, content: "Answer", model: null, attachments: [], + })).toBe(false); + expect(messages.findByIdInThreadIncludingInternal(THREAD_ID, messageId)).toBeNull(); + }); + + it("commits cloneable start, event checkpoint and terminal projection across a reload", async () => { + const start = startInput(); + expect(() => structuredClone(start)).not.toThrow(); + expect(writer.start(start).outcome).toBe("committed"); + expect(writer.start(start).outcome).toBe("duplicate"); + + const event = { + eventId: `${EXECUTION_ID}:item-1`, + routing: { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, itemId: "item-1" }, + sourceProviderId: "codex", + sourceIdentities: [], + payload: { + type: "item.recorded" as const, + item: { + id: "item-1", + threadId: THREAD_ID, + turnId: TURN_ID, + kind: "message" as const, + providerIdentities: [], + payload: { projection: "message", content: "Event" }, + createdAt: NOW, + updatedAt: NOW, + }, + }, + }; + expect(writer.append({ threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: [event] }).outcome) + .toBe("committed"); + expect(writer.append({ threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: [event] }).outcome) + .toBe("duplicate"); + + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = finishInput(staged); + expect(() => structuredClone(finish)).not.toThrow(); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect((await writer.finish(finish)).outcome).toBe("terminal-outcome-confirmed"); + expect(published.flat()).toContain(`${EXECUTION_ID}:turn.completed`); + + db.close(true); + db = openDatabase({ dbPath: path }); + const checkpoint = db.prepare("SELECT phase, terminal_outcome, last_accepted_sequence, last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID) as { phase: string; terminal_outcome: string; last_accepted_sequence: number; last_durable_sequence: number }; + expect(checkpoint).toMatchObject({ phase: "completed", terminal_outcome: "completed" }); + expect(checkpoint.last_accepted_sequence).toBe(checkpoint.last_durable_sequence); + expect(checkpoint.last_accepted_sequence).toBeGreaterThan(4); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, staged.id)).toMatchObject({ is_internal: false, outcome: "completed" }); + }); + + it("rolls back a failed user-message projection with canonical start", () => { + db.prepare("UPDATE threads SET user_completed_at = ? WHERE id = ?").run(NOW, THREAD_ID); + db.run("CREATE TRIGGER fail_user_message BEFORE INSERT ON messages BEGIN SELECT RAISE(ABORT, 'message unavailable'); END"); + expect(() => writer.start({ ...startInput(), reopenThread: true })).toThrow("message unavailable"); + expect(db.prepare("SELECT user_completed_at FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ user_completed_at: NOW }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_ingest_checkpoints").get()).toEqual({ count: 0 }); + expect(published).toEqual([]); + }); + + it("reuses a queued user message while reopening the thread and answering a plan question", () => { + const messages = new MessageRepo(db); + const queued = messages.create(THREAD_ID, "user", "Queued answer", 1); + const plan = messages.create(THREAD_ID, "assistant", "Question", 2); + db.prepare("UPDATE threads SET user_completed_at = ? WHERE id = ?").run(NOW, THREAD_ID); + const input: DataOnlyParentTurnStartInput = { + ...startInput(), + userMessage: { kind: "existing", messageId: queued.id }, + reopenThread: true, + answeredPlanQuestionMessageId: plan.id, + }; + + expect(writer.start(structuredClone(input)).outcome).toBe("committed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'user'").get()).toEqual({ count: 1 }); + expect(db.prepare("SELECT user_completed_at FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ user_completed_at: null }); + expect(new PlanQuestionAnswersRepo(db).isAnswered(plan.id)).toBe(true); + }); + + it("does not confirm a terminal checkpoint when assistant publication fails", async () => { + writer.start(startInput()); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + db.run("CREATE TRIGGER fail_assistant_outcome BEFORE UPDATE OF outcome ON messages BEGIN SELECT RAISE(ABORT, 'outcome unavailable'); END"); + const finish = finishInput(staged); + await expect(writer.finish(finish)).rejects.toThrow("outcome unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).find((message) => message.id === staged.id)) + .toMatchObject({ is_internal: true, outcome: null }); + expect(published.flat()).not.toContain(`${EXECUTION_ID}:turn.completed`); + + db.run("DROP TRIGGER fail_assistant_outcome"); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, staged.id)).toMatchObject({ is_internal: false, outcome: "completed" }); + }); + + it("stages cloneable terminal rows once and publishes only after finish", async () => { + writer.start(startInput()); + const input = terminalProjectionInput(); + expect(() => structuredClone(input)).not.toThrow(); + new ToolCallRecordRepo(db).create({ + toolCallId: "tool-1", messageId: "user-1", toolName: "Read", inputSummary: "earlier", + outputSummary: "", status: "running", startedAt: NOW, sortOrder: 1, + }); + + const first = writer.stageTerminalProjection(input); + expect(first.messageId).toMatch(/^[0-9a-f]{64}$/); + expect(first.toolCallCount).toBe(1); + expect(db.prepare("SELECT is_internal, outcome FROM messages WHERE id = ?").get(first.messageId!)) + .toEqual({ is_internal: 1, outcome: null }); + expect(db.prepare("SELECT message_id, status, completed_at FROM tool_call_records WHERE id = 'tool-1'").get()) + .toEqual({ message_id: first.messageId, status: "completed", completed_at: input.endedAt }); + expect(db.prepare("SELECT message_id, is_final_response FROM thought_segments WHERE id = 'thought-1'").get()) + .toEqual({ message_id: first.messageId, is_final_response: 1 }); + expect(db.prepare("SELECT message_id, ended_at FROM hook_executions WHERE id = 'hook-1'").get()) + .toEqual({ message_id: first.messageId, ended_at: input.endedAt }); + expect(published.flat()).not.toContain(`${EXECUTION_ID}:turn.completed`); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalParentTurnWrite(db, (events) => { + published.push(events.map((event) => event.eventId)); + }); + const repeated = writer.stageTerminalProjection(input); + expect(repeated.messageId).toBe(first.messageId); + expect(repeated.toolCallCount).toBe(1); + expect(() => structuredClone(repeated.projection)).not.toThrow(); + expect(db.prepare("SELECT COUNT(*) AS count FROM tool_call_records WHERE message_id = ?").get(first.messageId!)) + .toEqual({ count: 1 }); + const finish = { ...finishInput(first.projection.message!), projection: first.projection }; + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(published.flat()).toContain(`${EXECUTION_ID}:turn.completed`); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, first.messageId!)) + .toMatchObject({ is_internal: false, outcome: "completed", attachments: input.assistant.attachments }); + expect(writer.stageTerminalProjection(input).messageId).toBe(first.messageId); + }); + + it("keeps an assigned public assistant identity through writer staging and reload", async () => { + writer.start(startInput()); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + const input = terminalProjectionInput(); + input.assistant.messageId = messageId; + const staged = writer.stageTerminalProjection(input); + expect(staged.messageId).toBe(messageId); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalParentTurnWrite(db, (events) => { + published.push(events.map((event) => event.eventId)); + }); + expect(writer.stageTerminalProjection(input).messageId).toBe(messageId); + const finish = { ...finishInput(staged.projection.message!), projection: { kind: "writer-staged" as const, messageId } }; + expect(() => writer.finish({ ...finish, projection: { kind: "writer-staged", messageId: "0".repeat(64) } })) + .toThrow("Staged assistant projection not found"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, messageId)).toMatchObject({ + is_internal: false, outcome: "completed", content: input.assistant.content, + }); + }); + + it("commits selected diff evidence with the assigned terminal assistant", async () => { + writer.start(startInput()); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + const input = terminalProjectionInput(); + input.assistant.messageId = messageId; + writer.stageTerminalProjection(input); + const patch = "diff --git a/a.txt b/a.txt\nindex 1..2\n--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-old\n+new\n"; + const finish: DataOnlyParentTurnFinishInput = { + ...finishInput(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, messageId)!), + projection: { kind: "writer-staged", messageId }, + selectedTurnDiff: { thread_id: THREAD_ID, source: "native", patch, revision: 2 }, + }; + expect(() => writer.finish({ ...finish, selectedTurnDiff: { ...finish.selectedTurnDiff!, thread_id: "other" } })) + .toThrow("Invalid selected turn diff"); + db.run("CREATE TRIGGER fail_turn_diff BEFORE INSERT ON turn_diff_snapshots BEGIN SELECT RAISE(ABORT, 'diff unavailable'); END"); + await expect(writer.finish(finish)).rejects.toThrow("diff unavailable"); + expect(db.prepare("SELECT is_internal FROM messages WHERE id = ?").get(messageId)).toEqual({ is_internal: 1 }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + db.run("DROP TRIGGER fail_turn_diff"); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(db.prepare("SELECT message_id, source, patch, revision FROM turn_diff_snapshots WHERE message_id = ?") + .get(messageId)).toEqual({ message_id: messageId, source: "native", patch, revision: 2 }); + expect(db.prepare("SELECT count(*) AS count FROM turn_diff_snapshots").get()).toEqual({ count: 1 }); + }); + + it("commits exact-execution file evidence with the terminal assistant or rolls it all back", async () => { + writer.start(startInput()); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + const input = terminalProjectionInput(); + input.assistant.messageId = messageId; + const staged = writer.stageTerminalProjection(input); + const patch = "diff --git a/a.txt b/a.txt\nindex 1..2\n--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-old\n+new\n"; + const fileEffects = { + revision: 1, fileCount: 1, additions: 1, deletions: 1, + effects: [{ path: "a.txt", kind: "edited" as const, scope: "workspace" as const, + additions: 1, deletions: 1, binary: false, toolCallIds: ["tool-1"] }], + }; + const fileEvidence: PreparedExecutionFileEvidence = { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, deliveryAttempt: 2, + fileEffects, filesChanged: ["a.txt"], + snapshot: { + threadId: THREAD_ID, executionId: EXECUTION_ID, refBefore: "a".repeat(40), + refAfter: "b".repeat(40), filesChanged: ["a.txt"], fileEffects, worktreePath: null, + }, + selectedTurnDiff: { thread_id: THREAD_ID, source: "native", patch, revision: 2 }, + }; + const finish: DataOnlyParentTurnFinishInput = { + ...finishInput(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, messageId)!), + projection: { kind: "writer-staged", messageId }, deliveryAttempt: 2, fileEvidence, + }; + expect(() => writer.finish({ ...finish, fileEvidence: { ...fileEvidence, executionId: "old-execution" } })) + .toThrow("Invalid execution file evidence"); + expect(() => writer.finish({ ...finish, deliveryAttempt: 1 })).toThrow("Invalid execution file evidence"); + db.run("CREATE TRIGGER fail_snapshot BEFORE INSERT ON turn_snapshots BEGIN SELECT RAISE(ABORT, 'snapshot unavailable'); END"); + await expect(writer.finish(structuredClone(finish))).rejects.toThrow("snapshot unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT is_internal FROM messages WHERE id = ?").get(messageId)).toEqual({ is_internal: 1 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM turn_diff_snapshots").get()).toEqual({ count: 0 }); + db.run("DROP TRIGGER fail_snapshot"); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(db.prepare("SELECT message_id, thread_id, files_changed, ref_before, ref_after FROM turn_snapshots WHERE message_id = ?") + .get(messageId)).toEqual({ + message_id: messageId, thread_id: THREAD_ID, files_changed: '["a.txt"]', + ref_before: "a".repeat(40), ref_after: "b".repeat(40), + }); + expect(db.prepare("SELECT source, patch FROM turn_diff_snapshots WHERE message_id = ?").get(messageId)) + .toEqual({ source: "native", patch }); + expect(db.prepare("SELECT has_file_changes FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ has_file_changes: 1 }); + expect((await writer.finish({ ...finish, projection: staged.projection })).outcome).toBe("terminal-outcome-confirmed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM turn_snapshots").get()).toEqual({ count: 1 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM turn_diff_snapshots").get()).toEqual({ count: 1 }); + }); + + it("refuses to reuse a public assistant row as a staged turn projection", () => { + writer.start(startInput()); + const input = terminalProjectionInput(); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + input.assistant.messageId = messageId; + new MessageRepo(db).createAssistantIdempotent({ + id: messageId, threadId: THREAD_ID, content: input.assistant.content, + sequence: 2, model: input.assistant.model, attachments: [...input.assistant.attachments], + }); + + expect(() => writer.stageTerminalProjection(input)).toThrow("already public"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + }); + + it("rolls back all staged rows when a narrative write fails", async () => { + writer.start(startInput()); + const input = terminalProjectionInput(); + db.run("CREATE TRIGGER fail_hook BEFORE INSERT ON hook_executions BEGIN SELECT RAISE(ABORT, 'hook unavailable'); END"); + expect(() => writer.stageTerminalProjection(input)).toThrow("hook unavailable"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant'").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM tool_call_records").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM thought_segments").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(published.flat()).not.toContain(`${EXECUTION_ID}:turn.completed`); + + db.run("DROP TRIGGER fail_hook"); + const staged = writer.stageTerminalProjection(input); + expect(staged.messageId).not.toBeNull(); + expect(staged.toolCallCount).toBe(1); + }); +}); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts new file mode 100644 index 000000000..322c72a80 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts @@ -0,0 +1,175 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { AgentEventType, type PlanQuestion } from "@mcode/contracts"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { PlanRepo } from "../../planning/persistence/plan-repo.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; +import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; +import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; +import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; +import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; +import { ExecutionLivePublicationRelease } from "../execution-live-publication-release.js"; +import { ExecutionPlanQuestionRelease } from "../execution-plan-question-release.js"; +import type { DataOnlyParentTurnStartInput } from "../canonical-parent-turn-write.js"; + +const THREAD_ID = "thread-plan"; +const TURN_ID = "turn-plan"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000091"; +const NOW = "2026-09-24T10:00:00.000Z"; +const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID } as const; +const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "lease-plan" } as const; +const questions = [{ id: "q1", category: "AUTH", question: "Which login?", options: [ + { id: "o1", title: "Passkey", description: "Use passkeys.", recommended: true }, + { id: "o2", title: "Password", description: "Use passwords." }, +] }]; +const planOutput = { title: "Login plan", contentMd: "# Login plan\n## Build\nUse passkeys.", + sectionsJson: '[{"id":"s1","title":"Build","level":2}]', changeSummary: null }; + +function seedThread(db: Database): void { + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-plan", "Workspace", "C:/fixture", NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "workspace-plan", "Thread", "main", "codex", NOW, NOW); +} + +function startInput(): DataOnlyParentTurnStartInput { + return { + thread: { id: THREAD_ID, workspaceId: "workspace-plan", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "user-plan", content: "Plan login", sequence: 1 }, + }; +} + +function operation(ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation { + return { operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation }; +} + +describe("Codex live plan projections on the sole writer", () => { + let directory: string; + let path: string; + let db: Database; + let writer: CanonicalExecutionSemanticWriter; + + beforeEach(async () => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-plan-projection-")); + path = NodePath.join(directory, "mcode.db"); + db = openDatabase({ dbPath: path }); + seedThread(db); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect((await writer.transact(operation(1, { kind: "begin", providerId: "codex", input: startInput() }))).kind) + .toBe("committed"); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + it("replays the question publication receipt after the writer restarts", async () => { + const block = `\`\`\`plan-questions\n${JSON.stringify(questions)}\n\`\`\``; + const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + isFinalResponse: true, delta: block }; + const write = { ...operation(2, { kind: "live-event", text: { kind: "append", inputs: [ + { ...execution, sequence: 1, text: block }, + ] }, planQuestions: questions }), livePublication: [{ after: "writer" as const, event }] }; + const published: Array<{ threadId: string; questions: readonly PlanQuestion[] }> = []; + const release = new ExecutionPlanQuestionRelease((threadId, batch) => published.push({ threadId, questions: batch })); + expect(published).toEqual([]); + const receipt = await writer.transact(write); + expect(receipt).toMatchObject({ kind: "committed", planQuestions: { + publicationId: "lease-plan:2:plan-questions", threadId: THREAD_ID, questions, + }, livePublication: [{ event }] }); + release.release(write, receipt); + release.release(write, receipt); + expect(published).toEqual([{ threadId: THREAD_ID, questions }]); + const compound = { ...write, mutation: { + kind: "append-events" as const, phase: "running", nativeCursor: null, events: [], + parentLive: { text: write.mutation.text, planQuestions: questions }, + } }; + expect(new ExecutionPlanQuestionRelease(() => {}).validate(compound, receipt)).toEqual(receipt.kind === "committed" + ? receipt.planQuestions : null); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(write)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-plan:2")).toEqual({ count: 1 }); + }); + + it("publishes questions through the worker port only after its receipt", async () => { + const block = `\`\`\`plan-questions\n${JSON.stringify(questions)}\n\`\`\``; + const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + isFinalResponse: true, delta: block }; + const write = { ...operation(2, { kind: "live-event", text: { kind: "append", inputs: [ + { ...execution, sequence: 1, text: block }, + ] }, planQuestions: questions }), livePublication: [{ after: "writer" as const, event }] }; + const published: string[] = []; + const registry = new AgentEventPublicationRegistry(); + registry.bind((item) => published.push(`agent:${item.type}`)); + const client = new CanonicalAgentWriterClient(path); + const port = new CanonicalExecutionWriterPort(client, () => {}, + new ExecutionLivePublicationRelease(registry), + new ExecutionPlanQuestionRelease(() => published.push("plan.questions"))); + try { + expect(published).toEqual([]); + expect((await port.transact(write)).kind).toBe("committed"); + expect(published).toEqual(["agent:textDelta", "plan.questions"]); + expect((await port.transact(write)).kind).toBe("committed"); + expect(published).toEqual(["agent:textDelta", "plan.questions"]); + } finally { + await client.close(); + } + }); + + it("persists a plan with its staged assistant before acknowledging the message", async () => { + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-plan"); + const content = "# Login plan\n## Build\nUse passkeys."; + const event = { type: AgentEventType.Message, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + messageId, content, model: null, tokens: null }; + const write = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" }, + message: { precedingMessageId: "user-plan", messageId, content, model: null, attachments: [] }, + planOutput, + }), livePublication: [{ after: "writer" as const, event }] }; + db.run("CREATE TRIGGER fail_plan_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' BEGIN SELECT RAISE(ABORT, 'plan receipt unavailable'); END"); + await expect(writer.transact(write)).rejects.toThrow("plan receipt unavailable"); + expect(new PlanRepo(db).listByThread(THREAD_ID)).toEqual([]); + expect(db.prepare("SELECT id FROM messages WHERE id = ?").get(messageId)).toBeNull(); + db.run("DROP TRIGGER fail_plan_receipt"); + + const receipt = await writer.transact(write); + expect(receipt).toMatchObject({ kind: "committed", planOutput: { + threadId: THREAD_ID, messageId, title: planOutput.title, version: 1, + }, livePublication: [{ event }] }); + expect(db.prepare("SELECT is_internal FROM messages WHERE id = ?").get(messageId)).toEqual({ is_internal: 1 }); + expect(new PlanRepo(db).getByMessageId(messageId)).toEqual(receipt.kind === "committed" ? receipt.planOutput : null); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(write)).toEqual(receipt); + expect(new PlanRepo(db).listByThread(THREAD_ID)).toHaveLength(1); + expect(await writer.transact({ ...write, mutation: { ...write.mutation, planOutput: { + ...planOutput, title: "Different plan", + } } })).toEqual({ kind: "conflict", operationId: "lease-plan:2" }); + }); + + it("rejects detached or malformed plan data without advancing the execution", async () => { + const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: true, delta: "text" }; + const base = { ...operation(2, { kind: "live-event", text: { kind: "append", inputs: [ + { ...execution, sequence: 1, text: "text" }, + ] } }), livePublication: [{ after: "writer" as const, event }] }; + expect(await writer.transact({ ...base, mutation: { ...base.mutation, + planOutput } })).toEqual({ kind: "conflict", operationId: "lease-plan:2" }); + expect(await writer.transact({ ...base, mutation: { ...base.mutation, + planQuestions: [{ ...questions[0], options: [] }] } })).toEqual({ kind: "conflict", operationId: "lease-plan:2" }); + expect(new PlanRepo(db).listByThread(THREAD_ID)).toEqual([]); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-plan:2")).toEqual({ count: 0 }); + }); +}); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts index 0d86a14e2..1edf36c69 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts @@ -41,6 +41,7 @@ import { type TurnOutcome, } from "@mcode/contracts"; import { broadcast } from "../../../application/transport/push.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; import { canonicalAgentEvents, canonicalAgentIngestCheckpoints, @@ -56,6 +57,7 @@ import { ACTIVE_TURN_WRITE_BATCH_LIMITS, runBoundedWriteBatches, runBoundedWriteBatchesSync, + type RunBoundedWriteBatchesInput, type WriteBatchResult, } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { assertActiveTurnRecoveryRetention } from "../turns/active-turn-recovery-retention-policy.js"; @@ -188,6 +190,13 @@ export type CanonicalAgentBatchedCommitResult = Omit this.eventStore.commit(this.toEventStoreCommitInput(input))) + : this.eventStore.commit(this.toEventStoreCommitInput(input)); } private commitInsideTransaction(input: CanonicalAgentCommitInput): CanonicalAgentCommitResult { - return this.eventStore.applyWithinTransaction(this.toEventStoreCommitInput(input)); + return serverWorkTrace + ? serverWorkTrace.measure("canonical-write", input.threadId, input.executionId, + () => this.eventStore.applyWithinTransaction(this.toEventStoreCommitInput(input))) + : this.eventStore.applyWithinTransaction(this.toEventStoreCommitInput(input)); } private toEventStoreCommitInput(input: CanonicalAgentCommitInput): CanonicalAgentEventStoreInput { @@ -1741,6 +1760,26 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab return true; } + /** The writer worker yields between committed recovery batches so other SQLite writers can proceed. */ + async recordParentNarrativeRecoveryYielding( + input: ParentNarrativeRecoveryCommitInput, + ): Promise { + const turn = this.loadTurnByExecution(input.executionId); + if (!turn) return false; + const thread = this.loadThread(turn.threadId); + if (!thread) throw new Error(`Canonical parent thread not found: ${turn.threadId}`); + if (input.items.length === 0 && (input.discardedItemIds?.length ?? 0) === 0) return true; + const batch = this.parentNarrativeRecoveryBatchInput(input, thread, turn, new Date().toISOString()); + if (!batch) return true; + await runBoundedWriteBatches({ + ...batch, + beginImmediate: true, + // A macrotask-only yield can reacquire SQLite before another connection's busy waiter wakes. + yieldControl: () => new Promise((resolve) => setTimeout(resolve, 2)), + }); + return true; + } + /** Completes the canonical-to-display startup migration before provider recovery begins. */ async materializeConversationDisplay(): Promise { await this.displayMaterializer.runToCompletion(); @@ -1752,10 +1791,20 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab turn: AgentTurn, now: string, ): void { + const batch = this.parentNarrativeRecoveryBatchInput(input, thread, turn, now); + if (batch) runBoundedWriteBatchesSync(batch); + } + + private parentNarrativeRecoveryBatchInput( + input: ParentNarrativeRecoveryCommitInput, + thread: AgentThread, + turn: AgentTurn, + now: string, + ): RunBoundedWriteBatchesInput | null { const checkpoint = this.loadCheckpoint(input.executionId); if (!checkpoint) throw new Error(`Canonical parent checkpoint was not found: ${input.executionId}`); - if (checkpoint.terminalOutcome) return; - const operations = [ + if (checkpoint.terminalOutcome) return null; + const operations: ParentNarrativeRecoveryOperation[] = [ ...input.items.map((item) => ({ kind: "persist" as const, item })), ...(input.discardedItemIds ?? []).map((itemId) => ({ kind: "discard" as const, itemId })), ]; @@ -1766,7 +1815,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab operations.length, operations.reduce((total, operation) => total + byteLength(operation), 0), ); - runBoundedWriteBatchesSync({ + return { db: this.db, items: operations, limits: ACTIVE_TURN_WRITE_BATCH_LIMITS, @@ -1778,7 +1827,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab } this.discardParentNarrativeRecoveryItem(operation.itemId, thread, turn); }, - }); + }; } private persistParentNarrativeRecoveryItem( @@ -1984,15 +2033,16 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab }); } - /** Persist a terminal parent turn in bounded transactions and confirm it only in the final batch. */ + /** Persist a terminal parent turn in bounded transactions; the writer-local hook joins each batch transaction. */ async finishParentTurnBatched( input: CanonicalParentTurnFinishInput, + onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): Promise { const checkpoint = this.loadCheckpoint(input.executionId); if (checkpoint?.terminalOutcome) { return this.confirmedParentTerminalBatch(checkpoint, input.threadId); } - return this.writeParentTerminalBatches(input, checkpoint); + return this.writeParentTerminalBatches(input, checkpoint, onBatchWrite); } private confirmedParentTerminalBatch( @@ -2015,6 +2065,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab private async writeParentTerminalBatches( input: CanonicalParentTurnFinishInput, checkpoint: CanonicalAgentCheckpoint | null, + onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): Promise { const projection = input.projectTurn(); const endedAt = new Date().toISOString(); @@ -2042,7 +2093,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab terminalRevision, terminalEventId, ), - onBatchFinishing: () => this.finishParentTerminalBatch(state, input, terminalRevision), + onBatchFinishing: () => this.finishParentTerminalBatch(state, input, terminalRevision, onBatchWrite), onBatchCommitted: () => this.publishParentTerminalBatch(state), }); return this.parentTerminalBatchResult(state, writeBatches); @@ -2091,16 +2142,42 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab state: ParentTerminalBatchState, input: CanonicalParentTurnFinishInput, ): void { - state.modelState = this.loadState(input.threadId, input.executionId); - state.checkpoint = this.loadCheckpoint(input.executionId); + this.loadParentTerminalBatchState(state, input); state.acceptedAt = new Date().toISOString(); - state.acceptedSequence = state.checkpoint?.lastAcceptedSequence ?? 0; state.changed = false; state.wrote = false; state.terminal = false; state.ignoredTerminal = false; } + private loadParentTerminalBatchState( + state: ParentTerminalBatchState, + input: CanonicalParentTurnFinishInput, + ): void { + const modelState = createAgentModelState(); + const thread = this.loadThread(input.threadId); + if (thread) modelState.threads[thread.id] = thread; + const turnRow = this.loadCommitTurnStatement.get({ id: input.turnId, threadId: input.threadId }); + if (turnRow) { + const turn = this.turnFromRow(turnRow); + modelState.turns[turn.id] = turn; + } + state.checkpoint = this.loadCheckpoint(input.executionId); + state.acceptedSequence = state.checkpoint?.lastAcceptedSequence + ?? this.lastAcceptedSequence(input.threadId, input.executionId); + if (state.acceptedSequence > 0) { + modelState.lastAcceptedSequenceByExecution[input.executionId] = state.acceptedSequence; + } + // Every draft costs at least one row, bounding the sequences this transaction can accept. + this.addSequenceCollisions( + modelState, + input.executionId, + state.acceptedSequence + 1, + state.acceptedSequence + ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows, + ); + state.modelState = modelState; + } + private writeParentTerminalBatchDraft( state: ParentTerminalBatchState, input: CanonicalParentTurnFinishInput, @@ -2212,6 +2289,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab state: ParentTerminalBatchState, input: CanonicalParentTurnFinishInput, terminalRevision: number, + onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): void { const modelState = this.parentTerminalBatchModelState(state); if (!state.wrote) { @@ -2224,6 +2302,11 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab this.displayMaterializer.materializeItems(this.parentAssistantItemIds(input.turnId)); this.retireParentNarrativeRecovery(input.turnId); } + onBatchWrite?.({ + durableSequence: state.acceptedSequence, + publishedSequences: state.pendingPublication.map((event) => event.acceptedSequence), + terminal: state.terminal, + }); state.latest = this.committedParentTerminalBatchResult(state, input.threadId, terminalRevision); } @@ -2954,15 +3037,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab const acceptedSequences = events.map((event) => event.acceptedSequence); const firstAcceptedSequence = Math.min(...acceptedSequences); const lastAcceptedSequence = Math.max(...acceptedSequences); - const existing = this.loadCommitSequenceCollisionStatement.all({ - executionId: first.routing.executionId, - minSequence: firstAcceptedSequence, - maxSequence: lastAcceptedSequence, - }); - for (const event of existing) { - state.appliedEventIds[event.eventId] = true; - state.acceptedInputEventIds[`${first.routing.executionId}:${event.acceptedSequence}`] = event.eventId; - } + this.addSequenceCollisions(state, first.routing.executionId, firstAcceptedSequence, lastAcceptedSequence); const acceptedSequence = checkpoint?.lastAcceptedSequence ?? this.lastAcceptedSequence( first.routing.threadId, @@ -2973,6 +3048,19 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab } } + private addSequenceCollisions( + state: AgentModelState, + executionId: string, + minSequence: number, + maxSequence: number, + ): void { + const existing = this.loadCommitSequenceCollisionStatement.all({ executionId, minSequence, maxSequence }); + for (const event of existing) { + state.appliedEventIds[event.eventId] = true; + state.acceptedInputEventIds[`${executionId}:${event.acceptedSequence}`] = event.eventId; + } + } + private lastAcceptedSequence(threadId: string, executionId: string): number { const row = this.loadLastAcceptedSequenceStatement.get({ threadId, executionId }); return row?.acceptedSequence ?? 0; diff --git a/apps/server/src/features/agents/canonical/canonical-agent-event-sink.ts b/apps/server/src/features/agents/canonical/canonical-agent-event-sink.ts deleted file mode 100644 index d8ccbff9a..000000000 --- a/apps/server/src/features/agents/canonical/canonical-agent-event-sink.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { injectable } from "tsyringe"; - -import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; - -export * from "./canonical-agent-boundary.js"; - -/** Temporary compatibility façade for callers that have not yet moved to the named boundary. */ -@injectable() -export class CanonicalAgentEventSink extends CanonicalAgentBoundary {} diff --git a/apps/server/src/features/agents/canonical/canonical-agent-event-store.ts b/apps/server/src/features/agents/canonical/canonical-agent-event-store.ts index cded4bd95..6577b8595 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-event-store.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-event-store.ts @@ -13,7 +13,7 @@ import { import type { CanonicalAgentCommitResult, CanonicalAgentEventDraft, -} from "./canonical-agent-event-sink.js"; +} from "./canonical-agent-boundary.js"; import { decideCanonicalExecutionLifecycle } from "./canonical-execution-lifecycle.js"; /** Provider-neutral durable progress for one canonical execution. */ diff --git a/apps/server/src/features/agents/canonical/canonical-agent-read-repository.ts b/apps/server/src/features/agents/canonical/canonical-agent-read-repository.ts index da99e12c9..b9cb74689 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-read-repository.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-read-repository.ts @@ -12,7 +12,7 @@ import { type CanonicalAgentRevision, type Message, } from "@mcode/contracts"; -import type { CanonicalAgentCheckpoint } from "./canonical-agent-event-sink.js"; +import type { CanonicalAgentCheckpoint } from "./canonical-agent-boundary.js"; import { CanonicalConversationProjectionReader, type CanonicalConversationProjection, diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts new file mode 100644 index 000000000..e7724e0cc --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -0,0 +1,367 @@ +import * as NodePath from "node:path"; +import * as NodeCrypto from "node:crypto"; +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; +import type { + CanonicalParentNarrativeClassificationReceipt, + CanonicalParentNarrativeRecoveryReceipt, + CanonicalProviderWriteInput, + CanonicalProviderWriteReceipt, + CanonicalWriterRequest, + CanonicalWriterResponse, +} from "./canonical-agent-writer-protocol.js"; +import { SEMANTIC_PUBLICATION_PAGE_SIZE } from "./canonical-agent-writer-protocol.js"; +import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; +import type { ExecutionSemanticOperation, ExecutionWriteReceipt } from "../execution/execution-worker-handler.js"; +import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; + +const MAX_PENDING_WRITES = 64; +const MAX_WORKER_ATTEMPTS = 3; +const MAX_PENDING_ACKNOWLEDGEMENTS = 64; +const MAX_ACK_RETRIES_BEFORE_WRITE = 8; +const MAX_ACK_RETRIES_ON_CLOSE = 16; +const WRITER_EXECUTION_ID = "writer:init"; + +interface PendingRequest { + request: CanonicalWriterRequest; + onPublication?: (events: readonly CanonicalAgentEventEnvelope[]) => void; + resolve(response: CanonicalWriterResponse): void; + reject(error: Error): void; +} + +interface PendingAcknowledgement { + executionId: string; + operationId: string; +} + +class CanonicalWriterWorkerLost extends Error {} + +/** Sends cloneable canonical commands to a dedicated SQLite worker with bounded admission and crash retries. */ +export class CanonicalAgentWriterClient { + private worker: Worker | undefined; + private workerReady: Promise; + private restart: Promise | undefined; + private retryingAcknowledgements: Promise | undefined; + private readonly pending = new Map(); + private readonly pendingAcknowledgements = new Map(); + private generation = 0; + private closing = false; + private stopping = false; + + constructor( + private readonly dbPath: string, + private readonly createWorker: () => Worker = defaultCreateWorker, + ) { + if (!NodePath.isAbsolute(dbPath)) throw new Error("Canonical writer database path must be absolute"); + this.workerReady = this.startWorker(); + } + + /** Waits until the worker has opened the already-migrated database. */ + whenReady(): Promise { + return this.workerReady; + } + + /** Resolves with committed envelopes for main-loop publication; it never publishes them itself. */ + async commit(operationId: string, input: CanonicalProviderWriteInput): Promise { + if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); + const response = await this.sendWithRetry({ + kind: "commit", requestId: NodeCrypto.randomUUID(), operationId, executionId: input.executionId, input, + }); + if (response.kind !== "committed") throw new Error("Canonical writer returned an unexpected response"); + return response.receipt; + } + + /** Commits one execution operation and delivers bounded pages after each durable write. */ + async transactSemantic( + operation: ExecutionSemanticOperation, + onPublication: (events: readonly CanonicalAgentEventEnvelope[]) => void, + ): Promise { + if (!operation.operationId || !operation.execution.executionId) { + throw new Error("Semantic writer operation and execution IDs are required"); + } + const response = await this.sendWithRetry({ + kind: "semantic-transact", requestId: NodeCrypto.randomUUID(), + operationId: operation.operationId, executionId: operation.execution.executionId, operation, + }, false, onPublication); + if (response.kind !== "semantic-transacted") throw new Error("Canonical writer returned an unexpected response"); + return response.receipt; + } + + /** Reconcile a lost execution and deliver bounded committed publication pages. */ + async interruptWorkerLoss( + input: LostExecutionInterruption, + onPublication: (events: readonly CanonicalAgentEventEnvelope[]) => void, + ): Promise { + const operationId = `${input.lease.leaseId}:worker-lost`; + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); + const response = await this.sendWithRetry({ + kind: "semantic-worker-loss", requestId: NodeCrypto.randomUUID(), operationId, + executionId: input.execution.executionId, input, + }, false, onPublication); + if (response.kind !== "semantic-transacted") throw new Error("Canonical writer returned an unexpected response"); + return response.receipt; + } + + /** Resolves after recovery writes finish; a lost reply replays the durable receipt. */ + async recordParentNarrativeRecovery( + operationId: string, + input: ParentNarrativeRecoveryCommitInput, + ): Promise { + if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); + const response = await this.sendWithRetry({ + kind: "record-parent-narrative-recovery", requestId: NodeCrypto.randomUUID(), operationId, + executionId: input.executionId, input, + }); + if (response.kind !== "parent-narrative-recovery-recorded") { + throw new Error("Canonical writer returned an unexpected response"); + } + return response.receipt; + } + + /** Atomically persists recovery and resets provisional assistant text; the caller retires its journal after receipt. */ + async classifyParentNarrativeRecovery( + operationId: string, + input: ParentNarrativeRecoveryCommitInput, + ): Promise { + if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); + const response = await this.sendWithRetry({ + kind: "classify-parent-narrative-recovery", requestId: NodeCrypto.randomUUID(), operationId, + executionId: input.executionId, input, + }); + if (response.kind !== "parent-narrative-recovery-classified") { + throw new Error("Canonical writer returned an unexpected response"); + } + return response.receipt; + } + + /** Releases a receipt after its caller has finished publication or journal discard. Never acknowledge before that work. */ + async acknowledgeOperation(executionId: string, operationId: string): Promise { + if (!executionId || !operationId) throw new Error("Canonical writer operation and execution IDs are required"); + if (this.closing || this.stopping) throw new Error("Canonical writer closed"); + const key = acknowledgementKey(executionId, operationId); + try { + await this.sendAcknowledgement({ executionId, operationId }); + this.pendingAcknowledgements.delete(key); + } catch (error) { + if (!this.pendingAcknowledgements.has(key)) { + if (this.pendingAcknowledgements.size >= MAX_PENDING_ACKNOWLEDGEMENTS) { + throw new Error("Canonical writer acknowledgement retry queue is full", { cause: error }); + } + this.pendingAcknowledgements.set(key, { executionId, operationId }); + } + throw error; + } + } + + /** Number of handled operations whose receipt deletion still needs a retry. */ + get pendingAcknowledgementCount(): number { + return this.pendingAcknowledgements.size; + } + + /** Closes the database after accepted writes settle, then releases the worker. */ + async close(): Promise { + if (this.closing || this.stopping) return; + this.closing = true; + let acknowledgementError: Error | undefined; + try { + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_ON_CLOSE); + if (this.pendingAcknowledgements.size > 0) { + acknowledgementError = new Error( + `Canonical writer closed with ${this.pendingAcknowledgements.size} unacknowledged operations`, + ); + } + await this.workerReady; + if (this.worker) { + await this.sendRaw({ + kind: "close", requestId: NodeCrypto.randomUUID(), operationId: "writer:close", + executionId: WRITER_EXECUTION_ID, + }); + } + } catch { + // A failed worker is already closed; caller writes have their own visible failures. + } finally { + this.stopping = true; + this.loseWorker(new Error("Canonical writer closed")); + } + if (acknowledgementError) throw acknowledgementError; + } + + private sendAcknowledgement(acknowledgement: PendingAcknowledgement, allowDuringClose = false): Promise { + return this.sendWithRetry({ + kind: "ack-operation", requestId: NodeCrypto.randomUUID(), ...acknowledgement, + }, allowDuringClose).then((response) => { + if (response.kind !== "operation-acknowledged") { + throw new Error("Canonical writer returned an unexpected response"); + } + }); + } + + private retryPendingAcknowledgements(limit: number): Promise { + if (this.pendingAcknowledgements.size === 0) return Promise.resolve(); + if (!this.retryingAcknowledgements) { + this.retryingAcknowledgements = this.drainAcknowledgements(limit) + .finally(() => { this.retryingAcknowledgements = undefined; }); + } + return this.retryingAcknowledgements; + } + + private async drainAcknowledgements(limit: number): Promise { + for (const [key, acknowledgement] of [...this.pendingAcknowledgements].slice(0, limit)) { + try { + await this.sendAcknowledgement(acknowledgement, this.closing); + this.pendingAcknowledgements.delete(key); + } catch { + if (this.pendingAcknowledgements.get(key) !== acknowledgement) continue; + this.pendingAcknowledgements.delete(key); + this.pendingAcknowledgements.set(key, acknowledgement); + } + } + } + + private async sendWithRetry( + request: CanonicalWriterRequest, + allowDuringClose = false, + onPublication?: (events: readonly CanonicalAgentEventEnvelope[]) => void, + ): Promise { + let publishedCount = 0; + for (let attempt = 1; attempt <= MAX_WORKER_ATTEMPTS; attempt++) { + this.assertCanSend(allowDuringClose); + try { + await this.workerReady; + this.assertCanSend(allowDuringClose); + let attemptCount = 0; + const deliver = onPublication ? (events: readonly CanonicalAgentEventEnvelope[]) => { + const skipped = Math.min(events.length, Math.max(0, publishedCount - attemptCount)); + if (skipped < events.length) onPublication(events.slice(skipped)); + attemptCount += events.length; + publishedCount = Math.max(publishedCount, attemptCount); + } : undefined; + return await this.sendRaw(request, deliver); + } catch (error) { + if (!(error instanceof CanonicalWriterWorkerLost) || attempt === MAX_WORKER_ATTEMPTS) throw error; + await this.restartAfterLoss(allowDuringClose); + } + } + throw new Error("Canonical writer retry limit reached"); + } + + private assertCanSend(allowDuringClose: boolean): void { + if (this.stopping || (this.closing && !allowDuringClose)) throw new Error("Canonical writer closed"); + } + + private async restartAfterLoss(allowDuringClose: boolean): Promise { + try { + await this.restartWorker(allowDuringClose); + } catch (error) { + if (!(error instanceof CanonicalWriterWorkerLost)) throw error; + } + } + + private async restartWorker(allowDuringClose: boolean): Promise { + if (this.stopping || (this.closing && !allowDuringClose)) throw new Error("Canonical writer closed"); + if (!this.restart) { + this.restart = (async () => { + if (!this.worker) this.workerReady = this.startWorker(); + await this.workerReady; + })().finally(() => { this.restart = undefined; }); + } + await this.restart; + } + + private async startWorker(): Promise { + const generation = ++this.generation; + const worker = this.createWorker(); + this.worker = worker; + worker.onmessage = (message: MessageEvent) => { + if (this.generation === generation) this.receive(message.data); + }; + worker.onerror = () => this.loseWorker(new CanonicalWriterWorkerLost("Canonical writer worker failed"), generation); + worker.addEventListener("close", () => { + this.loseWorker(new CanonicalWriterWorkerLost("Canonical writer worker closed"), generation); + }); + const response = await this.sendRaw({ + kind: "open", requestId: NodeCrypto.randomUUID(), operationId: "writer:open", + executionId: WRITER_EXECUTION_ID, dbPath: this.dbPath, + }); + if (response.kind !== "opened") throw new Error("Canonical writer did not open its database"); + } + + private sendRaw( + request: CanonicalWriterRequest, + onPublication?: (events: readonly CanonicalAgentEventEnvelope[]) => void, + ): Promise { + const worker = this.worker; + if (!worker) return Promise.reject(new CanonicalWriterWorkerLost("Canonical writer worker closed")); + if (request.kind !== "open" && request.kind !== "close" && this.pending.size >= MAX_PENDING_WRITES) { + return Promise.reject(new Error("Canonical writer admission is full")); + } + return new Promise((resolve, reject) => { + this.pending.set(request.requestId, { request, onPublication, resolve, reject }); + try { + worker.postMessage(request); + } catch { + this.loseWorker(new CanonicalWriterWorkerLost("Canonical writer request could not be sent")); + } + }); + } + + private receive(response: CanonicalWriterResponse): void { + const pending = this.pending.get(response.requestId); + if (!pending) return; + if (pending.request.operationId !== response.operationId + || pending.request.executionId !== response.executionId) { + this.loseWorker(new Error("Canonical writer response identity mismatch")); + return; + } + if (response.kind === "semantic-publication") { + this.receivePublication(pending, response); + return; + } + this.pending.delete(response.requestId); + if (response.kind === "failed") { + pending.reject(new Error(`Canonical writer ${response.reason}`)); + return; + } + pending.resolve(response); + } + + private receivePublication( + pending: PendingRequest, + response: Extract, + ): void { + if (!pending.onPublication || response.events.length === 0 + || response.events.length > SEMANTIC_PUBLICATION_PAGE_SIZE) { + this.loseWorker(new Error("Canonical writer publication page is invalid")); + return; + } + try { + pending.onPublication(response.events); + } catch (error) { + this.pending.delete(response.requestId); + pending.reject(error instanceof Error ? error : new Error(String(error))); + } + } + + private loseWorker(error: Error, generation = this.generation): void { + if (generation !== this.generation) return; + const worker = this.worker; + this.worker = undefined; + worker?.terminate(); + for (const pending of this.pending.values()) pending.reject(error); + this.pending.clear(); + } +} + +function defaultCreateWorker(): Worker { + const workerFile = import.meta.url.endsWith(".cjs") + ? "./canonical-agent-writer.worker.cjs" + : "./canonical-agent-writer.worker.ts"; + const workerOptions = { type: "module", smol: true } satisfies WorkerOptions & { smol: boolean }; + return new Worker(new URL(workerFile, import.meta.url), workerOptions); +} + +function acknowledgementKey(executionId: string, operationId: string): string { + return JSON.stringify([executionId, operationId]); +} diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts new file mode 100644 index 000000000..81b5a09cb --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -0,0 +1,66 @@ +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; +import type { ExecutionSemanticOperation, ExecutionWriteReceipt } from "../execution/execution-worker-handler.js"; +import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; +import type { + CanonicalAgentCommitInput, + CanonicalAgentCommitResult, + CanonicalAgentEventDraft, + ParentNarrativeRecoveryCommitInput, +} from "./canonical-agent-boundary.js"; + +/** Upper bound for one committed semantic publication message. */ +export const SEMANTIC_PUBLICATION_PAGE_SIZE = 64; + +/** Cloneable provider batch accepted by the SQLite writer. Compatibility callbacks stay with their owner. */ +export type CanonicalProviderWriteInput = Pick< + CanonicalAgentCommitInput, + "threadId" | "turnId" | "executionId" | "phase" | "nativeCursor" +> & { events: readonly CanonicalAgentEventDraft[] }; + +/** Acknowledgement returned only after the canonical SQLite transaction has committed. */ +export interface CanonicalProviderWriteReceipt { + outcome: CanonicalAgentCommitResult["outcome"]; + conversationRevision: number; + rosterRevision: number; + acceptedThrough: number; + durableThrough: number; + events: readonly CanonicalAgentEventEnvelope[]; +} + +/** Acknowledges completed recovery writes; false means the execution was not found. */ +export interface CanonicalParentNarrativeRecoveryReceipt { + recorded: boolean; +} + +/** Both writes are acknowledged only after their shared transaction commits. */ +export interface CanonicalParentNarrativeClassificationReceipt { + recorded: true; + reset: true; +} + +interface Correlation { + requestId: string; + operationId: string; + executionId: string; +} + +export type CanonicalWriterRequest = + | (Correlation & { kind: "open"; dbPath: string }) + | (Correlation & { kind: "commit"; input: CanonicalProviderWriteInput }) + | (Correlation & { kind: "semantic-transact"; operation: ExecutionSemanticOperation }) + | (Correlation & { kind: "semantic-worker-loss"; input: LostExecutionInterruption }) + | (Correlation & { kind: "record-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) + | (Correlation & { kind: "classify-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) + | (Correlation & { kind: "ack-operation" }) + | (Correlation & { kind: "close" }); + +export type CanonicalWriterResponse = + | (Correlation & { kind: "opened" }) + | (Correlation & { kind: "committed"; receipt: CanonicalProviderWriteReceipt }) + | (Correlation & { kind: "semantic-publication"; events: readonly CanonicalAgentEventEnvelope[] }) + | (Correlation & { kind: "semantic-transacted"; receipt: ExecutionWriteReceipt }) + | (Correlation & { kind: "parent-narrative-recovery-recorded"; receipt: CanonicalParentNarrativeRecoveryReceipt }) + | (Correlation & { kind: "parent-narrative-recovery-classified"; receipt: CanonicalParentNarrativeClassificationReceipt }) + | (Correlation & { kind: "operation-acknowledged" }) + | (Correlation & { kind: "closed" }) + | (Correlation & { kind: "failed"; reason: "open-failed" | "write-failed" | "operation-conflict" | "receipt-capacity" }); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts new file mode 100644 index 000000000..2b773c788 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts @@ -0,0 +1,205 @@ +import type { Database } from "bun:sqlite"; +import * as NodeCrypto from "node:crypto"; +import { CanonicalAgentEventEnvelopeSchema } from "@mcode/contracts"; +import { and, count, eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/bun-sqlite"; +import { z } from "zod"; +import { + canonicalAgentEvents, + canonicalWriterOperationReceipts, +} from "../../../runtime/persistence/sqlite/schema.js"; +import type { CanonicalWriterRequest, CanonicalWriterResponse } from "./canonical-agent-writer-protocol.js"; + +type WriteRequest = Extract; +type AtomicWriteRequest = Extract; +type WriteResponse = Extract; + +const storedReceiptSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("committed"), + receipt: z.object({ + outcome: z.enum(["committed", "duplicate", "conflict", "terminal-outcome-confirmed", "ingest-overflow"]), + conversationRevision: z.number().int(), + rosterRevision: z.number().int(), + acceptedThrough: z.number().int(), + durableThrough: z.number().int(), + eventIds: z.array(z.string()), + }), + }), + z.object({ + kind: z.literal("parent-narrative-recovery-recorded"), + receipt: z.object({ recorded: z.boolean() }), + }), + z.object({ + kind: z.literal("parent-narrative-recovery-classified"), + receipt: z.object({ recorded: z.literal(true), reset: z.literal(true) }), + }), +]); + +/** An operation ID may be retried only with the same kind and semantic input. */ +export class CanonicalWriterOperationConflict extends Error {} + +/** The caller must acknowledge handled receipts before admitting more writes for an execution. */ +export class CanonicalWriterReceiptCapacity extends Error {} + +export const MAX_UNACKNOWLEDGED_RECEIPTS_PER_EXECUTION = 16_384; + +/** Atomically stores compact receipts so a restarted worker can replay lost acknowledgements. */ +export class CanonicalAgentWriterReceipts { + private readonly orm; + + constructor(private readonly db: Database) { + this.orm = drizzle(db); + } + + execute(request: AtomicWriteRequest, apply: () => WriteResponse): WriteResponse { + const inputHash = fingerprint(request); + return this.db.transaction(() => { + const existing = this.loadReceipt(request); + if (existing) return this.replayMatching(request, inputHash, existing); + this.assertCapacity(request.executionId); + const response = apply(); + this.orm.insert(canonicalWriterOperationReceipts).values({ + executionId: request.executionId, + operationId: request.operationId, + kind: request.kind, + inputHash, + receiptJson: compactReceipt(response), + }).run(); + return response; + })(); + } + + /** Recovery upserts are repeatable, so its bounded batches commit before a short final receipt transaction. */ + async executeBatchedRecovery( + request: Extract, + apply: () => Promise, + ): Promise { + const inputHash = fingerprint(request); + const existing = this.loadReceipt(request); + if (existing) return this.replayMatching(request, inputHash, existing); + this.assertCapacity(request.executionId); + const response = await apply(); + if (response.kind !== "parent-narrative-recovery-recorded") { + throw new Error("Canonical writer returned the wrong recovery response"); + } + if (!response.receipt.recorded) return response; + return this.db.transaction(() => { + const committed = this.loadReceipt(request); + if (committed) return this.replayMatching(request, inputHash, committed); + this.assertCapacity(request.executionId); + this.orm.insert(canonicalWriterOperationReceipts).values({ + executionId: request.executionId, + operationId: request.operationId, + kind: request.kind, + inputHash, + receiptJson: compactReceipt(response), + }).run(); + return response; + })(); + } + + /** Removes a receipt only after the caller has completed its dependent publication or journal discard. */ + acknowledge(executionId: string, operationId: string): void { + this.orm.delete(canonicalWriterOperationReceipts).where(and( + eq(canonicalWriterOperationReceipts.executionId, executionId), + eq(canonicalWriterOperationReceipts.operationId, operationId), + )).run(); + } + + private loadReceipt(request: WriteRequest) { + return this.orm.select().from(canonicalWriterOperationReceipts) + .where(and( + eq(canonicalWriterOperationReceipts.executionId, request.executionId), + eq(canonicalWriterOperationReceipts.operationId, request.operationId), + )).get(); + } + + private replayMatching(request: WriteRequest, inputHash: string, existing: NonNullable>): WriteResponse { + if (existing.kind !== request.kind || existing.inputHash !== inputHash) { + throw new CanonicalWriterOperationConflict("Canonical writer operation ID was reused with different input"); + } + return this.replay(request, existing.receiptJson); + } + + private assertCapacity(executionId: string): void { + const outstanding = this.orm.select({ count: count() }).from(canonicalWriterOperationReceipts) + .where(eq(canonicalWriterOperationReceipts.executionId, executionId)).get()?.count ?? 0; + if (outstanding >= MAX_UNACKNOWLEDGED_RECEIPTS_PER_EXECUTION) { + throw new CanonicalWriterReceiptCapacity("Canonical writer receipt capacity reached"); + } + } + + private replay(request: WriteRequest, receiptJson: string): WriteResponse { + const stored = storedReceiptSchema.parse(JSON.parse(receiptJson)); + const correlation = { + requestId: request.requestId, + operationId: request.operationId, + executionId: request.executionId, + }; + if (stored.kind === "committed") { + return { + ...correlation, + kind: "committed", + receipt: { + outcome: stored.receipt.outcome, + conversationRevision: stored.receipt.conversationRevision, + rosterRevision: stored.receipt.rosterRevision, + acceptedThrough: stored.receipt.acceptedThrough, + durableThrough: stored.receipt.durableThrough, + events: stored.receipt.eventIds.map((eventId) => this.loadEvent(request.executionId, eventId)), + }, + }; + } + if (stored.kind === "parent-narrative-recovery-recorded") { + return { ...correlation, kind: stored.kind, receipt: stored.receipt }; + } + return { ...correlation, kind: stored.kind, receipt: stored.receipt }; + } + + private loadEvent(executionId: string, eventId: string) { + const row = this.orm.select({ envelopeJson: canonicalAgentEvents.envelopeJson }) + .from(canonicalAgentEvents) + .where(and( + eq(canonicalAgentEvents.executionId, executionId), + eq(canonicalAgentEvents.eventId, eventId), + )) + .get(); + if (!row) throw new Error(`Canonical writer receipt event was not found: ${eventId}`); + return CanonicalAgentEventEnvelopeSchema.parse(JSON.parse(row.envelopeJson)); + } +} + +function compactReceipt(response: WriteResponse): string { + if (response.kind !== "committed") { + return JSON.stringify({ kind: response.kind, receipt: response.receipt }); + } + const { events, ...receipt } = response.receipt; + return JSON.stringify({ + kind: response.kind, + receipt: { ...receipt, eventIds: events.map((event) => event.eventId) }, + }); +} + +function fingerprint(request: WriteRequest): string { + if (!request.operationId || request.operationId.length > 256 || !request.executionId) { + throw new Error("Canonical writer operation and execution IDs are required and bounded"); + } + const input = stableJson({ kind: request.kind, input: request.input }); + return NodeCrypto.createHash("sha256").update(input).digest("hex"); +} + +function stableJson(value: unknown): string { + return JSON.stringify(sortJsonValue(JSON.parse(JSON.stringify(value)))); +} + +function sortJsonValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJsonValue); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, sortJsonValue(entry)])); +} diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts new file mode 100644 index 000000000..d472ff0a8 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -0,0 +1,310 @@ +import "reflect-metadata"; +import { gc } from "bun"; +import { Database } from "bun:sqlite"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import { applySQLiteConnectionPolicy } from "../../../runtime/persistence/sqlite/sqlite-connection-policy.js"; +import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; +import { CanonicalExecutionSemanticWriter } from "./canonical-execution-semantic-writer.js"; +import { isGroupableAppend, selectAppendGroup, type QueuedCanonicalWrite } from "./canonical-append-group.js"; +import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; +import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; +import { CanonicalAgentWriterReceipts, CanonicalWriterOperationConflict, CanonicalWriterReceiptCapacity } from "./canonical-agent-writer-receipts.js"; +import type { + CanonicalParentNarrativeClassificationReceipt, + CanonicalProviderWriteInput, + CanonicalWriterRequest, + CanonicalWriterResponse, +} from "./canonical-agent-writer-protocol.js"; +import { SEMANTIC_PUBLICATION_PAGE_SIZE } from "./canonical-agent-writer-protocol.js"; +import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; + +let db: Database | undefined; +let boundary: CanonicalAgentBoundary | undefined; +let assistantTextCheckpoints: ParentAssistantTextCheckpointService | undefined; +let receipts: CanonicalAgentWriterReceipts | undefined; +let semanticWriter: CanonicalExecutionSemanticWriter | undefined; +let semanticPublication: Pick | undefined; + +function openDatabase(dbPath: string): void { + if (boundary || !NodePath.isAbsolute(dbPath) || !NodeFS.existsSync(dbPath)) { + throw new Error("Canonical writer requires an existing absolute database path"); + } + const connection = new Database(dbPath, { strict: true, readwrite: true }); + try { + applySQLiteConnectionPolicy(connection, true); + boundary = new CanonicalAgentBoundary(connection, () => {}); + assistantTextCheckpoints = new ParentAssistantTextCheckpointService(connection); + receipts = new CanonicalAgentWriterReceipts(connection); + semanticWriter = new CanonicalExecutionSemanticWriter(connection, (events) => { + const correlation = semanticPublication; + if (!correlation) throw new Error("Semantic publication has no active request"); + publishSemantic(correlation, events); + }); + db = connection; + } catch (error) { + boundary = undefined; + assistantTextCheckpoints = undefined; + receipts = undefined; + semanticWriter = undefined; + connection.close(true); + throw error; + } +} + +function classifyParentNarrativeRecovery( + input: ParentNarrativeRecoveryCommitInput, +): CanonicalParentNarrativeClassificationReceipt { + const connection = db; + const canonical = boundary; + const checkpoints = assistantTextCheckpoints; + if (!connection || !canonical || !checkpoints) throw new Error("Canonical writer has not opened its database"); + return connection.transaction(() => { + const recorded = canonical.recordParentNarrativeRecovery(input); + if (!recorded) throw new Error("Canonical parent turn was not found"); + const reset = checkpoints.resetInTransaction(input.executionId); + if (!reset) throw new Error("Provisional assistant text checkpoint was not reset"); + return { recorded, reset }; + })(); +} + +function assertRouting(input: CanonicalProviderWriteInput, executionId: string): void { + if (input.executionId !== executionId || !input.threadId || !input.turnId || !input.phase + || !Array.isArray(input.events) || input.events.length === 0) { + throw new Error("Invalid canonical writer request"); + } + if (!input.events.every((event) => eventMatchesRouting(event, input, executionId))) { + throw new Error("Canonical writer event routing mismatch"); + } +} + +function eventMatchesRouting( + event: CanonicalProviderWriteInput["events"][number], + input: CanonicalProviderWriteInput, + executionId: string, +): boolean { + return event.routing.threadId === input.threadId + && event.routing.executionId === executionId + && (event.routing.turnId === undefined || event.routing.turnId === input.turnId); +} + +async function handle(request: CanonicalWriterRequest): Promise { + const correlation = { + requestId: request.requestId, + operationId: request.operationId, + executionId: request.executionId, + }; + if (request.kind === "open") { + try { + openDatabase(request.dbPath); + return { ...correlation, kind: "opened" }; + } catch { + return { ...correlation, kind: "failed", reason: "open-failed" }; + } + } + if (request.kind === "close") { + boundary = undefined; + assistantTextCheckpoints = undefined; + receipts = undefined; + semanticWriter = undefined; + db?.close(true); + db = undefined; + return { ...correlation, kind: "closed" }; + } + if (request.kind === "ack-operation") { + try { + if (!receipts) throw new Error("Canonical writer has not opened its database"); + receipts.acknowledge(request.executionId, request.operationId); + return { ...correlation, kind: "operation-acknowledged" }; + } catch { + return { ...correlation, kind: "failed", reason: "write-failed" }; + } + } + if (request.kind === "semantic-transact" || request.kind === "semantic-worker-loss") { + return handleSemanticWrite(request, correlation); + } + return handleWrite(request, correlation); +} + +async function handleSemanticWrite( + request: Extract, + correlation: Pick, +): Promise { + try { + const writer = semanticWriter; + if (!writer || semanticPublication) throw new Error("Semantic writer is not ready"); + if (request.kind === "semantic-transact" + ? request.operation.operationId !== request.operationId + || request.operation.execution.executionId !== request.executionId + : `${request.input.lease.leaseId}:worker-lost` !== request.operationId + || request.input.execution.executionId !== request.executionId) { + throw new Error("Semantic writer request identity mismatch"); + } + semanticPublication = correlation; + const receipt = request.kind === "semantic-transact" + ? await withSQLiteBusyRetry(() => writer.transact(request.operation)) + : await withSQLiteBusyRetry(() => writer.interruptWorkerLoss(request.input)); + return { ...correlation, kind: "semantic-transacted", receipt }; + } catch { + return { ...correlation, kind: "failed", reason: "write-failed" }; + } finally { + semanticPublication = undefined; + } +} + +function publishSemantic( + correlation: Pick, + events: Parameters[0], +): void { + for (let offset = 0; offset < events.length; offset += SEMANTIC_PUBLICATION_PAGE_SIZE) { + globalThis.postMessage({ ...correlation, kind: "semantic-publication", + events: events.slice(offset, offset + SEMANTIC_PUBLICATION_PAGE_SIZE) } satisfies CanonicalWriterResponse); + } +} + +async function withSQLiteBusyRetry(work: () => Promise | T): Promise { + const delaysMs = [10, 30, 90, 270, 500]; + for (const delayMs of delaysMs) { + try { + return await work(); + } catch (error) { + if (!isSQLiteBusy(error)) throw error; + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } + return work(); +} + +function isSQLiteBusy(error: unknown): boolean { + return error instanceof Error && "code" in error + && typeof error.code === "string" && error.code.startsWith("SQLITE_BUSY"); +} + +async function handleWrite( + request: Extract, + correlation: Pick, +): Promise { + try { + const canonical = boundary; + if (!canonical || !receipts) throw new Error("Canonical writer has not opened its database"); + if (request.kind === "record-parent-narrative-recovery") { + if (request.input.executionId !== request.executionId) { + throw new Error("Canonical writer recovery execution mismatch"); + } + return await receipts.executeBatchedRecovery(request, async () => ({ + ...correlation, + kind: "parent-narrative-recovery-recorded", + receipt: { recorded: await canonical.recordParentNarrativeRecoveryYielding(request.input) }, + })); + } + return receipts.execute(request, () => applyWrite(request, correlation, canonical)); + } catch (error) { + return { + ...correlation, + kind: "failed", + reason: error instanceof CanonicalWriterOperationConflict ? "operation-conflict" + : error instanceof CanonicalWriterReceiptCapacity ? "receipt-capacity" : "write-failed", + }; + } +} + +function applyWrite( + request: Extract, + correlation: Pick, + canonical: CanonicalAgentBoundary, +): Extract { + if (request.kind === "classify-parent-narrative-recovery") { + if (request.input.executionId !== request.executionId) { + throw new Error("Canonical writer classification execution mismatch"); + } + const receipt = classifyParentNarrativeRecovery(request.input); + return { ...correlation, kind: "parent-narrative-recovery-classified", receipt }; + } + assertRouting(request.input, request.executionId); + const result = canonical.commit(request.input); + const { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events } = result; + return { + ...correlation, + kind: "committed", + receipt: { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events }, + }; +} + +const requestQueue: QueuedCanonicalWrite[] = []; +let draining = false; + +function failRequest(request: CanonicalWriterRequest): void { + globalThis.postMessage({ requestId: request.requestId, operationId: request.operationId, + executionId: request.executionId, kind: "failed", reason: "write-failed" } satisfies CanonicalWriterResponse); +} + +async function handleAppendGroup(requests: Extract[]): Promise { + const first = requests[0]; + if (!first) return 0; + let results: ReturnType; + try { + const writer = semanticWriter; + if (!writer || semanticPublication) throw new Error("Semantic writer is not ready"); + results = await withSQLiteBusyRetry( + () => writer.transactAppendGroup(requests.map((request) => request.operation))); + } catch { + // A rolled-back peer must not inherit another execution's write failure. + // Replaying semantic operations is safe because their durable receipts fence duplicates. + for (const request of requests) await handleSingleRequest(request); + return requests.length; + } + for (const [index, result] of results.entries()) { + const request = requests[index]; + if (!request) throw new Error("Semantic append group lost its request"); + const correlation = { requestId: request.requestId, operationId: request.operationId, executionId: request.executionId }; + for (const events of result.publications) publishSemantic(correlation, events); + globalThis.postMessage({ ...correlation, kind: "semantic-transacted", receipt: result.receipt } satisfies CanonicalWriterResponse); + } + return results.length; +} + +async function handleSingleRequest(request: CanonicalWriterRequest): Promise { + const response = await handle(request); + globalThis.postMessage(response); + if (requestQueue.length === 1 && response.kind === "semantic-transacted" && response.receipt.kind === "committed" + && request.kind === "semantic-transact" && (request.operation.mutation.kind === "finish" + || request.operation.mutation.kind === "finish-live-event")) gc(true); +} + +async function drainRequests(): Promise { + const queued = requestQueue[0]; + if (!queued) { draining = false; return; } + const request = queued.request; + let consumed = 1; + try { + const group = selectAppendGroup(requestQueue); + if (group.length > 1) consumed = await handleAppendGroup(group); + else await handleSingleRequest(request); + } catch (error) { + console.error("Canonical writer request failed unexpectedly", error); + failRequest(request); + } finally { + requestQueue.splice(0, consumed); + if (requestQueue.length > 0) setImmediate(() => { void drainRequests(); }); + else draining = false; + } +} + +globalThis.onmessage = (message: MessageEvent): void => { + const request = message.data; + try { + requestQueue.push({ request, + bytes: request.kind === "semantic-transact" && isGroupableAppend(request.operation) + ? Buffer.byteLength(JSON.stringify(request.operation), "utf8") : 0 }); + } catch { + failRequest(request); + return; + } + if (!draining) { + draining = true; + // Collect already-ready messages without waiting for a group to fill. + setImmediate(() => { void drainRequests(); }); + } +}; + +process.on("exit", () => db?.close(true)); diff --git a/apps/server/src/features/agents/canonical/canonical-append-group.ts b/apps/server/src/features/agents/canonical/canonical-append-group.ts new file mode 100644 index 000000000..4e014b3e3 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-append-group.ts @@ -0,0 +1,37 @@ +import type { ExecutionSemanticOperation } from "../execution/execution-worker-handler.js"; +import type { CanonicalWriterRequest } from "./canonical-agent-writer-protocol.js"; + +/** Bounds one physical commit without waiting for more append requests. */ +export const APPEND_GROUP_LIMITS = { operations: 8, bytes: 512 * 1024, elapsedMs: 16 } as const; + +/** Reclassification removes a recovery journal after committing, so it remains a separate write. */ +export function isGroupableAppend(operation: ExecutionSemanticOperation): boolean { + return operation.mutation.kind === "append-events" + && operation.mutation.parentLive?.text.kind !== "reclassify"; +} + +/** Request bytes are measured once when the request enters the worker queue. */ +export interface QueuedCanonicalWrite { + readonly request: CanonicalWriterRequest; + readonly bytes: number; +} + +/** Select a FIFO prefix of independent appends; controls and repeated identities are barriers. */ +export function selectAppendGroup(queue: readonly QueuedCanonicalWrite[]): Extract[] { + const requests: Extract[] = []; + const executions = new Set(); + let bytes = 0; + for (const queued of queue) { + const request = queued.request; + if (request.kind !== "semantic-transact" || !isGroupableAppend(request.operation) + || request.operationId !== request.operation.operationId + || request.executionId !== request.operation.execution.executionId + || executions.has(request.executionId) + || requests.length === APPEND_GROUP_LIMITS.operations + || bytes + queued.bytes > APPEND_GROUP_LIMITS.bytes) break; + requests.push(request); + executions.add(request.executionId); + bytes += queued.bytes; + } + return requests; +} diff --git a/apps/server/src/features/agents/canonical/canonical-codex-collaboration-coordinator.ts b/apps/server/src/features/agents/canonical/canonical-codex-collaboration-coordinator.ts index 92787418d..02220e8a2 100644 --- a/apps/server/src/features/agents/canonical/canonical-codex-collaboration-coordinator.ts +++ b/apps/server/src/features/agents/canonical/canonical-codex-collaboration-coordinator.ts @@ -38,7 +38,7 @@ import type { CanonicalAgentCommitInput, CanonicalAgentCommitResult, CanonicalAgentEventDraft, -} from "./canonical-agent-event-sink.js"; +} from "./canonical-agent-boundary.js"; /** Generic persistence operations required by the Codex child-thread protocol. */ export interface CanonicalCodexCollaborationOperations { diff --git a/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts b/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts new file mode 100644 index 000000000..4231be344 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts @@ -0,0 +1,160 @@ +import type { Database } from "bun:sqlite"; +import * as NodeUtil from "node:util"; +import { ProviderIdSchema, type AgentEvent, type ProviderId } from "@mcode/contracts"; + +import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import type { CodexLiveReduction, CodexLiveWriterIntent } from "../execution/codex-live-event-reducer.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; +import type { DataOnlyParentTerminalProjectionInput } from "./canonical-parent-turn-write.js"; +import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; + +type Reduced = Extract; +type SystemEvent = Extract; +/** System intents accepted by the execution-bound live writer. */ +export type CodexSystemWriterIntent = Extract; +type ErrorTerminalIntent = Extract; + +function errorTerminalIntent(reduction: Reduced, event: Extract): ErrorTerminalIntent { + const error = reduction.writer.find((intent) => intent.kind === "turn-error"); + const terminal = reduction.writer.find((intent) => intent.kind === "terminal-projection"); + if (error?.kind !== "turn-error" || error.error !== event.error + || terminal?.kind !== "terminal-projection" || terminal.source !== "error" || terminal.outcome !== "errored") { + throw new Error("Codex error needs an errored terminal projection and end time"); + } + return terminal; +} + +/** Cloneable result of writer-local system projection or error terminal preparation. */ +export type CanonicalCodexSystemErrorResult = + | SystemProjectionResult + | { readonly kind: "error-terminal"; readonly event: Extract; readonly after: "terminal"; readonly input: DataOnlyParentTerminalProjectionInput }; + +type SystemProjectionResult = { readonly kind: "system"; readonly event: SystemEvent; readonly after: "writer" | "terminal" }; + +function expectedSystemIntentKind(event: SystemEvent): CodexSystemWriterIntent["kind"] | null { + if (event.subtype === "provider.session.started") return "notice-session"; + if (event.subtype.startsWith("provider.notice.") && event.message) return "system-notice"; + if (event.subtype.startsWith("sdk_session_id:") || event.subtype === "sdk_session_invalidated") return "session-cursor"; + return null; +} + +/** Check that a bound system event carries exactly its reducer-owned projection intent. */ +export function matchesCodexSystemIntents(event: SystemEvent, intents: readonly CodexSystemWriterIntent[]): boolean { + const expected = expectedSystemIntentKind(event); + if (!Array.isArray(intents) || intents.length !== (expected ? 1 : 0)) return false; + if (!expected) return true; + return intents[0]?.kind === expected && NodeUtil.isDeepStrictEqual(intents[0].event, event) + && (expected !== "system-notice" || !event.messageId); +} + +function isSystemWriterIntent(intent: CodexLiveWriterIntent): intent is CodexSystemWriterIntent { + return intent.kind === "notice-session" || intent.kind === "system-notice" || intent.kind === "session-cursor"; +} + +/** Projects only execution-bound Codex system events and prepares errors for the terminal owner. */ +export class CanonicalCodexSystemErrorProjection { + private readonly threads: ThreadRepo; + private readonly messages: MessageRepo; + private readonly canonical: CanonicalAgentBoundary; + + constructor(private readonly db: Database) { + this.threads = new ThreadRepo(db); + this.messages = new MessageRepo(db); + this.canonical = new CanonicalAgentBoundary(db, () => {}); + } + + /** Call inside the canonical append transaction, before its receipt and publication. Errors still require terminal staging and finish. */ + project(reduction: CodexLiveReduction, endedAt?: string): CanonicalCodexSystemErrorResult { + if (reduction.kind !== "reduced") throw new Error("Unsupported Codex event cannot be projected"); + const event = reduction.publication.event; + if (event.threadId !== reduction.execution.threadId || event.turnExecutionId !== reduction.execution.executionId) { + throw new Error("Codex event lacks exact execution ownership"); + } + if (event.type === "system") { + if (!reduction.writer.every(isSystemWriterIntent)) throw new Error("Codex system has a non-system intent"); + return this.projectBoundSystem(reduction.execution, "codex", event, reduction.writer, reduction.publication.after); + } + if (event.type === "error") return this.prepareError(reduction, event, endedAt); + throw new Error(`Codex ${event.type} needs another feature owner`); + } + + /** Apply one bound system event and return the event that the committed receipt must publish. */ + projectBoundSystem( + execution: ExecutionIdentity, + providerId: string, + event: SystemEvent, + intents: readonly CodexSystemWriterIntent[], + after: "writer" | "terminal", + ): SystemProjectionResult { + if (event.threadId !== execution.threadId || event.turnExecutionId !== execution.executionId) { + throw new Error("Codex system event lacks exact execution ownership"); + } + if (!matchesCodexSystemIntents(event, intents)) { + throw new Error("Codex system intents do not match the event"); + } + const provider = ProviderIdSchema.parse(providerId); + if (provider !== "codex" && provider !== "claude" && provider !== "cursor") { + throw new Error(`Provider ${provider} has no execution-bound system projection`); + } + return this.db.transaction(() => { + let published: SystemEvent = { ...event }; + for (const intent of intents) { + published = this.applySystemIntent(execution.executionId, provider, intent, published); + } + return structuredClone({ kind: "system", event: published, after } satisfies SystemProjectionResult); + })(); + } + + private applySystemIntent(executionId: string, providerId: ProviderId, intent: CodexSystemWriterIntent, event: SystemEvent): SystemEvent { + switch (intent.kind) { + case "notice-session": + if (event.subtype !== "provider.session.started") throw new Error("Codex notice session subtype is invalid"); + this.messages.beginNoticeSession(event.threadId, event.systemNotice?.sessionId); + return event; + case "system-notice": { + if (!event.subtype.startsWith("provider.notice.") || !event.message) throw new Error("Codex notice is invalid"); + const sequence = this.messages.getLatestSequenceIncludingInternal(event.threadId) + 1; + const message = this.messages.createSystemNotice(event.threadId, event.message ?? "", sequence, event.systemNotice); + return { ...event, messageId: message.id }; + } + case "session-cursor": + this.applyCursor(executionId, providerId, event); + return event; + } + } + + private applyCursor(executionId: string, providerId: ProviderId, event: SystemEvent): void { + if (event.subtype === "sdk_session_invalidated") { + if (!this.threads.clearSdkSessionId(event.threadId)) throw new Error("Codex cursor thread is missing"); + return; + } + if (!event.subtype.startsWith("sdk_session_id:")) throw new Error("Codex cursor subtype is invalid"); + const cursor = event.subtype.slice("sdk_session_id:".length); + if (!cursor) return; + if (!this.threads.updateSdkSessionId(event.threadId, cursor) + || !this.canonical.recordNativeCursor(executionId, { + providerId, scope: providerId === "codex" ? "thread" : "session", value: cursor, provenance: "native", + })) throw new Error("Codex cursor could not be persisted for this execution"); + } + + private prepareError(reduction: Reduced, event: Extract, endedAt?: string): CanonicalCodexSystemErrorResult { + if (reduction.publication.after !== "terminal" || !endedAt || !Number.isFinite(Date.parse(endedAt))) { + throw new Error("Codex error needs an errored terminal projection and end time"); + } + const terminal = errorTerminalIntent(reduction, event); + return structuredClone({ + kind: "error-terminal", + event, + after: "terminal", + input: { + threadId: reduction.execution.threadId, + executionId: reduction.execution.executionId, + outcome: "errored", + endedAt, + assistant: terminal.assistant, + narrative: terminal.narrative, + }, + } satisfies CanonicalCodexSystemErrorResult); + } +} diff --git a/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts b/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts new file mode 100644 index 000000000..eae4d84cc --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts @@ -0,0 +1,43 @@ +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; + +import { CodexCollaborationEventAdapter } from "../collaboration/adapters/codex-collaboration-event-adapter.js"; +import type { CodexCollaborationDurability } from "../collaboration/codex-collaboration-durability.js"; +import type { ProjectedCommittedProviderEvent } from "../execution/execution-worker-handler.js"; +import { processProviderEventWorkerTask } from "../../providers/composition/provider-event-worker-protocol.js"; +import type { ProviderEventProjection } from "../../providers/composition/provider-event-adapter.js"; + +/** Runs one Codex projection in a savepoint and retains only its complete effects. */ +export type CommitCodexProjection = (project: () => ProviderEventProjection) => ProviderEventProjection; + +/** Interprets committed runtime envelopes beside the SQLite connection that owns their effects. */ +export class CanonicalCommittedProviderProjector { + private readonly codex: CodexCollaborationEventAdapter; + + constructor(durability: CodexCollaborationDurability, private readonly commitCodex: CommitCodexProjection) { + this.codex = new CodexCollaborationEventAdapter(durability); + } + + /** Return cloneable, provider-neutral deliveries after any Codex child writes have committed. */ + project(envelopes: readonly CanonicalAgentEventEnvelope[]): ProjectedCommittedProviderEvent[] { + const projected: ProjectedCommittedProviderEvent[] = []; + for (const envelope of envelopes) { + if (envelope.payload.type !== "item.recorded" + || envelope.payload.item.payload.projection !== "providerRuntimeEvent") continue; + const outcome = processProviderEventWorkerTask({ kind: "canonical-commit", envelope }); + if (outcome.status !== "accepted" || !outcome.event.canonicalReceipt) { + throw new Error(`Committed provider envelope cannot be interpreted: ${envelope.eventId}`); + } + const interpretation = outcome.event.providerId === "codex" + ? this.commitCodex(() => this.codex.project(outcome.event)) + : { status: "forward" as const, event: outcome.event.event }; + if (interpretation.status !== "forward") continue; + projected.push({ + providerId: outcome.event.providerId, + sourceKind: "canonical-commit", + event: interpretation.event, + canonicalReceipt: outcome.event.canonicalReceipt, + }); + } + return projected; + } +} diff --git a/apps/server/src/features/agents/canonical/canonical-context-compaction-projection.ts b/apps/server/src/features/agents/canonical/canonical-context-compaction-projection.ts new file mode 100644 index 000000000..888521e14 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-context-compaction-projection.ts @@ -0,0 +1,56 @@ +import type { Database } from "bun:sqlite"; + +import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import type { ProjectedCommittedProviderEvent } from "../execution/execution-worker-handler.js"; + +/** Applies Codex context and compaction effects on the canonical writer connection. */ +export class CanonicalContextCompactionProjection { + private readonly threads: ThreadRepo; + private readonly messages: MessageRepo; + + constructor(db: Database) { + this.threads = new ThreadRepo(db); + this.messages = new MessageRepo(db); + } + + /** Return the compaction state after ordered events; null means an unsupported terminal arrived. */ + apply(threadId: string, compacting: boolean, events: readonly ProjectedCommittedProviderEvent[]): boolean | null { + let active = compacting; + for (const { providerId, event } of events) { + if (providerId !== "codex") continue; + const next = this.applyEvent(threadId, active, event); + if (next === null) return null; + active = next; + } + return active; + } + + private applyEvent(threadId: string, active: boolean, event: ProjectedCommittedProviderEvent["event"]): boolean | null { + switch (event.type) { + case "contextEstimate": + if (event.totalProcessedTokens !== undefined && !active) this.recordUsage(threadId, event.tokensIn, event.contextWindow); + return active; + case "turnComplete": + if (active) return null; + this.recordUsage(threadId, event.tokensIn, event.contextWindow); + return active; + case "compacting": + if (!event.active) this.persistDivider(threadId); + return event.active; + case "compactSummary": + this.threads.updateCompactSummary(threadId, event.summary); + return false; + default: return active; + } + } + + private recordUsage(threadId: string, tokensIn: number, contextWindow?: number): void { + if (tokensIn > 0) this.threads.updateContextUsage(threadId, tokensIn, contextWindow); + } + + private persistDivider(threadId: string): void { + const sequence = this.messages.getLatestSequenceIncludingInternal(threadId) + 1; + this.messages.create(threadId, "system", "Context compacted", sequence); + } +} diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts new file mode 100644 index 000000000..5a4973fbf --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -0,0 +1,1798 @@ +import type { Database } from "bun:sqlite"; +import * as NodeCrypto from "node:crypto"; +import * as NodeUtil from "node:util"; +import { + AgentEventSchema, + AgentEventType, + CanonicalAgentEventEnvelopeSchema, + ParentNarrativeRecoveryItemSchema, + PlanQuestionBatchSchema, + PlanRecordSchema, + PlanSectionNavSchema, + ProviderIdSchema, + ProviderIdentitySchema, + TurnOutcomeSchema, + type ProviderIdentity, + type ParentNarrativeRecoveryItem, + type TurnOutcome, +} from "@mcode/contracts"; +import { z } from "zod"; + +import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; +import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; +import { sameExecution, sameLease } from "../execution/execution-mailbox-protocol.js"; +import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; +import type { + ExecutionProviderCommitReceipt, + ExecutionLivePublicationIntent, + ProjectedCommittedProviderEvent, + ExecutionSemanticOperation, + ExecutionSemanticWriter, + ExecutionWriteReceipt, + ExecutionLivePublicationReceipt, + ExecutionPlanQuestionsReceipt, + ExecutionTerminalPersistenceReceipt, +} from "../execution/execution-worker-handler.js"; +import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; +import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; +import { APPEND_GROUP_LIMITS, isGroupableAppend } from "./canonical-append-group.js"; +import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; +import type { ProviderEventProjection } from "../../providers/composition/provider-event-adapter.js"; +import { CanonicalCodexSystemErrorProjection, matchesCodexSystemIntents } from "./canonical-codex-system-error-projection.js"; +import { CanonicalContextCompactionProjection } from "./canonical-context-compaction-projection.js"; +import { CanonicalParentTurnWrite, type DataOnlyParentLiveMessageInput, type DataOnlyParentTurnFinishInput } from "./canonical-parent-turn-write.js"; +import { TaskRepo } from "../orchestration/persistence/task-repo.js"; +import { PlanRepo } from "../planning/persistence/plan-repo.js"; +import { HookExecutionRepo } from "../events/persistence/hook-execution-repo.js"; +import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; +import { + ParentAssistantTextCheckpointService, + PARENT_ASSISTANT_TEXT_QUEUE_POLICY, + type ParentAssistantTextCheckpointInput, + type ParentAssistantTextCheckpointResult, +} from "../turns/parent-assistant-text-checkpoint-service.js"; + +const HEAD_ID = "semantic:head"; +const HEAD_KIND = "semantic-head"; +const PUBLICATION_KIND = "semantic-publication"; +const ACKNOWLEDGED_PUBLICATION_KIND = "semantic-publication-acknowledged"; +const PENDING_FINISH_KIND = "semantic:finish-pending"; +const PUBLICATION_CHUNK_SIZE = 64; +const PUBLICATION_CHUNK_PAGE_SIZE = 16; +const MAX_BUFFERED_PUBLICATION_EVENTS = 2_048; +const MAX_LIVE_PUBLICATION_EVENTS = 64; +const MAX_LIVE_PUBLICATION_BYTES = 256 * 1024; +const MAX_LIVE_RECEIPT_BYTES = 512 * 1024; +const LIVE_PUBLICATION_KINDS: ReadonlySet = new Set([ + "begin", "append-events", "finish", "finish-live-event", "live-event", "post-terminal-event", +]); + +class RejectedCodexProjection extends Error { + constructor(readonly diagnostic: Extract["diagnostic"]) { + super("Codex projection rejected"); + } +} +const LIVE_RECOVERY_KINDS: ReadonlySet = new Set([ + "append-assistant-text", "narrative-delta", "live-event", +]); +const narrativeDeltaSchema = z.object({ + executionId: z.string(), + items: z.array(ParentNarrativeRecoveryItemSchema()), + discardedItemIds: z.array(z.string().regex(/^(toolCall|narrationSegment|hook):.+$/)).optional(), +}); +const storedTaskSchema = z.object({ + id: z.string().max(256).optional(), + content: z.string().min(1).max(16 * 1024), + status: z.enum(["pending", "in_progress", "completed", "cancelled"]), + activeForm: z.string().max(4096).optional(), + group: z.string().max(128).optional(), +}).strict(); +const taskIntentsSchema = z.array(z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("upsert-group"), group: z.string().max(128), + tasks: z.array(storedTaskSchema).max(256) }).strict(), + z.object({ kind: z.literal("append-task"), task: storedTaskSchema }).strict(), + z.object({ kind: z.literal("update-task"), id: z.string().min(1).max(256), + group: z.string().max(128), patch: storedTaskSchema.pick({ status: true, content: true, activeForm: true }).partial() }).strict(), + z.object({ kind: z.literal("remove-task"), id: z.string().min(1).max(256), group: z.string().max(128) }).strict(), +])).max(16); +const planOutputSchema = z.object({ + title: z.string().trim().min(1).max(512), + contentMd: z.string().min(1).max(256 * 1024), + sectionsJson: z.string().max(64 * 1024), + changeSummary: z.string().max(4096).nullable(), +}).strict(); +const planSectionsSchema = z.array(PlanSectionNavSchema()).max(128); +const storedPublicationSequencesSchema = z.array(z.union([ + z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + z.object({ executionId: z.string(), sequence: z.number().int().positive().max(Number.MAX_SAFE_INTEGER) }), +])) + .min(1).max(PUBLICATION_CHUNK_SIZE); +const projectedProviderEventSchema = z.object({ + providerId: ProviderIdSchema, + sourceKind: z.literal("canonical-commit"), + event: AgentEventSchema(), + canonicalReceipt: z.object({ + eventId: z.string(), + sourceSequence: z.number().int().optional(), + acceptedSequence: z.number().int(), + durableRevision: z.number().int(), + serverTimestamps: z.object({ acceptedAt: z.string(), persistedAt: z.string().optional() }), + }), +}); +const storedHeadSchema = z.object({ + execution: z.object({ threadId: z.string(), turnId: z.string(), executionId: z.string() }), + providerId: z.string(), + lease: z.object({ + ownerEpoch: z.number().int(), + workerIndex: z.number().int(), + workerGeneration: z.number().int(), + leaseId: z.string(), + }), + ordinal: z.number().int(), + durableRevision: z.number().int(), + terminal: z.boolean(), + stopRequestId: z.string().nullable(), + stopWatermark: z.number().int().nullable(), + providerOutcome: TurnOutcomeSchema.nullable(), + assignedMessageId: z.string().regex(/^[0-9a-f]{64}$/).nullable().optional(), + compacting: z.boolean().default(false), + ended: z.boolean().optional(), +}); +const storedReceiptSchema = z.object({ + kind: z.literal("committed"), + operationId: z.string(), + durableRevision: z.number().int(), + publicationVersion: z.literal(1), + livePublication: z.array(z.object({ + publicationId: z.string().regex(/^[1-9]\d*$/).max(16), + after: z.enum(["writer", "terminal"]), + event: AgentEventSchema(), + })).min(1).max(MAX_LIVE_PUBLICATION_EVENTS).optional(), + providerCommit: z.object({ + outcome: z.enum(["committed", "duplicate", "conflict", "terminal-outcome-confirmed", "ingest-overflow"]), + conversationRevision: z.number().int(), + rosterRevision: z.number().int(), + acceptedThrough: z.number().int(), + durableThrough: z.number().int(), + eventCount: z.number().int().nonnegative(), + }).optional(), + providerEvents: z.array(projectedProviderEventSchema).optional(), + assistantTextCheckpoint: z.object({ + outcome: z.literal("committed"), + durableThrough: z.number().int().nonnegative(), + committedItems: z.number().int().positive(), + committedBytes: z.number().int().positive(), + }).optional(), + planQuestions: z.object({ + publicationId: z.string(), threadId: z.string(), + questions: PlanQuestionBatchSchema().shape.questions, + }).optional(), + planOutput: PlanRecordSchema().optional(), +}); +const storedOperationSchema = z.object({ + kind: z.string(), + input_hash: z.string(), + receipt_json: z.string(), +}); +const pendingFinishSchema = storedReceiptSchema.pick({ providerCommit: true }).extend({ + providerIdentities: z.array(ProviderIdentitySchema), +}); +const storedPublicationChunkPageSchema = z.array(storedOperationSchema.extend({ operation_id: z.string() })) + .max(PUBLICATION_CHUNK_PAGE_SIZE); +const storedEnvelopeRowSchema = z.object({ envelope_json: z.string() }); +const durableSequenceRowSchema = z.object({ last_durable_sequence: z.number().int() }); +const livePublicationHeadSchema = z.object({ last_sequence: z.number().int().positive().max(Number.MAX_SAFE_INTEGER) }); + +interface SemanticHead { + readonly execution: ExecutionIdentity; + readonly providerId: string; + readonly lease: ExecutionLease; + readonly ordinal: number; + readonly durableRevision: number; + readonly terminal: boolean; + readonly stopRequestId: string | null; + readonly stopWatermark: number | null; + readonly providerOutcome: TurnOutcome | null; + readonly assignedMessageId?: string | null; + readonly compacting: boolean; + readonly ended?: boolean; +} + +type StoredOperation = z.infer; + +class SemanticConflict extends Error {} + +/** The revoked ownership and recovery incident for a worker that cannot prove its live turn. */ +export interface LostExecutionInterruption { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly reason: string; + readonly recoveryIncidentId: string; +} + +/** + * Adapts execution mutations to one writer-local SQLite connection. + * Outer writes reserve the WAL writer before reading because the main connection may write concurrently. + */ +export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter { + private readonly turns: CanonicalParentTurnWrite; + private readonly compoundTurns: CanonicalParentTurnWrite; + private readonly providerProjector: CanonicalCommittedProviderProjector; + private readonly contextCompaction: CanonicalContextCompactionProjection; + private readonly systemProjection: CanonicalCodexSystemErrorProjection; + private readonly tasks: TaskRepo; + private readonly plans: PlanRepo; + private readonly threads: ThreadRepo; + private readonly hooks: HookExecutionRepo; + private readonly assistantText: ParentAssistantTextCheckpointService; + private readonly canonical: CanonicalAgentBoundary; + private readonly findOperation: ReturnType; + private readonly listPublicationChunks: ReturnType; + private readonly acknowledgePublication: ReturnType; + private readonly findPublishedEvent: ReturnType; + private readonly findDurableSequence: ReturnType; + private readonly advanceLivePublication: ReturnType; + private readonly insertOperation: ReturnType; + private readonly commitPendingFinish: ReturnType; + private readonly updateHead: ReturnType; + private readonly updateCheckpointPhase: ReturnType; + private readonly updateCheckpoint: ReturnType; + private bufferedPublication: Parameters[0][] | null = null; + private bufferedPublicationCount = 0; + private groupedPublication: Parameters[0][] | null = null; + + constructor(private readonly db: Database, private readonly publish: CanonicalAgentEventPublisher) { + this.turns = new CanonicalParentTurnWrite(db, (events) => { + if (this.bufferedPublication) this.bufferPublication(events); + else this.publishCommitted(events); + }); + // Compound terminal batches publish from their durable sequence log only after the final receipt commits. + this.compoundTurns = new CanonicalParentTurnWrite(db, () => {}); + const codexBoundary = new CanonicalAgentBoundary(db, (events) => { + if (!this.bufferedPublication) throw new Error("Codex projection requires a semantic transaction"); + this.bufferPublication(events); + }); + this.canonical = codexBoundary; + this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary, (project) => this.commitCodexProjection(project)); + this.contextCompaction = new CanonicalContextCompactionProjection(db); + this.systemProjection = new CanonicalCodexSystemErrorProjection(db); + this.tasks = new TaskRepo(db); + this.plans = new PlanRepo(db); + this.threads = new ThreadRepo(db); + this.hooks = new HookExecutionRepo(db); + this.assistantText = new ParentAssistantTextCheckpointService(db); + this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); + this.listPublicationChunks = db.prepare("SELECT operation_id, kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id > ? AND operation_id < ? ORDER BY operation_id LIMIT ?"); + this.acknowledgePublication = db.prepare("UPDATE canonical_writer_operation_receipts SET kind = ? WHERE execution_id = ? AND operation_id = ? AND kind = ? AND input_hash = ?"); + this.findPublishedEvent = db.prepare("SELECT envelope_json FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence = ?"); + this.findDurableSequence = db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?"); + this.advanceLivePublication = db.prepare(`INSERT INTO canonical_writer_live_publication_heads (thread_id, last_sequence) + VALUES (?, ?) ON CONFLICT(thread_id) DO UPDATE SET last_sequence = last_sequence + excluded.last_sequence + WHERE last_sequence <= 9007199254740991 - excluded.last_sequence RETURNING last_sequence`); + this.insertOperation = db.prepare("INSERT INTO canonical_writer_operation_receipts (execution_id, operation_id, kind, input_hash, receipt_json) VALUES (?, ?, ?, ?, ?)"); + this.commitPendingFinish = db.prepare("UPDATE canonical_writer_operation_receipts SET kind = ?, receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ? AND input_hash = ?"); + this.updateHead = db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ?"); + this.updateCheckpointPhase = db.prepare("UPDATE canonical_agent_ingest_checkpoints SET phase = ?, updated_at = ? WHERE execution_id = ? AND terminal_outcome IS NULL"); + this.updateCheckpoint = db.prepare("UPDATE canonical_agent_ingest_checkpoints SET phase = ?, native_cursor_json = ?, updated_at = ? WHERE execution_id = ? AND terminal_outcome IS NULL"); + } + + /** Commit a supported operation with a durable receipt, or reject it without changing canonical state. */ + async transact(operation: ExecutionSemanticOperation): Promise { + if (operation.mutation.kind === "append-events") return this.transactAppend(operation); + if (!validOperation(operation)) return conflict(operation); + const hash = fingerprint(operation); + const existing = this.existingReceipt(operation, hash); + if (existing) { + this.retireReplayedText(operation, existing); + return existing; + } + try { + return await this.applySupported(operation, hash); + } catch (error) { + if (error instanceof SemanticConflict) return conflict(operation); + throw error; + } + } + + /** Fence a lost worker and apply the existing interrupted-turn recovery contract. */ + interruptWorkerLoss(input: LostExecutionInterruption): ExecutionWriteReceipt { + const operation = workerLossOperation(input); + if (!validLostExecutionInput(input, operation)) return conflict(operation); + const hash = fingerprint(input); + const existing = this.loadOperation(input.execution.executionId, operation.operationId); + if (existing) { + const receipt = this.replay(operation, hash, existing); + if (receipt.kind === "committed") this.turns.retireInterruptedText(input.execution.executionId); + return receipt; + } + const head = this.loadHead(input.execution.executionId); + if (!head) return { ...conflict(operation), recoveryState: "not-started" }; + if (!sameExecutionAndLease(head, input)) return conflict(operation); + if (head.terminal) return { ...conflict(operation), recoveryState: "already-terminal" }; + this.turns.importRecoveryJournals(); + try { + const receipt = this.withBufferedPublication(() => this.db.transaction( + () => this.writeLostInterruption(input, operation, hash), + ).immediate()); + this.turns.retireInterruptedText(input.execution.executionId); + return receipt; + } catch (error) { + if (error instanceof SemanticConflict) return conflict(operation); + throw error; + } + } + + private writeLostInterruption( + input: LostExecutionInterruption, + operation: ExecutionSemanticOperation, + hash: string, + ): ExecutionWriteReceipt { + const current = this.loadHead(input.execution.executionId); + if (!current || current.terminal || !sameExecutionAndLease(current, input)) throw new SemanticConflict(); + const result = this.turns.interruptLostExecution({ ...input.execution, + reason: input.reason, recoveryIncidentId: input.recoveryIncidentId, + ...(current.assignedMessageId ? { assignedMessageId: current.assignedMessageId } : {}) }); + const receipt = committed(operation, result.durableThrough); + const sequences = this.bufferedPublication?.flatMap((batch) => batch.map((event) => event.acceptedSequence)) ?? []; + this.storePublicationChunks(input.execution.executionId, hash, sequences); + this.storeHead({ ...current, ordinal: current.ordinal + 1, + durableRevision: receipt.durableRevision, terminal: true }); + return this.storeReceipt(operation, hash, receipt); + } + + private async applySupported(operation: ExecutionSemanticOperation, hash: string): Promise { + if (LIVE_RECOVERY_KINDS.has(operation.mutation.kind)) return this.applyLiveRecovery(operation, hash); + if (isFinishMutation(operation.mutation)) return await this.finish(operation, hash); + switch (operation.mutation.kind) { + case "begin": return this.begin(operation, hash); + case "checkpoint": + case "stop-requested": + case "provider-outcome": return this.control(operation, hash); + case "stage-terminal": return this.stageTerminal(operation, hash); + case "post-terminal-event": return this.postTerminalEvent(operation, hash); + default: return conflict(operation); + } + } + + private existingReceipt(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt | null { + const row = this.loadOperation(operation.execution.executionId, operation.operationId); + if (!row) return null; + if (row.kind !== PENDING_FINISH_KIND) return this.replay(operation, hash, row); + return isFinishMutation(operation.mutation) && row.input_hash === hash ? null : conflict(operation); + } + + private begin(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "begin" || operation.ordinal !== 1 || this.loadHead(operation.execution.executionId)) { + return conflict(operation); + } + if (mutation.providerId !== mutation.input.thread.providerId + || !validPublicationProvider(operation, mutation.providerId) + || mutation.input.thread.id !== operation.execution.threadId + || mutation.input.turnId !== operation.execution.turnId + || mutation.input.executionId !== operation.execution.executionId) return conflict(operation); + return this.withBufferedPublication(() => this.db.transaction(() => { + if (this.loadHead(operation.execution.executionId)) throw new SemanticConflict(); + const result = this.turns.start(mutation.input); + if (result.outcome !== "committed") throw new SemanticConflict(); + const receipt = committed(operation, result.durableThrough); + const head: SemanticHead = { + execution: operation.execution, + providerId: mutation.providerId, + lease: operation.lease, + ordinal: operation.ordinal, + durableRevision: receipt.durableRevision, + terminal: false, + stopRequestId: null, + stopWatermark: null, + providerOutcome: null, + compacting: false, + }; + this.insertOperation.run(operation.execution.executionId, HEAD_ID, HEAD_KIND, fingerprint(head.lease), JSON.stringify(head)); + this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); + return this.storeReceipt(operation, hash, receipt); + }).immediate()); + } + + private append(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "append-events" || !mutation.phase || mutation.phase.length > 64 + || mutation.events.length === 0 + || mutation.events.some((event) => event.routing.threadId !== operation.execution.threadId + || event.routing.turnId !== operation.execution.turnId + || event.routing.executionId !== operation.execution.executionId)) return conflict(operation); + const receipt = this.withBufferedPublication(() => this.db.transaction(() => { + const head = this.requireNextHead(operation); + if (!validPublicationProvider(operation, head.providerId)) throw new SemanticConflict(); + const result = this.turns.append({ + ...operation.execution, + phase: mutation.phase, + nativeCursor: mutation.nativeCursor, + events: mutation.events, + }); + if (result.outcome !== "committed") throw new SemanticConflict(); + const providerEvents = this.providerProjector.project(result.events); + const compacting = this.contextCompaction.apply(operation.execution.threadId, head.compacting, providerEvents); + if (compacting === null) throw new SemanticConflict(); + const live = this.appendParentLive(operation, head, providerEvents); + const checkpoint = durableSequenceRowSchema.parse(this.findDurableSequence.get(operation.execution.executionId)); + const providerCommit: ExecutionProviderCommitReceipt = { + outcome: result.outcome, + conversationRevision: result.conversationRevision, + rosterRevision: result.rosterRevision, + acceptedThrough: result.acceptedThrough, + durableThrough: result.durableThrough, + eventCount: result.events.length, + }; + const receipt = committed(operation, checkpoint.last_durable_sequence, providerCommit, live.providerEvents, + live.textResult, undefined, live.planQuestions, live.planOutput); + this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, compacting, + ...(mutation.parentLive?.message ? { assignedMessageId: mutation.parentLive.message.messageId } : {}) }); + const publication = this.bufferedPublication?.flat() ?? []; + // A Codex child write has its own execution sequence, even when the parent event caused it. + this.storePublicationChunks(operation.execution.executionId, hash, publication.map((event) => ({ + executionId: event.routing.executionId, + sequence: event.acceptedSequence, + }))); + return this.storeReceipt(operation, hash, receipt, live.publication); + }).immediate()); + if (mutation.parentLive?.text.kind === "reclassify") { + this.assistantText.discardRecoveryJournal(operation.execution.executionId); + } + return receipt; + } + + /** Commit a bounded prefix of independent appends, returning publications only after physical commit. */ + transactAppendGroup(operations: readonly ExecutionSemanticOperation[]) { + if (this.groupedPublication || operations.length === 0 || operations.length > APPEND_GROUP_LIMITS.operations + || operations.some((operation) => !isGroupableAppend(operation)) + || new Set(operations.map((operation) => operation.execution.executionId)).size !== operations.length) { + throw new Error("Invalid semantic append group"); + } + try { + return this.db.transaction(() => { + const results: { operation: ExecutionSemanticOperation; receipt: ExecutionWriteReceipt; + publications: Parameters[0][] }[] = []; + const startedAt = performance.now(); + for (const operation of operations) { + const publications: Parameters[0][] = []; + this.groupedPublication = publications; + const receipt = this.transactAppend(operation); + results.push({ operation, receipt, publications }); + if (performance.now() - startedAt >= APPEND_GROUP_LIMITS.elapsedMs) break; + } + return results; + }).immediate(); + } finally { + this.groupedPublication = null; + } + } + + private transactAppend(operation: ExecutionSemanticOperation): ExecutionWriteReceipt { + if (!validOperation(operation)) return conflict(operation); + const hash = fingerprint(operation); + const existing = this.existingReceipt(operation, hash); + if (existing) { + this.retireReplayedText(operation, existing); + return existing; + } + try { + return this.append(operation, hash); + } catch (error) { + if (error instanceof SemanticConflict) return conflict(operation); + throw error; + } + } + + private retireReplayedText(operation: ExecutionSemanticOperation, receipt: ExecutionWriteReceipt): void { + if (receipt.kind !== "committed") return; + if (isFinishMutation(operation.mutation)) this.assistantText.retire(operation.execution.executionId); + if (parentLiveOperation(operation)?.mutation.text.kind === "reclassify") { + this.assistantText.discardRecoveryJournal(operation.execution.executionId); + } + } + + private appendParentLive( + operation: ExecutionSemanticOperation, + head: SemanticHead, + providerEvents: readonly ProjectedCommittedProviderEvent[], + ) { + const live = parentLiveOperation(operation); + if (!live) return { providerEvents, textResult: undefined, planQuestions: undefined, + planOutput: undefined, publication: undefined }; + const publication = liveEventPublication(live); + const matches = providerEvents.filter((projected) => publication + && sameParentLiveEvent(projected.event, publication.event)); + if (matches.length !== 1) throw new SemanticConflict(); + this.requireUnfinishedCheckpoint(operation.execution); + return { + ...this.applyLiveFeatureEffects(live, head), + publication: this.projectLiveSystem(live, head.providerId), + providerEvents: providerEvents.filter((projected) => projected !== matches[0]), + }; + } + + private commitCodexProjection(project: () => ProviderEventProjection): ProviderEventProjection { + const pending = this.bufferedPublication; + if (!pending) throw new Error("Codex projection requires a semantic transaction"); + const pendingLength = pending.length; + const pendingCount = this.bufferedPublicationCount; + try { + return this.db.transaction(() => { + const result = project(); + if (result.status === "rejected") throw new RejectedCodexProjection(result.diagnostic); + return result; + })(); + } catch (error) { + pending.length = pendingLength; + this.bufferedPublicationCount = pendingCount; + if (!(error instanceof RejectedCodexProjection)) throw error; + // The adapter's diagnostic was part of the rejected savepoint. + this.canonical.recordCodexChildRoutingDiagnostic(error.diagnostic); + return { status: "rejected", diagnostic: error.diagnostic }; + } + } + + private appendAssistantText(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "append-assistant-text") return conflict(operation); + return this.db.transaction(() => { + const head = this.requireNextHead(operation); + const result = this.assistantText.appendChunk(mutation.inputs); + if (result.outcome !== "committed") throw new SemanticConflict(); + // Assistant text has its own durable sequence; it does not advance the canonical event revision. + const receipt = committed(operation, head.durableRevision, undefined, undefined, result); + this.storeHead({ ...head, ordinal: operation.ordinal }); + return this.storeReceipt(operation, hash, receipt); + }).immediate(); + } + + private recordNarrativeDelta(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "narrative-delta") return conflict(operation); + return this.db.transaction(() => { + const head = this.requireNextHead(operation); + this.requireUnfinishedCheckpoint(operation.execution); + this.persistNarrativeDelta(mutation.input); + const receipt = committed(operation, head.durableRevision); + this.storeHead({ ...head, ordinal: operation.ordinal }); + return this.storeReceipt(operation, hash, receipt); + }).immediate(); + } + + private applyLiveRecovery(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + if (operation.mutation.kind === "append-assistant-text") return this.appendAssistantText(operation, hash); + if (operation.mutation.kind === "narrative-delta") return this.recordNarrativeDelta(operation, hash); + return this.recordLiveEvent(operation, hash); + } + + private recordLiveEvent(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "live-event") return conflict(operation); + const receipt = this.db.transaction(() => { + const head = this.requireNextHead(operation); + if (!validPublicationProvider(operation, head.providerId)) throw new SemanticConflict(); + this.requireUnfinishedCheckpoint(operation.execution); + const { textResult, planQuestions, planOutput } = this.applyLiveFeatureEffects(operation, head); + const livePublication = this.projectLiveSystem(operation, head.providerId); + const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult, + undefined, planQuestions, planOutput); + this.storeHead({ ...head, ordinal: operation.ordinal, + ...(mutation.message ? { assignedMessageId: mutation.message.messageId } : {}) }); + return this.storeReceipt(operation, hash, committedReceipt, livePublication); + }).immediate(); + if (mutation.text.kind === "reclassify") { + this.assistantText.discardRecoveryJournal(operation.execution.executionId); + } + return receipt; + } + + private projectLiveSystem(operation: ExecutionSemanticOperation, providerId: string): readonly ExecutionLivePublicationIntent[] | undefined { + const mutation = operation.mutation; + if (mutation.kind !== "live-event" || mutation.systemIntents === undefined) return undefined; + const publication = operation.livePublication?.length === 1 ? operation.livePublication[0] : undefined; + if (publication?.event.type !== "system") throw new SemanticConflict(); + const result = this.systemProjection.projectBoundSystem( + operation.execution, providerId, publication.event, mutation.systemIntents, publication.after, + ); + return [{ after: result.after, event: result.event }]; + } + + private applyLiveFeatureEffects( + operation: ExecutionSemanticOperation, + head: SemanticHead, + ) { + const mutation = operation.mutation; + if (mutation.kind !== "live-event") throw new SemanticConflict(); + this.stageLiveMessage(operation.execution, head, mutation.message); + const textResult = this.applyLiveText(mutation.text, operation.execution.executionId); + if (mutation.narrative) this.persistNarrativeDelta(mutation.narrative); + if (mutation.taskIntents) this.applyTaskIntents(operation.execution.threadId, mutation.taskIntents); + const planOutput = this.persistLivePlanOutput(operation.execution.threadId, mutation); + if (mutation.text.kind === "reclassify" && !this.assistantText.resetInTransaction(operation.execution.executionId)) { + throw new SemanticConflict(); + } + const planQuestions = mutation.planQuestions ? { + publicationId: `${operation.operationId}:plan-questions`, + threadId: operation.execution.threadId, + questions: mutation.planQuestions, + } satisfies ExecutionPlanQuestionsReceipt : undefined; + return { textResult, planQuestions, planOutput }; + } + + private stageLiveMessage( + execution: ExecutionIdentity, + head: SemanticHead, + message: DataOnlyParentLiveMessageInput | undefined, + ): void { + if (!message) return; + if (head.assignedMessageId && head.assignedMessageId !== message.messageId) throw new SemanticConflict(); + if (!this.turns.stageLiveAssistant(execution, message)) throw new SemanticConflict(); + } + + private persistLivePlanOutput( + threadId: string, + mutation: Extract, + ) { + return mutation.planOutput && mutation.message + ? this.persistPlanOutput(threadId, mutation.message.messageId, mutation.planOutput) + : undefined; + } + + private applyLiveText( + text: Extract["text"], + executionId: string, + ): ParentAssistantTextCheckpointResult | undefined { + if (text.kind === "unchanged") return undefined; + if (text.kind === "reclassify") { + if (this.assistantText.restore(executionId) !== text.expectedText) throw new SemanticConflict(); + return undefined; + } + const result = this.assistantText.appendChunk(text.kind === "append" ? text.inputs : [text.input]); + if (result.outcome !== "committed") throw new SemanticConflict(); + return result; + } + + private applyTaskIntents(threadId: string, intents: readonly TaskToolWriteIntent[]): void { + for (const intent of intents) { + switch (intent.kind) { + case "upsert-group": this.tasks.upsertGroup(threadId, intent.group, intent.tasks); break; + case "append-task": this.tasks.appendTask(threadId, intent.task); break; + case "update-task": this.tasks.updateTask(threadId, intent.id, intent.patch, intent.group); break; + case "remove-task": this.tasks.removeTask(threadId, intent.id, intent.group); break; + } + } + } + + private persistPlanOutput( + threadId: string, + messageId: string, + output: NonNullable["planOutput"]>, + ) { + if (this.plans.getByMessageId(messageId)) throw new SemanticConflict(); + return this.plans.create(threadId, messageId, output.title, output.contentMd, + output.sectionsJson, output.changeSummary); + } + + private requireUnfinishedCheckpoint(execution: ExecutionIdentity): void { + const checkpoint = this.canonical.loadCheckpoint(execution.executionId); + if (!checkpoint || checkpoint.threadId !== execution.threadId + || checkpoint.turnId !== execution.turnId || checkpoint.terminalOutcome !== null) { + throw new SemanticConflict(); + } + } + + private persistNarrativeDelta(input: Extract["input"]): void { + if (!this.canonical.recordParentNarrativeRecovery(input)) throw new SemanticConflict(); + } + + private async finish(operation: ExecutionSemanticOperation, hash: string): Promise { + const mutation = operation.mutation; + if (!isFinishMutation(mutation) || !validFinish(operation, mutation)) return conflict(operation); + if (!this.validFinishHead(operation, mutation.input.providerId)) return conflict(operation); + const providerCommit = mutation.kind === "finish-live-event" + ? this.prepareLiveFinish(operation, hash, mutation) : undefined; + if (mutation.kind === "finish") { + this.reserveFinish(operation, hash); + this.publishStoredEvents(operation.execution.executionId, hash); + } + const turns = mutation.kind === "finish-live-event" ? this.compoundTurns : this.turns; + const input = this.terminalFinishInput(operation, mutation); + const result = await turns.finish(input, (batch) => { + this.storePublicationChunks(operation.execution.executionId, hash, batch.publishedSequences); + if (batch.terminal) { + const current = this.requireNextHead(operation); + if (current.providerId !== mutation.input.providerId) throw new SemanticConflict(); + const receipt = committed(operation, batch.durableSequence, providerCommit, providerCommit ? [] : undefined); + const status = mutation.outcome === "cancelled" ? "interrupted" : mutation.outcome; + if (!this.threads.updateStatus(operation.execution.threadId, status)) throw new SemanticConflict(); + this.storeHead({ ...current, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, terminal: true }); + this.storeReceipt(operation, hash, receipt); + } + }); + if (result.outcome !== "committed") return conflict(operation); + return this.finishedReceipt(operation, hash); + } + + private finishedReceipt(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const stored = this.loadOperation(operation.execution.executionId, operation.operationId); + if (!stored) return conflict(operation); + const receipt = this.readReceipt(operation, hash, stored); + if (operation.mutation.kind === "finish-live-event" && receipt.kind === "committed") { + this.publishStoredEvents(operation.execution.executionId, hash, true); + } + if (receipt.kind === "committed") this.assistantText.retire(operation.execution.executionId); + return receipt; + } + + private terminalFinishInput( + operation: ExecutionSemanticOperation, + mutation: Extract, + ): DataOnlyParentTurnFinishInput { + if (mutation.kind === "finish") return mutation.input; + const pending = this.loadOperation(operation.execution.executionId, operation.operationId); + if (!pending || pending.kind !== PENDING_FINISH_KIND) throw new SemanticConflict(); + const preparation = pendingFinishSchema.parse(JSON.parse(pending.receipt_json)); + return { ...mutation.input, providerIdentities: preparation.providerIdentities }; + } + + private terminalProviderIdentities(threadId: string, providerId: string): ProviderIdentity[] { + const canonicalThread = this.canonical.loadThread(threadId); + if (!canonicalThread || canonicalThread.providerId !== providerId) throw new SemanticConflict(); + const thread = this.threads.findById(threadId); + if (!thread?.sdk_session_id || thread.provider !== providerId) return [...canonicalThread.providerIdentities]; + return [{ providerId: canonicalThread.providerId, scope: providerId === "codex" ? "thread" : "session", + value: thread.sdk_session_id, provenance: "native" }]; + } + + private prepareLiveFinish( + operation: ExecutionSemanticOperation, + hash: string, + mutation: Extract, + ): ExecutionProviderCommitReceipt | undefined { + if (!validLiveFinish(operation.execution, mutation)) throw new SemanticConflict(); + const prepared = this.withBufferedPublication(() => this.db.transaction(() => { + const head = this.requireNextHead(operation); + const pending = this.loadOperation(operation.execution.executionId, operation.operationId); + if (pending) { + this.reserveFinish(operation, hash); + return committed(operation, head.durableRevision, pendingFinishSchema.parse(JSON.parse(pending.receipt_json)).providerCommit); + } + if (head.assignedMessageId && head.assignedMessageId !== mutation.projection.assistant.messageId) { + throw new SemanticConflict(); + } + const next = this.recordProviderOutcome(head, mutation.outcome); + const evidence = this.appendTerminalEvidence(operation, hash, mutation, next); + const staged = this.turns.stageTerminalProjection(mutation.projection); + this.storeHead({ ...evidence.head, assignedMessageId: staged.messageId }); + this.reserveFinish(operation, hash, evidence.providerCommit, + this.terminalProviderIdentities(operation.execution.threadId, head.providerId)); + return committed(operation, head.durableRevision, evidence.providerCommit); + }).immediate(), false); + if (prepared.kind !== "committed") throw new SemanticConflict(); + return prepared.providerCommit; + } + + private appendTerminalEvidence( + operation: ExecutionSemanticOperation, + hash: string, + mutation: Extract, + head: SemanticHead, + ): { head: SemanticHead; providerCommit: ExecutionProviderCommitReceipt | undefined } { + if (!mutation.providerEvent) return { head, providerCommit: undefined }; + const checkpoint = this.canonical.loadCheckpoint(operation.execution.executionId); + const result = this.turns.append({ ...operation.execution, ...mutation.providerEvent, + nativeCursor: mutation.providerEvent.nativeCursor ?? checkpoint?.nativeCursor ?? null }); + if (result.outcome !== "committed") throw new SemanticConflict(); + const projected = this.providerProjector.project(result.events); + if (!matchesTerminalProviderPublication(operation, projected)) throw new SemanticConflict(); + const compacting = this.contextCompaction.apply(operation.execution.threadId, head.compacting, projected); + if (compacting === null) throw new SemanticConflict(); + const publicationEvents = this.bufferedPublication?.flat() ?? []; + this.storePublicationChunks(operation.execution.executionId, hash, + publicationEvents.map((event) => event.acceptedSequence)); + return { head: { ...head, compacting, durableRevision: result.durableThrough }, providerCommit: { + outcome: result.outcome, conversationRevision: result.conversationRevision, rosterRevision: result.rosterRevision, + acceptedThrough: result.acceptedThrough, durableThrough: result.durableThrough, eventCount: result.events.length, + } }; + } + + private postTerminalEvent(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "post-terminal-event") return conflict(operation); + return this.withBufferedPublication(() => this.db.transaction(() => { + const head = this.requireTerminalHead(operation); + this.requirePostTerminalPublication(operation, head); + const hookDrafts = this.persistTerminalHooks(operation, head, mutation.hooks ?? []); + const providerCommit = this.appendPostTerminalEvidence(operation, hookDrafts); + const live = parentLiveOperation(operation); + const publication = this.terminalHookPublication(operation) + ?? (live ? this.projectLiveSystem(live, head.providerId) : undefined); + return this.storePostTerminalReceipt(operation, hash, head, providerCommit, publication); + }).immediate()); + } + + private requirePostTerminalPublication(operation: ExecutionSemanticOperation, head: SemanticHead): void { + const event = operation.livePublication?.[0]?.event; + if (!event || !validPublicationProvider(operation, head.providerId)) throw new SemanticConflict(); + const checkpoint = this.canonical.loadCheckpoint(operation.execution.executionId); + if (!checkpoint?.terminalOutcome || !matchesLateOutcome(event, checkpoint.terminalOutcome)) throw new SemanticConflict(); + } + + private terminalHookPublication(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationIntent[] | undefined { + const mutation = operation.mutation; + const publication = operation.livePublication?.[0]; + if (mutation.kind !== "post-terminal-event" || publication?.event.type !== "hookCompleted") return undefined; + const hook = mutation.hooks?.[0]; + if (!hook) return undefined; + const message = this.canonical.loadTerminalProjection(operation.execution.turnId).message; + if (!message) throw new SemanticConflict(); + return [{ after: "terminal", event: { ...publication.event, + persistedMessageId: message.id, persistedHookId: hook.record.id } }]; + } + + private storePostTerminalReceipt(operation: ExecutionSemanticOperation, hash: string, head: SemanticHead, + providerCommit: ExecutionProviderCommitReceipt | undefined, publication: readonly ExecutionLivePublicationIntent[] | undefined): ExecutionWriteReceipt { + const receipt = committed(operation, providerCommit?.durableThrough ?? head.durableRevision, + providerCommit, providerCommit ? [] : undefined); + this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, + ...(operation.livePublication?.[0]?.event.type === "ended" ? { ended: true } : {}) }); + const events = this.bufferedPublication?.flat() ?? []; + this.storePublicationChunks(operation.execution.executionId, hash, events.map((entry) => entry.acceptedSequence)); + return this.storeReceipt(operation, hash, receipt, publication); + } + + private requireTerminalHead(operation: ExecutionSemanticOperation): SemanticHead { + const head = this.loadHead(operation.execution.executionId); + if (!head?.terminal || head.ended || head.ordinal + 1 !== operation.ordinal + || !sameExecutionAndLease(head, operation)) throw new SemanticConflict(); + return head; + } + + private persistTerminalHooks( + operation: ExecutionSemanticOperation, + head: SemanticHead, + hooks: readonly Extract[], + ): Extract["events"] { + if (hooks.length === 0) return []; + const message = this.canonical.loadTerminalProjection(operation.execution.turnId).message; + if (!message) throw new SemanticConflict(); + const drafts = []; + for (const hook of hooks) { + const record = hook.record; + const existing = this.db.prepare("SELECT message_id FROM hook_executions WHERE id = ?").get(record.id); + if (existing && !NodeUtil.isDeepStrictEqual(existing, { message_id: message.id })) throw new SemanticConflict(); + if (record.message_id && record.message_id !== message.id) throw new SemanticConflict(); + this.hooks.bulkCreate([{ id: record.id, messageId: message.id, hookName: record.hook_name, + toolName: record.tool_name, phase: record.phase, payload: record.payload, durationMs: record.duration_ms, + didBlock: record.did_block, startedAt: record.started_at, endedAt: record.ended_at, sortOrder: record.sort_order }], true); + drafts.push(terminalHookDraft(operation, head.providerId, { ...record, message_id: message.id })); + } + return drafts; + } + + private appendPostTerminalEvidence( + operation: ExecutionSemanticOperation, + hooks: Extract["events"], + ): ExecutionProviderCommitReceipt | undefined { + const mutation = operation.mutation; + if (mutation.kind !== "post-terminal-event") throw new SemanticConflict(); + const source = mutation.providerEvent; + const events = [...source ? source.events : [], ...hooks]; + if (events.length === 0) return undefined; + const result = this.turns.append({ ...operation.execution, events, + phase: source ? source.phase : "post-terminal", nativeCursor: null }); + if (result.outcome !== "committed") throw new SemanticConflict(); + const projected = this.providerProjector.project(result.events); + if (mutation.providerEvent && !matchesPostTerminalProviderPublication(operation, projected)) throw new SemanticConflict(); + return { outcome: result.outcome, conversationRevision: result.conversationRevision, rosterRevision: result.rosterRevision, + acceptedThrough: result.acceptedThrough, durableThrough: result.durableThrough, + eventCount: source ? source.events.length : 0 }; + } + + private stageTerminal(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "stage-terminal" || mutation.input.threadId !== operation.execution.threadId + || mutation.input.executionId !== operation.execution.executionId) return conflict(operation); + return this.db.transaction(() => { + const head = this.requireNextHead(operation); + if (head.providerOutcome !== mutation.input.outcome) throw new SemanticConflict(); + if (head.assignedMessageId && head.assignedMessageId !== mutation.input.assistant.messageId) { + throw new SemanticConflict(); + } + this.turns.stageTerminalProjection(mutation.input); + const receipt = committed(operation, head.durableRevision); + this.storeHead({ ...head, ordinal: operation.ordinal, + assignedMessageId: mutation.input.assistant.messageId ?? head.assignedMessageId ?? null }); + return this.storeReceipt(operation, hash, receipt); + }).immediate(); + } + + private control(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + return this.db.transaction(() => { + const head = this.requireNextHead(operation); + const next = this.applyControlMutation(head, operation); + const receipt = committed(operation, head.durableRevision); + this.storeHead({ ...next, ordinal: operation.ordinal }); + return this.storeReceipt(operation, hash, receipt); + }).immediate(); + } + + private applyControlMutation(head: SemanticHead, operation: ExecutionSemanticOperation): SemanticHead { + const mutation = operation.mutation; + switch (mutation.kind) { + case "stop-requested": return this.recordStop(head, operation, mutation); + case "provider-outcome": return this.recordProviderOutcome(head, mutation.outcome); + case "checkpoint": return this.recordCheckpoint(head, operation.execution.executionId, mutation); + default: throw new SemanticConflict(); + } + } + + private recordStop( + head: SemanticHead, + operation: ExecutionSemanticOperation, + mutation: Extract, + ): SemanticHead { + if (!mutation.requestId || head.stopRequestId || head.providerOutcome + || mutation.lastAdmittedOrdinal !== operation.ordinal - 1) throw new SemanticConflict(); + if (this.updateCheckpointPhase.run("stopping", new Date().toISOString(), operation.execution.executionId).changes !== 1) { + throw new SemanticConflict(); + } + return { ...head, stopRequestId: mutation.requestId, stopWatermark: mutation.lastAdmittedOrdinal }; + } + + private recordProviderOutcome(head: SemanticHead, outcome: TurnOutcome): SemanticHead { + if (head.providerOutcome || head.stopRequestId && outcome !== "cancelled") throw new SemanticConflict(); + return { ...head, providerOutcome: outcome }; + } + + private recordCheckpoint( + head: SemanticHead, + executionId: string, + mutation: Extract, + ): SemanticHead { + const cursor = mutation.nativeCursor === null ? null : JSON.stringify(mutation.nativeCursor); + if (!mutation.phase || mutation.phase.length > 64 || cursor === undefined + || this.updateCheckpoint.run(mutation.phase, cursor, new Date().toISOString(), executionId).changes !== 1) { + throw new SemanticConflict(); + } + return head; + } + + private hasStagedTerminalPredecessor(operation: ExecutionSemanticOperation): boolean { + const previousId = `${operation.lease.leaseId}:${operation.ordinal - 1}`; + return this.loadOperation(operation.execution.executionId, previousId)?.kind === "semantic:stage-terminal"; + } + + private validStagedFinish(operation: ExecutionSemanticOperation, head: SemanticHead): boolean { + const mutation = operation.mutation; + if (mutation.kind !== "finish" || !("kind" in mutation.input.projection)) return true; + if (head.providerOutcome !== mutation.outcome || !this.hasStagedTerminalPredecessor(operation)) return false; + return head.assignedMessageId + ? head.assignedMessageId === mutation.input.projection.messageId + : mutation.input.projection.messageId === undefined; + } + + private validFinishHead(operation: ExecutionSemanticOperation, providerId: string): boolean { + const head = this.loadHead(operation.execution.executionId); + return Boolean(head && nextHead(head, operation) && head.providerId === providerId + && validPublicationProvider(operation, head.providerId) && this.validStagedFinish(operation, head)); + } + + private requireNextHead(operation: ExecutionSemanticOperation): SemanticHead { + const head = this.loadHead(operation.execution.executionId); + if (!head || !nextHead(head, operation)) throw new SemanticConflict(); + return head; + } + + private loadHead(executionId: string): SemanticHead | null { + const row = this.loadOperation(executionId, HEAD_ID); + if (!row) return null; + if (row.kind !== HEAD_KIND) throw new SemanticConflict(); + return storedHeadSchema.parse(JSON.parse(row.receipt_json)); + } + + private loadOperation(executionId: string, operationId: string): StoredOperation | null { + return storedOperationSchema.nullable().parse(this.findOperation.get(executionId, operationId)); + } + + private storeHead(head: SemanticHead): void { + this.updateHead.run(JSON.stringify(head), head.execution.executionId, HEAD_ID, HEAD_KIND); + } + + private reserveFinish(operation: ExecutionSemanticOperation, hash: string, providerCommit?: ExecutionProviderCommitReceipt, + providerIdentities?: readonly ProviderIdentity[]): void { + const existing = this.loadOperation(operation.execution.executionId, operation.operationId); + if (existing) { + if (existing.kind !== PENDING_FINISH_KIND || existing.input_hash !== hash) throw new SemanticConflict(); + return; + } + this.insertOperation.run(operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, + JSON.stringify({ providerCommit, providerIdentities })); + } + + private storePublicationChunks( + executionId: string, + hash: string, + sequences: readonly (number | { executionId: string; sequence: number })[], + ): void { + for (let offset = 0; offset < sequences.length; offset += PUBLICATION_CHUNK_SIZE) { + const chunk = sequences.slice(offset, offset + PUBLICATION_CHUNK_SIZE); + const first = chunk[0]; + if (first === undefined) continue; + const id = publicationChunkId(hash, typeof first === "number" ? first : offset + 1); + const json = JSON.stringify(chunk); + const existing = this.loadOperation(executionId, id); + if (existing) { + if (existing.kind !== PUBLICATION_KIND || existing.input_hash !== hash || existing.receipt_json !== json) { + throw new SemanticConflict(); + } + continue; + } + this.insertOperation.run(executionId, id, PUBLICATION_KIND, hash, json); + } + } + + private storeReceipt( + operation: ExecutionSemanticOperation, + hash: string, + receipt: Extract, + publicationOverride?: readonly ExecutionLivePublicationIntent[], + ): Extract { + const publication = this.numberLivePublication(operation, publicationOverride); + const storedReceipt = publication ? { ...receipt, livePublication: publication } : receipt; + const receiptJson = JSON.stringify({ ...storedReceipt, publicationVersion: 1 }); + if (operation.livePublication && Buffer.byteLength(receiptJson, "utf8") > MAX_LIVE_RECEIPT_BYTES) { + throw new SemanticConflict(); + } + if (isFinishMutation(operation.mutation)) { + const updated = this.commitPendingFinish.run( + `semantic:${operation.mutation.kind}`, receiptJson, operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, + ); + if (updated.changes !== 1) throw new SemanticConflict(); + return storedReceipt; + } + this.insertOperation.run( + operation.execution.executionId, + operation.operationId, + `semantic:${operation.mutation.kind}`, + hash, + receiptJson, + ); + return storedReceipt; + } + + private numberLivePublication( + operation: ExecutionSemanticOperation, + publicationOverride?: readonly ExecutionLivePublicationIntent[], + ): readonly ExecutionLivePublicationReceipt[] | undefined { + const intents = publicationOverride ?? livePublicationFor(operation); + if (!intents) return undefined; + const row = livePublicationHeadSchema.safeParse( + this.advanceLivePublication.get(operation.execution.threadId, intents.length), + ); + if (!row.success) throw new SemanticConflict(); + const first = row.data.last_sequence - intents.length + 1; + return intents.map((intent, index) => ({ ...intent, publicationId: String(first + index) })); + } + + private replay(operation: ExecutionSemanticOperation, hash: string, stored: StoredOperation): ExecutionWriteReceipt { + const receipt = this.readReceipt(operation, hash, stored); + if (receipt.kind === "committed") { + this.publishStoredEvents(operation.execution.executionId, hash, operation.mutation.kind === "finish-live-event"); + } + return receipt; + } + + private readReceipt(operation: ExecutionSemanticOperation, hash: string, stored: StoredOperation): ExecutionWriteReceipt { + if (stored.kind !== `semantic:${operation.mutation.kind}` || stored.input_hash !== hash) return conflict(operation); + if (operation.livePublication && Buffer.byteLength(stored.receipt_json, "utf8") > MAX_LIVE_RECEIPT_BYTES) { + throw new Error("Live publication receipt exceeds its size limit"); + } + const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); + if (!validReceiptPublicationIds(receipt.livePublication) + || !this.matchesLivePublicationReceipt(operation, receipt.livePublication)) { + throw new Error("Live publication receipt does not match its operation"); + } + if (receipt.operationId !== operation.operationId || !Number.isSafeInteger(receipt.durableRevision)) { + return conflict(operation); + } + const committedReceipt = committed(operation, receipt.durableRevision, receipt.providerCommit, receipt.providerEvents, + receipt.assistantTextCheckpoint, receipt.livePublication, receipt.planQuestions, receipt.planOutput); + return operation.mutation.kind === "finish-live-event" + ? { ...committedReceipt, terminalPersistence: this.terminalPersistence(operation) } : committedReceipt; + } + + private terminalPersistence(operation: ExecutionSemanticOperation): ExecutionTerminalPersistenceReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "finish-live-event") throw new SemanticConflict(); + const persisted = this.canonical.loadTerminalProjection(operation.execution.turnId); + const message = persisted.message; + if (message && !matchesCommittedTerminalMessage(message, operation.execution, mutation.outcome)) { + throw new SemanticConflict(); + } + const fileEvidence = mutation.input.fileEvidence; + return { + messageId: message?.id ?? null, + outcome: mutation.outcome, + toolCallCount: persisted.toolCallCount, + filesChanged: [...fileEvidence?.filesChanged ?? []], + ...(fileEvidence ? { fileEffects: fileEvidence.fileEffects } : {}), + }; + } + + private matchesLivePublicationReceipt( + operation: ExecutionSemanticOperation, + actual: readonly ExecutionLivePublicationReceipt[] | undefined, + ): boolean { + const hookPublication = this.terminalHookPublication(operation); + if (hookPublication) { + return publicationFingerprint(actual) === publicationFingerprint(hookPublication); + } + const mutation = parentLiveOperation(operation)?.mutation; + if (mutation?.systemIntents?.[0]?.kind === "system-notice") { + return this.matchesGeneratedNoticeReceipt(operation, actual); + } + return publicationFingerprint(actual) === publicationFingerprint(livePublicationFor(operation)); + } + + private matchesGeneratedNoticeReceipt( + operation: ExecutionSemanticOperation, + actual: readonly ExecutionLivePublicationReceipt[] | undefined, + ): boolean { + const expected = livePublicationFor(operation)?.[0]; + const published = actual?.[0]; + if (actual?.length !== 1 || expected?.event.type !== "system" || expected.event.messageId + || !isGeneratedNoticePublication(published)) return false; + return fingerprint({ after: published.after, event: { ...published.event, messageId: undefined } }) + === fingerprint(expected); + } + + private publishStoredEvents(executionId: string, hash: string, acknowledge = false): void { + const prefix = publicationChunkPrefix(hash); + let cursor = prefix; + while (true) { + const chunks = storedPublicationChunkPageSchema.parse(this.listPublicationChunks.all( + executionId, cursor, `${prefix}~`, PUBLICATION_CHUNK_PAGE_SIZE, + )); + if (chunks.length === 0) return; + for (const chunk of chunks) { + this.publishStoredChunk(executionId, hash, chunk, acknowledge); + } + cursor = chunks[chunks.length - 1]!.operation_id; + } + } + + private publishStoredChunk( + executionId: string, + hash: string, + chunk: z.infer[number], + acknowledge: boolean, + ): void { + if (acknowledge && chunk.kind === ACKNOWLEDGED_PUBLICATION_KIND && chunk.input_hash === hash) return; + if (chunk.kind !== PUBLICATION_KIND || chunk.input_hash !== hash) throw new Error("Semantic publication chunk mismatch"); + const sequences = storedPublicationSequencesSchema.parse(JSON.parse(chunk.receipt_json)); + const events = sequences.map((sequence) => typeof sequence === "number" + ? this.loadPublishedEvent(executionId, sequence) + : this.loadPublishedEvent(sequence.executionId, sequence.sequence)); + this.publishCommitted(events); + if (acknowledge) this.acknowledgePublication.run(ACKNOWLEDGED_PUBLICATION_KIND, executionId, + chunk.operation_id, PUBLICATION_KIND, hash); + } + + private loadPublishedEvent(executionId: string, sequence: number) { + const row = storedEnvelopeRowSchema.nullable().parse(this.findPublishedEvent.get(executionId, sequence)); + if (!row) throw new Error(`Semantic publication event missing at ${executionId}:${sequence}`); + return CanonicalAgentEventEnvelopeSchema.parse(JSON.parse(row.envelope_json)); + } + + // Only the synchronous begin and append transactions use this buffer. Finish publishes after each committed batch. + private bufferPublication(events: Parameters[0]): void { + if (!this.bufferedPublication || this.bufferedPublicationCount + events.length > MAX_BUFFERED_PUBLICATION_EVENTS) { + throw new Error("Semantic publication buffer exceeded its limit"); + } + this.bufferedPublication.push(events); + this.bufferedPublicationCount += events.length; + } + + private withBufferedPublication(write: () => ExecutionWriteReceipt, publishAfterCommit = true): ExecutionWriteReceipt { + if (this.bufferedPublication) throw new Error("Nested semantic publication buffer"); + const pending: Parameters[0][] = []; + this.bufferedPublication = pending; + this.bufferedPublicationCount = 0; + try { + const result = write(); + this.bufferedPublication = null; + if (publishAfterCommit) for (const events of pending) this.publishCommitted(events); + return result; + } finally { + this.bufferedPublication = null; + this.bufferedPublicationCount = 0; + } + } + + private publishCommitted(events: Parameters[0]): void { + if (this.groupedPublication) this.groupedPublication.push(events); + else this.publish(events); + } +} + +function validOperation(operation: ExecutionSemanticOperation): boolean { + return operation.operationId === `${operation.lease.leaseId}:${operation.ordinal}` + && operation.operationId !== HEAD_ID && operation.operationId.length <= 256 + && Number.isSafeInteger(operation.ordinal) && operation.ordinal > 0 + && validIdentity(operation.execution) && validLease(operation.lease) + && validSemanticMutationInput(operation) + && validLivePublication(operation); +} + +function validLivePublication(operation: ExecutionSemanticOperation): boolean { + const publication = operation.livePublication; + if (publication === undefined) return true; + if (!validLivePublicationShape(publication, operation.mutation.kind)) return false; + const barrier = isFinishMutation(operation.mutation) || operation.mutation.kind === "post-terminal-event" ? "terminal" : "writer"; + let bytes = 0; + for (const intent of publication) { + if (!validLivePublicationIntent(intent, operation, barrier)) return false; + bytes += Buffer.byteLength(JSON.stringify(intent), "utf8"); + if (bytes > MAX_LIVE_PUBLICATION_BYTES) return false; + } + return true; +} + +function validLivePublicationShape( + publication: readonly ExecutionLivePublicationIntent[], + mutationKind: ExecutionSemanticOperation["mutation"]["kind"], +): boolean { + return Array.isArray(publication) && publication.length > 0 && publication.length <= MAX_LIVE_PUBLICATION_EVENTS + && publication.every((intent) => intent.event.publicationId === undefined) + && LIVE_PUBLICATION_KINDS.has(mutationKind) + && (mutationKind === "live-event" || mutationKind === "post-terminal-event" ? publication.length === 1 : true); +} + +function validLivePublicationIntent( + intent: NonNullable[number], + operation: ExecutionSemanticOperation, + barrier: "writer" | "terminal", +): boolean { + return intent.after === barrier && AgentEventSchema().safeParse(intent.event).success + && intent.event.threadId === operation.execution.threadId + && intent.event.turnExecutionId === operation.execution.executionId + && (operation.mutation.kind !== "begin" || intent.event.type === "turnStarted") + && validTurnStartedPublication(intent.event, operation.mutation) + && (barrier === "terminal" || !isTerminalLiveEvent(intent.event.type)); +} + +function isTerminalLiveEvent(type: ExecutionLivePublicationIntent["event"]["type"]): boolean { + return type === "turnComplete" || type === "error" || type === "ended"; +} + +function validPublicationProvider(operation: ExecutionSemanticOperation, providerId: string): boolean { + return operation.livePublication === undefined || ["codex", "claude", "cursor"].includes(providerId); +} + +function validTurnStartedPublication(event: ExecutionLivePublicationIntent["event"], mutation: ExecutionSemanticOperation["mutation"]): boolean { + return event.type !== "turnStarted" || mutation.kind === "begin" + || mutation.kind === "append-events" && mutation.parentLive !== undefined; +} + +function validPostTerminalInput(operation: ExecutionSemanticOperation): boolean { + const mutation = operation.mutation; + const publication = operation.livePublication; + if (mutation.kind !== "post-terminal-event" || publication?.length !== 1) return false; + const event = publication[0]?.event; + if (!event || !AgentEventSchema().safeParse(event).success) return false; + return validTerminalProviderEvent(operation.execution, mutation.providerEvent) + && validPostTerminalEffects(mutation, event) + && Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; +} + +function validPostTerminalEffects( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], +): boolean { + const hooks = mutation.hooks ?? []; + if (!Array.isArray(hooks) || hooks.length > 1) return false; + if (event.type === "system") { + return hooks.length === 0 && validPostTerminalSystem(event, mutation.systemIntents); + } + if (mutation.systemIntents !== undefined) return false; + if (event.type === "ended" || event.type === "hookProgress") return hooks.length === 0; + return validLateHookRecords(event, hooks); +} + +function validPostTerminalSystem( + event: Extract, + intents: Extract["systemIntents"], +): boolean { + return Array.isArray(intents) && intents.every((intent) => intent.kind !== "session-cursor") + && matchesCodexSystemIntents(event, intents); +} + +function validLateHookRecords( + event: ExecutionLivePublicationIntent["event"], + hooks: readonly Extract[], +): boolean { + if (event.type !== "hookStarted" && event.type !== "hookCompleted") return false; + if (event.type === "hookCompleted" && (event.persistedMessageId || event.persistedHookId)) return false; + if (event.type === "hookStarted" && hooks.length !== 1) return false; + return hooks.every((hook) => ParentNarrativeRecoveryItemSchema().safeParse(hook).success + && hook.kind === "hook" && hook.record.hook_name === event.hookName + && validLateHookRecord(event, hook.record)); +} + +function validLateHookRecord( + event: Extract, + record: Extract["record"], +): boolean { + if (!Number.isSafeInteger(record.sort_order) || record.sort_order < 0 || !Number.isFinite(Date.parse(record.started_at))) return false; + if (event.type === "hookStarted") return record.ended_at === null && record.phase === "stop"; + return record.ended_at !== null && Number.isFinite(Date.parse(record.ended_at)) + && record.duration_ms === event.durationMs && record.did_block === event.didBlock; +} + +function matchesLateOutcome(event: ExecutionLivePublicationIntent["event"], outcome: TurnOutcome): boolean { + return event.type !== "ended" || event.outcome === undefined + || normalizeTerminalOutcome(event.outcome) === normalizeTerminalOutcome(outcome); +} + +function matchesPostTerminalProviderPublication( + operation: ExecutionSemanticOperation, + projected: readonly ProjectedCommittedProviderEvent[], +): boolean { + const event = operation.livePublication?.[0]?.event; + return projected.length === 1 && event !== undefined + && (event.type === "ended" ? sameTerminalEvent(projected[0]?.event, event) + : NodeUtil.isDeepStrictEqual(projected[0]?.event, event)); +} + +function terminalHookDraft( + operation: ExecutionSemanticOperation, + providerId: string, + record: Extract["record"], +): Extract["events"][number] { + const itemId = `hook:${record.id}`; + return { eventId: `${operation.execution.executionId}:post-terminal:${operation.ordinal}:${itemId}`, + routing: { ...operation.execution, itemId }, sourceProviderId: providerId, sourceIdentities: [], + payload: { type: "item.recorded", item: { id: itemId, threadId: operation.execution.threadId, + turnId: operation.execution.turnId, kind: "system", providerIdentities: [], + payload: { projection: "hook", record }, createdAt: record.started_at, updatedAt: record.ended_at ?? record.started_at } } }; +} + +function validSemanticMutationInput(operation: ExecutionSemanticOperation): boolean { + switch (operation.mutation.kind) { + case "post-terminal-event": return validPostTerminalInput(operation); + case "finish-live-event": return validLiveFinish(operation.execution, operation.mutation) + && validTerminalPublicationOutcome(operation.mutation, operation.livePublication); + case "append-assistant-text": return validAssistantTextInput(operation.mutation.inputs, operation.execution); + case "narrative-delta": return validNarrativeDeltaInput(operation.mutation.input, operation.execution); + case "live-event": return validLiveEventInput(operation); + case "append-events": return validAppendParentInput(operation); + default: return true; + } +} + +function validAppendParentInput(operation: ExecutionSemanticOperation): boolean { + if (operation.mutation.kind === "append-events" && operation.mutation.parentLive !== undefined) { + const live = parentLiveOperation(operation); + return live !== null && validLiveEventInput(live) + && Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; + } + return true; +} + +function parentLiveOperation(operation: ExecutionSemanticOperation): + (ExecutionSemanticOperation & { mutation: Extract }) | null { + const mutation = operation.mutation; + if (mutation.kind === "live-event") return { ...operation, mutation }; + if (mutation.kind === "post-terminal-event") { + return { ...operation, mutation: { kind: "live-event", text: { kind: "unchanged" }, systemIntents: mutation.systemIntents } }; + } + return mutation.kind === "append-events" && mutation.parentLive + ? { ...operation, mutation: { kind: "live-event", ...mutation.parentLive } } : null; +} + +function sameParentLiveEvent( + source: ExecutionLivePublicationIntent["event"], + publication: ExecutionLivePublicationIntent["event"], +): boolean { + if (source.type === "message" && publication.type === "message") { + return NodeUtil.isDeepStrictEqual({ ...source, messageId: undefined }, { ...publication, messageId: undefined }); + } + return NodeUtil.isDeepStrictEqual(source, publication); +} + +function validLiveEventInput(operation: ExecutionSemanticOperation): boolean { + const mutation = operation.mutation; + if (mutation.kind !== "live-event") return false; + const publication = liveEventPublication(operation); + if (!publication) return false; + const event = publication.event; + if (!validLiveSystemMutation(mutation, event)) return false; + if (!validLiveTextPayload(mutation, operation.execution) + || !AgentEventSchema().safeParse(event).success + || !validLiveEventAssociation(mutation, event, operation.execution)) return false; + if (mutation.narrative && !validNarrativeDeltaInput(mutation.narrative, operation.execution)) return false; + if (!validLiveFeatureIntents(mutation, event, operation.execution.threadId)) return false; + return validLiveEventBudget(operation, mutation); +} + +function validLiveFeatureIntents( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], + threadId: string, +): boolean { + return validLiveTaskIntents(mutation.taskIntents, event) + && validLivePlanProjection(mutation, event, threadId); +} + +function validLiveSystemMutation( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], +): boolean { + if (event.type !== "system") return mutation.systemIntents === undefined; + return Array.isArray(mutation.systemIntents) && mutation.systemIntents.length <= 1 + && mutation.text.kind === "unchanged" && mutation.narrative === undefined && mutation.taskIntents === undefined + && matchesCodexSystemIntents(event, mutation.systemIntents); +} + +function validLivePlanProjection( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], + threadId: string, +): boolean { + if (mutation.planQuestions !== undefined) { + if (event.type !== "textDelta" || mutation.planOutput !== undefined + || !PlanQuestionBatchSchema().safeParse({ threadId, questions: mutation.planQuestions }).success) return false; + } + if (mutation.planOutput === undefined) return true; + if (event.type !== "message" || !mutation.message || !planOutputSchema.safeParse(mutation.planOutput).success) return false; + try { + return planSectionsSchema.safeParse(JSON.parse(mutation.planOutput.sectionsJson)).success; + } catch { + return false; + } +} + +function validLiveTaskIntents( + intents: readonly TaskToolWriteIntent[] | undefined, + event: ExecutionLivePublicationIntent["event"], +): boolean { + if (!intents) return true; + if (!taskIntentsSchema.safeParse(intents).success) return false; + return intents.length === 0 || event.type === "toolUse" || event.type === "toolResult"; +} + +function liveEventPublication(operation: ExecutionSemanticOperation): ExecutionLivePublicationIntent | null { + const publication = operation.livePublication; + return Array.isArray(publication) && publication.length === 1 && publication[0]?.after === "writer" + ? publication[0] : null; +} + +function validLiveEventBudget( + operation: ExecutionSemanticOperation, + mutation: Extract, +): boolean { + const narrativeRows = mutation.narrative + ? mutation.narrative.items.length + (mutation.narrative.discardedItemIds?.length ?? 0) : 0; + const textRows = mutation.text.kind === "append" ? mutation.text.inputs.length + : mutation.text.kind === "unchanged" ? 0 : 1; + const rows = textRows + narrativeRows + (mutation.taskIntents?.length ?? 0) + + Number(Boolean(mutation.message)) + Number(Boolean(mutation.planOutput)); + if (rows > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + return Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; +} + +function validLiveTextPayload( + mutation: Extract, + execution: ExecutionIdentity, +): boolean { + if (!mutation.text) return false; + if (mutation.text.kind === "append") return validAssistantTextInput(mutation.text.inputs, execution); + if (mutation.text.kind === "promote") return validAssistantTextInput([mutation.text.input], execution); + if (mutation.text.kind === "reclassify") return validReclassification(mutation.text.expectedText, mutation.narrative); + return mutation.text.kind === "unchanged"; +} + +function validReclassification( + expectedText: string, + narrative: Extract["narrative"], +): boolean { + if (typeof expectedText !== "string" || !expectedText) return false; + return expectedText.trim().length === 0 || Boolean(narrative?.items.some((item) => + item.kind === "narrationSegment" && item.record.text === expectedText)); +} + +function validLiveTextAssociation( + text: Extract["text"], + event: ExecutionLivePublicationIntent["event"], +): boolean { + switch (text.kind) { + case "unchanged": return unchangedTextEvent(event); + case "append": return validAppendEvent(event, text.inputs); + case "reclassify": return event.type === "assistantMessageBoundary" && event.isFinalResponse === false + && text.expectedText.length > 0; + case "promote": return event.type === "assistantMessageBoundary" && event.isFinalResponse === true; + } +} + +function validLiveEventAssociation( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], + execution: ExecutionIdentity, +): boolean { + if (mutation.message) { + return mutation.text.kind === "unchanged" && validLiveMessageAssociation(mutation.message, event, execution); + } + return event.type !== AgentEventType.Message && validLiveTextAssociation(mutation.text, event); +} + +function validLiveMessageAssociation( + message: DataOnlyParentLiveMessageInput, + event: ExecutionLivePublicationIntent["event"], + execution: ExecutionIdentity, +): boolean { + return event.type === AgentEventType.Message && validLiveMessageIdentity(message) + && validLiveMessageBody(message) && event.threadId === execution.threadId + && event.messageId === message.messageId && event.content === message.content + && (event.model ?? null) === message.model + && JSON.stringify(event.attachments ?? []) === JSON.stringify(message.attachments); +} + +function validLiveMessageIdentity(message: DataOnlyParentLiveMessageInput): boolean { + return typeof message.precedingMessageId === "string" && message.precedingMessageId.length > 0 + && typeof message.messageId === "string" && /^[0-9a-f]{64}$/.test(message.messageId); +} + +function validLiveMessageBody(message: DataOnlyParentLiveMessageInput): boolean { + if (typeof message.content !== "string" || !Array.isArray(message.attachments)) return false; + return (message.model === null || typeof message.model === "string") + && (message.content.trim().length > 0 || message.attachments.length > 0); +} + +function validNarrativeEvent(event: ExecutionLivePublicationIntent["event"]): boolean { + return event.type === "textDelta" ? event.isFinalResponse === false + : [ + "assistantMessageBoundary", "toolUse", "toolResult", "hookStarted", "hookCompleted", + "modelFallback", "toolInputDelta", "toolProgress", "providerUnavailable", "hookProgress", + "apiRetry", "rateLimited", "quotaUpdate", "goalUpdated", "goalCleared", "mcpServerStartupStatus", + ].includes(event.type); +} + +function validAppendEvent( + event: ExecutionLivePublicationIntent["event"], + inputs: readonly ParentAssistantTextCheckpointInput[], +): boolean { + return event.type === "textDelta" && event.isFinalResponse !== false + && inputs.map((input) => input.text).join("") === event.delta; +} + +function validNarrativeDeltaInput( + input: Extract["input"], + execution: ExecutionIdentity, +): boolean { + if (!input || input.executionId !== execution.executionId || !Array.isArray(input.items)) return false; + const discarded = input.discardedItemIds ?? []; + if (!Array.isArray(discarded)) return false; + const count = input.items.length + discarded.length; + if (count === 0 || count > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + return Buffer.byteLength(JSON.stringify(input), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes + && narrativeDeltaSchema.safeParse(input).success; +} + +function validAssistantTextInput( + inputs: readonly ParentAssistantTextCheckpointInput[], + execution: ExecutionIdentity, +): boolean { + if (!Array.isArray(inputs) || inputs.length === 0 + || inputs.length > PARENT_ASSISTANT_TEXT_QUEUE_POLICY.maxQueuedEvents) return false; + let bytes = 0; + const first = inputs[0]; + if (!first || !validAssistantTextEntry(first, execution)) return false; + const firstSequence = first.sequence; + for (const [index, input] of inputs.entries()) { + if (!validAssistantTextEntry(input, execution) || input.sequence !== firstSequence + index) return false; + bytes += Buffer.byteLength(input.text, "utf8"); + if (bytes > PARENT_ASSISTANT_TEXT_QUEUE_POLICY.maxChunkBytes) return false; + } + return bytes > 0; +} + +function validAssistantTextEntry(input: ParentAssistantTextCheckpointInput | undefined, execution: ExecutionIdentity): boolean { + if (!input) return false; + return sameExecution(input, execution) && typeof input.text === "string" + && Buffer.byteLength(input.text, "utf8") > 0 + && Number.isSafeInteger(input.sequence) && input.sequence > 0; +} + +function validIdentity(identity: ExecutionIdentity): boolean { + return Boolean(identity.threadId && identity.turnId && identity.executionId); +} + +function validLease(lease: ExecutionLease): boolean { + return Boolean(lease.leaseId) && Number.isSafeInteger(lease.ownerEpoch) && lease.ownerEpoch >= 0 + && Number.isSafeInteger(lease.workerIndex) && lease.workerIndex >= 0 + && Number.isSafeInteger(lease.workerGeneration) && lease.workerGeneration >= 0; +} + +function validFinish( + operation: ExecutionSemanticOperation, + mutation: Extract, +): boolean { + return mutation.outcome === mutation.input.outcome + && sameExecution(mutation.input, operation.execution); +} + +function unchangedTextEvent(event: ExecutionLivePublicationIntent["event"]): boolean { + return ["system", "turnStarted", "generatedAttachment", "contextEstimate", "compacting", "compactSummary"].includes(event.type) + || validNarrativeEvent(event); +} + +function isFinishMutation(mutation: ExecutionSemanticOperation["mutation"]): mutation is Extract< + ExecutionSemanticOperation["mutation"], { kind: "finish" | "finish-live-event" } +> { + return mutation.kind === "finish" || mutation.kind === "finish-live-event"; +} + +function validLiveFinish( + execution: ExecutionIdentity, + mutation: Extract, +): boolean { + return mutation.projection.threadId === execution.threadId + && mutation.projection.executionId === execution.executionId + && mutation.projection.outcome === mutation.outcome + && "kind" in mutation.input.projection + && mutation.input.projection.kind === "writer-staged" + && mutation.input.projection.messageId === mutation.projection.assistant.messageId + && validTerminalProviderEvent(execution, mutation.providerEvent); +} + +function validTerminalProviderEvent( + execution: ExecutionIdentity, + input: Extract["providerEvent"], +): boolean { + return input === undefined || Boolean(input.phase && input.phase.length <= 64 && input.events.length > 0 + && input.events.every((event) => sameExecution(event.routing, execution))); +} + +function sameTerminalEvent( + source: ExecutionLivePublicationIntent["event"] | undefined, + publication: ExecutionLivePublicationIntent["event"], +): boolean { + if (!source || !isTerminalLiveEvent(source.type)) return false; + const normalized = source.type === "ended" && source.outcome === "cancelled" + ? { ...source, outcome: "interrupted" } : source; + return NodeUtil.isDeepStrictEqual(normalized, publication); +} + +function matchesTerminalProviderPublication( + operation: ExecutionSemanticOperation, + projected: readonly ProjectedCommittedProviderEvent[], +): boolean { + const publication = operation.livePublication?.[0]; + return projected.length === 1 && operation.livePublication?.length === 1 && publication !== undefined + && sameTerminalEvent(projected[0]?.event, publication.event); +} + +function validTerminalPublicationOutcome( + mutation: Extract, + publications: readonly ExecutionLivePublicationIntent[] | undefined, +): boolean { + if (publications?.length !== 1) return false; + const event = publications[0]?.event; + switch (event?.type) { + case "turnComplete": return mutation.outcome === "completed"; + case "error": return mutation.outcome === "errored" && mutation.input.error === event.error; + case "ended": return event.outcome === undefined + || normalizeTerminalOutcome(event.outcome) === normalizeTerminalOutcome(mutation.outcome); + default: return false; + } +} + +function normalizeTerminalOutcome(outcome: TurnOutcome): Exclude { + return outcome === "cancelled" ? "interrupted" : outcome; +} + +function nextHead(head: SemanticHead, operation: ExecutionSemanticOperation): boolean { + const lease = head.lease; + const candidate = operation.lease; + return !head.terminal && head.execution.threadId === operation.execution.threadId + && head.execution.turnId === operation.execution.turnId + && head.ordinal + 1 === operation.ordinal + && sameLease(lease, candidate); +} + +function validLostExecutionInput(input: LostExecutionInterruption, operation: ExecutionSemanticOperation): boolean { + return validIdentity(input.execution) && validLease(input.lease) + && Boolean(input.reason && input.recoveryIncidentId) && operation.operationId.length <= 256; +} + +function sameExecutionAndLease(head: SemanticHead, input: Pick): boolean { + return sameExecution(head.execution, input.execution) && sameLease(head.lease, input.lease); +} + +function workerLossOperation(input: LostExecutionInterruption): ExecutionSemanticOperation { + return { operationId: `${input.lease.leaseId}:worker-lost`, execution: input.execution, + lease: input.lease, ordinal: 0, + mutation: { kind: "worker-lost", reason: input.reason, recoveryIncidentId: input.recoveryIncidentId } }; +} + +function committed( + operation: ExecutionSemanticOperation, + durableRevision: number, + providerCommit?: ExecutionProviderCommitReceipt, + providerEvents?: readonly ProjectedCommittedProviderEvent[], + assistantTextCheckpoint?: ParentAssistantTextCheckpointResult, + livePublication?: readonly ExecutionLivePublicationReceipt[], + planQuestions?: ExecutionPlanQuestionsReceipt, + planOutput?: ReturnType, +): Extract { + return { + kind: "committed", operationId: operation.operationId, durableRevision, + ...(providerCommit ? { providerCommit } : {}), + ...(providerEvents ? { providerEvents } : {}), + ...(assistantTextCheckpoint ? { assistantTextCheckpoint } : {}), + ...(livePublication ? { livePublication } : {}), + ...(planQuestions ? { planQuestions } : {}), + ...(planOutput ? { planOutput } : {}), + }; +} + +function matchesCommittedTerminalMessage( + message: NonNullable["message"]>, + execution: ExecutionIdentity, + outcome: TurnOutcome, +): boolean { + return message.thread_id === execution.threadId && message.role === "assistant" + && !message.is_internal && message.outcomeExecutionId === execution.executionId + && message.outcome === outcome; +} + +function livePublicationFor(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationIntent[] | undefined { + return operation.livePublication?.map((intent) => ({ + after: intent.after, + event: AgentEventSchema().parse(intent.event), + })); +} + +function isGeneratedNoticePublication( + receipt: ExecutionLivePublicationReceipt | undefined, +): receipt is ExecutionLivePublicationReceipt & { readonly event: Extract & { readonly messageId: string } } { + return receipt?.event.type === "system" && typeof receipt.event.messageId === "string" + && z.string().uuid().safeParse(receipt.event.messageId).success; +} + +function validReceiptPublicationIds(publication: readonly ExecutionLivePublicationReceipt[] | undefined): boolean { + if (!publication) return true; + const first = Number(publication[0]?.publicationId); + return Number.isSafeInteger(first) && first > 0 + && publication.every((entry, index) => entry.publicationId === String(first + index)); +} + +function conflict(operation: ExecutionSemanticOperation): Extract { + return { kind: "conflict", operationId: operation.operationId }; +} + +function publicationFingerprint(publications: readonly ExecutionLivePublicationIntent[] | undefined): string { + return fingerprint(publications?.map(({ after, event }) => ({ after, event })) ?? null); +} + +function fingerprint(value: unknown): string { + return NodeCrypto.createHash("sha256").update(JSON.stringify(sortJson(value))).digest("hex"); +} + +function publicationChunkPrefix(hash: string): string { + return `semantic:publication:${hash}:`; +} + +function publicationChunkId(hash: string, firstSequence: number): string { + return `${publicationChunkPrefix(hash)}${firstSequence.toString().padStart(16, "0")}`; +} + +function sortJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJson); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)) + .map(([key, entry]) => [key, sortJson(entry)])); +} diff --git a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts new file mode 100644 index 000000000..07e965e58 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts @@ -0,0 +1,37 @@ +import type { + ExecutionSemanticOperation, + ExecutionSemanticWriter, + ExecutionWriteReceipt, +} from "../execution/execution-worker-handler.js"; +import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; +import { CanonicalAgentWriterClient } from "./canonical-agent-writer-client.js"; +import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; +import type { ExecutionLivePublicationRelease } from "./execution-live-publication-release.js"; +import type { ExecutionPlanQuestionRelease } from "./execution-plan-question-release.js"; + +/** Bridges an execution worker to the sole SQLite writer and publishes only committed events. */ +export class CanonicalExecutionWriterPort implements ExecutionSemanticWriter { + constructor( + private readonly writer: CanonicalAgentWriterClient, + private readonly publish: CanonicalAgentEventPublisher, + private readonly livePublication?: ExecutionLivePublicationRelease, + private readonly planQuestions?: ExecutionPlanQuestionRelease, + ) {} + + /** Resolve after the durable writer receipt and its canonical event publication. */ + async transact(operation: ExecutionSemanticOperation): Promise { + const receipt = await this.writer.transactSemantic(operation, this.publish); + if (receipt.kind === "committed" && receipt.planQuestions && !this.planQuestions) { + throw new Error("Plan-question publication is not bound"); + } + this.planQuestions?.validate(operation, receipt); + this.livePublication?.release(operation, receipt); + this.planQuestions?.release(operation, receipt); + return receipt; + } + + /** Reconcile a lost worker through the sole writer and publish its committed interruption. */ + async interruptWorkerLoss(input: LostExecutionInterruption): Promise { + return this.writer.interruptWorkerLoss(input, this.publish); + } +} diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-lifecycle.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-lifecycle.ts index b9d455de9..6b909b5b1 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-lifecycle.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-lifecycle.ts @@ -27,7 +27,7 @@ import type { CanonicalAgentCommitResult, CanonicalAgentEventDraft, CanonicalProviderContinuationInput, -} from "./canonical-agent-event-sink.js"; +} from "./canonical-agent-boundary.js"; /** Canonical alias for the parent-turn start durability input. */ export type CanonicalParentTurnStartInput = ParentTurnStartInput; diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts new file mode 100644 index 000000000..598b27f27 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -0,0 +1,651 @@ +import type { Database } from "bun:sqlite"; +import * as NodeCrypto from "node:crypto"; +import { + ParentNarrativeRecoveryItemSchema, + TurnFileEffectSummarySchema, + type ParentNarrativeRecoveryItem, + type StoredAttachment, + type TurnOutcome, +} from "@mcode/contracts"; +import { z } from "zod"; + +import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import { NarrativeStore } from "../conversation/narrative/narrative-store.js"; +import { ThoughtSegmentRepo } from "../conversation/narrative/persistence/thought-segment-repo.js"; +import { HookExecutionRepo } from "../events/persistence/hook-execution-repo.js"; +import { PlanQuestionAnswersRepo } from "../planning/persistence/plan-question-answers-repo.js"; +import { ToolCallRecordRepo } from "../tools/persistence/tool-call-record-repo.js"; +import { deriveTurnAssistantMessageId } from "../turns/turn-assistant-message-id.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; +import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; +import { TURN_DIFF_MAX_BYTES, parseTurnDiff } from "../turns/turn-diff-patch.js"; +import { TurnDiffRepo } from "../turns/persistence/turn-diff-repo.js"; +import { TurnSnapshotRepo } from "../turns/persistence/turn-snapshot-repo.js"; +import type { PreparedExecutionFileEvidence } from "../turns/turn-execution-file-evidence.js"; +import type { SelectedTurnDiff } from "../turns/turn-diff-service.js"; +import type { + ParentTurnFinishInput, + ParentTurnProjection, + ParentTurnStartInput, +} from "../turns/parent-turn-durability.js"; +import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; +import { + CanonicalAgentBoundary, + type CanonicalAgentBatchedCommitResult, + type CanonicalAgentCommitInput, + type CanonicalAgentCommitResult, + type CanonicalAgentEventDraft, + type CanonicalAgentEventPublisher, + type CanonicalTerminalBatchWrite, +} from "./canonical-agent-boundary.js"; + +type CreateMessageArgument = Parameters; +const stagedAssistantSchema = z.object({ role: z.string(), content: z.string() }); + +/** User-message data the writer can project inside the canonical start transaction. */ +export type ParentUserMessageWrite = + | { kind: "existing"; messageId: string } + | { + kind: "create"; + messageId?: string; + content: string; + sequence: number; + attachments?: CreateMessageArgument[4]; + replyToMessageId?: CreateMessageArgument[5]; + quotedText?: CreateMessageArgument[6]; + mentions?: CreateMessageArgument[9]; + previewAnnotations?: CreateMessageArgument[10]; + origin?: CreateMessageArgument[11]; + selectedTextComments?: CreateMessageArgument[13]; + }; + +/** Project prepared user-message data on the transaction's SQLite connection. */ +export function projectParentUserMessage(messages: MessageRepo, threadId: string, userMessage: ParentUserMessageWrite) { + if (userMessage.kind === "existing") { + const existing = messages.findByIdInThread(threadId, userMessage.messageId); + if (!existing || existing.role !== "user") { + throw new Error(`Queued user message not found: ${userMessage.messageId}`); + } + return existing; + } + return messages.create( + threadId, + "user", + userMessage.content, + userMessage.sequence, + userMessage.attachments, + userMessage.replyToMessageId, + userMessage.quotedText, + undefined, + undefined, + userMessage.mentions, + userMessage.previewAnnotations, + userMessage.origin, + userMessage.messageId, + userMessage.selectedTextComments, + ); +} + +/** Values for an atomic start; no caller closure enters the writer. */ +export interface DataOnlyParentTurnStartInput extends Omit { + userMessage: ParentUserMessageWrite; + reopenThread?: boolean; + answeredPlanQuestionMessageId?: string; +} + +/** Values for one canonical event and checkpoint transaction. */ +export interface DataOnlyParentEventInput extends Omit { + events: readonly CanonicalAgentEventDraft[]; +} + +/** A staged assistant and narrative to confirm in the terminal transaction. */ +export interface DataOnlyParentTurnFinishInput extends Omit { + projection: ParentTurnProjection | { readonly kind: "writer-staged"; readonly messageId?: string }; + deliveryAttempt?: number; + selectedTurnDiff?: SelectedTurnDiff; + fileEvidence?: PreparedExecutionFileEvidence; +} + +/** Cloneable terminal data whose compatibility rows are staged on the writer connection. */ +export interface DataOnlyParentTerminalProjectionInput { + threadId: string; + executionId: string; + outcome: TurnOutcome; + endedAt: string; + assistant: { content: string; model: string | null; attachments: readonly StoredAttachment[]; messageId?: string }; + narrative: readonly ParentNarrativeRecoveryItem[]; +} + +/** Assistant body and assigned ID that must exist before its live message is published. */ +export interface DataOnlyParentLiveMessageInput { + readonly precedingMessageId: string; + readonly messageId: string; + readonly content: string; + readonly model: string | null; + readonly attachments: readonly StoredAttachment[]; +} + +/** A staged projection ready for the canonical terminal commit. */ +export interface StagedParentTerminalProjection { + projection: ParentTurnProjection; + messageId: string | null; + toolCallCount: number; +} + +/** Recovery data for an execution whose owning worker has been lost. */ +export interface LostParentExecutionInput { + threadId: string; + turnId: string; + executionId: string; + reason: string; + recoveryIncidentId: string; + assignedMessageId?: string; +} + +/** Writer-local semantic operations with cloneable inputs and durable receipts. */ +export class CanonicalParentTurnWrite { + private readonly db: Database; + private readonly canonical: CanonicalAgentBoundary; + private readonly messages: MessageRepo; + private readonly narrative: NarrativeStore; + private readonly threads: ThreadRepo; + private readonly planAnswers: PlanQuestionAnswersRepo; + private readonly stagedAssistant: ReturnType; + private readonly assistantTextCheckpoints: ParentAssistantTextCheckpointService; + private readonly turnDiffs: TurnDiffRepo; + private readonly turnSnapshots: TurnSnapshotRepo; + private readonly markThreadFilesChanged: ReturnType; + + constructor(db: Database, publish: CanonicalAgentEventPublisher) { + this.db = db; + this.canonical = new CanonicalAgentBoundary(db, publish); + this.messages = new MessageRepo(db); + this.narrative = new NarrativeStore( + this.messages, + new ToolCallRecordRepo(db), + new ThoughtSegmentRepo(db), + new HookExecutionRepo(db), + db, + ); + this.threads = new ThreadRepo(db); + this.planAnswers = new PlanQuestionAnswersRepo(db); + this.stagedAssistant = db.prepare("SELECT role, content FROM messages WHERE id = ? AND thread_id = ?"); + this.assistantTextCheckpoints = new ParentAssistantTextCheckpointService(db); + this.turnDiffs = new TurnDiffRepo(db); + this.turnSnapshots = new TurnSnapshotRepo(db); + this.markThreadFilesChanged = db.prepare("UPDATE threads SET has_file_changes = 1 WHERE id = ? AND has_file_changes = 0"); + } + + /** Import fsynced text before the interruption transaction reads its durable prefix. */ + importRecoveryJournals(): void { + this.assistantTextCheckpoints.importRecoveryJournals(); + } + + /** Apply the existing interrupted-turn contract on the writer connection. Caller owns the transaction. */ + interruptLostExecution(input: LostParentExecutionInput): CanonicalAgentCommitResult { + const turn = this.canonical.loadTurnByExecution(input.executionId); + const checkpoint = this.canonical.loadCheckpoint(input.executionId); + if (!isUnfinishedLostTurn(turn, checkpoint, input)) { + throw new Error(`Unfinished parent execution not found: ${input.executionId}`); + } + const existingAssistant = this.canonical.loadTerminalProjection(input.turnId).message; + const assignedAssistant = this.assignedAssistantForLoss(input, existingAssistant); + const recoveredNarrative = this.canonical.loadParentNarrativeRecovery(input.turnId); + const text = existingAssistant || assignedAssistant ? "" : this.assistantTextCheckpoints.restore(input.executionId); + const assistant = existingAssistant ?? assignedAssistant ?? this.stageRecoveredAssistant(input, text, recoveredNarrative.length); + this.canonical.markUnresolvedCodexChildDeliveriesUnknown(input.executionId); + const result = this.canonical.interruptUnfinishedExecution( + input.executionId, + input.reason, + assistant ?? undefined, + recoveredNarrative.length > 0 + ? (message, narrative) => this.narrative.persistRecoveredNarrative(message.id, narrative) + : undefined, + recoveredNarrative, + input.recoveryIncidentId, + ); + if (result.outcome !== "committed") throw new Error(`Parent interruption did not commit: ${input.executionId}`); + this.threads.updateStatus(input.threadId, "interrupted"); + return result; + } + + private assignedAssistantForLoss( + input: LostParentExecutionInput, + terminal: ReturnType["message"], + ) { + if (!input.assignedMessageId) return null; + const assigned = this.messages.findByIdInThreadIncludingInternal(input.threadId, input.assignedMessageId); + if (!assigned || assigned.role !== "assistant" || !assigned.is_internal + || terminal && terminal.id !== assigned.id) { + throw new Error(`Assigned assistant message was not staged: ${input.assignedMessageId}`); + } + return assigned; + } + + /** Retire provisional text only after the canonical interruption and receipt commit. */ + retireInterruptedText(executionId: string): void { + if (!this.assistantTextCheckpoints.retire(executionId)) { + throw new Error(`Interrupted assistant text checkpoint was not retired: ${executionId}`); + } + } + + private stageRecoveredAssistant(input: LostParentExecutionInput, text: string, narrativeCount: number) { + if (text.length === 0 && narrativeCount === 0) return null; + const id = deriveTurnAssistantMessageId(input.threadId, `recovery:${input.executionId}`); + const existing = this.messages.findByIdInThreadIncludingInternal(input.threadId, id); + if (existing) return existing; + return this.messages.createAssistantIdempotent({ + id, + threadId: input.threadId, + content: text, + sequence: this.messages.getLatestSequenceIncludingInternal(input.threadId) + 1, + model: this.threads.findById(input.threadId)?.model ?? null, + isInternal: true, + }); + } + + /** Commit the user message, optional thread reopen and plan answer with canonical start. */ + start(input: DataOnlyParentTurnStartInput): CanonicalAgentCommitResult { + return this.canonical.startParentTurn({ + ...input, + projectUserMessage: () => { + if (input.reopenThread && !this.threads.reopen(input.thread.id)) { + throw new Error(`Thread not found: ${input.thread.id}`); + } + const message = projectParentUserMessage(this.messages, input.thread.id, input.userMessage); + if (input.answeredPlanQuestionMessageId) { + this.planAnswers.markAnswered(input.answeredPlanQuestionMessageId, input.thread.id); + } + return message; + }, + }); + } + + /** Commit a plain canonical event batch and its checkpoint before publication. */ + append(input: DataOnlyParentEventInput): CanonicalAgentCommitResult { + return this.canonical.commit(input); + } + + /** Stage the exact live body on this writer connection for terminal reuse. */ + stageLiveAssistant(execution: ExecutionIdentity, input: DataOnlyParentLiveMessageInput): boolean { + if (!this.canStageLiveAssistant(execution, input)) return false; + const existing = this.messages.findByIdInThreadIncludingInternal(execution.threadId, input.messageId); + if (existing) return Boolean(existing.is_internal && this.matchesAssistantBody(existing, input)); + return this.createLiveAssistant(execution.threadId, input); + } + + private canStageLiveAssistant(execution: ExecutionIdentity, input: DataOnlyParentLiveMessageInput): boolean { + const turn = this.canonical.loadTurnByExecution(execution.executionId); + const checkpoint = this.canonical.loadCheckpoint(execution.executionId); + return Boolean(turn && turn.id === execution.turnId && turn.threadId === execution.threadId + && checkpoint?.turnId === execution.turnId && checkpoint.terminalOutcome === null) + && deriveTurnAssistantMessageId(execution.threadId, input.precedingMessageId) === input.messageId; + } + + private createLiveAssistant(threadId: string, input: DataOnlyParentLiveMessageInput): boolean { + const preceding = this.messages.findByIdInThreadIncludingInternal(threadId, input.precedingMessageId); + if (!preceding || !this.messages.isLatestNonSystemMessage(threadId, preceding.id)) return false; + const sequence = this.messages.getLatestSequenceIncludingInternal(threadId) + 1; + this.messages.createAssistantIdempotent({ + id: input.messageId, + threadId, + content: input.content, + sequence, + model: input.model, + attachments: [...input.attachments], + isInternal: true, + }); + const staged = this.messages.findByIdInThreadIncludingInternal(threadId, input.messageId); + return Boolean(staged?.is_internal && this.matchesAssistantBody(staged, input)); + } + + /** Stage an internal assistant and its terminal narrative without publishing them. */ + stageTerminalProjection(input: DataOnlyParentTerminalProjectionInput): StagedParentTerminalProjection { + if (!input.threadId || !input.executionId || !Number.isFinite(Date.parse(input.endedAt))) { + throw new Error("Invalid parent terminal projection identity or end time"); + } + if (input.assistant.messageId !== undefined && !/^[0-9a-f]{64}$/.test(input.assistant.messageId)) { + throw new Error("Invalid parent assistant message identity"); + } + const narrative = input.narrative.map((item) => ParentNarrativeRecoveryItemSchema().parse(item)); + return this.db.transaction(() => this.stageTerminalProjectionInTransaction(input, narrative))(); + } + + private stageTerminalProjectionInTransaction( + input: DataOnlyParentTerminalProjectionInput, + snapshot: readonly ParentNarrativeRecoveryItem[], + ): StagedParentTerminalProjection { + const turn = this.terminalProjectionTurn(input); + if (turn.terminal) return this.persistedTerminalProjection(input, turn.turnId); + if (!hasTerminalSubstance(input, snapshot)) { + return { projection: { message: null, narrative: [] }, messageId: null, toolCallCount: 0 }; + } + const message = this.stageAssistant(input); + const settled = settleTerminalNarrative(snapshot, message.id, input.outcome, input.endedAt, input.assistant.content); + this.narrative.persistRecoveredNarrative(message.id, settled, true); + return this.projectionForMessage(message); + } + + private terminalProjectionTurn(input: DataOnlyParentTerminalProjectionInput): { turnId: string; terminal: boolean } { + const turn = this.canonical.loadTurnByExecution(input.executionId); + const checkpoint = this.canonical.loadCheckpoint(input.executionId); + if (!turn || turn.threadId !== input.threadId || checkpoint?.turnId !== turn.id) { + throw new Error(`Canonical parent execution not found: ${input.executionId}`); + } + if (checkpoint.terminalOutcome && checkpoint.terminalOutcome !== input.outcome) { + throw new Error(`Canonical parent execution has a different terminal outcome: ${input.executionId}`); + } + return { turnId: turn.id, terminal: checkpoint.terminalOutcome !== null }; + } + + private persistedTerminalProjection(input: DataOnlyParentTerminalProjectionInput, turnId: string): StagedParentTerminalProjection { + const persisted = this.canonical.loadTerminalProjection(turnId); + if (input.assistant.messageId && persisted.message?.id !== input.assistant.messageId) { + throw new Error(`Canonical terminal projection has a different assistant identity: ${input.executionId}`); + } + if (!persisted.message) { + if (input.assistant.content.trim() || input.assistant.attachments.length > 0 || input.narrative.length > 0) { + throw new Error(`Canonical terminal projection is empty: ${input.executionId}`); + } + return { projection: { message: null, narrative: [] }, messageId: null, toolCallCount: 0 }; + } + this.assertAssistantMatches(persisted.message, input); + return this.projectionForMessage(persisted.message); + } + + private stageAssistant(input: DataOnlyParentTerminalProjectionInput) { + const id = input.assistant.messageId ?? deriveTurnAssistantMessageId(input.threadId, `execution:${input.executionId}`); + const existing = this.messages.findByIdInThreadIncludingInternal(input.threadId, id); + if (existing) { + if (!existing.is_internal) throw new Error(`Staged assistant projection is already public: ${id}`); + this.assertAssistantMatches(existing, input); + return existing; + } + const sequence = this.messages.getLatestSequenceIncludingInternal(input.threadId) + 1; + return this.messages.createAssistantIdempotent({ + id, + threadId: input.threadId, + content: input.assistant.content, + sequence, + model: input.assistant.model, + attachments: [...input.assistant.attachments], + isInternal: true, + }); + } + + private assertAssistantMatches( + message: NonNullable>, + input: DataOnlyParentTerminalProjectionInput, + ): void { + if (message.role !== "assistant" || message.content !== input.assistant.content + || message.model !== input.assistant.model + || JSON.stringify(message.attachments ?? []) !== JSON.stringify(input.assistant.attachments)) { + throw new Error(`Staged assistant projection conflicts with terminal input: ${input.executionId}`); + } + } + + private matchesAssistantBody( + message: NonNullable>, + input: DataOnlyParentLiveMessageInput, + ): boolean { + return message.role === "assistant" && message.content === input.content + && message.model === input.model + && JSON.stringify(message.attachments ?? []) === JSON.stringify(input.attachments); + } + + private projectionForMessage(message: NonNullable>): StagedParentTerminalProjection { + const narrative = this.narrative.loadForMessages([message]); + return { + projection: { message, narrative }, + messageId: message.id, + toolCallCount: narrative.filter((entry) => entry.kind === "toolCall").length, + }; + } + + /** Confirm the terminal checkpoint and publish the staged assistant in one transaction. */ + finish( + input: DataOnlyParentTurnFinishInput, + onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, + ): Promise { + const staged = "kind" in input.projection + ? this.loadStagedTerminalProjection(input.threadId, input.executionId, input.projection.messageId) + : input.projection; + this.assertStagedAssistant(input.threadId, staged); + const fileEvidence = input.fileEvidence; + const selectedTurnDiff = selectTerminalDiff(input, staged.message !== null); + const projection: ParentTurnProjection = { + message: staged.message + ? { + ...staged.message, + is_internal: false, + outcome: input.outcome, + outcomeExecutionId: input.executionId, + } + : null, + narrative: staged.narrative, + }; + return this.canonical.finishParentTurnBatched({ + ...input, + projectTurn: () => projection, + finalizeCompatibility: () => { + if (!projection.message) { + this.markEmptyTerminalFileChanges(input.threadId, fileEvidence); + return; + } + this.messages.setAssistantOutcome(projection.message.id, input.outcome, input.executionId); + this.messages.publishAssistant(projection.message.id); + if (fileEvidence?.snapshot) { + const snapshot = fileEvidence.snapshot; + this.turnSnapshots.create({ + messageId: projection.message.id, + threadId: snapshot.threadId, + refBefore: snapshot.refBefore, + refAfter: snapshot.refAfter, + filesChanged: [...snapshot.filesChanged], + fileEffects: snapshot.fileEffects, + worktreePath: null, + }); + if (fileEvidence.fileEffects.fileCount > 0 || fileEvidence.filesChanged.length > 0) { + this.markThreadFilesChanged.run(input.threadId); + } + } + if (selectedTurnDiff) this.turnDiffs.create({ + id: NodeCrypto.randomUUID(), message_id: projection.message.id, ...selectedTurnDiff, + }); + }, + }, onBatchWrite); + } + + private markEmptyTerminalFileChanges(threadId: string, evidence?: PreparedExecutionFileEvidence): void { + if (evidence && (evidence.fileEffects.fileCount > 0 || evidence.filesChanged.length > 0)) { + this.markThreadFilesChanged.run(threadId); + } + } + + private loadStagedTerminalProjection(threadId: string, executionId: string, messageId?: string): ParentTurnProjection { + if (messageId !== undefined && !/^[0-9a-f]{64}$/.test(messageId)) { + throw new Error("Invalid staged assistant message identity"); + } + const id = messageId ?? deriveTurnAssistantMessageId(threadId, `execution:${executionId}`); + const message = this.messages.findByIdInThreadIncludingInternal(threadId, id); + if (!message && messageId) throw new Error(`Staged assistant projection not found: ${messageId}`); + if (!message) return { message: null, narrative: [] }; + if (message.role !== "assistant" || !message.is_internal) { + throw new Error(`Staged assistant projection is not private: ${id}`); + } + return { message, narrative: this.narrative.loadForMessages([message]) }; + } + + private assertStagedAssistant(threadId: string, projection: ParentTurnProjection): void { + const projected = projection.message; + if (!projected) return; + const staged = stagedAssistantSchema.nullable().parse(this.stagedAssistant.get(projected.id, threadId)); + if (!staged || staged.role !== "assistant" || staged.content !== projected.content) { + throw new Error(`Staged assistant projection not found: ${projected.id}`); + } + } +} + +function selectTerminalDiff(input: DataOnlyParentTurnFinishInput, hasAssistant: boolean): SelectedTurnDiff | null | undefined { + if (input.fileEvidence) assertExecutionFileEvidence(input, hasAssistant); + const selected = input.fileEvidence?.selectedTurnDiff ?? input.selectedTurnDiff; + if (selected && (input.outcome !== "completed" || !hasAssistant + || !validSelectedTurnDiff(selected, input.threadId))) { + throw new Error("Invalid selected turn diff for terminal assistant"); + } + return selected; +} + +function assertExecutionFileEvidence(input: DataOnlyParentTurnFinishInput, hasAssistant: boolean): void { + const evidence = input.fileEvidence; + if (!evidence || input.selectedTurnDiff || (!hasAssistant && evidence.selectedTurnDiff) + || !validExecutionFileEvidence(evidence, input.threadId, input.turnId, + input.executionId, input.deliveryAttempt)) { + throw new Error("Invalid execution file evidence for terminal assistant"); + } +} + +function validExecutionFileEvidence( + evidence: PreparedExecutionFileEvidence, + threadId: string, + turnId: string, + executionId: string, + deliveryAttempt: number | undefined, +): boolean { + if (!sameTerminalFileOwner(evidence, threadId, turnId, executionId, deliveryAttempt) + || !TurnFileEffectSummarySchema().safeParse(evidence.fileEffects).success + || evidence.fileEffects.fileCount !== evidence.fileEffects.effects.length) return false; + const paths = evidence.fileEffects.effects.filter((effect) => effect.scope === "workspace") + .map((effect) => effect.path); + if (JSON.stringify(evidence.filesChanged) !== JSON.stringify(paths)) return false; + const snapshot = evidence.snapshot; + if (!snapshot) return evidence.fileEffects.fileCount === 0 && paths.length === 0; + return validExecutionSnapshot(snapshot, evidence, paths); +} + +function sameTerminalFileOwner( + evidence: PreparedExecutionFileEvidence, + threadId: string, + turnId: string, + executionId: string, + deliveryAttempt: number | undefined, +): boolean { + return Number.isSafeInteger(deliveryAttempt) && deliveryAttempt !== undefined && deliveryAttempt >= 1 + && evidence.threadId === threadId && evidence.turnId === turnId + && evidence.executionId === executionId && evidence.deliveryAttempt === deliveryAttempt; +} + +function validExecutionSnapshot( + snapshot: NonNullable, + evidence: PreparedExecutionFileEvidence, + paths: readonly string[], +): boolean { + return snapshot.threadId === evidence.threadId && snapshot.executionId === evidence.executionId + && snapshot.worktreePath === null && isSnapshotRef(snapshot.refBefore) && isSnapshotRef(snapshot.refAfter) + && JSON.stringify(snapshot.filesChanged) === JSON.stringify(paths) + && JSON.stringify(snapshot.fileEffects) === JSON.stringify(evidence.fileEffects); +} + +function isSnapshotRef(ref: string): boolean { + return ref === "" || /^[0-9a-f]{40}$|^[0-9a-f]{64}$/.test(ref); +} + +function validSelectedTurnDiff(selected: SelectedTurnDiff, threadId: string): boolean { + if (selected.thread_id !== threadId || !Number.isSafeInteger(selected.revision) || selected.revision < 0) return false; + if (selected.source === "git") return selected.patch === null; + if (selected.source !== "native" && selected.source !== "tracked") return false; + return validSelectedPatch(selected.source, selected.patch); +} + +function validSelectedPatch(source: "native" | "tracked", patch: string | null): boolean { + if (typeof patch !== "string" || Buffer.byteLength(patch, "utf8") > TURN_DIFF_MAX_BYTES) return false; + return source === "native" && patch.length === 0 || parseTurnDiff(patch) !== null; +} + +function isUnfinishedLostTurn( + turn: ReturnType, + checkpoint: ReturnType, + input: LostParentExecutionInput, +): boolean { + return Boolean(turn && checkpoint && turn.id === input.turnId + && checkpoint.threadId === input.threadId && checkpoint.turnId === input.turnId + && checkpoint.terminalOutcome === null); +} + +function settleTerminalNarrative( + items: readonly ParentNarrativeRecoveryItem[], + messageId: string, + outcome: TurnOutcome, + endedAt: string, + assistantContent: string, +): ParentNarrativeRecoveryItem[] { + const finalText = assistantContent.trim(); + let lastThoughtOrder = -Infinity; + for (const item of items) { + if (item.kind === "narrationSegment") lastThoughtOrder = Math.max(lastThoughtOrder, item.record.sort_order); + } + return items.map((item) => { + if (item.kind === "toolCall") return settleToolCall(item, messageId, outcome, endedAt); + if (item.kind === "hook") return settleHook(item, messageId, endedAt); + return settleThought(item, messageId, finalText, lastThoughtOrder); + }); +} + +function hasTerminalSubstance( + input: DataOnlyParentTerminalProjectionInput, + snapshot: readonly ParentNarrativeRecoveryItem[], +): boolean { + return Boolean(input.assistant.content.trim()) || input.assistant.attachments.length > 0 || snapshot.length > 0; +} + +function settleToolCall( + item: Extract, + messageId: string, + outcome: TurnOutcome, + endedAt: string, +): ParentNarrativeRecoveryItem { + const status = item.record.status === "running" ? terminalToolStatus(outcome) : item.record.status; + return { kind: "toolCall", record: { + ...item.record, + message_id: messageId, + status, + completed_at: item.record.status === "running" ? endedAt : item.record.completed_at, + } }; +} + +function settleHook( + item: Extract, + messageId: string, + endedAt: string, +): ParentNarrativeRecoveryItem { + const durationMs = item.record.ended_at + ? item.record.duration_ms + : Math.max(0, Date.parse(endedAt) - Date.parse(item.record.started_at)); + return { kind: "hook", record: { + ...item.record, + message_id: messageId, + duration_ms: durationMs, + ended_at: item.record.ended_at ?? endedAt, + } }; +} + +function settleThought( + item: Extract, + messageId: string, + finalText: string, + lastThoughtOrder: number, +): ParentNarrativeRecoveryItem { + const thoughtText = item.record.text.trim(); + const isFinalResponse = finalText.length > 0 && thoughtText.length > 0 + && (thoughtText === finalText || (item.record.sort_order === lastThoughtOrder && finalText.endsWith(thoughtText))); + return { kind: "narrationSegment", record: { + ...item.record, + message_id: messageId, + is_final_response: isFinalResponse ? 1 : item.record.is_final_response, + } }; +} + +function terminalToolStatus(outcome: TurnOutcome): "completed" | "failed" | "cancelled" { + if (outcome === "errored" || outcome === "interrupted") return "failed"; + return outcome === "cancelled" ? "cancelled" : "completed"; +} diff --git a/apps/server/src/features/agents/canonical/execution-live-publication-release.ts b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts new file mode 100644 index 000000000..424073dd3 --- /dev/null +++ b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts @@ -0,0 +1,128 @@ +import { AgentEventSchema, type AgentEvent } from "@mcode/contracts"; +import * as NodeUtil from "node:util"; + +import type { + ExecutionLivePublicationReceipt, ExecutionSemanticOperation, ExecutionWriteReceipt, +} from "../execution/execution-worker-handler.js"; + +const MAX_RECENT_PUBLICATIONS = 8_192; +const MAX_RECEIPT_EVENTS = 64; +const MAX_RECEIPT_BYTES = 512 * 1024; + +/** The bound public AgentEvent bridge, normally AgentEventPublicationRegistry. */ +export interface LiveAgentEventPublisher { + isBound(): boolean; + publish(event: AgentEvent): void; +} + +/** Releases committed live events with stable identities for transport replay. */ +export class ExecutionLivePublicationRelease { + private readonly published = new Set(); + + constructor(private readonly publisher: LiveAgentEventPublisher) {} + + /** Validate the entire receipt before exposing any event in its original order. */ + release(operation: ExecutionSemanticOperation, receipt: ExecutionWriteReceipt): void { + if (receipt.kind !== "committed" || !receipt.livePublication) return; + const events = this.validate(operation, receipt); + if (!this.publisher.isBound()) throw new Error("Live AgentEvent publisher is not bound"); + for (const entry of events) { + if (this.published.has(entry.key)) continue; + this.publisher.publish(entry.event); + // Recent suppression saves repeat host work. The durable wire identity handles older replays. + this.published.delete(entry.key); + this.published.add(entry.key); + if (this.published.size > MAX_RECENT_PUBLICATIONS) { + this.published.delete(this.published.values().next().value as string); + } + } + } + + private validate( + operation: ExecutionSemanticOperation, + receipt: Extract, + ) { + const entries = receipt.livePublication ?? []; + if (receipt.operationId !== operation.operationId || !operation.livePublication + || entries.length !== operation.livePublication.length || entries.length > MAX_RECEIPT_EVENTS + || Buffer.byteLength(JSON.stringify(entries), "utf8") > MAX_RECEIPT_BYTES + || !contiguousPublicationIds(entries)) { + throw new Error("Live AgentEvent publication receipt is invalid"); + } + return entries.map((entry, index) => this.validateEntry(operation, entry, index)); + } + + private validateEntry(operation: ExecutionSemanticOperation, entry: ExecutionLivePublicationReceipt, index: number) { + const expected = operation.livePublication?.[index]; + if (!expected || !samePublicationHeader(operation, entry.after, expected.after)) { + throw new Error("Live AgentEvent publication receipt is invalid"); + } + const parsed = AgentEventSchema().safeParse(entry.event); + const original = AgentEventSchema().safeParse(expected.event); + if (!parsed.success || !original.success) throw new Error("Live AgentEvent publication receipt is invalid"); + if (parsed.data.threadId !== operation.execution.threadId + || parsed.data.turnExecutionId !== operation.execution.executionId + || !samePublishedEvent(operation, parsed.data, original.data)) { + throw new Error("Live AgentEvent publication receipt is invalid"); + } + return { key: JSON.stringify([operation.execution.executionId, entry.publicationId]), + event: { ...parsed.data, publicationId: entry.publicationId } }; + } +} + +function samePublicationHeader( + operation: ExecutionSemanticOperation, + after: "writer" | "terminal", + expectedBarrier: "writer" | "terminal", +): boolean { + return after === expectedBarrier && after === barrierFor(operation.mutation.kind); +} + +function contiguousPublicationIds(entries: readonly ExecutionLivePublicationReceipt[]): boolean { + const first = Number(entries[0]?.publicationId); + return Number.isSafeInteger(first) && first > 0 + && entries.every((entry, index) => entry.publicationId === String(first + index)); +} + +function samePublishedEvent(operation: ExecutionSemanticOperation, actual: AgentEvent, expected: AgentEvent): boolean { + // Writer receipts may arrive from a live object or JSON replay. Compare the same wire shape in both cases. + const wireActual = AgentEventSchema().parse(JSON.parse(JSON.stringify(actual))); + const wireExpected = AgentEventSchema().parse(JSON.parse(JSON.stringify(expected))); + if (NodeUtil.isDeepStrictEqual(wireActual, wireExpected)) return true; + if (isPersistedTerminalHook(operation, wireActual)) { + const { persistedMessageId: _messageId, persistedHookId: _hookId, ...original } = wireActual; + return NodeUtil.isDeepStrictEqual(original, wireExpected); + } + if (!isGeneratedSystemNotice(operation, wireActual, wireExpected)) return false; + const { messageId: _generatedId, ...withoutGeneratedId } = wireActual; + return NodeUtil.isDeepStrictEqual(withoutGeneratedId, wireExpected); +} + +function isPersistedTerminalHook(operation: ExecutionSemanticOperation, actual: AgentEvent): + actual is Extract { + return operation.mutation.kind === "post-terminal-event" && actual.type === "hookCompleted" + && Boolean(actual.persistedMessageId) + && typeof actual.persistedHookId === "string" + && actual.persistedHookId === operation.mutation.hooks?.[0]?.record.id; +} + +function isGeneratedSystemNotice( + operation: ExecutionSemanticOperation, + actual: AgentEvent, + expected: AgentEvent, +): actual is Extract { + const systemIntents = systemIntentsFor(operation.mutation); + return systemIntents?.[0]?.kind === "system-notice" + && actual.type === "system" && expected.type === "system" + && !expected.messageId && Boolean(actual.messageId); +} + +function systemIntentsFor(mutation: ExecutionSemanticOperation["mutation"]) { + if (mutation.kind === "live-event" || mutation.kind === "post-terminal-event") return mutation.systemIntents; + return mutation.kind === "append-events" ? mutation.parentLive?.systemIntents : undefined; +} + +function barrierFor(kind: ExecutionSemanticOperation["mutation"]["kind"]): "writer" | "terminal" | null { + if (kind === "begin" || kind === "append-events" || kind === "live-event") return "writer"; + return kind === "finish" || kind === "finish-live-event" || kind === "post-terminal-event" ? "terminal" : null; +} diff --git a/apps/server/src/features/agents/canonical/execution-plan-question-release.ts b/apps/server/src/features/agents/canonical/execution-plan-question-release.ts new file mode 100644 index 000000000..6a46bfaf3 --- /dev/null +++ b/apps/server/src/features/agents/canonical/execution-plan-question-release.ts @@ -0,0 +1,55 @@ +import type { PlanQuestion } from "@mcode/contracts"; +import { PlanQuestionBatchSchema } from "@mcode/contracts"; +import * as NodeUtil from "node:util"; + +import type { + ExecutionPlanQuestionsReceipt, ExecutionSemanticOperation, ExecutionWriteReceipt, +} from "../execution/execution-worker-handler.js"; + +const MAX_RELEASED_QUESTIONS = 8_192; + +/** Publishes a parsed plan-question batch only from its committed writer receipt. */ +export class ExecutionPlanQuestionRelease { + private readonly released = new Set(); + + constructor(private readonly publish: (threadId: string, questions: readonly PlanQuestion[]) => void) {} + + /** Check question data before another publication channel releases the event. */ + validate(operation: ExecutionSemanticOperation, receipt: ExecutionWriteReceipt): ExecutionPlanQuestionsReceipt | null { + if (receipt.kind !== "committed" || !receipt.planQuestions) return null; + const batch = receipt.planQuestions; + if (!matchesOperation(operation, receipt, batch) || !PlanQuestionBatchSchema().safeParse({ + threadId: batch.threadId, questions: batch.questions, + }).success || !NodeUtil.isDeepStrictEqual(batch.questions, planQuestionsFor(operation))) { + throw new Error("Plan-question publication receipt is invalid"); + } + return batch; + } + + /** Publish once per host lifetime after the writer acknowledges the event. */ + release(operation: ExecutionSemanticOperation, receipt: ExecutionWriteReceipt): void { + const batch = this.validate(operation, receipt); + if (!batch) return; + const key = JSON.stringify([operation.execution.executionId, batch.publicationId]); + if (this.released.has(key)) return; + if (this.released.size >= MAX_RELEASED_QUESTIONS) throw new Error("Plan-question publication tracking is full"); + this.publish(batch.threadId, batch.questions); + this.released.add(key); + } +} + +function matchesOperation( + operation: ExecutionSemanticOperation, + receipt: Extract, + batch: ExecutionPlanQuestionsReceipt, +): boolean { + return receipt.operationId === operation.operationId && Boolean(planQuestionsFor(operation)) + && operation.livePublication?.[0]?.event.type === "textDelta" + && batch.publicationId === `${operation.operationId}:plan-questions` + && batch.threadId === operation.execution.threadId; +} + +function planQuestionsFor(operation: ExecutionSemanticOperation): readonly PlanQuestion[] | undefined { + if (operation.mutation.kind === "live-event") return operation.mutation.planQuestions; + return operation.mutation.kind === "append-events" ? operation.mutation.parentLive?.planQuestions : undefined; +} diff --git a/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts b/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts index 432417ee3..206c0e454 100644 --- a/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts +++ b/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts @@ -2,30 +2,57 @@ import "reflect-metadata"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import * as NodeEvents from "node:events"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { Database } from "bun:sqlite"; +import type { WebSocket } from "ws"; import { container } from "tsyringe"; +import type { AgentEvent, IAgentProvider, IProviderRegistry, TurnRequest } from "@mcode/contracts"; import { setupContainer } from "../../../../application/composition/container.js"; +import { WorkerOwnedTurnRuntime } from "../../execution/worker-owned-turn-runtime.js"; +import { ExecutionThreadWorkerPort } from "../../execution/execution-worker-port.js"; +import { CanonicalAgentWriterClient } from "../../canonical/canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../../canonical/canonical-execution-writer-port.js"; +import { ExecutionMailboxOwner } from "../../execution/execution-mailbox-owner.js"; +import { ExecutionMailboxScheduler } from "../../execution/execution-mailbox-scheduler.js"; +import { ExecutionProviderEventOwnership } from "../../execution/execution-provider-event-ownership.js"; +import { ExecutionWorkerLossCoordinator } from "../../execution/execution-worker-loss-coordinator.js"; +import { ExecutionFileEvidenceCoordinator } from "../../execution/execution-file-evidence-coordinator.js"; import { AgentService } from "../../orchestration/agent-service.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; +import { TurnRuntimeController } from "../../orchestration/turn-runtime-controller.js"; import { ThreadCreationCoordinator } from "../../turns/thread-creation-coordinator.js"; import { TURN_FEATURE_EFFECTS, TurnFeatureEffects } from "../../turns/turn-feature-effects.js"; +import { WorkspaceRepo } from "../../../projects/persistence/workspace-repo.js"; +import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { ProviderAvailabilityService } from "../../../providers/availability/provider-availability-service.js"; +import { addClient, removeClient } from "../../../../application/transport/push.js"; describe("AgentService container composition", () => { let database: Database | undefined; + let workerRuntime: WorkerOwnedTurnRuntime | undefined; let temporaryDirectory: string | undefined; + let pushClient: WebSocket | undefined; const previousDatabasePath = process.env.MCODE_DB_PATH; - beforeEach(() => { + beforeEach(async () => { container.reset(); temporaryDirectory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-agent-service-composition-")); process.env.MCODE_DB_PATH = NodePath.join(temporaryDirectory, "mcode.db"); setupContainer(temporaryDirectory); database = container.resolve("Database"); + workerRuntime = container.resolve(WorkerOwnedTurnRuntime); + await workerRuntime.whenReady(); }); - afterEach(() => { - database?.close(); + afterEach(async () => { + if (pushClient) removeClient(pushClient); + pushClient = undefined; + await workerRuntime?.close(); + workerRuntime = undefined; + database?.close(true); database = undefined; container.reset(); if (previousDatabasePath === undefined) delete process.env.MCODE_DB_PATH; @@ -36,9 +63,26 @@ describe("AgentService container composition", () => { it("resolves AgentService through the production container with explicit feature effects", () => { expect(container.resolve(TURN_FEATURE_EFFECTS)).toBeInstanceOf(TurnFeatureEffects); + expect(container.resolve(ExecutionFileEvidenceCoordinator)) + .toBe(container.resolve(ExecutionFileEvidenceCoordinator)); expect(container.resolve(AgentService)).toBeInstanceOf(AgentService); }); + it("starts one writer and a fixed execution pool from the opened database", async () => { + workerRuntime = container.resolve(WorkerOwnedTurnRuntime); + await workerRuntime.whenReady(); + + expect(container.resolve(CanonicalAgentWriterClient)).toBe(workerRuntime.writer); + expect(container.resolve(CanonicalExecutionWriterPort)).toBe(workerRuntime.writerPort); + expect(container.resolve(ExecutionWorkerLossCoordinator)).toBe(workerRuntime.workerLoss); + expect(container.resolve(ExecutionMailboxScheduler)).toBe(workerRuntime.scheduler); + expect(container.resolve(ExecutionMailboxOwner)).toBe(workerRuntime.owner); + expect(container.resolve(ExecutionProviderEventOwnership)).toBe(workerRuntime.providerEvents); + expect(workerRuntime.scheduler.depth()).toMatchObject({ + pending: 0, activeExecutions: 0, workers: [{ index: 0 }, { index: 1 }, { index: 2 }, { index: 3 }], + }); + }); + it("resolves ThreadService when a worktree thread reaches branch validation", async () => { const coordinator = container.resolve(ThreadCreationCoordinator); @@ -52,4 +96,347 @@ describe("AgentService container composition", () => { permissionMode: "default", })).rejects.toThrow("Branch name contains invalid characters: invalid branch"); }); + + it("commits a Codex turn through its execution owner and releases after Ended", async () => { + const published: AgentEvent[] = []; + const pushes: Array<{ channel: string; data: unknown }> = []; + pushClient = capturePushes(pushes); + let providerError: unknown; + const provider = fakeCodexProvider(async (request) => { + try { + await submitCodexEvent(workerRuntime!, request, 1, { type: "textDelta", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, + delta: "owned answer", isFinalResponse: true }); + await submitCodexEvent(workerRuntime!, request, 2, { type: "turnComplete", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, + providerId: "codex", reason: "end_turn", costUsd: null, tokensIn: 1, tokensOut: 2, + totalProcessedTokens: 3, contextWindow: undefined, cacheReadTokens: undefined }); + await submitCodexEvent(workerRuntime!, request, 3, { type: "ended", + threadId: request.threadId, turnExecutionId: request.turnExecutionId }); + } catch (error) { + providerError = error; + throw error; + } + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-test", temporaryDirectory!); + const threads = container.resolve(ThreadRepo); + const thread = threads.create(workspace.id, "Worker turn", "direct", "main", true, "claude"); + threads.updateSettings(thread.id, { thinking: true }); + + await container.resolve(AgentService).sendMessage({ + threadId: thread.id, content: "hello", provider: "codex", model: "gpt-5.6-luna", + permissionMode: "full", reasoningLevel: "high", codexFastMode: false, + }); + await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); + + expect(providerError).toBeUndefined(); + expect(threads.findById(thread.id)).toMatchObject({ + provider: "codex", model: "gpt-5.6-luna", permission_mode: "full", + reasoning_level: "high", codex_fast_mode: false, thinking: true, + }); + expect(pushes.filter((push) => push.channel === "thread.modelUpdated")).toEqual([ + { type: "push", channel: "thread.modelUpdated", data: { threadId: thread.id, model: "gpt-5.6-luna", provider: "codex" } }, + ]); + expect(published.map((event) => event.type)).toEqual(expect.arrayContaining([ + "turnStarted", "textDelta", "turnComplete", "ended", + ])); + expect(container.resolve(TurnRuntimeController).snapshot(thread.id)?.phase).toBe("completed"); + expect(workerRuntime?.scheduler.depth().activeExecutions).toBe(0); + expect(pushes.filter((push) => push.channel === "turn.persisted")).toEqual([ + expect.objectContaining({ data: expect.objectContaining({ threadId: thread.id, + outcome: "completed", executionId: expect.any(String), toolCallCount: 0 }) }), + ]); + expect(pushes.filter((push) => push.channel === "thread.status")).toEqual([ + expect.objectContaining({ data: { threadId: thread.id, status: "completed" } }), + ]); + }); + + it("returns from Stop after a durable worker terminal that preserves partial text", async () => { + const published: AgentEvent[] = []; + const pushes: Array<{ channel: string; data: unknown }> = []; + pushClient = capturePushes(pushes); + const stopProvider = vi.fn(async () => undefined); + const provider = fakeCodexProvider(async (request) => { + await submitCodexEvent(workerRuntime!, request, 1, { type: "textDelta", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, + delta: "partial answer", isFinalResponse: true }); + }, stopProvider); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-stop", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Worker stop", "direct", "main", true, "codex"); + const service = container.resolve(AgentService); + + await service.sendMessage({ threadId: thread.id, content: "hello", permissionMode: "default" }); + const stopped = await service.stopSession(thread.id); + + expect(stopped.status).toBe("cancelled"); + expect(stopProvider).toHaveBeenCalledWith(`mcode-${thread.id}`); + expect(published.some((event) => event.type === "ended" && event.outcome === "interrupted")).toBe(true); + expect(container.resolve(MessageRepo).listByThread(thread.id, 10).messages + .some((message) => message.role === "assistant" && message.content === "partial answer")).toBe(true); + expect(workerRuntime?.owner.current(thread.id)).toBeUndefined(); + expect(pushes.filter((push) => push.channel === "turn.persisted")).toEqual([ + expect.objectContaining({ data: expect.objectContaining({ threadId: thread.id, + outcome: "cancelled", executionId: expect.any(String) }) }), + ]); + expect(pushes.filter((push) => push.channel === "thread.status")).toEqual([ + expect.objectContaining({ data: { threadId: thread.id, status: "interrupted" } }), + ]); + }); + + it("confirms a stopped turn without assistant text through its durable terminal push", async () => { + const pushes: Array<{ channel: string; data: unknown }> = []; + pushClient = capturePushes(pushes); + registerFakeCodex(fakeCodexProvider(async () => undefined)); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind(() => undefined); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-empty-stop", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Empty worker stop", "direct", "main", true, "codex"); + const service = container.resolve(AgentService); + + await service.sendMessage({ threadId: thread.id, content: "hello", permissionMode: "default" }); + const stopped = await service.stopSession(thread.id); + + expect(stopped.status).toBe("cancelled"); + expect(container.resolve(MessageRepo).listByThread(thread.id, 10).messages.map((message) => message.role)) + .toEqual(["user"]); + expect(pushes.filter((push) => push.channel === "turn.persisted")).toEqual([ + expect.objectContaining({ data: expect.objectContaining({ threadId: thread.id, + messageId: null, toolCallCount: 0, outcome: "cancelled", executionId: stopped.turnExecutionId }) }), + ]); + expect(pushes.filter((push) => push.channel === "thread.status")).toEqual([ + expect.objectContaining({ data: { threadId: thread.id, status: "interrupted" } }), + ]); + expect(workerRuntime?.owner.current(thread.id)).toBeUndefined(); + }); + + it("publishes a provider cancelled Ended event from the worker terminal receipt", async () => { + const published: AgentEvent[] = []; + const provider = fakeCodexProvider(async (request) => { + await submitCodexEvent(workerRuntime!, request, 1, { type: "ended", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, outcome: "cancelled" }); + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-cancelled", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Provider cancelled", "direct", "main", true, "codex"); + + await container.resolve(AgentService).sendMessage({ threadId: thread.id, + content: "hello", permissionMode: "default" }); + await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); + + expect(published.some((event) => event.type === "ended" && event.outcome === "interrupted")).toBe(true); + }); + + it("finishes the exact owned turn when canonical delivery fails after provider send", async () => { + let sent: TurnRequest | undefined; + let reportFailure: ((routing: { threadId: string; turnId: string; + executionId: string; deliveryAttempt: number }, error: Error) => void | Promise) | undefined; + const provider = fakeCodexProvider(async (request) => { + sent = request; + }); + Object.assign(provider, { + setCanonicalTurnDeliveryFailureHandler: (handler: NonNullable) => { + reportFailure = handler; + }, + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-delivery", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Delivery failure", "direct", "main", true, "codex"); + + await container.resolve(AgentService).sendMessage({ threadId: thread.id, + content: "hello", permissionMode: "default" }); + if (!sent || !reportFailure) throw new Error("Expected an exact canonical delivery route"); + const routing = { threadId: sent.threadId, turnId: sent.turnId, + executionId: sent.turnExecutionId, deliveryAttempt: 1 }; + await reportFailure({ ...routing, deliveryAttempt: 2 }, new Error("stale attempt")); + expect(workerRuntime?.owner.current(thread.id)).toBeDefined(); + await reportFailure(routing, new Error("canonical sink failed")); + await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); + + expect(container.resolve(TurnRuntimeController).snapshot(thread.id)?.phase).toBe("errored"); + expect(published.some((event) => event.type === "error" && event.error === "canonical sink failed")).toBe(true); + }); + + it("admits seven concurrent owned turns and releases each terminal", async () => { + const provider = fakeCodexProvider(async (request) => { + await submitCodexEvent(workerRuntime!, request, 1, { type: "turnComplete", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, + providerId: "codex", reason: "end_turn", costUsd: null, tokensIn: 1, tokensOut: 1 }); + await submitCodexEvent(workerRuntime!, request, 2, { type: "ended", + threadId: request.threadId, turnExecutionId: request.turnExecutionId }); + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-concurrent", temporaryDirectory!); + const threads = Array.from({ length: 7 }, (_, index) => + container.resolve(ThreadRepo).create(workspace.id, `Concurrent ${index}`, "direct", "main", true, "codex")); + const service = container.resolve(AgentService); + + await Promise.all(threads.map((thread) => service.sendMessage({ threadId: thread.id, + content: "hello", permissionMode: "default" }))); + await waitFor(() => threads.every((thread) => workerRuntime?.owner.current(thread.id) === undefined)); + + expect(published.filter((event) => event.type === "turnStarted")).toHaveLength(7); + expect(published.filter((event) => event.type === "turnComplete")).toHaveLength(7); + expect(workerRuntime?.scheduler.depth().activeExecutions).toBe(0); + }); + + it("interrupts and releases a rejected provider event whose ordinal cannot be replayed", async () => { + let reportFailure: ((routing: { threadId: string; turnId: string; + executionId: string; deliveryAttempt: number }, error: Error) => void | Promise) | undefined; + const provider = fakeCodexProvider(async (request) => { + try { + await submitCodexEvent(workerRuntime!, request, 1, { type: "turnStarted", + threadId: request.threadId, turnExecutionId: request.turnExecutionId }); + } catch (error) { + if (!reportFailure) throw error; + await reportFailure({ threadId: request.threadId, turnId: request.turnId, + executionId: request.turnExecutionId, deliveryAttempt: 1 }, error as Error); + } + }); + Object.assign(provider, { setCanonicalTurnDeliveryFailureHandler: (handler: NonNullable) => { + reportFailure = handler; + } }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind(() => undefined); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-rejected", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Rejected event", "direct", "main", true, "codex"); + + await container.resolve(AgentService).sendMessage({ threadId: thread.id, + content: "hello", permissionMode: "default" }); + await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); + + expect(container.resolve(TurnRuntimeController).snapshot(thread.id)?.phase).toBe("interrupted"); + expect(workerRuntime?.scheduler.depth().activeExecutions).toBe(0); + }); + + it("releases a crashed execution while a peer remains runnable", async () => { + const provider = fakeCodexProvider(async (request) => { + void request; + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind(() => undefined); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-crash", temporaryDirectory!); + const threads = container.resolve(ThreadRepo); + const lost = threads.create(workspace.id, "Lost worker", "direct", "main", true, "codex"); + const peer = threads.create(workspace.id, "Peer worker", "direct", "main", true, "codex"); + const service = container.resolve(AgentService); + await service.sendMessage({ threadId: lost.id, content: "one", permissionMode: "default" }); + await service.sendMessage({ threadId: peer.id, content: "two", permissionMode: "default" }); + const assignment = requireValue(workerRuntime?.owner.current(lost.id), "Expected lost execution owner"); + const peerAssignment = requireValue(workerRuntime?.owner.current(peer.id), "Expected peer execution owner"); + expect(assignment.lease.workerIndex).not.toBe(peerAssignment.lease.workerIndex); + + const ports = (workerRuntime as unknown as { initialWorkers: ExecutionThreadWorkerPort[] }).initialWorkers; + const crashed = requireValue(ports[assignment.lease.workerIndex], "Lost worker port was not found"); + crashed.terminate(); + crashed.onclose?.(); + await waitFor(() => workerRuntime?.owner.current(lost.id) === undefined); + expect(await workerRuntime?.workerLoss.waitForRecovery(assignment.lease.workerIndex)) + .toMatchObject({ kind: "recovered" }); + + expect(container.resolve(TurnRuntimeController).snapshot(lost.id)?.phase).toBe("interrupted"); + expect(workerRuntime?.owner.current(peer.id)?.execution.executionId).toBe(peerAssignment.execution.executionId); + expect((await service.stopSession(peer.id)).status).toBe("cancelled"); + }); + + function registerFakeCodex(provider: IAgentProvider): void { + container.registerInstance("IProviderRegistry", { + resolve: () => provider, resolveAll: () => [provider], shutdown: () => undefined, + }); + container.registerInstance(ProviderAvailabilityService, { + assertUsable: () => undefined, + } as ProviderAvailabilityService); + } }); + +function fakeCodexProvider( + sendTurn: (request: TurnRequest) => Promise, + stopSession: (sessionId: string) => Promise = async () => undefined, +): IAgentProvider { + return Object.assign(new NodeEvents.EventEmitter(), { + id: "codex" as const, + descriptor: { id: "codex" }, + supportsCompletion: false, + sessionForkOnResume: "unsupported" as const, + maxInputCharactersPerTurn: 16000, + sendTurn, + stopSession, + shutdown: () => undefined, + listModels: async () => [], + setCanonicalTurnEventDeliveryEnabled: () => undefined, + setCanonicalTurnDeliveryFailureHandler: () => undefined, + fenceCanonicalTurnEvents: async () => undefined, + retireCanonicalTurnEvents: async () => undefined, + }) as IAgentProvider; +} + +async function submitCodexEvent( + runtime: WorkerOwnedTurnRuntime, + request: TurnRequest, + sequence: number, + event: AgentEvent, +): Promise { + const route = runtime.providerEvents.resolve(request.turnExecutionId, "codex"); + if (route.kind !== "worker") throw new Error("Codex turn did not bind its worker route"); + const itemId = `codex:${request.turnExecutionId}:item:${sequence}`; + const eventId = `codex:${request.turnExecutionId}:event:${sequence}`; + const timestamp = new Date().toISOString(); + await route.submit({ threadId: request.threadId, turnId: request.turnId, + executionId: request.turnExecutionId, phase: "running", deliveryAttempt: 1, batchId: eventId, + events: [{ eventId, sourceProviderId: "codex", sourceIdentities: [], sourceSequence: sequence, + providerTimestamp: timestamp, + routing: { threadId: request.threadId, turnId: request.turnId, + executionId: request.turnExecutionId, itemId }, + payload: { type: "item.recorded", item: { id: itemId, threadId: request.threadId, + turnId: request.turnId, kind: "system", providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent: { event } }, + createdAt: timestamp, updatedAt: timestamp } } }], + }); +} + +async function waitFor(predicate: () => boolean): Promise { + for (let index = 0; index < 300; index++) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error("Timed out waiting for worker turn release"); +} + +function capturePushes(pushes: Array<{ channel: string; data: unknown }>): WebSocket { + const client = { OPEN: 1, readyState: 1, bufferedAmount: 0, + send: (payload: string, complete: (error?: Error) => void) => { + const message = JSON.parse(payload) as { channel: string; data: unknown }; + pushes.push(message); + complete(); + } } as unknown as WebSocket; + addClient(client); + return client; +} + +function requireValue(value: T | undefined, message: string): T { + if (value === undefined) throw new Error(message); + return value; +} diff --git a/apps/server/src/features/agents/composition/register-agents.ts b/apps/server/src/features/agents/composition/register-agents.ts index 1b7dca5e8..d5ce7b35b 100644 --- a/apps/server/src/features/agents/composition/register-agents.ts +++ b/apps/server/src/features/agents/composition/register-agents.ts @@ -1,3 +1,5 @@ +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; import { Lifecycle, instanceCachingFactory, type DependencyContainer } from "tsyringe"; import type { HostRuntime } from "@mcode/shared/node/host-runtime"; import { broadcast } from "../../../application/transport/push.js"; @@ -6,7 +8,6 @@ import { AgentPermissionService, AgentService, CanonicalAgentBoundary, - CanonicalAgentEventSink, ParentAssistantTextCheckpointService, publishCanonicalAgentEvents, TurnRecoveryService, @@ -38,6 +39,15 @@ import { AgentTurnCommandPort, } from "../orchestration/agent-turn-command-port.js"; import { AgentEventPublicationRegistry } from "../orchestration/agent-event-publication-registry.js"; +import { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../canonical/canonical-execution-writer-port.js"; +import { ExecutionMailboxOwner } from "../execution/execution-mailbox-owner.js"; +import { ExecutionMailboxScheduler } from "../execution/execution-mailbox-scheduler.js"; +import { ExecutionProviderEventOwnership } from "../execution/execution-provider-event-ownership.js"; +import { ExecutionWorkerLossCoordinator } from "../execution/execution-worker-loss-coordinator.js"; +import { ExecutionFileEvidenceCoordinator } from "../execution/execution-file-evidence-coordinator.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution/execution-worker-handler.js"; +import { WorkerOwnedTurnRuntime } from "../execution/worker-owned-turn-runtime.js"; import { AgentEventPublicationRuntimePort, AgentReliabilityPort, @@ -83,6 +93,36 @@ import { TurnRuntimeController } from "../orchestration/turn-runtime-controller. /** Register agent orchestration, event, recovery, and planning services. */ export function registerAgentServices(container: DependencyContainer): void { + container.register(WorkerOwnedTurnRuntime, { + useFactory: instanceCachingFactory((c) => { + const dbPath = c.resolve("Database").filename; + if (!dbPath || dbPath === ":memory:") { + throw new Error("Execution workers require a file-backed database"); + } + return new WorkerOwnedTurnRuntime(NodePath.resolve(dbPath), c.resolve(AgentEventPublicationRegistry)); + }), + }); + container.register("WorkerOwnedTurnRuntime", { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime), + }); + container.register(CanonicalAgentWriterClient, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).writer, + }); + container.register(CanonicalExecutionWriterPort, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).writerPort, + }); + container.register(ExecutionWorkerLossCoordinator, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).workerLoss, + }); + container.register>(ExecutionMailboxScheduler, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).scheduler, + }); + container.register(ExecutionMailboxOwner, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).owner, + }); + container.register(ExecutionProviderEventOwnership, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).providerEvents, + }); container.register(TurnDiffService, { useFactory: instanceCachingFactory((c) => new TurnDiffService( new TurnDiffRepo(c.resolve("Database")), @@ -114,9 +154,6 @@ export function registerAgentServices(container: DependencyContainer): void { container.register(SUBAGENT_LIFECYCLE_DURABILITY, { useFactory: (c) => c.resolve(CanonicalAgentBoundary), }); - container.register(CanonicalAgentEventSink, { - useFactory: (c) => c.resolve(CanonicalAgentBoundary) as CanonicalAgentEventSink, - }); container.register( ParentAssistantTextCheckpointService, { useClass: ParentAssistantTextCheckpointService }, @@ -182,6 +219,18 @@ export function registerAgentServices(container: DependencyContainer): void { ); }), }); + container.register(ExecutionFileEvidenceCoordinator, { + useFactory: instanceCachingFactory((c) => new ExecutionFileEvidenceCoordinator( + c.resolve(TURN_FILE_TRACKER), + c.resolve(TurnDiffService), + )), + }); + container.register("ExecutionFileEvidenceCoordinator", { + useFactory: (c) => c.resolve(ExecutionFileEvidenceCoordinator), + }); + container.register("WorkerTurnSnapshotService", { + useFactory: (c) => c.resolve(SnapshotService), + }); container.register(TURN_FINALIZER, { useFactory: instanceCachingFactory((c) => new TurnFinalizer( c.resolve(MessageRepo), @@ -223,6 +272,7 @@ export function registerAgentServices(container: DependencyContainer): void { c.resolve(WorkspaceEnvironmentService), c.resolve(FileService), c.resolve("HostRuntime").platform, + c.resolve(WorkerOwnedTurnRuntime).owner, )), }); container.register(TurnConversationProjectionService, { useClass: TurnConversationProjectionService }, { lifecycle: Lifecycle.Singleton }); diff --git a/apps/server/src/features/agents/conversation/migrations/__tests__/conversation-display-materializer.test.ts b/apps/server/src/features/agents/conversation/migrations/__tests__/conversation-display-materializer.test.ts index f89a4abb8..5dafa480f 100644 --- a/apps/server/src/features/agents/conversation/migrations/__tests__/conversation-display-materializer.test.ts +++ b/apps/server/src/features/agents/conversation/migrations/__tests__/conversation-display-materializer.test.ts @@ -69,6 +69,29 @@ function insertItem( `).run(id, THREAD_ID, TURN_ID, JSON.stringify(payload), createdAt, createdAt); } +function recoveryThought(messageId?: string, id = "thought-1"): Record { + return { + projection: "narrativeRecovery", + narrative: { + kind: "narrationSegment", + record: { + id, + message_id: messageId, + text: "Working", + started_at: NOW, + sort_order: 0, + }, + }, + }; +} + +function directThought(id: string, messageId: string): Record { + return { + projection: "narrationSegment", + record: { id, message_id: messageId, text: id, started_at: NOW, sort_order: 0 }, + }; +} + describe("ConversationDisplayMaterializer", () => { let db: Database; @@ -81,6 +104,175 @@ describe("ConversationDisplayMaterializer", () => { db.close(); }); + it("keeps explicit anchors while choosing the earliest visible assistant for mixed child rows", () => { + insertItem(db, "hidden", { + projection: "message", message: { ...message("hidden-1", "assistant", 0), is_internal: true }, + }); + insertItem(db, "prompt", { projection: "message", message: message("prompt-1", "user", 1) }); + insertItem(db, "first", { projection: "message", message: message("assistant-a", "assistant", 2) }); + insertItem(db, "tied", { projection: "message", message: message("assistant-z", "assistant", 2) }); + insertItem(db, "last", { projection: "message", message: message("assistant-last", "assistant", 3) }); + insertItem(db, "direct-1", directThought("explicit-1", "assistant-last")); + insertItem(db, "direct-2", directThought("explicit-2", "assistant-last")); + insertItem(db, "recovery", recoveryThought(undefined, "implicit-1")); + insertItem(db, "child-reasoning", { projection: "codexChildReasoning", nativeItemId: "reasoning", content: "Thinking" }); + insertItem(db, "child-call", { projection: "codexChildToolCall", nativeItemId: "tool", toolName: "Read", toolInput: {} }); + insertItem(db, "child-result", { projection: "codexChildToolResult", nativeItemId: "tool", output: "contents" }); + insertItem(db, "hook", { + projection: "hook", + record: { id: "hook-1", message_id: "assistant-last", hook_name: "Stop", phase: "stop", started_at: NOW }, + }); + + new ConversationDisplayMaterializer(db).materializeItems(["last"]); + + expect(db.prepare("SELECT id, message_id FROM thought_segments WHERE id NOT LIKE 'codex-child-%' ORDER BY id").all()).toEqual([ + { id: "explicit-1", message_id: "assistant-last" }, + { id: "explicit-2", message_id: "assistant-last" }, + { id: "implicit-1", message_id: "assistant-a" }, + ]); + expect(db.prepare("SELECT message_id, text FROM thought_segments WHERE id LIKE 'codex-child-%'").all()) + .toEqual([{ message_id: "assistant-a", text: "Thinking" }]); + expect(db.prepare("SELECT message_id, output_summary, status FROM tool_call_records").all()) + .toEqual([{ message_id: "assistant-a", output_summary: "contents", status: "completed" }]); + expect(db.prepare("SELECT id, message_id FROM hook_executions").all()) + .toEqual([{ id: "hook-1", message_id: "assistant-last" }]); + expect(db.prepare("SELECT id FROM messages ORDER BY id").all()) + .toEqual([{ id: "assistant-a" }, { id: "assistant-last" }]); + }); + + it("preserves message updates between different resolutions of a shared canonical message ID", () => { + insertItem(db, "a-explicit-source", { + projection: "message", message: { ...message("shared", "assistant", 3), content: "Explicit source" }, + }); + insertItem(db, "b-anchor-source", { + projection: "message", message: { ...message("shared", "assistant", 2), content: "Anchor source" }, + }); + insertItem(db, "child-a", directThought("explicit-a", "shared")); + insertItem(db, "child-b", { projection: "codexChildReasoning", nativeItemId: "reasoning", content: "Thinking" }); + insertItem(db, "child-c", directThought("explicit-c", "shared")); + const materializer = new ConversationDisplayMaterializer(db); + materializer.materializeItems(["a-explicit-source"]); + expect(db.prepare("SELECT content FROM messages WHERE id = 'shared'").get()).toEqual({ content: "Explicit source" }); + + insertItem(db, "child-d", recoveryThought(undefined, "implicit")); + materializer.materializeItems(["a-explicit-source"]); + expect(db.prepare("SELECT content FROM messages WHERE id = 'shared'").get()).toEqual({ content: "Explicit source" }); + }); + + it("rolls back a failed child traversal and resolves current messages when the same materializer retries", () => { + insertItem(db, "assistant", { projection: "message", message: message("assistant-1", "assistant", 2) }); + insertItem(db, "a-recovery", recoveryThought()); + insertItem(db, "b-direct", directThought("direct-1", "assistant-1")); + insertItem(db, "c-invalid", { + projection: "narrationSegment", + record: { id: "invalid-1", message_id: "assistant-1", text: "Retried", started_at: "" }, + }); + const materializer = new ConversationDisplayMaterializer(db); + const materialize = db.transaction(() => materializer.materializeItems(["assistant"])); + + expect(materialize).toThrow("Canonical thought started_at is required"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM thought_segments").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_conversation_display_mappings").get()).toEqual({ count: 0 }); + + db.prepare("UPDATE canonical_agent_items SET payload_json = json_set(payload_json, '$.record.started_at', ?) WHERE id = 'c-invalid'").run(NOW); + db.prepare("UPDATE canonical_agent_items SET payload_json = json_set(payload_json, '$.message.content', 'New answer') WHERE id = 'assistant'").run(); + materialize(); + + expect(db.prepare("SELECT content FROM messages WHERE id = 'assistant-1'").get()).toEqual({ content: "New answer" }); + expect(db.prepare("SELECT id, message_id FROM thought_segments ORDER BY id").all()).toEqual([ + { id: "direct-1", message_id: "assistant-1" }, + { id: "invalid-1", message_id: "assistant-1" }, + { id: "thought-1", message_id: "assistant-1" }, + ]); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_conversation_display_mappings").get()).toEqual({ count: 4 }); + }); + + it("resolves later canonical visibility and content changes without reusing a prior traversal", () => { + insertItem(db, "first", { projection: "message", message: message("assistant-first", "assistant", 2) }); + insertItem(db, "last", { projection: "message", message: message("assistant-last", "assistant", 3) }); + insertItem(db, "recovery", recoveryThought()); + insertItem(db, "direct", directThought("direct-1", "assistant-last")); + const materializer = new ConversationDisplayMaterializer(db); + materializer.materializeItems(["last"]); + expect(db.prepare("SELECT message_id FROM thought_segments WHERE id = 'thought-1'").get()) + .toEqual({ message_id: "assistant-first" }); + + db.prepare("UPDATE canonical_agent_items SET payload_json = json_set(payload_json, '$.message.is_internal', 1) WHERE id = 'first'").run(); + db.prepare("UPDATE canonical_agent_items SET payload_json = json_set(payload_json, '$.message.content', 'Updated answer') WHERE id = 'last'").run(); + materializer.materializeItems(["last"]); + + expect(db.prepare("SELECT id, message_id FROM thought_segments ORDER BY id").all()).toEqual([ + { id: "direct-1", message_id: "assistant-last" }, + { id: "thought-1", message_id: "assistant-last" }, + ]); + expect(db.prepare("SELECT content FROM messages WHERE id = 'assistant-last'").get()).toEqual({ content: "Updated answer" }); + + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET terminal_outcome = NULL, phase = 'running'").run(); + insertItem(db, "prompt", { projection: "message", message: message("prompt-1", "user", 1) }); + materializer.materializeItems(["recovery", "direct"]); + expect(db.prepare("SELECT DISTINCT message_id FROM thought_segments").all()).toEqual([{ message_id: "prompt-1" }]); + }); + + it.each(["completed", "cancelled"])("keeps recovered narrative on the prompt until the turn is %s", (outcome) => { + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET terminal_outcome = NULL, phase = 'running'").run(); + insertItem(db, "prompt-item", { projection: "message", message: message("prompt-1", "user", 1) }); + insertItem(db, "assistant-item", { projection: "message", message: message("assistant-1", "assistant", 2) }); + insertItem(db, "recovery-item", recoveryThought()); + + new ConversationDisplayMaterializer(db).materializeItems(["recovery-item"]); + + expect(db.prepare("SELECT id, content FROM messages").all()).toEqual([{ id: "prompt-1", content: "Question" }]); + expect(db.prepare("SELECT id, message_id, text FROM thought_segments").all()).toEqual([ + { id: "thought-1", message_id: "prompt-1", text: "Working" }, + ]); + + const recoveredMaterializer = new ConversationDisplayMaterializer(db); + recoveredMaterializer.materializeItems(["recovery-item"]); + expect(db.prepare("SELECT message_id FROM thought_segments").all()).toEqual([{ message_id: "prompt-1" }]); + + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET terminal_outcome = ?, phase = ?").run(outcome, outcome); + recoveredMaterializer.materializeItems(["assistant-item"]); + + expect(db.prepare("SELECT id, message_id, text FROM thought_segments").all()).toEqual([ + { id: "thought-1", message_id: "assistant-1", text: "Working" }, + ]); + expect(db.prepare("SELECT target_kind, target_id FROM canonical_conversation_display_mappings WHERE source_item_id = 'recovery-item'").get()) + .toEqual({ target_kind: "narrationSegment", target_id: "thought-1" }); + }); + + it("defers recovery narrative with no visible anchor until a terminal message arrives", () => { + insertItem(db, "recovery-item", recoveryThought()); + const materializer = new ConversationDisplayMaterializer(db); + + materializer.materializeItems(["recovery-item"]); + + expect(db.prepare("SELECT COUNT(*) AS count FROM thought_segments").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_conversation_display_mappings").get()).toEqual({ count: 0 }); + + insertItem(db, "assistant-item", { projection: "message", message: message("assistant-1", "assistant", 2) }); + materializer.materializeItems(["assistant-item"]); + + expect(db.prepare("SELECT id, message_id FROM thought_segments").all()).toEqual([ + { id: "thought-1", message_id: "assistant-1" }, + ]); + }); + + it.each(["missing", "hidden"])("rejects recovery narrative with an explicit %s message", (visibility) => { + insertItem(db, "prompt-item", { projection: "message", message: message("prompt-1", "user", 1) }); + if (visibility === "hidden") { + insertItem(db, "hidden-item", { + projection: "message", + message: { ...message("invalid-1", "user", 2), is_internal: true }, + }); + } + insertItem(db, "recovery-item", recoveryThought("invalid-1")); + + expect(() => new ConversationDisplayMaterializer(db).materializeItems(["recovery-item"])) + .toThrow(`Canonical narrative item recovery-item references ${visibility === "hidden" ? "a hidden" : "missing"} message invalid-1`); + expect(db.prepare("SELECT COUNT(*) AS count FROM thought_segments").get()).toEqual({ count: 0 }); + }); + it("defers unanchored canonical child rows until a terminal assistant can display them", async () => { insertItem(db, "child-tool-item", { projection: "codexChildToolCall", diff --git a/apps/server/src/features/agents/conversation/migrations/__tests__/legacy-conversation-migration.test.ts b/apps/server/src/features/agents/conversation/migrations/__tests__/legacy-conversation-migration.test.ts index a1d1bc445..35ae8ae14 100644 --- a/apps/server/src/features/agents/conversation/migrations/__tests__/legacy-conversation-migration.test.ts +++ b/apps/server/src/features/agents/conversation/migrations/__tests__/legacy-conversation-migration.test.ts @@ -5,7 +5,7 @@ import * as NodeURL from "node:url"; import type { Database } from "bun:sqlite"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { openMemoryDatabase } from "../../../../../runtime/persistence/sqlite/database.js"; -import { CanonicalAgentEventSink } from "../../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../../canonical/canonical-agent-boundary.js"; import { LEGACY_CONVERSATION_MIGRATION_MAX_BYTES, LEGACY_CONVERSATION_MIGRATION_VERSION, @@ -60,7 +60,7 @@ describe("LegacyConversationMigration", () => { provenance: "native", }]), }); - const projection = new CanonicalAgentEventSink(db, vi.fn()) + const projection = new CanonicalAgentBoundary(db, vi.fn()) .loadConversationProjection("thread-v1", 10); expect(projection.messages).toEqual([ expect.objectContaining({ @@ -144,7 +144,7 @@ describe("LegacyConversationMigration", () => { WHERE thread_id = 'thread-child-v1' AND json_extract(payload_json, '$.projection') = 'toolCall' `).get()).toEqual({ count: 1 }); - const restoredSink = new CanonicalAgentEventSink(db, vi.fn()); + const restoredSink = new CanonicalAgentBoundary(db, vi.fn()); expect(restoredSink.loadThread("thread-child-v1")).toMatchObject({ parentThreadId: "thread-child-parent-v1", rootThreadId: "thread-child-parent-v1", diff --git a/apps/server/src/features/agents/conversation/migrations/conversation-display-materializer.ts b/apps/server/src/features/agents/conversation/migrations/conversation-display-materializer.ts index 2b0bfd68b..6c9946632 100644 --- a/apps/server/src/features/agents/conversation/migrations/conversation-display-materializer.ts +++ b/apps/server/src/features/agents/conversation/migrations/conversation-display-materializer.ts @@ -51,6 +51,12 @@ interface DisplayToolRecord { sortOrder: number; } +interface NarrativeDisplayResolver { + childAnchor(): string | null; + recoveryAnchor(): string | null; + displayMessageId(value: unknown, row: CanonicalItemRow): string | null; +} + /** * Writes display-table projections for canonical conversation items before the * server admits provider work. The mapping stores identities only, so canonical @@ -63,6 +69,10 @@ export class ConversationDisplayMaterializer { private readonly insertThought: ReturnType; private readonly insertHook: ReturnType; private readonly mapSource: ReturnType; + private readonly findChildMessage: ReturnType; + private readonly findCanonicalMessage: ReturnType; + private readonly findDisplayMessage: ReturnType; + private readonly findTerminalCheckpoint: ReturnType; constructor(private readonly db: Database) { this.orm = drizzle(db); @@ -71,6 +81,10 @@ export class ConversationDisplayMaterializer { this.insertThought = this.buildInsertThought(); this.insertHook = this.buildInsertHook(); this.mapSource = this.buildMapSource(); + this.findChildMessage = this.buildFindChildMessage(); + this.findCanonicalMessage = this.buildFindCanonicalMessage(); + this.findDisplayMessage = this.buildFindDisplayMessage(); + this.findTerminalCheckpoint = this.buildFindTerminalCheckpoint(); } private buildInsertMessage() { @@ -377,9 +391,9 @@ export class ConversationDisplayMaterializer { return bounded; } - private materializeRow(row: CanonicalItemRow): void { + private materializeRow(row: CanonicalItemRow, resolver?: NarrativeDisplayResolver): void { const payload = parsePayload(row); - const target = this.materializePayload(row, payload); + const target = this.materializePayload(row, payload, resolver); if (!target) return; this.mapSource.run({ sourceItemId: row.id, @@ -393,16 +407,17 @@ export class ConversationDisplayMaterializer { private materializePayload( row: CanonicalItemRow, payload: NarrativePayload, + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { switch (payload.projection) { case "message": return this.materializeMessagePayload(row, payload.message); - case "toolCall": return this.materializeDirectNarrative(row, payload, "toolCall"); - case "narrationSegment": return this.materializeDirectNarrative(row, payload, "narrationSegment"); - case "hook": return this.materializeDirectNarrative(row, payload, "hook"); - case "narrativeRecovery": return this.materializeRecoveryPayload(row, payload.narrative); - case "codexChildReasoning": return this.materializeChildReasoning(row, payload); + case "toolCall": return this.materializeDirectNarrative(row, payload, "toolCall", resolver); + case "narrationSegment": return this.materializeDirectNarrative(row, payload, "narrationSegment", resolver); + case "hook": return this.materializeDirectNarrative(row, payload, "hook", resolver); + case "narrativeRecovery": return this.materializeRecoveryPayload(row, payload.narrative, resolver); + case "codexChildReasoning": return this.materializeChildReasoning(row, payload, resolver); case "codexChildToolCall": - case "codexChildToolResult": return this.materializeChildTool(row, payload); + case "codexChildToolResult": return this.materializeChildTool(row, payload, resolver); default: return null; } } @@ -410,8 +425,9 @@ export class ConversationDisplayMaterializer { private materializeRecoveryPayload( row: CanonicalItemRow, narrative: NarrativePayload["narrative"], + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { - return narrative?.record ? this.materializeRecoveryNarrative(row, narrative) : null; + return narrative?.record ? this.materializeRecoveryNarrative(row, narrative, resolver) : null; } private materializeMessagePayload( @@ -428,9 +444,10 @@ export class ConversationDisplayMaterializer { row: CanonicalItemRow, payload: NarrativePayload, kind: "toolCall" | "narrationSegment" | "hook", + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { if (!payload.record) return null; - const record = this.withDisplayMessage(payload.record, row); + const record = this.withDisplayMessage(payload.record, row, resolver); return record ? this.materializeNarrativeRecord(kind, record) : null; } @@ -454,17 +471,21 @@ export class ConversationDisplayMaterializer { private materializeRecoveryNarrative( row: CanonicalItemRow, narrative: { kind?: string; record?: Record }, + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { const record = { ...narrative.record! }; requiredString(record.id, "recovery narrative id"); + let visibleRecord: Record | null; if (typeof record.message_id !== "string" || record.message_id.length === 0) { // Recovery snapshots intentionally omit a staged assistant message. Keep // their visible rows on the prompt until a terminal assistant can own them. - const anchor = this.childAnchor(row); + const anchor = resolver ? resolver.recoveryAnchor() : this.recoveryAnchor(row); if (!anchor) return null; record.message_id = anchor; + visibleRecord = record; + } else { + visibleRecord = this.withDisplayMessage(record, row, resolver); } - const visibleRecord = this.withDisplayMessage(record, row); if (!visibleRecord) return null; return narrative.kind === "toolCall" || narrative.kind === "narrationSegment" || narrative.kind === "hook" ? this.materializeNarrativeRecord(narrative.kind, visibleRecord) @@ -569,8 +590,9 @@ export class ConversationDisplayMaterializer { private materializeChildReasoning( row: CanonicalItemRow, payload: NarrativePayload, + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { - const messageId = this.childAnchor(row); + const messageId = resolver ? resolver.childAnchor() : this.childAnchor(row); if (!messageId) return null; const id = `codex-child-reasoning:${hashCodexKey(`${payload.nativeItemId ?? row.payloadJson}:${row.id}`)}`; const existing = this.orm.select({ sortOrder: thoughtSegments.sortOrder }) @@ -593,8 +615,9 @@ export class ConversationDisplayMaterializer { private materializeChildTool( row: CanonicalItemRow, payload: NarrativePayload, + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { - const messageId = this.childAnchor(row); + const messageId = resolver ? resolver.childAnchor() : this.childAnchor(row); if (!messageId) return null; const nativeItemId = payload.nativeItemId ?? row.payloadJson; const id = `codex-child-tool:${hashCodexKey(nativeItemId)}`; @@ -659,7 +682,19 @@ export class ConversationDisplayMaterializer { } private childAnchor(row: CanonicalItemRow): string | null { - const childMessage = this.orm.select({ + const childMessage = this.findChildMessage.get({ threadId: row.threadId, turnId: row.turnId }); + // Historical child events may be complete while their provider never + // emitted a visible message. They have no display projection yet, but the + // canonical row remains intact and a later terminal message reanchors it. + if (!childMessage) return null; + const payload = parsePayload(childMessage); + if (!payload.message) throw new Error(`Canonical child anchor ${childMessage.id} has no message payload`); + this.materializeMessage(payload.message); + return payload.message.id; + } + + private buildFindChildMessage() { + return this.orm.select({ id: canonicalAgentItems.id, threadId: canonicalAgentItems.threadId, turnId: canonicalAgentItems.turnId, @@ -668,8 +703,8 @@ export class ConversationDisplayMaterializer { updatedAt: canonicalAgentItems.updatedAt, }).from(canonicalAgentItems) .where(and( - eq(canonicalAgentItems.threadId, row.threadId), - eq(canonicalAgentItems.turnId, row.turnId), + eq(canonicalAgentItems.threadId, placeholder("threadId")), + eq(canonicalAgentItems.turnId, placeholder("turnId")), eq(canonicalAgentItems.kind, "message"), sql`json_extract(${canonicalAgentItems.payloadJson}, '$.projection') = 'message'`, sql`COALESCE(json_extract(${canonicalAgentItems.payloadJson}, '$.message.is_internal'), 0) = 0`, @@ -688,45 +723,24 @@ export class ConversationDisplayMaterializer { sql`json_extract(${canonicalAgentItems.payloadJson}, '$.message.id') ASC`, ) .limit(1) - .get(); - // Historical child events may be complete while their provider never - // emitted a visible message. They have no display projection yet, but the - // canonical row remains intact and a later terminal message reanchors it. - if (!childMessage) return null; - const payload = parsePayload(childMessage); - if (!payload.message) throw new Error(`Canonical child anchor ${childMessage.id} has no message payload`); - this.materializeMessage(payload.message); - return payload.message.id; + .prepare(); } private withDisplayMessage( record: Record, row: CanonicalItemRow, + resolver?: NarrativeDisplayResolver, ): Record | null { - const messageId = this.displayMessageId(record.message_id, row); + const messageId = resolver + ? resolver.displayMessageId(record.message_id, row) + : this.displayMessageId(record.message_id, row); return messageId ? { ...record, message_id: messageId } : null; } private displayMessageId(value: unknown, row: CanonicalItemRow): string | null { const messageId = requiredString(value, "narrative message_id"); - const exists = this.orm.select({ id: messages.id }).from(messages) - .where(eq(messages.id, messageId)) - .get(); - const source = this.orm.select({ - id: canonicalAgentItems.id, - threadId: canonicalAgentItems.threadId, - turnId: canonicalAgentItems.turnId, - payloadJson: canonicalAgentItems.payloadJson, - createdAt: canonicalAgentItems.createdAt, - updatedAt: canonicalAgentItems.updatedAt, - }).from(canonicalAgentItems) - .where(and( - eq(canonicalAgentItems.threadId, row.threadId), - sql`json_extract(${canonicalAgentItems.payloadJson}, '$.projection') = 'message'`, - sql`json_extract(${canonicalAgentItems.payloadJson}, '$.message.id') = ${messageId}`, - )) - .limit(1) - .get(); + const exists = this.findDisplayMessage.get({ messageId }); + const source = this.findCanonicalMessage.get({ threadId: row.threadId, messageId }); if (!source) { if (exists) return messageId; throw new Error(`Canonical narrative item ${row.id} references missing message ${messageId}`); @@ -741,6 +755,30 @@ export class ConversationDisplayMaterializer { throw new Error(`Canonical narrative item ${row.id} references a hidden message ${messageId}`); } + private buildFindDisplayMessage() { + return this.orm.select({ id: messages.id }).from(messages) + .where(eq(messages.id, placeholder("messageId"))) + .prepare(); + } + + private buildFindCanonicalMessage() { + return this.orm.select({ + id: canonicalAgentItems.id, + threadId: canonicalAgentItems.threadId, + turnId: canonicalAgentItems.turnId, + payloadJson: canonicalAgentItems.payloadJson, + createdAt: canonicalAgentItems.createdAt, + updatedAt: canonicalAgentItems.updatedAt, + }).from(canonicalAgentItems) + .where(and( + eq(canonicalAgentItems.threadId, placeholder("threadId")), + sql`json_extract(${canonicalAgentItems.payloadJson}, '$.projection') = 'message'`, + sql`json_extract(${canonicalAgentItems.payloadJson}, '$.message.id') = ${placeholder("messageId")}`, + )) + .limit(1) + .prepare(); + } + private nextChildThoughtSortOrder(messageId: string): number { return this.orm.select({ count: count() }).from(thoughtSegments) .where(and( @@ -766,14 +804,18 @@ export class ConversationDisplayMaterializer { } private turnHasTerminalCheckpoint(turnId: string): boolean { + return this.findTerminalCheckpoint.get({ turnId }) != null; + } + + private buildFindTerminalCheckpoint() { return this.orm.select({ executionId: canonicalAgentIngestCheckpoints.executionId }) .from(canonicalAgentIngestCheckpoints) .where(and( - eq(canonicalAgentIngestCheckpoints.turnId, turnId), + eq(canonicalAgentIngestCheckpoints.turnId, placeholder("turnId")), isNotNull(canonicalAgentIngestCheckpoints.terminalOutcome), )) .limit(1) - .get() != null; + .prepare(); } private reanchorTurnChildren(row: CanonicalItemRow): void { @@ -796,7 +838,33 @@ export class ConversationDisplayMaterializer { )) .orderBy(asc(canonicalAgentItems.createdAt), asc(canonicalAgentItems.id)) .all(); - for (const child of children) this.materializeRow(child); + const resolver = this.reanchorMessageResolver(row); + for (const child of children) this.materializeRow(child, resolver); + } + + private recoveryAnchor(row: CanonicalItemRow): string | null { + const anchor = this.childAnchor(row); + return anchor ? this.displayMessageId(anchor, row) : null; + } + + private reanchorMessageResolver(row: CanonicalItemRow): NarrativeDisplayResolver { + // Resolution also updates the display message. Only consecutive identical + // lookups can skip that write when canonical sources share a message ID. + let previous: { key: string; messageId: string | null } | undefined; + const resolve = (key: string, read: () => string | null): string | null => { + if (previous?.key === key) return previous.messageId; + const messageId = read(); + previous = { key, messageId }; + return messageId; + }; + return { + childAnchor: () => resolve("child", () => this.childAnchor(row)), + recoveryAnchor: () => resolve("recovery", () => this.recoveryAnchor(row)), + displayMessageId: (value, child) => { + const messageId = requiredString(value, "narrative message_id"); + return resolve(`message:${messageId}`, () => this.displayMessageId(messageId, child)); + }, + }; } } diff --git a/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts b/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts index 1ce417e1d..7dc10b0ca 100644 --- a/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts +++ b/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts @@ -7,6 +7,7 @@ import { ToolCallRecordRepo } from "../../../tools/persistence/tool-call-record- import { ThoughtSegmentRepo } from "../persistence/thought-segment-repo.js"; import { HookExecutionRepo } from "../../../events/persistence/hook-execution-repo.js"; import { NarrativeStore } from "../narrative-store.js"; +import { NarrativeTurnState } from "../narrative-turn-state.js"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "../../../turns/parent-assistant-text-checkpoint-service.js"; import { @@ -638,6 +639,100 @@ describe("NarrativeStore write seam (server-side traps)", () => { return { toolCallId, toolName, toolInput: {}, parentToolCallId }; } + it("emits late Agent identity as data without a repository", () => { + const state = new NarrativeTurnState(THREAD); + state.beginTurn(THREAD); + state.resetTurnCounters(THREAD); + state.bufferToolCall(THREAD, { + toolCallId: "late-agent", + toolName: "Agent", + toolInput: { codexCollabKind: "spawnAgent", agentPath: "/root/worker" }, + }); + state.prepareNarrativePersistence(THREAD, "m1", "", "completed"); + state.bufferToolCall(THREAD, { + toolCallId: "late-agent", + toolName: "Agent", + toolInput: { + codexCollabKind: "spawnAgent", + agentPath: "/root/worker", + nativeThreadId: "native-late-agent", + }, + }); + + expect(state.takeEffects()).toEqual([{ + kind: "update-subagent-identity", + toolCallId: "late-agent", + messageId: "m1", + identityKey: encodeSubagentAliasDetailTarget("native-late-agent"), + }]); + }); + + it("keeps two execution states for the same thread independent", () => { + const firstExecution = { threadId: THREAD, turnId: "turn-1", executionId: "execution-1" }; + const secondExecution = { threadId: THREAD, turnId: "turn-2", executionId: "execution-2" }; + const first = new NarrativeTurnState(firstExecution); + const second = new NarrativeTurnState(secondExecution); + first.beginTurn(THREAD); + first.resetTurnCounters(THREAD); + second.beginTurn(THREAD); + second.resetTurnCounters(THREAD); + first.openOrExtendThought(THREAD, "First turn"); + first.bufferToolCall(THREAD, toolUse("read-1", "Read")); + + expect(first.recoverySnapshot(THREAD).map((item) => item.record.sort_order)).toEqual([0, 1]); + expect(first.execution).toEqual(firstExecution); + expect(second.execution).toEqual(secondExecution); + expect(() => first.assertExecution(secondExecution)).toThrow("different execution"); + expect(() => first.assertExecution(firstExecution)).not.toThrow(); + expect(second.recoverySnapshot(THREAD)).toEqual([]); + expect(second.nextSortOrder(THREAD)).toBe(0); + expect(() => first.bufferToolCall("another-thread", toolUse("read-2", "Read"))) + .toThrow("Narrative turn belongs to thread-1"); + expect(first.getBufferedToolCalls(THREAD)).toHaveLength(1); + }); + + it("keeps the live store's thread adapters independent", () => { + const otherThread = "thread-2"; + store.beginTurn(THREAD); + store.resetTurnCounters(THREAD); + store.beginTurn(otherThread); + store.resetTurnCounters(otherThread); + store.bufferToolCall(THREAD, toolUse("read-1", "Read")); + + expect(store.getBufferedToolCalls(otherThread)).toEqual([]); + expect(store.nextSortOrder(otherThread)).toBe(0); + expect(store.nextSortOrder(THREAD)).toBe(1); + }); + + it("does not retain empty adapters for completed threads without narrative state", () => { + for (let index = 0; index < 50; index += 1) { + const threadId = `completed-thread-${index}`; + store.updateBufferedToolCallOutput(threadId, "missing-tool", "done", false); + expect(store.stageNarrationSegment(threadId, "")).toBeNull(); + expect(store.prepareNarrativePersistence(threadId, "m1", "", "completed")) + .toEqual({ toolCalls: [], thoughts: [], hooks: [] }); + store.clearTurn(threadId); + } + + expect(Reflect.get(store, "narrativeStates").size).toBe(0); + store.beginTurn("completed-thread-0"); + store.resetTurnCounters("completed-thread-0"); + store.bufferToolCall("completed-thread-0", toolUse("new-tool", "Read")); + expect(store.getBufferedToolCalls("completed-thread-0")).toHaveLength(1); + }); + + it("keeps the completed turn's sort counter for late hooks, then resets it on the next turn", () => { + store.beginTurn(THREAD); + store.resetTurnCounters(THREAD); + store.bufferToolCall(THREAD, toolUse("read-1", "Read")); + store.clearTurn(THREAD); + + expect(store.nextSortOrder(THREAD)).toBe(1); + store.beginTurn(THREAD); + store.resetTurnCounters(THREAD); + expect(store.nextSortOrder(THREAD)).toBe(0); + }); + it("persists a long active turn in order across bounded transactions", async () => { seedAssistantMessage("m1", "done", 1); store.beginTurn(THREAD); @@ -676,6 +771,45 @@ describe("NarrativeStore write seam (server-side traps)", () => { expect(() => store.recoverySnapshot(THREAD)).toThrow("active-turn byte limit"); }); + it("keeps mixed live recovery and persisted narrative in the same order", () => { + seedAssistantMessage("m1", "Done", 1); + store.beginTurn(THREAD); + store.resetTurnCounters(THREAD); + store.openOrExtendThought(THREAD, "I will check."); + store.closeOpenThought(THREAD); + store.bufferToolCall(THREAD, toolUse("agent", "Agent")); + store.bufferToolCall(THREAD, toolUse("child", "Read", "agent")); + store.updateBufferedToolCallOutput(THREAD, "child", "Found it", false); + store.openHook(THREAD, { + hookName: "AfterTool", + toolName: "Read", + phase: "post-tool", + payload: "{}", + sortOrder: store.nextSortOrder(THREAD), + }); + + const live = store.recoverySnapshot(THREAD); + expect(live.map((item) => [item.kind, item.record.sort_order])).toEqual([ + ["narrationSegment", 0], + ["toolCall", 1], + ["toolCall", 2], + ["hook", 3], + ]); + expect(live.find((item) => item.kind === "toolCall" && item.record.id === "child")) + .toMatchObject({ record: { parent_tool_call_id: "agent", output_summary: "Found it" } }); + + store.persistNarrative(THREAD, "m1", "Done", "completed"); + + expect(store.load(THREAD) + .filter((entry) => entry.kind !== "assistantMessage") + .map((entry) => [entry.kind, entry.sortOrder])).toEqual([ + ["narrationSegment", 0], + ["toolCall", 1], + ["toolCall", 2], + ["hook", 3], + ]); + }); + it("rejects individually valid recovery records that exceed the retained byte budget together", () => { store.beginTurn(THREAD); store.resetTurnCounters(THREAD); diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts index d950e467c..5ed92a688 100644 --- a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts +++ b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts @@ -10,11 +10,11 @@ * race two hydration streams (the old `message.list` + `narrative.list` pair) * and Tool calls never render before the assistant message body. * - * Write seam: this store owns the per-turn buffers (tool calls, the + * Write seam: NarrativeTurnState owns the per-turn buffers (tool calls, the * `agentCallStack`, the open/closed thought segments, hook executions, and the - * shared sort counter) and the enrichment + classification + persistence logic - * that AgentService used to inline. The six narrative-pipeline traps documented - * in `docs/internals/narrative-pipeline.md` are enforced here: + * shared sort counter). This store adds SQLite persistence and applies emitted + * data-only effects. The six narrative-pipeline traps documented in + * `docs/internals/narrative-pipeline.md` are enforced here: * * - Trap 1: {@link bufferToolCall} prefers the SDK `parent_tool_use_id` and only * falls back to {@link getCurrentParentToolCallId} when exactly one Agent on @@ -34,7 +34,6 @@ * the persisted rows verbatim and changes no counts. */ import { injectable, inject } from "tsyringe"; -import * as NodeCrypto from "node:crypto"; import type { Database } from "bun:sqlite"; import { drizzle, type BunSQLiteDatabase } from "drizzle-orm/bun-sqlite"; import { and, asc, desc, eq, gt, gte, lt, ne, sql, type SQL, type SQLWrapper } from "drizzle-orm"; @@ -47,20 +46,11 @@ import { import { logger } from "@mcode/shared"; import { NarrativeEntrySchema, - resolveBrowserNarrativeTool, - resolveSubagentDuration, - resolveSubagentMetadata, - resolveSubagentPrompt, - encodeCanonicalSubagentDetailTarget, - encodeSubagentAliasDetailTarget, - createSubagentPresentation, - mergeSubagentPresentation, type Message, type NarrativeEntry, type NarrativeDetailCursor, type ParentNarrativeRecoveryItem, type TurnRange, - type SubagentPresentation, } from "@mcode/contracts"; import { MessageRepo } from "../persistence/message-repo.js"; import { @@ -77,7 +67,17 @@ import { } from "../../events/persistence/hook-execution-repo.js"; import type { TurnOutcome } from "../../turns/turn-outcome.js"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../runtime/persistence/sqlite/bounded-write-batches.js"; -import { assertActiveTurnRecoveryRetention } from "../../turns/active-turn-recovery-retention-policy.js"; +import { + NarrativeTurnState, + type PreparedNarrativePersistence, + type NarrativeTurnStateEffect, +} from "./narrative-turn-state.js"; +export type { + BufferedToolCall, + BufferToolCallEvent, + OpenHook, + StagedNarrationSegment, +} from "./narrative-turn-state.js"; /** Default number of recent messages hydrated when no range is supplied. */ const DEFAULT_LOAD_LIMIT = 200; @@ -121,82 +121,12 @@ function compareNarrativeEntries(left: NarrativeEntry, right: NarrativeEntry): n || compareSqliteBinaryIds(narrativeEntryId(left), narrativeEntryId(right)); } -/** Buffered tool call with raw input preserved for deferred summarization. */ -export interface BufferedToolCall extends CreateToolCallRecordInput { - _rawToolInput?: Record; - _subagentPresentation?: SubagentPresentation; -} - -/** Extracts bounded provider metadata that must survive a persisted Agent card. */ -function persistedSubagentMetadata(input: Record) { - return { - subagentPrompt: resolveSubagentPrompt(input.prompt), - subagentType: resolveSubagentMetadata(input.subagentType), - subagentAgentId: resolveSubagentMetadata(input.agentId), - subagentDurationMs: resolveSubagentDuration(input.durationMs), - }; -} - -function persistedSubagentIdentityKey(presentation: SubagentPresentation | undefined): string | undefined { - if (!presentation) return undefined; - if (presentation.detail.kind === "canonical-child") { - return encodeCanonicalSubagentDetailTarget(presentation.detail.threadId); - } - return presentation.detail.kind === "canonical-alias" - ? encodeSubagentAliasDetailTarget(presentation.detail.identityKey) - : undefined; -} - -/** In-flight thought segment accumulated from consecutive textDelta events. */ -interface OpenThought { - id: string; - text: string; - startedAt: string; - sortOrder: number; -} - -/** In-flight hook execution awaiting its paired HookCompleted. */ -export interface OpenHook { - id: string; - hookName: string; - toolName: string | null; - phase: string; - payload: string; - startedAt: string; - sortOrder: number; -} - -/** One narration record staged for a durable ownership transfer. */ -export interface StagedNarrationSegment { - id: string; - text: string; - startedAt: string; - endedAt: string; - sortOrder: number; - openThoughtId?: string; -} - -/** Tool-use event shape consumed by {@link NarrativeStore.bufferToolCall}. */ -export interface BufferToolCallEvent { - toolCallId: string; - toolName: string; - toolInput: Record; - parentToolCallId?: string; - subagentPresentation?: SubagentPresentation; -} - /** Result of persisting a turn's narrative rows. */ export interface PersistNarrativeResult { /** Number of buffered tool calls written (drives the turn.persisted count). */ toolCallCount: number; } -interface PreparedNarrativePersistence { - toolCalls: BufferedToolCall[]; - thoughts: CreateThoughtSegmentInput[]; - hooks: CreateHookExecutionInput[]; -} - interface PendingNarrativePersistence extends PreparedNarrativePersistence {} interface PersistedNarrativeRows { @@ -207,41 +137,8 @@ interface PersistedNarrativeRows { type RecoveredToolCallItem = Extract; -/** Bounds persisted shell commands while retaining enough text for readable expansion. */ -const MAX_PERSISTED_COMMAND_CHARS = 4096; - -const NARRATIVE_INPUT_SUMMARIZERS: Record) => string> = { - read: fileInputSummary, - edit: fileInputSummary, - write: fileInputSummary, - move: renameInputSummary, - rename: renameInputSummary, - bash: commandInputSummary, - shell: commandInputSummary, - terminal: commandInputSummary, - command_execution: commandInputSummary, - grep: patternInputSummary, - glob: patternInputSummary, - agent: agentInputSummary, -}; - @injectable() export class NarrativeStore { - /** Per-thread buffer of tool calls accumulated during the current turn. */ - private turnToolCalls = new Map(); - /** Stack of active Agent tool call IDs per thread (for nesting inference). */ - private agentCallStack = new Map(); - /** Per-thread sort counter shared across tool calls, thoughts, and hooks. */ - private turnSortCounters = new Map(); - /** In-flight thought segment being accumulated from textDelta events, per thread. */ - private turnOpenThought = new Map(); - /** Closed thought segments awaiting persistence at turn end, per thread. */ - private turnThoughts = new Map(); - /** In-flight hook executions keyed by hookName, per thread. */ - private turnOpenHooks = new Map>(); - /** Closed hook executions awaiting persistence at turn end, per thread. */ - private turnHooks = new Map(); - constructor( @inject(MessageRepo) _messageRepo: MessageRepo, @inject(ToolCallRecordRepo) private readonly toolCallRecordRepo: ToolCallRecordRepo, @@ -250,6 +147,117 @@ export class NarrativeStore { @inject("Database") private readonly db?: Database, ) {} + private applyStateEffect(effect: NarrativeTurnStateEffect): void { + this.toolCallRecordRepo.updateSubagentIdentity( + effect.toolCallId, effect.messageId, effect.identityKey, + ); + } + + private readonly narrativeStates = new Map(); + + private stateFor(threadId: string): NarrativeTurnState { + const existing = this.narrativeStates.get(threadId); + if (existing) return existing; + const state = new NarrativeTurnState(threadId, (effect) => this.applyStateEffect(effect)); + this.narrativeStates.set(threadId, state); + return state; + } + + beginTurn(...args: Parameters): ReturnType { + return this.stateFor(args[0]).beginTurn(...args); + } + + resetTurnCounters(...args: Parameters): ReturnType { + return this.stateFor(args[0]).resetTurnCounters(...args); + } + + nextSortOrder(...args: Parameters): ReturnType { + return this.stateFor(args[0]).nextSortOrder(...args); + } + + openOrExtendThought(...args: Parameters): ReturnType { + return this.stateFor(args[0]).openOrExtendThought(...args); + } + + closeOpenThought(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.closeOpenThought(...args); + } + + dropOpenThought(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.dropOpenThought(...args); + } + + takeOpenThought(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.takeOpenThought(...args) ?? ""; + } + + getCurrentParentToolCallId(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.getCurrentParentToolCallId(...args); + } + + bufferToolCall(...args: Parameters): ReturnType { + return this.stateFor(args[0]).bufferToolCall(...args); + } + + updateBufferedToolCallOutput(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.updateBufferedToolCallOutput(...args); + } + + clearAgentStackOnMessage(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.clearAgentStackOnMessage(...args); + } + + getBufferedToolCalls(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.getBufferedToolCalls(...args) ?? []; + } + + recoverySnapshot(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.recoverySnapshot(...args) ?? []; + } + + stageNarrationSegment(...args: Parameters): ReturnType { + return (this.narrativeStates.get(args[0]) ?? new NarrativeTurnState(args[0])) + .stageNarrationSegment(...args); + } + + recoverySnapshotWithStagedNarration(...args: Parameters): ReturnType { + return (this.narrativeStates.get(args[0]) ?? new NarrativeTurnState(args[0])) + .recoverySnapshotWithStagedNarration(...args); + } + + applyStagedNarrationSegment(...args: Parameters): ReturnType { + return this.stateFor(args[0]).applyStagedNarrationSegment(...args); + } + + hasBufferedNarrative(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.hasBufferedNarrative(...args) ?? false; + } + + openHook(...args: Parameters): ReturnType { + return this.stateFor(args[0]).openHook(...args); + } + + peekOpenHook(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.peekOpenHook(...args); + } + + removeOpenHook(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.removeOpenHook(...args); + } + + pushClosedHook(...args: Parameters): ReturnType { + return this.stateFor(args[0]).pushClosedHook(...args); + } + + prepareNarrativePersistence(...args: Parameters): ReturnType { + return (this.narrativeStates.get(args[0]) ?? new NarrativeTurnState(args[0])) + .prepareNarrativePersistence(...args); + } + + clearTurn(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.clearTurn(...args); + } + private ormInstance: BunSQLiteDatabase | null = null; private requireOrm(): BunSQLiteDatabase { @@ -571,753 +579,11 @@ export class NarrativeStore { for (const hook of hooks) entries.push({ kind: "hook", sequence, sortOrder: hook.sort_order, record: hook }); } - // ---------------------------------------------------------------------- - // Write seam - // ---------------------------------------------------------------------- - - /** - * Reset the volatile per-turn buffers at the START of a turn (Trap 3). Mirrors - * the seeding AgentService used to do in its `sendMessage`/turnStarted prelude. - * Note: the sort counter and Agent stack are reset separately via - * {@link resetTurnCounters} on the TurnStarted event so late hooks from the - * prior turn can still increment the old counter. - */ - beginTurn(threadId: string): void { - this.turnToolCalls.set(threadId, []); - this.turnOpenThought.set(threadId, null); - this.turnThoughts.set(threadId, []); - this.turnOpenHooks.set(threadId, new Map()); - this.turnHooks.set(threadId, []); - } - - /** - * Reset the per-turn sort counter and Agent stack. Called from the - * TurnStarted handler rather than {@link beginTurn} so a fresh counter is - * available for each new turn while late hooks from the prior turn can still - * increment the old one (see {@link clearTurn}). - */ - resetTurnCounters(threadId: string): void { - this.turnSortCounters.set(threadId, 0); - this.agentCallStack.set(threadId, []); - } - - /** Allocate the next shared sort order for the thread's current turn. */ - nextSortOrder(threadId: string): number { - const sortOrder = this.turnSortCounters.get(threadId) ?? 0; - this.turnSortCounters.set(threadId, sortOrder + 1); - return sortOrder; - } - - /** - * Open or extend the in-flight thought segment from a non-final `textDelta`. - * The sort order is allocated lazily on the first delta so consecutive deltas - * keep the same slot, taken BEFORE any following tool call's slot — matching - * the live client builder. Never touches the `agentCallStack` (Trap 2). - */ - openOrExtendThought(threadId: string, delta: string): void { - const open = this.turnOpenThought.get(threadId); - if (!open) { - const sortOrder = this.nextSortOrder(threadId); - this.turnOpenThought.set(threadId, { - id: NodeCrypto.randomUUID(), - text: delta, - startedAt: new Date().toISOString(), - sortOrder, - }); - } else { - open.text += delta; - } - } - - /** - * Close any in-flight thought segment for the thread and push it onto the - * closed-thoughts list. Called before a tool call begins (so the thought - * sorts strictly before the tool) and during turn-end drain. - */ - closeOpenThought(threadId: string): void { - const open = this.turnOpenThought.get(threadId); - if (!open) return; - const list = this.turnThoughts.get(threadId) ?? []; - list.push({ - id: open.id, - messageId: "", - text: open.text, - startedAt: open.startedAt, - endedAt: new Date().toISOString(), - sortOrder: open.sortOrder, - }); - this.turnThoughts.set(threadId, list); - this.turnOpenThought.set(threadId, null); - } - - /** - * Discards the open thought without persisting it. - * - * Called when `AssistantMessageBoundary` reports `isFinalResponse: true` — - * the streamed text was actually the final assistant response and will be - * persisted via the `Message` event, so keeping the matching thought row - * would duplicate the body as a ThoughtBlock in the narrative. - */ - dropOpenThought(threadId: string): void { - this.turnOpenThought.set(threadId, null); - } - - /** - * Move the open thought text out of NarrativeStore without persisting it. - * - * Used when an authoritative boundary retroactively classifies previously - * unknown deltas as the final assistant response. Ownership moves to - * TurnFinalizer so the same text is not buffered in both stores. - */ - takeOpenThought(threadId: string): string { - const open = this.turnOpenThought.get(threadId); - this.turnOpenThought.set(threadId, null); - return open?.text ?? ""; - } - - /** - * Get the current parent tool call ID for a thread's active Agent nesting. - * This is the fallback consulted by `index.ts` enrichment and - * {@link bufferToolCall} when the SDK omits `parent_tool_use_id` (Trap 1). - */ - getCurrentParentToolCallId(threadId: string): string | undefined { - return this.getStackDerivedParentFallback(threadId); - } - - /** - * A single running Agent on the stack (buffer `status === "running"`) can - * serve as a parent fallback when the SDK omits `parent_tool_use_id`. - * Zero or multiple running Agents means the fallback is ambiguous (parallel - * dispatch, nested agents, or coordinator work after children); return - * undefined so tools do not attach under the wrong subagent row. - */ - private getStackDerivedParentFallback(threadId: string): string | undefined { - const stack = this.agentCallStack.get(threadId) ?? []; - if (stack.length === 0) return undefined; - - const buffer = this.turnToolCalls.get(threadId) ?? []; - const runningAgentIds: string[] = []; - for (const agentId of stack) { - const row = buffer.find( - (b) => b.toolCallId === agentId && b.toolName === "Agent", - ); - if (row?.status === "running") { - runningAgentIds.push(agentId); - } - } - - return runningAgentIds.length === 1 ? runningAgentIds[0] : undefined; - } - - /** - * Buffer a tool call event for later persistence and return the parent tool - * call ID attributed to it. An explicit provider parent always wins. The - * stack fallback applies only to non-Agent calls when exactly one Agent is - * still running (Trap 1). Agent calls never inherit a stack-derived parent - * and are pushed onto the `agentCallStack` (Trap 2 push site). - */ - bufferToolCall(threadId: string, event: BufferToolCallEvent): string | undefined { - const buffer = this.turnToolCalls.get(threadId) ?? []; - const stack = this.agentCallStack.get(threadId) ?? []; - const parentToolCallId = this.resolveParentToolCallId(threadId, event); - this.logParentToolCallAttribution(threadId, event, parentToolCallId, stack.length); - const existing = buffer.find((tc) => tc.toolCallId === event.toolCallId); - if (existing) { - return this.updateExistingBufferedToolCall(existing, event, parentToolCallId); - } - return this.addBufferedToolCall( - threadId, - buffer, - stack, - event, - parentToolCallId, - ); - } - - private resolveParentToolCallId( - threadId: string, - event: BufferToolCallEvent, - ): string | undefined { - if (event.toolName === "Agent") return event.parentToolCallId; - return event.parentToolCallId ?? this.getStackDerivedParentFallback(threadId); - } - - private logParentToolCallAttribution( - threadId: string, - event: BufferToolCallEvent, - parentToolCallId: string | undefined, - stackDepth: number, - ): void { - if (event.toolName === "Agent" || !parentToolCallId) return; - logger.debug("bufferToolCall: parent attribution", { - threadId, - toolCallId: event.toolCallId, - toolName: event.toolName, - sdkParent: event.parentToolCallId ?? null, - stackDepth, - attributed: parentToolCallId, - source: event.parentToolCallId ? "sdk" : "stack-fallback", - }); - } - - private updateExistingBufferedToolCall( - existing: BufferedToolCall, - event: BufferToolCallEvent, - parentToolCallId: string | undefined, - ): string | undefined { - const shouldMergeDuplicate = this.shouldMergeDuplicateToolCall(existing, event); - if (shouldMergeDuplicate) { - this.mergeDuplicateToolCall(existing, event); - } - this.mergeParentToolCallId( - existing, - event.parentToolCallId, - parentToolCallId, - shouldMergeDuplicate, - ); - return existing.parentToolCallId; - } - - private shouldMergeDuplicateToolCall( - existing: BufferedToolCall, - event: BufferToolCallEvent, - ): boolean { - if (existing.toolName === "Agent" && event.toolName === "Agent") return true; - const rawToolInput = existing._rawToolInput ?? {}; - return existing.status === "running" && ( - Object.keys(rawToolInput).length === 0 || existing.toolName !== event.toolName - ); - } - - private mergeDuplicateToolCall(existing: BufferedToolCall, event: BufferToolCallEvent): void { - existing.toolName = event.toolName; - const mergedToolInput = { - ...existing._rawToolInput, - ...event.toolInput, - }; - existing._rawToolInput = mergedToolInput; - if (event.toolName === "Agent") { - existing._subagentPresentation = mergeSubagentPresentation( - existing._subagentPresentation, - event.subagentPresentation - ?? createSubagentPresentation(mergedToolInput, event.toolCallId), - event.toolCallId, - ); - this.applyAgentPresentation(existing, mergedToolInput, event.toolCallId); - } - } - - private mergeParentToolCallId( - existing: BufferedToolCall, - providerParentToolCallId: string | undefined, - inferredParentToolCallId: string | undefined, - mergedDuplicate: boolean, - ): void { - if (providerParentToolCallId) { - existing.parentToolCallId = providerParentToolCallId; - return; - } - if (mergedDuplicate && inferredParentToolCallId && !existing.parentToolCallId) { - existing.parentToolCallId = inferredParentToolCallId; - } - } - - private applyAgentPresentation( - toolCall: BufferedToolCall, - rawToolInput: Record, - toolCallId: string, - ): void { - const presentation = toolCall._subagentPresentation - ?? createSubagentPresentation(rawToolInput, toolCallId); - toolCall.displayName = presentation.hasExplicitIdentity ? presentation.displayName : undefined; - toolCall.providerAgentKey = presentation.providerAgentKey; - toolCall.subagentIdentityKey = persistedSubagentIdentityKey(presentation) ?? toolCall.subagentIdentityKey; - toolCall.subagentProviderName = this.subagentProviderName(presentation); - Object.assign(toolCall, persistedSubagentMetadata(rawToolInput)); - if (toolCall.messageId && toolCall.subagentIdentityKey) { - this.toolCallRecordRepo.updateSubagentIdentity( - toolCall.toolCallId!, - toolCall.messageId, - toolCall.subagentIdentityKey, - ); - } - toolCall.model = presentation.model; - toolCall.reasoningEffort = presentation.reasoningEffort; - } - - private addBufferedToolCall( - threadId: string, - buffer: BufferedToolCall[], - stack: string[], - event: BufferToolCallEvent, - parentToolCallId: string | undefined, - ): string | undefined { - const sortOrder = this.nextSortOrder(threadId); - if (event.toolName === "Agent") { - stack.push(event.toolCallId); - this.agentCallStack.set(threadId, stack); - } - const presentation = this.subagentPresentation(event); - buffer.push(this.createBufferedToolCall(event, presentation, sortOrder, parentToolCallId)); - this.turnToolCalls.set(threadId, buffer); - return parentToolCallId; - } - - private subagentPresentation(event: BufferToolCallEvent): SubagentPresentation | undefined { - if (event.toolName !== "Agent") return undefined; - return event.subagentPresentation ?? createSubagentPresentation(event.toolInput, event.toolCallId); - } - - private createBufferedToolCall( - event: BufferToolCallEvent, - presentation: SubagentPresentation | undefined, - sortOrder: number, - parentToolCallId: string | undefined, - ): BufferedToolCall { - const isAgent = event.toolName === "Agent"; - return { - toolCallId: event.toolCallId, - messageId: "", - toolName: event.toolName, - displayName: presentation?.hasExplicitIdentity ? presentation.displayName : undefined, - providerAgentKey: presentation?.providerAgentKey, - subagentIdentityKey: isAgent ? persistedSubagentIdentityKey(presentation) : undefined, - subagentProviderName: this.subagentProviderName(presentation), - ...(isAgent ? persistedSubagentMetadata(event.toolInput) : {}), - model: presentation?.model, - reasoningEffort: presentation?.reasoningEffort, - inputSummary: "", - outputSummary: "", - status: "running", - startedAt: new Date().toISOString(), - sortOrder, - parentToolCallId, - _rawToolInput: event.toolInput, - ...(presentation ? { _subagentPresentation: presentation } : {}), - }; - } - - private subagentProviderName(presentation: SubagentPresentation | undefined): string | undefined { - if (presentation?.detail.kind !== "transcript-unavailable") return undefined; - return presentation.detail.providerName; - } - - /** - * Update a buffered tool call with its output when its result arrives, and - * pop the call from the `agentCallStack` if it was an Agent (Trap 2 pop site). - */ - updateBufferedToolCallOutput( - threadId: string, - toolCallId: string, - output: string, - isError: boolean, - toolInput?: Record, - outputMeta?: { - outputTruncated?: boolean; - outputTotalBytes?: number; - outputArtifactPath?: string; - exitCode?: number; - }, - subagentPresentation?: SubagentPresentation, - ): void { - this.removeAgentFromStack(threadId, toolCallId); - const toolCall = this.latestBufferedToolCall(threadId, toolCallId); - if (!toolCall) return; - this.applyToolCallOutput(toolCall, output, isError, outputMeta); - this.mergeToolCallInput(toolCall, toolInput); - if (toolCall.toolName === "Agent") { - const incomingPresentation = subagentPresentation - ?? createSubagentPresentation(toolCall._rawToolInput ?? {}, toolCallId); - toolCall._subagentPresentation = mergeSubagentPresentation( - toolCall._subagentPresentation, - incomingPresentation, - toolCallId, - ); - this.applyAgentPresentation(toolCall, toolCall._rawToolInput ?? {}, toolCallId); - } - } - - private removeAgentFromStack(threadId: string, toolCallId: string): void { - const stack = this.agentCallStack.get(threadId) ?? []; - const stackIndex = stack.indexOf(toolCallId); - if (stackIndex < 0) return; - stack.splice(stackIndex, 1); - this.agentCallStack.set(threadId, stack); - logger.debug("updateBufferedToolCallOutput: popped Agent from stack", { - threadId, - toolCallId, - remainingDepth: stack.length, - }); - } - - private latestBufferedToolCall(threadId: string, toolCallId: string): BufferedToolCall | undefined { - const buffer = this.turnToolCalls.get(threadId) ?? []; - for (let index = buffer.length - 1; index >= 0; index -= 1) { - if (buffer[index].toolCallId === toolCallId) return buffer[index]; - } - return undefined; - } - - private applyToolCallOutput( - toolCall: BufferedToolCall, - output: string, - isError: boolean, - outputMeta: { - outputTruncated?: boolean; - outputTotalBytes?: number; - outputArtifactPath?: string; - exitCode?: number; - } | undefined, - ): void { - const outputLimit = resolveBrowserNarrativeTool(toolCall.toolName) ? 4_000 : 500; - toolCall.outputSummary = output.slice(0, outputLimit); - toolCall.outputTruncated = outputMeta?.outputTruncated === true; - this.applyOutputMetadata(toolCall, outputMeta); - toolCall.status = isError ? "failed" : "completed"; - toolCall.completedAt = new Date().toISOString(); - } - - private applyOutputMetadata( - toolCall: BufferedToolCall, - outputMeta: { - outputTotalBytes?: number; - outputArtifactPath?: string; - exitCode?: number; - } | undefined, - ): void { - delete toolCall.outputTotalBytes; - delete toolCall.outputArtifactPath; - delete toolCall.exitCode; - if (outputMeta?.outputTotalBytes != null) toolCall.outputTotalBytes = outputMeta.outputTotalBytes; - if (outputMeta?.outputArtifactPath) toolCall.outputArtifactPath = outputMeta.outputArtifactPath; - if (outputMeta?.exitCode !== undefined) toolCall.exitCode = outputMeta.exitCode; - } - - private mergeToolCallInput(toolCall: BufferedToolCall, toolInput: Record | undefined): void { - if (!toolInput || Object.keys(toolInput).length === 0) return; - toolCall._rawToolInput = { - ...toolCall._rawToolInput, - ...toolInput, - }; - } - - /** - * Clear the whole Agent stack when a final `Message` event arrives — the turn - * is over and any Agent calls still on the stack are implicitly done (Trap 2 - * end-of-turn clear). No-ops when the stack is already empty. - */ - clearAgentStackOnMessage(threadId: string): void { - const stack = this.agentCallStack.get(threadId); - if (stack && stack.length > 0) { - stack.length = 0; - } - } - - /** Snapshot of the thread's buffered tool calls (read-only inspection). */ - getBufferedToolCalls(threadId: string): readonly BufferedToolCall[] { - return this.turnToolCalls.get(threadId) ?? []; - } - - /** - * Snapshot the visible structured narrative for an unfinished turn without - * retaining provider protocol traffic or private raw tool input. - */ - recoverySnapshot(threadId: string): ParentNarrativeRecoveryItem[] { - const snapshot: ParentNarrativeRecoveryItem[] = []; - let bytes = 0; - bytes = this.appendBufferedToolCallRecoveryItems(snapshot, bytes, threadId); - for (const thought of this.turnThoughts.get(threadId) ?? []) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.thoughtRecoveryItem(thought), - threadId, - ); - } - const openThought = this.turnOpenThought.get(threadId); - if (openThought) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.openThoughtRecoveryItem(openThought), - threadId, - ); - } - for (const hook of this.turnHooks.get(threadId) ?? []) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.hookRecoveryItem(hook), - threadId, - ); - } - for (const hook of this.turnOpenHooks.get(threadId)?.values() ?? []) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.openHookRecoveryItem(hook), - threadId, - ); - } - return this.sortRecoverySnapshot(snapshot); - } - - private appendBufferedToolCallRecoveryItems( - snapshot: ParentNarrativeRecoveryItem[], - bytes: number, - threadId: string, - ): number { - for (const toolCall of this.turnToolCalls.get(threadId) ?? []) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.toolCallRecoveryItem(toolCall), - threadId, - ); - } - return bytes; - } - - private appendRecoverySnapshotItem( - snapshot: ParentNarrativeRecoveryItem[], - bytes: number, - item: ParentNarrativeRecoveryItem, - threadId: string, - ): number { - const nextBytes = Buffer.byteLength(JSON.stringify(item), "utf8"); - this.assertRecoveryItemFitsWriteBatch(nextBytes, threadId); - const retainedBytes = bytes + nextBytes; - assertActiveTurnRecoveryRetention(snapshot.length + 1, retainedBytes); - snapshot.push(item); - return retainedBytes; - } - - private toolCallRecoveryItem(toolCall: BufferedToolCall): ParentNarrativeRecoveryItem { - return { - kind: "toolCall", - record: { - ...this.toolCallRecoveryIdentityFields(toolCall), - ...this.toolCallRecoveryPresentationFields(toolCall), - ...this.toolCallRecoveryOutputFields(toolCall), - ...this.toolCallRecoveryStateFields(toolCall), - }, - }; - } - - private toolCallRecoveryIdentityFields(toolCall: BufferedToolCall) { - return { - id: this.requireRecoveryString(toolCall.toolCallId, "tool call id"), - message_id: this.requireRecoveryString(toolCall.messageId, "tool call message id"), - parent_tool_call_id: toolCall.parentToolCallId ?? null, - tool_name: toolCall.toolName, - display_name: toolCall.displayName ?? null, - provider_agent_key: toolCall.providerAgentKey ?? null, - subagent_identity_key: toolCall.subagentIdentityKey ?? null, - subagent_provider_name: toolCall.subagentProviderName ?? null, - }; - } - - private toolCallRecoveryPresentationFields(toolCall: BufferedToolCall) { - return { - subagent_prompt: toolCall.subagentPrompt ?? null, - subagent_type: toolCall.subagentType ?? null, - subagent_agent_id: toolCall.subagentAgentId ?? null, - subagent_duration_ms: toolCall.subagentDurationMs ?? null, - model: toolCall.model ?? null, - reasoning_effort: toolCall.reasoningEffort ?? null, - }; - } - - private toolCallRecoveryOutputFields(toolCall: BufferedToolCall) { - return { - input_summary: this.toolCallRecoveryInputSummary(toolCall), - output_summary: toolCall.outputSummary, - ...(toolCall.outputTruncated ? { output_truncated: 1 } : {}), - output_total_bytes: toolCall.outputTotalBytes ?? null, - output_artifact_path: toolCall.outputArtifactPath ?? null, - exit_code: toolCall.exitCode ?? null, - }; - } - - private toolCallRecoveryInputSummary(toolCall: BufferedToolCall): string { - if (toolCall.inputSummary) return toolCall.inputSummary; - return this.summarizeInput(toolCall.toolName, toolCall._rawToolInput ?? {}); - } - - private toolCallRecoveryStateFields(toolCall: BufferedToolCall) { - return { - status: toolCall.status, - started_at: this.requireRecoveryString(toolCall.startedAt, "tool call start time"), - completed_at: toolCall.completedAt ?? null, - sort_order: toolCall.sortOrder, - }; - } - - private thoughtRecoveryItem(thought: CreateThoughtSegmentInput): ParentNarrativeRecoveryItem { - return { - kind: "narrationSegment", - record: { - id: this.requireRecoveryString(thought.id, "thought id"), - message_id: this.requireRecoveryString(thought.messageId, "thought message id"), - text: this.requireRecoveryString(thought.text, "thought text"), - started_at: this.requireRecoveryString(thought.startedAt, "thought start time"), - ended_at: thought.endedAt ?? null, - sort_order: thought.sortOrder, - ...(thought.isFinalResponse ? { is_final_response: thought.isFinalResponse } : {}), - }, - }; - } - - private openThoughtRecoveryItem(openThought: OpenThought): ParentNarrativeRecoveryItem { - return { - kind: "narrationSegment", - record: { - id: openThought.id, - message_id: "", - text: openThought.text, - started_at: openThought.startedAt, - ended_at: null, - sort_order: openThought.sortOrder, - }, - }; - } - - private hookRecoveryItem(hook: CreateHookExecutionInput): ParentNarrativeRecoveryItem { - return { - kind: "hook", - record: { - id: this.requireRecoveryString(hook.id, "hook id"), - message_id: this.requireRecoveryString(hook.messageId, "hook message id"), - hook_name: hook.hookName, - tool_name: hook.toolName, - phase: hook.phase, - payload: hook.payload, - duration_ms: hook.durationMs ?? null, - did_block: hook.didBlock, - started_at: this.requireRecoveryString(hook.startedAt, "hook start time"), - ended_at: hook.endedAt ?? null, - sort_order: hook.sortOrder, - }, - }; - } - - private openHookRecoveryItem(hook: OpenHook): ParentNarrativeRecoveryItem { - return { - kind: "hook", - record: { - id: hook.id, - message_id: "", - hook_name: hook.hookName, - tool_name: hook.toolName, - phase: hook.phase, - payload: hook.payload, - duration_ms: null, - did_block: false, - started_at: hook.startedAt, - ended_at: null, - sort_order: hook.sortOrder, - }, - }; - } - - private sortRecoverySnapshot( - snapshot: ParentNarrativeRecoveryItem[], - ): ParentNarrativeRecoveryItem[] { - return snapshot.sort((left, right) => ( - left.record.sort_order - right.record.sort_order || left.record.id.localeCompare(right.record.id) - )); - } - - /** Stage narration for recovery without mutating the active turn buffer. */ - stageNarrationSegment(threadId: string, text: string): StagedNarrationSegment | null { - const open = this.turnOpenThought.get(threadId); - const endedAt = new Date().toISOString(); - if (open) { - return { - id: open.id, - text: `${open.text}${text}`, - startedAt: open.startedAt, - endedAt, - sortOrder: open.sortOrder, - openThoughtId: open.id, - }; - } - if (!text) return null; - return { - id: NodeCrypto.randomUUID(), - text, - startedAt: endedAt, - endedAt, - sortOrder: this.turnSortCounters.get(threadId) ?? 0, - }; - } - - /** Add staged narration to a recovery snapshot without mutating the active turn buffer. */ - recoverySnapshotWithStagedNarration( - threadId: string, - staged: StagedNarrationSegment, - ): ParentNarrativeRecoveryItem[] { - const snapshot = this.recoverySnapshot(threadId).filter((item) => ( - item.kind !== "narrationSegment" || item.record.id !== staged.id - )); - snapshot.push({ - kind: "narrationSegment", - record: { - id: staged.id, - message_id: "", - text: staged.text, - started_at: staged.startedAt, - ended_at: staged.endedAt, - sort_order: staged.sortOrder, - }, - }); - let bytes = 0; - for (const [index, item] of snapshot.entries()) { - const itemBytes = Buffer.byteLength(JSON.stringify(item), "utf8"); - this.assertRecoveryItemFitsWriteBatch(itemBytes, threadId); - bytes += itemBytes; - assertActiveTurnRecoveryRetention(index + 1, bytes); - } - return snapshot.sort((left, right) => ( - left.record.sort_order - right.record.sort_order || left.record.id.localeCompare(right.record.id) - )); - } - - /** Apply a narration record only after its recovery projection is durable. */ - applyStagedNarrationSegment(threadId: string, staged: StagedNarrationSegment): void { - if (staged.openThoughtId) { - const open = this.turnOpenThought.get(threadId); - if (!open || open.id !== staged.openThoughtId) { - throw new Error(`Staged narration no longer matches the open thought: ${staged.id}`); - } - this.turnOpenThought.set(threadId, null); - } else { - const nextSortOrder = this.turnSortCounters.get(threadId) ?? 0; - if (nextSortOrder <= staged.sortOrder) { - this.turnSortCounters.set(threadId, staged.sortOrder + 1); - } - } - const thoughts = this.turnThoughts.get(threadId) ?? []; - thoughts.push({ - id: staged.id, - messageId: "", - text: staged.text, - startedAt: staged.startedAt, - endedAt: staged.endedAt, - sortOrder: staged.sortOrder, - }); - this.turnThoughts.set(threadId, thoughts); - } - - private requireRecoveryString(value: string | undefined, field: string): string { - if (value === undefined) throw new Error(`Narrative recovery is missing ${field}`); - return value; - } - /** Persist a durable semantic snapshot against its recovered assistant row. */ persistRecoveredNarrative( messageId: string, items: readonly ParentNarrativeRecoveryItem[], + replaceExisting = false, ): void { const tools: CreateToolCallRecordInput[] = []; const thoughts: CreateThoughtSegmentInput[] = []; @@ -1327,15 +593,9 @@ export class NarrativeStore { if (item.kind === "narrationSegment") thoughts.push(this.recoveredThought(messageId, item)); if (item.kind === "hook") hooks.push(this.recoveredHook(messageId, item)); } - if (tools.length > 0) this.toolCallRecordRepo.bulkCreate(tools); - if (thoughts.length > 0) this.thoughtSegmentRepo.bulkCreate(thoughts); - if (hooks.length > 0) this.hookExecutionRepo.bulkCreate(hooks); - } - - private assertRecoveryItemFitsWriteBatch(byteLength: number, threadId: string): void { - if (byteLength > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes) { - throw new Error(`Parent narrative recovery item exceeds the active-turn byte limit: ${threadId}`); - } + if (tools.length > 0) this.toolCallRecordRepo.bulkCreate(tools, replaceExisting); + if (thoughts.length > 0) this.thoughtSegmentRepo.bulkCreate(thoughts, replaceExisting); + if (hooks.length > 0) this.hookExecutionRepo.bulkCreate(hooks, replaceExisting); } private recoveredToolCall( @@ -1443,67 +703,6 @@ export class NarrativeStore { }; } - /** - * True when the current turn has buffered at least one narrative contributor — - * a tool call, a narration segment (open or closed), or a hook (open or - * closed). Feeds the {@link TurnFinalizer.hasRecordableActivity} predicate so - * a turn with narrative but no assistant body still earns a persisted row. - */ - hasBufferedNarrative(threadId: string): boolean { - return this.narrativeBufferStates(threadId).some(Boolean); - } - - private narrativeBufferStates(threadId: string): boolean[] { - return [ - (this.turnToolCalls.get(threadId)?.length ?? 0) > 0, - Boolean(this.turnOpenThought.get(threadId)), - (this.turnThoughts.get(threadId)?.length ?? 0) > 0, - (this.turnOpenHooks.get(threadId)?.size ?? 0) > 0, - (this.turnHooks.get(threadId)?.length ?? 0) > 0, - ]; - } - - /** - * Record an in-flight hook execution (HookStarted). The caller supplies the - * already-allocated sort order (the late-hook path in AgentService allocates - * it before deciding routing). Returns the generated row id. - */ - openHook( - threadId: string, - hook: { hookName: string; toolName: string | null; phase: string; payload: string; sortOrder: number }, - ): string { - const map = this.turnOpenHooks.get(threadId) ?? new Map(); - const id = NodeCrypto.randomUUID(); - map.set(hook.hookName, { - id, - hookName: hook.hookName, - toolName: hook.toolName, - phase: hook.phase, - payload: hook.payload, - startedAt: new Date().toISOString(), - sortOrder: hook.sortOrder, - }); - this.turnOpenHooks.set(threadId, map); - return id; - } - - /** Look up (without removing) an open hook by name. */ - peekOpenHook(threadId: string, hookName: string): OpenHook | undefined { - return this.turnOpenHooks.get(threadId)?.get(hookName); - } - - /** Remove an open hook by name (after it has been completed or flushed). */ - removeOpenHook(threadId: string, hookName: string): void { - this.turnOpenHooks.get(threadId)?.delete(hookName); - } - - /** Push a completed hook execution onto the closed-hooks list for persistence. */ - pushClosedHook(threadId: string, hook: CreateHookExecutionInput): void { - const list = this.turnHooks.get(threadId) ?? []; - list.push(hook); - this.turnHooks.set(threadId, list); - } - /** * Persist the buffered narrative rows (tool calls, thoughts, hooks) for a * completed turn against `messageId`, tagging the final-response thought via @@ -1663,212 +862,4 @@ export class NarrativeStore { ): void { for (const item of items) persisted.add(identifier(item)); } - - private prepareNarrativePersistence( - threadId: string, - messageId: string, - messageContent: string, - outcome: TurnOutcome, - ): PreparedNarrativePersistence { - const toolCalls = this.prepareToolCallsForPersistence(threadId, messageId, outcome); - this.closeOpenThought(threadId); - this.closeOpenHooksForPersistence(threadId); - const thoughts = this.prepareThoughtsForPersistence(threadId, messageId, messageContent); - const hooks = this.prepareHooksForPersistence(threadId, messageId); - return { toolCalls, thoughts, hooks }; - } - - private prepareToolCallsForPersistence( - threadId: string, - messageId: string, - outcome: TurnOutcome, - ): BufferedToolCall[] { - const toolCalls = this.turnToolCalls.get(threadId) ?? []; - const settledAt = new Date().toISOString(); - for (const toolCall of toolCalls) { - toolCall.toolCallId ??= NodeCrypto.randomUUID(); - this.settleRunningToolCall(toolCall, outcome, settledAt); - toolCall.messageId = messageId; - this.prepareToolCallInput(toolCall); - } - return toolCalls; - } - - private settleRunningToolCall( - toolCall: BufferedToolCall, - outcome: TurnOutcome, - settledAt: string, - ): void { - if (toolCall.status !== "running") return; - toolCall.status = this.settledToolCallStatus(outcome); - toolCall.completedAt = settledAt; - } - - private settledToolCallStatus(outcome: TurnOutcome): BufferedToolCall["status"] { - if (outcome === "errored" || outcome === "interrupted") return "failed"; - if (outcome === "cancelled") return "cancelled"; - return "completed"; - } - - private prepareToolCallInput(toolCall: BufferedToolCall): void { - const rawToolInput = toolCall._rawToolInput; - if (toolCall.inputSummary || !rawToolInput) return; - if (toolCall.toolName === "Agent") { - this.applyAgentPersistenceMetadata(toolCall, rawToolInput); - } - toolCall.inputSummary = this.summarizeInput(toolCall.toolName, rawToolInput); - delete toolCall._rawToolInput; - } - - private applyAgentPersistenceMetadata( - toolCall: BufferedToolCall, - rawToolInput: Record, - ): void { - const presentation = createSubagentPresentation(rawToolInput, toolCall.toolCallId!); - const persistedPresentation = toolCall._subagentPresentation ?? presentation; - toolCall.displayName = persistedPresentation.hasExplicitIdentity - ? persistedPresentation.displayName - : undefined; - toolCall.providerAgentKey = persistedPresentation.providerAgentKey; - toolCall.subagentIdentityKey = persistedSubagentIdentityKey(persistedPresentation) - ?? toolCall.subagentIdentityKey; - toolCall.subagentProviderName = this.subagentProviderName(persistedPresentation); - Object.assign(toolCall, persistedSubagentMetadata(rawToolInput)); - toolCall.model = persistedPresentation.model; - toolCall.reasoningEffort = persistedPresentation.reasoningEffort; - } - - private closeOpenHooksForPersistence(threadId: string): void { - const openHookMap = this.turnOpenHooks.get(threadId); - if (!openHookMap || openHookMap.size === 0) return; - const list = this.turnHooks.get(threadId) ?? []; - const endedAt = new Date().toISOString(); - for (const open of openHookMap.values()) { - list.push({ - id: open.id, - messageId: "", - hookName: open.hookName, - toolName: open.toolName, - phase: open.phase, - payload: open.payload, - durationMs: Date.parse(endedAt) - Date.parse(open.startedAt), - didBlock: false, - startedAt: open.startedAt, - endedAt, - sortOrder: open.sortOrder, - }); - } - this.turnHooks.set(threadId, list); - openHookMap.clear(); - } - - private prepareThoughtsForPersistence( - threadId: string, - messageId: string, - messageContent: string, - ): CreateThoughtSegmentInput[] { - const bufferedThoughts = this.turnThoughts.get(threadId) ?? []; - for (const thought of bufferedThoughts) thought.id ??= NodeCrypto.randomUUID(); - const thoughts = bufferedThoughts.map((thought) => ({ ...thought, messageId })); - const message = messageContent.trim(); - if (message.length > 0 && thoughts.length > 0) { - this.markFinalResponseThoughts(thoughts, message); - } - return thoughts; - } - - private markFinalResponseThoughts( - thoughts: CreateThoughtSegmentInput[], - message: string, - ): void { - const maxSortOrder = Math.max(...thoughts.map((thought) => thought.sortOrder)); - for (const thought of thoughts) { - const text = thought.text.trim(); - if (text.length > 0 && this.isFinalResponseThought(thought, text, message, maxSortOrder)) { - thought.isFinalResponse = 1; - } - } - } - - private isFinalResponseThought( - thought: CreateThoughtSegmentInput, - text: string, - message: string, - maxSortOrder: number, - ): boolean { - if (text === message) return true; - return thought.sortOrder === maxSortOrder - && (thought.isFinalResponse === 1 || message.endsWith(text)); - } - - private prepareHooksForPersistence( - threadId: string, - messageId: string, - ): CreateHookExecutionInput[] { - const bufferedHooks = this.turnHooks.get(threadId) ?? []; - for (const hook of bufferedHooks) hook.id ??= NodeCrypto.randomUUID(); - return bufferedHooks.map((hook) => ({ ...hook, messageId })); - } - - /** - * Clear the per-turn narrative buffers this store owns. The sort counter and - * Agent stack are intentionally NOT cleared here — they are reset in the - * TurnStarted handler so late hooks that arrive after this point can still - * increment the completed turn's counter (mirrors the old clearTurnState). - */ - clearTurn(threadId: string): void { - this.turnToolCalls.delete(threadId); - this.turnOpenThought.delete(threadId); - this.turnThoughts.delete(threadId); - this.turnOpenHooks.delete(threadId); - this.turnHooks.delete(threadId); - } - - /** Generate a human-readable summary of tool input. */ - private summarizeInput(toolName: string, input: Record): string { - if (resolveBrowserNarrativeTool(toolName)) return JSON.stringify(input).slice(0, 4_000); - return (NARRATIVE_INPUT_SUMMARIZERS[toolName.toLowerCase()] ?? genericInputSummary)(input); - } -} - -function fileInputSummary(input: Record): string { - return String(firstProvidedInputValue(input, ["file_path", "filePath"]) ?? ""); -} - -function renameInputSummary(input: Record): string { - const source = firstProvidedInputValue(input, [ - "oldPath", "old_path", "oldFilePath", "sourcePath", "source_path", "source", "from", - ]); - const destination = firstProvidedInputValue(input, [ - "newPath", "new_path", "destinationPath", "destination_path", "destination", "to", "path", - "file_path", "filePath", "target_file", "targetFile", - ]); - return [source, destination] - .filter((value): value is string => typeof value === "string") - .join(" -> ") - .slice(0, 200); -} - -function commandInputSummary(input: Record): string { - return String(firstProvidedInputValue(input, ["command"]) ?? "").slice(0, MAX_PERSISTED_COMMAND_CHARS); -} - -function patternInputSummary(input: Record): string { - return String(firstProvidedInputValue(input, ["pattern"]) ?? ""); -} - -function agentInputSummary(input: Record): string { - return String(firstProvidedInputValue(input, ["description"]) ?? "").slice(0, 100); -} - -function genericInputSummary(input: Record): string { - return JSON.stringify(input).slice(0, 200); -} - -function firstProvidedInputValue(input: Record, keys: readonly string[]): unknown { - for (const key of keys) { - const value = input[key]; - if (value !== null && value !== undefined) return value; - } - return undefined; } diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts b/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts new file mode 100644 index 000000000..236e8fbaf --- /dev/null +++ b/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts @@ -0,0 +1,1217 @@ +import * as NodeCrypto from "node:crypto"; +import { logger } from "@mcode/shared"; +import { + createSubagentPresentation, mergeSubagentPresentation, resolveBrowserNarrativeTool, + resolveSubagentDuration, resolveSubagentMetadata, resolveSubagentPrompt, + encodeCanonicalSubagentDetailTarget, encodeSubagentAliasDetailTarget, + type ParentNarrativeRecoveryItem, type SubagentPresentation, +} from "@mcode/contracts"; +import type { CreateToolCallRecordInput } from "../../tools/persistence/tool-call-record-repo.js"; +import type { CreateThoughtSegmentInput } from "./persistence/thought-segment-repo.js"; +import type { CreateHookExecutionInput } from "../../events/persistence/hook-execution-repo.js"; +import type { TurnOutcome } from "../../turns/turn-outcome.js"; +import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../runtime/persistence/sqlite/bounded-write-batches.js"; +import { assertActiveTurnRecoveryRetention } from "../../turns/active-turn-recovery-retention-policy.js"; +import type { ExecutionIdentity } from "../../execution/execution-mailbox-protocol.js"; + +/** Buffered tool call with raw input preserved for deferred summarization. */ +export interface BufferedToolCall extends CreateToolCallRecordInput { + _rawToolInput?: Record; + _subagentPresentation?: SubagentPresentation; +} + +/** Extracts bounded provider metadata that must survive a persisted Agent card. */ +function persistedSubagentMetadata(input: Record) { + return { + subagentPrompt: resolveSubagentPrompt(input.prompt), + subagentType: resolveSubagentMetadata(input.subagentType), + subagentAgentId: resolveSubagentMetadata(input.agentId), + subagentDurationMs: resolveSubagentDuration(input.durationMs), + }; +} + +function persistedSubagentIdentityKey(presentation: SubagentPresentation | undefined): string | undefined { + if (!presentation) return undefined; + if (presentation.detail.kind === "canonical-child") { + return encodeCanonicalSubagentDetailTarget(presentation.detail.threadId); + } + return presentation.detail.kind === "canonical-alias" + ? encodeSubagentAliasDetailTarget(presentation.detail.identityKey) + : undefined; +} + +/** In-flight thought segment accumulated from consecutive textDelta events. */ +interface OpenThought { + id: string; + text: string; + startedAt: string; + sortOrder: number; +} + +/** In-flight hook execution awaiting its paired HookCompleted. */ +export interface OpenHook { + id: string; + hookName: string; + toolName: string | null; + phase: string; + payload: string; + startedAt: string; + sortOrder: number; +} + +/** One narration record staged for a durable ownership transfer. */ +export interface StagedNarrationSegment { + id: string; + text: string; + startedAt: string; + endedAt: string; + sortOrder: number; + openThoughtId?: string; +} + +/** Tool-use event shape consumed by {@link NarrativeTurnState.bufferToolCall}. */ +export interface BufferToolCallEvent { + toolCallId: string; + toolName: string; + toolInput: Record; + parentToolCallId?: string; + subagentPresentation?: SubagentPresentation; +} + +/** Complete narrative rows prepared for one terminal persistence pass. */ +export interface PreparedNarrativePersistence { + toolCalls: BufferedToolCall[]; + thoughts: CreateThoughtSegmentInput[]; + hooks: CreateHookExecutionInput[]; +} + +/** Bounds persisted shell commands while retaining enough text for readable expansion. */ +const MAX_PERSISTED_COMMAND_CHARS = 4096; + +const NARRATIVE_INPUT_SUMMARIZERS: Record) => string> = { + read: fileInputSummary, + edit: fileInputSummary, + write: fileInputSummary, + move: renameInputSummary, + rename: renameInputSummary, + bash: commandInputSummary, + shell: commandInputSummary, + terminal: commandInputSummary, + command_execution: commandInputSummary, + grep: patternInputSummary, + glob: patternInputSummary, + agent: agentInputSummary, +}; + +function fileInputSummary(input: Record): string { + return String(firstProvidedInputValue(input, ["file_path", "filePath"]) ?? ""); +} + +function renameInputSummary(input: Record): string { + const source = firstProvidedInputValue(input, [ + "oldPath", "old_path", "oldFilePath", "sourcePath", "source_path", "source", "from", + ]); + const destination = firstProvidedInputValue(input, [ + "newPath", "new_path", "destinationPath", "destination_path", "destination", "to", "path", + "file_path", "filePath", "target_file", "targetFile", + ]); + return [source, destination] + .filter((value): value is string => typeof value === "string") + .join(" -> ") + .slice(0, 200); +} + +function commandInputSummary(input: Record): string { + return String(firstProvidedInputValue(input, ["command"]) ?? "").slice(0, MAX_PERSISTED_COMMAND_CHARS); +} + +function patternInputSummary(input: Record): string { + return String(firstProvidedInputValue(input, ["pattern"]) ?? ""); +} + +function agentInputSummary(input: Record): string { + return String(firstProvidedInputValue(input, ["description"]) ?? "").slice(0, 100); +} + +function genericInputSummary(input: Record): string { + return JSON.stringify(input).slice(0, 200); +} + +function firstProvidedInputValue(input: Record, keys: readonly string[]): unknown { + for (const key of keys) { + const value = input[key]; + if (value !== null && value !== undefined) return value; + } + return undefined; +} + +/** Data-only update emitted when a buffered Agent gains its durable identity. */ +export type NarrativeTurnStateEffect = { + readonly kind: "update-subagent-identity"; + readonly toolCallId: string; + readonly messageId: string; + readonly identityKey: string; +}; + +/** Owns one identity's narrative buffers; the live store currently binds by thread. */ +export class NarrativeTurnState { + private turnToolCalls: BufferedToolCall[] = []; + private agentCallStack: string[] = []; + private turnSortCounters = 0; + private turnOpenThought: OpenThought | null = null; + private turnThoughts: CreateThoughtSegmentInput[] = []; + private turnOpenHooks = new Map(); + private turnHooks: CreateHookExecutionInput[] = []; + private readonly effectSink: ((effect: NarrativeTurnStateEffect) => void) | undefined; + private readonly pendingEffects: NarrativeTurnStateEffect[] = []; + readonly threadId: string; + readonly execution: ExecutionIdentity | undefined; + + constructor( + owner: string | ExecutionIdentity, + effectSink?: (effect: NarrativeTurnStateEffect) => void, + ) { + this.threadId = typeof owner === "string" ? owner : owner.threadId; + this.execution = typeof owner === "string" ? undefined : { ...owner }; + this.effectSink = effectSink; + } + + /** Fence a worker command to the exact turn attempt bound to this state. */ + assertExecution(execution: ExecutionIdentity): void { + const owner = this.execution; + if (!owner || owner.threadId !== execution.threadId + || owner.turnId !== execution.turnId || owner.executionId !== execution.executionId) { + throw new Error(`Narrative turn belongs to a different execution: ${execution.executionId}`); + } + } + + private assertThread(threadId: string): void { + if (threadId !== this.threadId) { + throw new Error(`Narrative turn belongs to ${this.threadId}, received ${threadId}`); + } + } + + /** Return any data-only effects emitted without a live persistence adapter. */ + takeEffects(): NarrativeTurnStateEffect[] { + return this.pendingEffects.splice(0); + } + + private emitEffect(effect: NarrativeTurnStateEffect): void { + if (this.effectSink) this.effectSink(effect); + else this.pendingEffects.push(effect); + } + + /** + * Reset the volatile per-turn buffers at the START of a turn (Trap 3). Mirrors + * the seeding AgentService used to do in its `sendMessage`/turnStarted prelude. + * Note: the sort counter and Agent stack are reset separately via + * {@link resetTurnCounters} on the TurnStarted event so late hooks from the + * prior turn can still increment the old counter. + */ + beginTurn(threadId: string): void { + this.assertThread(threadId); + this.turnToolCalls = []; + this.turnOpenThought = null; + this.turnThoughts = []; + this.turnOpenHooks = new Map(); + this.turnHooks = []; + } + + /** + * Reset the per-turn sort counter and Agent stack. Called from the + * TurnStarted handler rather than {@link beginTurn} so a fresh counter is + * available for each new turn while late hooks from the prior turn can still + * increment the old one (see {@link clearTurn}). + */ + resetTurnCounters(threadId: string): void { + this.assertThread(threadId); + this.turnSortCounters = 0; + this.agentCallStack = []; + } + + /** Allocate the next shared sort order for the thread's current turn. */ + nextSortOrder(threadId: string): number { + this.assertThread(threadId); + const sortOrder = this.turnSortCounters; + this.turnSortCounters = sortOrder + 1; + return sortOrder; + } + + /** + * Open or extend the in-flight thought segment from a non-final `textDelta`. + * The sort order is allocated lazily on the first delta so consecutive deltas + * keep the same slot, taken BEFORE any following tool call's slot — matching + * the live client builder. Never touches the `agentCallStack` (Trap 2). + */ + openOrExtendThought(threadId: string, delta: string): void { + this.assertThread(threadId); + const open = this.turnOpenThought; + if (!open) { + const sortOrder = this.nextSortOrder(threadId); + this.turnOpenThought = { + id: NodeCrypto.randomUUID(), + text: delta, + startedAt: new Date().toISOString(), + sortOrder, + }; + } else { + open.text += delta; + } + } + + /** + * Close any in-flight thought segment for the thread and push it onto the + * closed-thoughts list. Called before a tool call begins (so the thought + * sorts strictly before the tool) and during turn-end drain. + */ + closeOpenThought(threadId: string): void { + this.assertThread(threadId); + const open = this.turnOpenThought; + if (!open) return; + const list = this.turnThoughts; + list.push({ + id: open.id, + messageId: "", + text: open.text, + startedAt: open.startedAt, + endedAt: new Date().toISOString(), + sortOrder: open.sortOrder, + }); + this.turnThoughts = list; + this.turnOpenThought = null; + } + + /** + * Discards the open thought without persisting it. + * + * Called when `AssistantMessageBoundary` reports `isFinalResponse: true` — + * the streamed text was actually the final assistant response and will be + * persisted via the `Message` event, so keeping the matching thought row + * would duplicate the body as a ThoughtBlock in the narrative. + */ + dropOpenThought(threadId: string): void { + this.assertThread(threadId); + this.turnOpenThought = null; + } + + /** + * Move the open thought text out of NarrativeStore without persisting it. + * + * Used when an authoritative boundary retroactively classifies previously + * unknown deltas as the final assistant response. Ownership moves to + * TurnFinalizer so the same text is not buffered in both stores. + */ + takeOpenThought(threadId: string): string { + this.assertThread(threadId); + const open = this.turnOpenThought; + this.turnOpenThought = null; + return open?.text ?? ""; + } + + /** + * Get the current parent tool call ID for a thread's active Agent nesting. + * This is the fallback consulted by `index.ts` enrichment and + * {@link bufferToolCall} when the SDK omits `parent_tool_use_id` (Trap 1). + */ + getCurrentParentToolCallId(threadId: string): string | undefined { + this.assertThread(threadId); + return this.getStackDerivedParentFallback(threadId); + } + + /** + * A single running Agent on the stack (buffer `status === "running"`) can + * serve as a parent fallback when the SDK omits `parent_tool_use_id`. + * Zero or multiple running Agents means the fallback is ambiguous (parallel + * dispatch, nested agents, or coordinator work after children); return + * undefined so tools do not attach under the wrong subagent row. + */ + private getStackDerivedParentFallback(threadId: string): string | undefined { + this.assertThread(threadId); + const stack = this.agentCallStack; + if (stack.length === 0) return undefined; + + const buffer = this.turnToolCalls; + const runningAgentIds: string[] = []; + for (const agentId of stack) { + const row = buffer.find( + (b) => b.toolCallId === agentId && b.toolName === "Agent", + ); + if (row?.status === "running") { + runningAgentIds.push(agentId); + } + } + + return runningAgentIds.length === 1 ? runningAgentIds[0] : undefined; + } + + /** + * Buffer a tool call event for later persistence and return the parent tool + * call ID attributed to it. An explicit provider parent always wins. The + * stack fallback applies only to non-Agent calls when exactly one Agent is + * still running (Trap 1). Agent calls never inherit a stack-derived parent + * and are pushed onto the `agentCallStack` (Trap 2 push site). + */ + bufferToolCall(threadId: string, event: BufferToolCallEvent): string | undefined { + this.assertThread(threadId); + const buffer = this.turnToolCalls; + const stack = this.agentCallStack; + const parentToolCallId = this.resolveParentToolCallId(threadId, event); + this.logParentToolCallAttribution(threadId, event, parentToolCallId, stack.length); + const existing = buffer.find((tc) => tc.toolCallId === event.toolCallId); + if (existing) { + return this.updateExistingBufferedToolCall(existing, event, parentToolCallId); + } + return this.addBufferedToolCall( + threadId, + buffer, + stack, + event, + parentToolCallId, + ); + } + + private resolveParentToolCallId( + threadId: string, + event: BufferToolCallEvent, + ): string | undefined { + this.assertThread(threadId); + if (event.toolName === "Agent") return event.parentToolCallId; + return event.parentToolCallId ?? this.getStackDerivedParentFallback(threadId); + } + + private logParentToolCallAttribution( + threadId: string, + event: BufferToolCallEvent, + parentToolCallId: string | undefined, + stackDepth: number, + ): void { + this.assertThread(threadId); + if (event.toolName === "Agent" || !parentToolCallId) return; + logger.debug("bufferToolCall: parent attribution", { + threadId, + toolCallId: event.toolCallId, + toolName: event.toolName, + sdkParent: event.parentToolCallId ?? null, + stackDepth, + attributed: parentToolCallId, + source: event.parentToolCallId ? "sdk" : "stack-fallback", + }); + } + + private updateExistingBufferedToolCall( + existing: BufferedToolCall, + event: BufferToolCallEvent, + parentToolCallId: string | undefined, + ): string | undefined { + const shouldMergeDuplicate = this.shouldMergeDuplicateToolCall(existing, event); + if (shouldMergeDuplicate) { + this.mergeDuplicateToolCall(existing, event); + } + this.mergeParentToolCallId( + existing, + event.parentToolCallId, + parentToolCallId, + shouldMergeDuplicate, + ); + return existing.parentToolCallId; + } + + private shouldMergeDuplicateToolCall( + existing: BufferedToolCall, + event: BufferToolCallEvent, + ): boolean { + if (existing.toolName === "Agent" && event.toolName === "Agent") return true; + const rawToolInput = existing._rawToolInput ?? {}; + return existing.status === "running" && ( + Object.keys(rawToolInput).length === 0 || existing.toolName !== event.toolName + ); + } + + private mergeDuplicateToolCall(existing: BufferedToolCall, event: BufferToolCallEvent): void { + existing.toolName = event.toolName; + const mergedToolInput = { + ...existing._rawToolInput, + ...event.toolInput, + }; + existing._rawToolInput = mergedToolInput; + if (event.toolName === "Agent") { + existing._subagentPresentation = mergeSubagentPresentation( + existing._subagentPresentation, + event.subagentPresentation + ?? createSubagentPresentation(mergedToolInput, event.toolCallId), + event.toolCallId, + ); + this.applyAgentPresentation(existing, mergedToolInput, event.toolCallId); + } + } + + private mergeParentToolCallId( + existing: BufferedToolCall, + providerParentToolCallId: string | undefined, + inferredParentToolCallId: string | undefined, + mergedDuplicate: boolean, + ): void { + if (providerParentToolCallId) { + existing.parentToolCallId = providerParentToolCallId; + return; + } + if (mergedDuplicate && inferredParentToolCallId && !existing.parentToolCallId) { + existing.parentToolCallId = inferredParentToolCallId; + } + } + + private applyAgentPresentation( + toolCall: BufferedToolCall, + rawToolInput: Record, + toolCallId: string, + ): void { + const presentation = toolCall._subagentPresentation + ?? createSubagentPresentation(rawToolInput, toolCallId); + toolCall.displayName = presentation.hasExplicitIdentity ? presentation.displayName : undefined; + toolCall.providerAgentKey = presentation.providerAgentKey; + toolCall.subagentIdentityKey = persistedSubagentIdentityKey(presentation) ?? toolCall.subagentIdentityKey; + toolCall.subagentProviderName = this.subagentProviderName(presentation); + Object.assign(toolCall, persistedSubagentMetadata(rawToolInput)); + if (toolCall.messageId && toolCall.subagentIdentityKey) { + this.emitEffect({ + kind: "update-subagent-identity", + toolCallId: toolCall.toolCallId!, + messageId: toolCall.messageId, + identityKey: toolCall.subagentIdentityKey, + }); + } + toolCall.model = presentation.model; + toolCall.reasoningEffort = presentation.reasoningEffort; + } + + private addBufferedToolCall( + threadId: string, + buffer: BufferedToolCall[], + stack: string[], + event: BufferToolCallEvent, + parentToolCallId: string | undefined, + ): string | undefined { + this.assertThread(threadId); + const sortOrder = this.nextSortOrder(threadId); + if (event.toolName === "Agent") { + stack.push(event.toolCallId); + this.agentCallStack = stack; + } + const presentation = this.subagentPresentation(event); + buffer.push(this.createBufferedToolCall(event, presentation, sortOrder, parentToolCallId)); + this.turnToolCalls = buffer; + return parentToolCallId; + } + + private subagentPresentation(event: BufferToolCallEvent): SubagentPresentation | undefined { + if (event.toolName !== "Agent") return undefined; + return event.subagentPresentation ?? createSubagentPresentation(event.toolInput, event.toolCallId); + } + + private createBufferedToolCall( + event: BufferToolCallEvent, + presentation: SubagentPresentation | undefined, + sortOrder: number, + parentToolCallId: string | undefined, + ): BufferedToolCall { + const isAgent = event.toolName === "Agent"; + return { + toolCallId: event.toolCallId, + messageId: "", + toolName: event.toolName, + displayName: presentation?.hasExplicitIdentity ? presentation.displayName : undefined, + providerAgentKey: presentation?.providerAgentKey, + subagentIdentityKey: isAgent ? persistedSubagentIdentityKey(presentation) : undefined, + subagentProviderName: this.subagentProviderName(presentation), + ...(isAgent ? persistedSubagentMetadata(event.toolInput) : {}), + model: presentation?.model, + reasoningEffort: presentation?.reasoningEffort, + inputSummary: "", + outputSummary: "", + status: "running", + startedAt: new Date().toISOString(), + sortOrder, + parentToolCallId, + _rawToolInput: event.toolInput, + ...(presentation ? { _subagentPresentation: presentation } : {}), + }; + } + + private subagentProviderName(presentation: SubagentPresentation | undefined): string | undefined { + if (presentation?.detail.kind !== "transcript-unavailable") return undefined; + return presentation.detail.providerName; + } + + /** + * Update a buffered tool call with its output when its result arrives, and + * pop the call from the `agentCallStack` if it was an Agent (Trap 2 pop site). + */ + updateBufferedToolCallOutput( + threadId: string, + toolCallId: string, + output: string, + isError: boolean, + toolInput?: Record, + outputMeta?: { + outputTruncated?: boolean; + outputTotalBytes?: number; + outputArtifactPath?: string; + exitCode?: number; + }, + subagentPresentation?: SubagentPresentation, + ): void { + this.assertThread(threadId); + this.removeAgentFromStack(threadId, toolCallId); + const toolCall = this.latestBufferedToolCall(threadId, toolCallId); + if (!toolCall) return; + this.applyToolCallOutput(toolCall, output, isError, outputMeta); + this.mergeToolCallInput(toolCall, toolInput); + if (toolCall.toolName === "Agent") { + const incomingPresentation = subagentPresentation + ?? createSubagentPresentation(toolCall._rawToolInput ?? {}, toolCallId); + toolCall._subagentPresentation = mergeSubagentPresentation( + toolCall._subagentPresentation, + incomingPresentation, + toolCallId, + ); + this.applyAgentPresentation(toolCall, toolCall._rawToolInput ?? {}, toolCallId); + } + } + + private removeAgentFromStack(threadId: string, toolCallId: string): void { + this.assertThread(threadId); + const stack = this.agentCallStack; + const stackIndex = stack.indexOf(toolCallId); + if (stackIndex < 0) return; + stack.splice(stackIndex, 1); + this.agentCallStack = stack; + logger.debug("updateBufferedToolCallOutput: popped Agent from stack", { + threadId, + toolCallId, + remainingDepth: stack.length, + }); + } + + private latestBufferedToolCall(threadId: string, toolCallId: string): BufferedToolCall | undefined { + this.assertThread(threadId); + const buffer = this.turnToolCalls; + for (let index = buffer.length - 1; index >= 0; index -= 1) { + if (buffer[index].toolCallId === toolCallId) return buffer[index]; + } + return undefined; + } + + private applyToolCallOutput( + toolCall: BufferedToolCall, + output: string, + isError: boolean, + outputMeta: { + outputTruncated?: boolean; + outputTotalBytes?: number; + outputArtifactPath?: string; + exitCode?: number; + } | undefined, + ): void { + const outputLimit = resolveBrowserNarrativeTool(toolCall.toolName) ? 4_000 : 500; + toolCall.outputSummary = output.slice(0, outputLimit); + toolCall.outputTruncated = outputMeta?.outputTruncated === true; + this.applyOutputMetadata(toolCall, outputMeta); + toolCall.status = isError ? "failed" : "completed"; + toolCall.completedAt = new Date().toISOString(); + } + + private applyOutputMetadata( + toolCall: BufferedToolCall, + outputMeta: { + outputTotalBytes?: number; + outputArtifactPath?: string; + exitCode?: number; + } | undefined, + ): void { + delete toolCall.outputTotalBytes; + delete toolCall.outputArtifactPath; + delete toolCall.exitCode; + if (outputMeta?.outputTotalBytes != null) toolCall.outputTotalBytes = outputMeta.outputTotalBytes; + if (outputMeta?.outputArtifactPath) toolCall.outputArtifactPath = outputMeta.outputArtifactPath; + if (outputMeta?.exitCode !== undefined) toolCall.exitCode = outputMeta.exitCode; + } + + private mergeToolCallInput(toolCall: BufferedToolCall, toolInput: Record | undefined): void { + if (!toolInput || Object.keys(toolInput).length === 0) return; + toolCall._rawToolInput = { + ...toolCall._rawToolInput, + ...toolInput, + }; + } + + /** + * Clear the whole Agent stack when a final `Message` event arrives — the turn + * is over and any Agent calls still on the stack are implicitly done (Trap 2 + * end-of-turn clear). No-ops when the stack is already empty. + */ + clearAgentStackOnMessage(threadId: string): void { + this.assertThread(threadId); + const stack = this.agentCallStack; + if (stack.length > 0) { + stack.length = 0; + } + } + + /** Snapshot of the thread's buffered tool calls (read-only inspection). */ + getBufferedToolCalls(threadId: string): readonly BufferedToolCall[] { + this.assertThread(threadId); + return this.turnToolCalls; + } + + /** Project only the tool changed by a result; other narrative records remain unchanged. */ + toolRecoveryItem(threadId: string, toolCallId: string): Extract | null { + const toolCall = this.latestBufferedToolCall(threadId, toolCallId); + return toolCall ? this.toolCallRecoveryItem(toolCall) : null; + } + + /** + * Snapshot the visible structured narrative for an unfinished turn without + * retaining provider protocol traffic or private raw tool input. + */ + recoverySnapshot(threadId: string): ParentNarrativeRecoveryItem[] { + this.assertThread(threadId); + const snapshot: ParentNarrativeRecoveryItem[] = []; + let bytes = 0; + bytes = this.appendBufferedToolCallRecoveryItems(snapshot, bytes, threadId); + for (const thought of this.turnThoughts) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.thoughtRecoveryItem(thought), + threadId, + ); + } + const openThought = this.turnOpenThought; + if (openThought) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.openThoughtRecoveryItem(openThought), + threadId, + ); + } + for (const hook of this.turnHooks) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.hookRecoveryItem(hook), + threadId, + ); + } + for (const hook of this.turnOpenHooks.values()) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.openHookRecoveryItem(hook), + threadId, + ); + } + return this.sortRecoverySnapshot(snapshot); + } + + private appendBufferedToolCallRecoveryItems( + snapshot: ParentNarrativeRecoveryItem[], + bytes: number, + threadId: string, + ): number { + for (const toolCall of this.turnToolCalls) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.toolCallRecoveryItem(toolCall), + threadId, + ); + } + return bytes; + } + + private appendRecoverySnapshotItem( + snapshot: ParentNarrativeRecoveryItem[], + bytes: number, + item: ParentNarrativeRecoveryItem, + threadId: string, + ): number { + const nextBytes = Buffer.byteLength(JSON.stringify(item), "utf8"); + this.assertRecoveryItemFitsWriteBatch(nextBytes, threadId); + const retainedBytes = bytes + nextBytes; + assertActiveTurnRecoveryRetention(snapshot.length + 1, retainedBytes); + snapshot.push(item); + return retainedBytes; + } + + private toolCallRecoveryItem(toolCall: BufferedToolCall): Extract { + return { + kind: "toolCall", + record: { + ...this.toolCallRecoveryIdentityFields(toolCall), + ...this.toolCallRecoveryPresentationFields(toolCall), + ...this.toolCallRecoveryOutputFields(toolCall), + ...this.toolCallRecoveryStateFields(toolCall), + }, + }; + } + + private toolCallRecoveryIdentityFields(toolCall: BufferedToolCall) { + return { + id: this.requireRecoveryString(toolCall.toolCallId, "tool call id"), + message_id: this.requireRecoveryString(toolCall.messageId, "tool call message id"), + parent_tool_call_id: toolCall.parentToolCallId ?? null, + tool_name: toolCall.toolName, + display_name: toolCall.displayName ?? null, + provider_agent_key: toolCall.providerAgentKey ?? null, + subagent_identity_key: toolCall.subagentIdentityKey ?? null, + subagent_provider_name: toolCall.subagentProviderName ?? null, + }; + } + + private toolCallRecoveryPresentationFields(toolCall: BufferedToolCall) { + return { + subagent_prompt: toolCall.subagentPrompt ?? null, + subagent_type: toolCall.subagentType ?? null, + subagent_agent_id: toolCall.subagentAgentId ?? null, + subagent_duration_ms: toolCall.subagentDurationMs ?? null, + model: toolCall.model ?? null, + reasoning_effort: toolCall.reasoningEffort ?? null, + }; + } + + private toolCallRecoveryOutputFields(toolCall: BufferedToolCall) { + return { + input_summary: this.toolCallRecoveryInputSummary(toolCall), + output_summary: toolCall.outputSummary, + ...(toolCall.outputTruncated ? { output_truncated: 1 } : {}), + output_total_bytes: toolCall.outputTotalBytes ?? null, + output_artifact_path: toolCall.outputArtifactPath ?? null, + exit_code: toolCall.exitCode ?? null, + }; + } + + private toolCallRecoveryInputSummary(toolCall: BufferedToolCall): string { + if (toolCall.inputSummary) return toolCall.inputSummary; + return this.summarizeInput(toolCall.toolName, toolCall._rawToolInput ?? {}); + } + + private toolCallRecoveryStateFields(toolCall: BufferedToolCall) { + return { + status: toolCall.status, + started_at: this.requireRecoveryString(toolCall.startedAt, "tool call start time"), + completed_at: toolCall.completedAt ?? null, + sort_order: toolCall.sortOrder, + }; + } + + private thoughtRecoveryItem(thought: CreateThoughtSegmentInput): ParentNarrativeRecoveryItem { + return { + kind: "narrationSegment", + record: { + id: this.requireRecoveryString(thought.id, "thought id"), + message_id: this.requireRecoveryString(thought.messageId, "thought message id"), + text: this.requireRecoveryString(thought.text, "thought text"), + started_at: this.requireRecoveryString(thought.startedAt, "thought start time"), + ended_at: thought.endedAt ?? null, + sort_order: thought.sortOrder, + ...(thought.isFinalResponse ? { is_final_response: thought.isFinalResponse } : {}), + }, + }; + } + + private openThoughtRecoveryItem(openThought: OpenThought): ParentNarrativeRecoveryItem { + return { + kind: "narrationSegment", + record: { + id: openThought.id, + message_id: "", + text: openThought.text, + started_at: openThought.startedAt, + ended_at: null, + sort_order: openThought.sortOrder, + }, + }; + } + + private hookRecoveryItem(hook: CreateHookExecutionInput): ParentNarrativeRecoveryItem { + return { + kind: "hook", + record: { + id: this.requireRecoveryString(hook.id, "hook id"), + message_id: this.requireRecoveryString(hook.messageId, "hook message id"), + hook_name: hook.hookName, + tool_name: hook.toolName, + phase: hook.phase, + payload: hook.payload, + duration_ms: hook.durationMs ?? null, + did_block: hook.didBlock, + started_at: this.requireRecoveryString(hook.startedAt, "hook start time"), + ended_at: hook.endedAt ?? null, + sort_order: hook.sortOrder, + }, + }; + } + + private openHookRecoveryItem(hook: OpenHook): ParentNarrativeRecoveryItem { + return { + kind: "hook", + record: { + id: hook.id, + message_id: "", + hook_name: hook.hookName, + tool_name: hook.toolName, + phase: hook.phase, + payload: hook.payload, + duration_ms: null, + did_block: false, + started_at: hook.startedAt, + ended_at: null, + sort_order: hook.sortOrder, + }, + }; + } + + private sortRecoverySnapshot( + snapshot: ParentNarrativeRecoveryItem[], + ): ParentNarrativeRecoveryItem[] { + return snapshot.sort((left, right) => ( + left.record.sort_order - right.record.sort_order || left.record.id.localeCompare(right.record.id) + )); + } + + /** Stage narration for recovery without mutating the active turn buffer. */ + stageNarrationSegment(threadId: string, text: string): StagedNarrationSegment | null { + this.assertThread(threadId); + const open = this.turnOpenThought; + const endedAt = new Date().toISOString(); + if (open) { + return { + id: open.id, + text: `${open.text}${text}`, + startedAt: open.startedAt, + endedAt, + sortOrder: open.sortOrder, + openThoughtId: open.id, + }; + } + if (!text) return null; + return { + id: NodeCrypto.randomUUID(), + text, + startedAt: endedAt, + endedAt, + sortOrder: this.turnSortCounters, + }; + } + + /** Add staged narration to a recovery snapshot without mutating the active turn buffer. */ + recoverySnapshotWithStagedNarration( + threadId: string, + staged: StagedNarrationSegment, + ): ParentNarrativeRecoveryItem[] { + this.assertThread(threadId); + const snapshot = this.recoverySnapshot(threadId).filter((item) => ( + item.kind !== "narrationSegment" || item.record.id !== staged.id + )); + snapshot.push({ + kind: "narrationSegment", + record: { + id: staged.id, + message_id: "", + text: staged.text, + started_at: staged.startedAt, + ended_at: staged.endedAt, + sort_order: staged.sortOrder, + }, + }); + let bytes = 0; + for (const [index, item] of snapshot.entries()) { + const itemBytes = Buffer.byteLength(JSON.stringify(item), "utf8"); + this.assertRecoveryItemFitsWriteBatch(itemBytes, threadId); + bytes += itemBytes; + assertActiveTurnRecoveryRetention(index + 1, bytes); + } + return snapshot.sort((left, right) => ( + left.record.sort_order - right.record.sort_order || left.record.id.localeCompare(right.record.id) + )); + } + + /** Apply a narration record only after its recovery projection is durable. */ + applyStagedNarrationSegment(threadId: string, staged: StagedNarrationSegment): void { + this.assertThread(threadId); + if (staged.openThoughtId) { + const open = this.turnOpenThought; + if (!open || open.id !== staged.openThoughtId) { + throw new Error(`Staged narration no longer matches the open thought: ${staged.id}`); + } + this.turnOpenThought = null; + } else { + const nextSortOrder = this.turnSortCounters; + if (nextSortOrder <= staged.sortOrder) { + this.turnSortCounters = staged.sortOrder + 1; + } + } + const thoughts = this.turnThoughts; + thoughts.push({ + id: staged.id, + messageId: "", + text: staged.text, + startedAt: staged.startedAt, + endedAt: staged.endedAt, + sortOrder: staged.sortOrder, + }); + this.turnThoughts = thoughts; + } + + private requireRecoveryString(value: string | undefined, field: string): string { + if (value === undefined) throw new Error(`Narrative recovery is missing ${field}`); + return value; + } + + private assertRecoveryItemFitsWriteBatch(byteLength: number, threadId: string): void { + if (byteLength > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes) { + throw new Error(`Parent narrative recovery item exceeds the active-turn byte limit: ${threadId}`); + } + } + + /** + * True when the current turn has buffered at least one narrative contributor — + * a tool call, a narration segment (open or closed), or a hook (open or + * closed). Feeds the {@link TurnFinalizer.hasRecordableActivity} predicate so + * a turn with narrative but no assistant body still earns a persisted row. + */ + hasBufferedNarrative(threadId: string): boolean { + this.assertThread(threadId); + return this.narrativeBufferStates(threadId).some(Boolean); + } + + private narrativeBufferStates(threadId: string): boolean[] { + this.assertThread(threadId); + return [ + this.turnToolCalls.length > 0, + Boolean(this.turnOpenThought), + this.turnThoughts.length > 0, + this.turnOpenHooks.size > 0, + this.turnHooks.length > 0, + ]; + } + + /** + * Record an in-flight hook execution (HookStarted). The caller supplies the + * already-allocated sort order (the late-hook path in AgentService allocates + * it before deciding routing). Returns the generated row id. + */ + openHook( + threadId: string, + hook: { hookName: string; toolName: string | null; phase: string; payload: string; sortOrder: number }, + ): string { + this.assertThread(threadId); + const map = this.turnOpenHooks; + const id = NodeCrypto.randomUUID(); + map.set(hook.hookName, { + id, + hookName: hook.hookName, + toolName: hook.toolName, + phase: hook.phase, + payload: hook.payload, + startedAt: new Date().toISOString(), + sortOrder: hook.sortOrder, + }); + this.turnOpenHooks = map; + return id; + } + + /** Look up (without removing) an open hook by name. */ + peekOpenHook(threadId: string, hookName: string): OpenHook | undefined { + this.assertThread(threadId); + return this.turnOpenHooks.get(hookName); + } + + /** Remove an open hook by name (after it has been completed or flushed). */ + removeOpenHook(threadId: string, hookName: string): void { + this.assertThread(threadId); + this.turnOpenHooks.delete(hookName); + } + + /** Push a completed hook execution onto the closed-hooks list for persistence. */ + pushClosedHook(threadId: string, hook: CreateHookExecutionInput): void { + this.assertThread(threadId); + const list = this.turnHooks; + list.push(hook); + this.turnHooks = list; + } + + /** Settle one turn's buffered records into data-only persistence rows. */ + prepareNarrativePersistence( + threadId: string, + messageId: string, + messageContent: string, + outcome: TurnOutcome, + ): PreparedNarrativePersistence { + this.assertThread(threadId); + const toolCalls = this.prepareToolCallsForPersistence(threadId, messageId, outcome); + this.closeOpenThought(threadId); + this.closeOpenHooksForPersistence(threadId); + const thoughts = this.prepareThoughtsForPersistence(threadId, messageId, messageContent); + const hooks = this.prepareHooksForPersistence(threadId, messageId); + return { toolCalls, thoughts, hooks }; + } + + private prepareToolCallsForPersistence( + threadId: string, + messageId: string, + outcome: TurnOutcome, + ): BufferedToolCall[] { + this.assertThread(threadId); + const toolCalls = this.turnToolCalls; + const settledAt = new Date().toISOString(); + for (const toolCall of toolCalls) { + toolCall.toolCallId ??= NodeCrypto.randomUUID(); + this.settleRunningToolCall(toolCall, outcome, settledAt); + toolCall.messageId = messageId; + this.prepareToolCallInput(toolCall); + } + return toolCalls; + } + + private settleRunningToolCall( + toolCall: BufferedToolCall, + outcome: TurnOutcome, + settledAt: string, + ): void { + if (toolCall.status !== "running") return; + toolCall.status = this.settledToolCallStatus(outcome); + toolCall.completedAt = settledAt; + } + + private settledToolCallStatus(outcome: TurnOutcome): BufferedToolCall["status"] { + if (outcome === "errored" || outcome === "interrupted") return "failed"; + if (outcome === "cancelled") return "cancelled"; + return "completed"; + } + + private prepareToolCallInput(toolCall: BufferedToolCall): void { + const rawToolInput = toolCall._rawToolInput; + if (toolCall.inputSummary || !rawToolInput) return; + if (toolCall.toolName === "Agent") { + this.applyAgentPersistenceMetadata(toolCall, rawToolInput); + } + toolCall.inputSummary = this.summarizeInput(toolCall.toolName, rawToolInput); + delete toolCall._rawToolInput; + } + + private applyAgentPersistenceMetadata( + toolCall: BufferedToolCall, + rawToolInput: Record, + ): void { + const presentation = createSubagentPresentation(rawToolInput, toolCall.toolCallId!); + const persistedPresentation = toolCall._subagentPresentation ?? presentation; + toolCall.displayName = persistedPresentation.hasExplicitIdentity + ? persistedPresentation.displayName + : undefined; + toolCall.providerAgentKey = persistedPresentation.providerAgentKey; + toolCall.subagentIdentityKey = persistedSubagentIdentityKey(persistedPresentation) + ?? toolCall.subagentIdentityKey; + toolCall.subagentProviderName = this.subagentProviderName(persistedPresentation); + Object.assign(toolCall, persistedSubagentMetadata(rawToolInput)); + toolCall.model = persistedPresentation.model; + toolCall.reasoningEffort = persistedPresentation.reasoningEffort; + } + + private closeOpenHooksForPersistence(threadId: string): void { + this.assertThread(threadId); + const openHookMap = this.turnOpenHooks; + if (!openHookMap || openHookMap.size === 0) return; + const list = this.turnHooks; + const endedAt = new Date().toISOString(); + for (const open of openHookMap.values()) { + list.push({ + id: open.id, + messageId: "", + hookName: open.hookName, + toolName: open.toolName, + phase: open.phase, + payload: open.payload, + durationMs: Date.parse(endedAt) - Date.parse(open.startedAt), + didBlock: false, + startedAt: open.startedAt, + endedAt, + sortOrder: open.sortOrder, + }); + } + this.turnHooks = list; + openHookMap.clear(); + } + + private prepareThoughtsForPersistence( + threadId: string, + messageId: string, + messageContent: string, + ): CreateThoughtSegmentInput[] { + this.assertThread(threadId); + const bufferedThoughts = this.turnThoughts; + for (const thought of bufferedThoughts) thought.id ??= NodeCrypto.randomUUID(); + const thoughts = bufferedThoughts.map((thought) => ({ ...thought, messageId })); + const message = messageContent.trim(); + if (message.length > 0 && thoughts.length > 0) { + this.markFinalResponseThoughts(thoughts, message); + } + return thoughts; + } + + private markFinalResponseThoughts( + thoughts: CreateThoughtSegmentInput[], + message: string, + ): void { + const maxSortOrder = Math.max(...thoughts.map((thought) => thought.sortOrder)); + for (const thought of thoughts) { + const text = thought.text.trim(); + if (text.length > 0 && this.isFinalResponseThought(thought, text, message, maxSortOrder)) { + thought.isFinalResponse = 1; + } + } + } + + private isFinalResponseThought( + thought: CreateThoughtSegmentInput, + text: string, + message: string, + maxSortOrder: number, + ): boolean { + if (text === message) return true; + return thought.sortOrder === maxSortOrder + && (thought.isFinalResponse === 1 || message.endsWith(text)); + } + + private prepareHooksForPersistence( + threadId: string, + messageId: string, + ): CreateHookExecutionInput[] { + this.assertThread(threadId); + const bufferedHooks = this.turnHooks; + for (const hook of bufferedHooks) hook.id ??= NodeCrypto.randomUUID(); + return bufferedHooks.map((hook) => ({ ...hook, messageId })); + } + + /** + * Clear the per-turn narrative buffers this store owns. The sort counter and + * Agent stack are intentionally NOT cleared here — they are reset in the + * TurnStarted handler so late hooks that arrive after this point can still + * increment the completed turn's counter (mirrors the old clearTurnState). + */ + clearTurn(threadId: string): void { + this.assertThread(threadId); + this.turnToolCalls = []; + this.turnOpenThought = null; + this.turnThoughts = []; + this.turnOpenHooks = new Map(); + this.turnHooks = []; + } + + /** Generate a human-readable summary of tool input. */ + private summarizeInput(toolName: string, input: Record): string { + if (resolveBrowserNarrativeTool(toolName)) return JSON.stringify(input).slice(0, 4_000); + return (NARRATIVE_INPUT_SUMMARIZERS[toolName.toLowerCase()] ?? genericInputSummary)(input); + } +} diff --git a/apps/server/src/features/agents/conversation/narrative/persistence/thought-segment-repo.ts b/apps/server/src/features/agents/conversation/narrative/persistence/thought-segment-repo.ts index 040fbc195..2b95ee014 100644 --- a/apps/server/src/features/agents/conversation/narrative/persistence/thought-segment-repo.ts +++ b/apps/server/src/features/agents/conversation/narrative/persistence/thought-segment-repo.ts @@ -83,22 +83,22 @@ export class ThoughtSegmentRepo { } /** Insert multiple thought segment records in a single transaction. */ - bulkCreate(inputs: CreateThoughtSegmentInput[]): void { + bulkCreate(inputs: CreateThoughtSegmentInput[], replaceExisting = false): void { if (inputs.length === 0) return; this.orm.transaction((tx) => { for (const item of inputs) { - tx.insert(thoughtSegments) - .values({ - id: item.id ?? NodeCrypto.randomUUID(), - messageId: item.messageId, - text: item.text, - startedAt: item.startedAt, - endedAt: item.endedAt, - sortOrder: item.sortOrder, - isFinalResponse: item.isFinalResponse ?? 0, - }) - .onConflictDoNothing() - .run(); + const { id, ...rest } = { + id: item.id ?? NodeCrypto.randomUUID(), + messageId: item.messageId, + text: item.text, + startedAt: item.startedAt, + endedAt: item.endedAt, + sortOrder: item.sortOrder, + isFinalResponse: item.isFinalResponse ?? 0, + }; + const write = tx.insert(thoughtSegments).values({ id, ...rest }); + if (replaceExisting) write.onConflictDoUpdate({ target: thoughtSegments.id, set: rest }).run(); + else write.onConflictDoNothing().run(); } }); } diff --git a/apps/server/src/features/agents/conversation/persistence/__tests__/message-repo.test.ts b/apps/server/src/features/agents/conversation/persistence/__tests__/message-repo.test.ts index a1ecb3905..02693a3c7 100644 --- a/apps/server/src/features/agents/conversation/persistence/__tests__/message-repo.test.ts +++ b/apps/server/src/features/agents/conversation/persistence/__tests__/message-repo.test.ts @@ -164,6 +164,25 @@ describe("MessageRepo", () => { }); describe("listByThread", () => { + it("does not let hidden provider notices displace the visible conversation tail", () => { + const user = repo.create("thread-1", "user", "Stop this turn", 1); + repo.createSystemNotice("thread-1", "Provider warning", 2, { + kind: "warning", presentation: "timeline", scope: "turn", sessionId: "session-1", noticeKey: "warning-1", + }); + repo.createSystemNotice("thread-1", "Deprecated setting", 3, { + kind: "deprecation", presentation: "timeline", scope: "turn", sessionId: "session-1", noticeKey: "deprecation-1", + }); + + expect(repo.listByThread("thread-1", 2)).toMatchObject({ + messages: [{ id: user.id }], + hasMore: false, + }); + expect(repo.listByThreadAfter("thread-1", 2, 0)).toMatchObject({ + messages: [{ id: user.id }], + hasMore: false, + }); + }); + it("returns stored canonical legacy provenance without replacing it with the ambiguous fallback", () => { db.prepare(` INSERT INTO messages (id, thread_id, role, content, timestamp, sequence, origin_type, legacy_provenance) diff --git a/apps/server/src/features/agents/conversation/persistence/message-repo.ts b/apps/server/src/features/agents/conversation/persistence/message-repo.ts index e779949f8..7031635d6 100644 --- a/apps/server/src/features/agents/conversation/persistence/message-repo.ts +++ b/apps/server/src/features/agents/conversation/persistence/message-repo.ts @@ -181,6 +181,8 @@ function rowToMessage(row: MessageRow): Message { /** Session-scoped system notices are hidden from transcript reads. */ const notSessionNotice = sql`json_extract(${messages.systemNotice}, '$.scope') IS NOT 'session'`; +// The transcript hides these notices, so they cannot consume a bounded display page. +const notRoutineTranscriptNotice = sql`COALESCE(json_extract(${messages.systemNotice}, '$.kind'), '') NOT IN ('warning', 'configuration', 'deprecation', 'authentication-recovered')`; const DEFAULT_HISTORY_PAGE_SIZE = 100; const MAX_HISTORY_PAGE_SIZE = 500; @@ -593,6 +595,14 @@ export class MessageRepo { return row?.sequence ?? 0; } + /** Provider system notices may follow a user prompt without starting another conversation turn. */ + isLatestNonSystemMessage(threadId: string, messageId: string): boolean { + const latest = this.orm.select({ id: messages.id }).from(messages) + .where(and(eq(messages.threadId, threadId), inArray(messages.role, ["user", "assistant"]))) + .orderBy(desc(messages.sequence)).limit(1).get(); + return latest?.id === messageId; + } + /** Append stored attachments to an existing message, deduping by attachment id. */ appendAttachments(messageId: string, attachments: StoredAttachment[]): StoredAttachment[] { if (attachments.length === 0) return []; @@ -643,6 +653,7 @@ export class MessageRepo { ...(before != null ? [lt(messages.sequence, before)] : []), eq(messages.isInternal, 0), notSessionNotice, + notRoutineTranscriptNotice, ), "DESC", fetchLimit, @@ -700,6 +711,7 @@ export class MessageRepo { gt(messages.sequence, after), eq(messages.isInternal, 0), notSessionNotice, + notRoutineTranscriptNotice, ), "ASC", fetchLimit, @@ -1080,6 +1092,14 @@ export class MessageRepo { return row ? rowToMessage(row) : null; } + /** Find one private or published message for writer-owned turn materialization. */ + findByIdInThreadIncludingInternal(threadId: string, messageId: string): Message | null { + const row = this.orm.select().from(messages) + .where(and(eq(messages.id, messageId), eq(messages.threadId, threadId))) + .get(); + return row ? rowToMessage(row) : null; + } + /** Look up a single message by its primary key. */ findById(id: string): Message | undefined { const row = this.orm diff --git a/apps/server/src/features/agents/conversation/read-model/__tests__/conversation-page.test.ts b/apps/server/src/features/agents/conversation/read-model/__tests__/conversation-page.test.ts index d03209c9f..6c077eaab 100644 --- a/apps/server/src/features/agents/conversation/read-model/__tests__/conversation-page.test.ts +++ b/apps/server/src/features/agents/conversation/read-model/__tests__/conversation-page.test.ts @@ -55,7 +55,7 @@ function createDeps(db: Database) { } describe("loadConversationPage", () => { - it("replays current-session notices beyond the bounded tail and retains old turn notices in history", () => { + it("replays current-session notices beyond the bounded tail without spending message slots on hidden notices", () => { const db = openMemoryDatabase(); seedThread(db); const deps = createDeps(db); @@ -95,7 +95,7 @@ describe("loadConversationPage", () => { const tail = loadConversationTail(deps, { threadId: "thread-1", limit: 2 }); expect(page.messages.map((message) => message.content)).toEqual([ - "Old security notice", "Late old-session warning", "Security notice", "Warning notice", "Model rerouted", "Tail message one", "Tail message two", + "Old security notice", "Security notice", "Model rerouted", "Tail message one", "Tail message two", ]); expect(tail.messages.map((message) => message.content)).toEqual(["Tail message one", "Tail message two"]); expect(page.sessionNotices.map((message) => message.content)).toEqual([ @@ -110,8 +110,10 @@ describe("loadConversationPage", () => { "SELECT COUNT(*) AS count FROM messages WHERE content = 'Fix config again'", ).get()).toEqual({ count: 0 }); expect(deps.messageRepo.listByThread("thread-1", 20).messages.map((message) => message.content)).toEqual([ - "Old security notice", "Late old-session warning", "Security notice", "Warning notice", "Model rerouted", "Tail message one", "Tail message two", + "Old security notice", "Security notice", "Model rerouted", "Tail message one", "Tail message two", ]); + expect(db.prepare("SELECT content FROM messages WHERE content IN ('Late old-session warning', 'Warning notice') ORDER BY sequence").all()) + .toEqual([{ content: "Late old-session warning" }, { content: "Warning notice" }]); db.close(); }); @@ -191,7 +193,9 @@ describe("loadConversationPage", () => { repo.beginNoticeSession("thread-1", "session-1"); for (let i = 0; i < 23; i++) repo.createSystemNotice("thread-1", `Warning ${i}`, i, { kind: "warning", presentation: "timeline", scope: "turn", sessionId: "session-1", noticeKey: `warning-${i}` }); expect(repo.listSessionNotices("thread-1").map((message) => message.content)).toEqual(Array.from({ length: 20 }, (_, i) => `Warning ${i + 3}`)); - expect(repo.listByThread("thread-1", 23).messages).toHaveLength(23); + expect(repo.listByThread("thread-1", 23).messages).toEqual([]); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get("thread-1")) + .toEqual({ count: 23 }); db.close(); }); diff --git a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts new file mode 100644 index 000000000..5c973002d --- /dev/null +++ b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts @@ -0,0 +1,88 @@ +import * as NodePerfHooks from "node:perf_hooks"; +import { describe, expect, it } from "vitest"; +import { eventApplyType, ServerWorkTrace, type ServerWorkTraceReport } from "../server-work-trace.js"; + +describe("ServerWorkTrace", () => { + it("attributes aggregate work in a stalled tick and bounds content-free output", () => { + const reports: ServerWorkTraceReport[] = []; + const trace = new ServerWorkTrace((report) => reports.push(report)); + const started = NodePerfHooks.performance.now(); + trace.tick(started + 20); + for (let index = 0; index < 40; index += 1) { + trace.record("event-apply", "thread-a", "execution-a", 12, "textDelta"); + } + trace.record("canonical-write", "thread-b", "execution-b", 30); + trace.record("event-apply", "secret prompt with spaces", "secret output with spaces", 1, "other"); + for (let index = 0; index < 80; index += 1) { + trace.record("worker-wait", `thread-${index}`, "execution-c", 1); + } + trace.record("event-apply", "thread-overflow", "execution-d", 9, "toolResult"); + trace.record("narrative-checkpoint", "thread-overflow", "execution-d", 7); + trace.record("narrative-prepare", "thread-overflow", "execution-d", 4); + trace.record("narrative-persist", "thread-overflow", "execution-d", 2); + trace.record("narrative-confirm", "thread-overflow", "execution-d", 1); + trace.tick(started + 260); + + expect(reports).toHaveLength(1); + const report = reports[0]; + expect(report?.kind).toBe("server-work-stall"); + if (report?.kind !== "server-work-stall") return; + expect(report.delayMs).toBeCloseTo(220, 8); + expect(report.samples).toHaveLength(64); + expect(report.overflowCount).toBe(24); + expect(report.eventApplyByType.textDelta).toEqual({ count: 40, totalMs: 480, maxMs: 12 }); + expect(report.eventApplyByType.toolResult).toEqual({ count: 1, totalMs: 9, maxMs: 9 }); + expect(report.eventApplyByType.other).toEqual({ count: 1, totalMs: 1, maxMs: 1 }); + expect(report.narrativeCheckpoint).toEqual({ count: 1, totalMs: 7, maxMs: 7 }); + expect(report.narrativeCheckpointByStep).toEqual({ + "narrative-prepare": { count: 1, totalMs: 4, maxMs: 4 }, + "narrative-persist": { count: 1, totalMs: 2, maxMs: 2 }, + "narrative-confirm": { count: 1, totalMs: 1, maxMs: 1 }, + }); + expect(report.samples[0]).toEqual({ + phase: "event-apply", threadId: "thread-a", executionId: "execution-a", + count: 40, totalMs: 480, maxMs: 12, + }); + expect(report.samples[1]).toEqual({ + phase: "canonical-write", threadId: "thread-b", executionId: "execution-b", + count: 1, totalMs: 30, maxMs: 30, + }); + expect(report.samples[2]).toMatchObject({ threadId: null, executionId: null }); + expect(JSON.stringify(report)).not.toMatch(/prompt|toolInput|output|secret/); + + trace.tick(started + 280); + expect(reports).toHaveLength(1); + }); + + it("classifies only the fixed event type across threads", () => { + const reports: ServerWorkTraceReport[] = []; + const trace = new ServerWorkTrace((report) => reports.push(report)); + const started = NodePerfHooks.performance.now(); + trace.tick(started + 20); + const types = ["textDelta", "toolUse", "toolResult", "assistantMessageBoundary", "unrecognized"]; + types.forEach((type, index) => { + trace.record("event-apply", `thread-${index}`, "execution-a", index + 1, eventApplyType(type)); + trace.record("narrative-checkpoint", `thread-${index}`, "execution-a", index + 2); + trace.record("narrative-prepare", `thread-${index}`, "execution-a", index + 1); + trace.record("narrative-persist", `thread-${index}`, "execution-a", 1); + trace.record("narrative-confirm", `thread-${index}`, "execution-a", 1); + }); + trace.tick(started + 220); + const report = reports[0]; + expect(report?.kind).toBe("server-work-stall"); + if (report?.kind !== "server-work-stall") return; + expect(report.eventApplyByType).toEqual({ + textDelta: { count: 1, totalMs: 1, maxMs: 1 }, + toolUse: { count: 1, totalMs: 2, maxMs: 2 }, + toolResult: { count: 1, totalMs: 3, maxMs: 3 }, + assistantMessageBoundary: { count: 1, totalMs: 4, maxMs: 4 }, + other: { count: 1, totalMs: 5, maxMs: 5 }, + }); + expect(report.narrativeCheckpoint).toEqual({ count: 5, totalMs: 20, maxMs: 6 }); + expect(report.narrativeCheckpointByStep).toEqual({ + "narrative-prepare": { count: 5, totalMs: 15, maxMs: 5 }, + "narrative-persist": { count: 5, totalMs: 5, maxMs: 1 }, + "narrative-confirm": { count: 5, totalMs: 5, maxMs: 1 }, + }); + }); +}); diff --git a/apps/server/src/features/agents/diagnostics/server-work-trace.ts b/apps/server/src/features/agents/diagnostics/server-work-trace.ts new file mode 100644 index 000000000..d6c25f9cf --- /dev/null +++ b/apps/server/src/features/agents/diagnostics/server-work-trace.ts @@ -0,0 +1,203 @@ +import * as NodePerfHooks from "node:perf_hooks"; +import * as NodeWorkerThreads from "node:worker_threads"; +import { logger } from "@mcode/shared"; + +/** Fixed names keep trace output free of event bodies and provider text. */ +export type ServerWorkPhase = + | "provider-callback" | "worker-admission" | "worker-wait" | "mailbox-wait" + | "canonical-write" | "event-apply" | "narrative-checkpoint" + | "assistant-text-write" + | "narrative-prepare" | "narrative-persist" | "narrative-confirm" + | "finalization" | "publication" + | "terminal-create" | "thread-create" | "agent-send"; + +/** Fixed event categories; unknown and future event types stay in other. */ +export type EventApplyType = "textDelta" | "toolUse" | "toolResult" | "assistantMessageBoundary" | "other"; + +/** Classify only the event discriminant, never an event payload. */ +export function eventApplyType(type: string): EventApplyType { + switch (type) { + case "textDelta": + case "toolUse": + case "toolResult": + case "assistantMessageBoundary": + return type; + default: + return "other"; + } +} + +interface EventApplyTotals { + count: number; + totalMs: number; + maxMs: number; +} + +type NarrativeCheckpointStep = "narrative-prepare" | "narrative-persist" | "narrative-confirm"; + +function emptyNarrativeStepTotals(): Record { + return { + "narrative-prepare": { count: 0, totalMs: 0, maxMs: 0 }, + "narrative-persist": { count: 0, totalMs: 0, maxMs: 0 }, + "narrative-confirm": { count: 0, totalMs: 0, maxMs: 0 }, + }; +} + +function emptyEventApplyTotals(): Record { + return { + textDelta: { count: 0, totalMs: 0, maxMs: 0 }, + toolUse: { count: 0, totalMs: 0, maxMs: 0 }, + toolResult: { count: 0, totalMs: 0, maxMs: 0 }, + assistantMessageBoundary: { count: 0, totalMs: 0, maxMs: 0 }, + other: { count: 0, totalMs: 0, maxMs: 0 }, + }; +} + +interface WorkSample { + phase: ServerWorkPhase; + threadId: string | null; + executionId: string | null; + count: number; + totalMs: number; + maxMs: number; +} + +/** One bounded report for work accumulated while the server loop could not run. */ +export type ServerWorkTraceReport = { + kind: "server-work-stall"; + at: number; + delayMs: number; + windowMs: number; + samples: WorkSample[]; + eventApplyByType: Record; + narrativeCheckpoint: EventApplyTotals; + narrativeCheckpointByStep: Record; + overflowCount: number; +} | { + kind: "server-work-slow-operation"; + at: number; + sample: WorkSample; +}; + +const MAX_KEYS = 64; +const INTERVAL_MS = 20; +const REPORT_DELAY_MS = 100; +const TRACE_ID = /^[A-Za-z0-9_-]{1,128}$/; + +function safeId(value: string | undefined): string | null { + return value && TRACE_ID.test(value) ? value : null; +} + +/** Aggregates fixed-phase timings in memory and emits only delayed loop windows. */ +export class ServerWorkTrace { + private readonly samples = new Map(); + private eventApplyByType = emptyEventApplyTotals(); + private narrativeCheckpoint: EventApplyTotals = { count: 0, totalMs: 0, maxMs: 0 }; + private narrativeCheckpointByStep = emptyNarrativeStepTotals(); + private overflowCount = 0; + private lastTick = NodePerfHooks.performance.now(); + private timer: ReturnType | undefined; + + constructor(private readonly emit: (report: ServerWorkTraceReport) => void) {} + + /** Begin the lightweight event-loop probe. */ + start(): void { + if (this.timer) return; + this.lastTick = NodePerfHooks.performance.now(); + this.timer = setInterval(() => this.tick(), INTERVAL_MS); + this.timer.unref(); + } + + /** Stop the probe and discard its pending window. */ + stop(): void { + if (this.timer) clearInterval(this.timer); + this.timer = undefined; + this.samples.clear(); + this.eventApplyByType = emptyEventApplyTotals(); + this.narrativeCheckpoint = { count: 0, totalMs: 0, maxMs: 0 }; + this.narrativeCheckpointByStep = emptyNarrativeStepTotals(); + this.overflowCount = 0; + } + + /** Measure synchronous work without retaining its input or result. */ + measure(phase: ServerWorkPhase, threadId: string | undefined, executionId: string | undefined, work: () => T, applyType?: EventApplyType): T { + const started = NodePerfHooks.performance.now(); + try { + return work(); + } finally { + this.record(phase, threadId, executionId, NodePerfHooks.performance.now() - started, applyType); + } + } + + /** Record an asynchronous wait or a measured operation after completion. */ + record(phase: ServerWorkPhase, threadId: string | undefined, executionId: string | undefined, durationMs: number, applyType?: EventApplyType): void { + if (phase === "event-apply" && applyType) { + const total = this.eventApplyByType[applyType]; + total.count += 1; + total.totalMs += durationMs; + total.maxMs = Math.max(total.maxMs, durationMs); + } + if (phase === "narrative-checkpoint") { + this.narrativeCheckpoint.count += 1; + this.narrativeCheckpoint.totalMs += durationMs; + this.narrativeCheckpoint.maxMs = Math.max(this.narrativeCheckpoint.maxMs, durationMs); + } + if (phase === "narrative-prepare" || phase === "narrative-persist" || phase === "narrative-confirm") { + const total = this.narrativeCheckpointByStep[phase]; + total.count += 1; + total.totalMs += durationMs; + total.maxMs = Math.max(total.maxMs, durationMs); + } + const safeThreadId = safeId(threadId); + const safeExecutionId = safeId(executionId); + if (this.isSlowOperation(phase, durationMs)) { + this.emit({ kind: "server-work-slow-operation", at: Date.now(), sample: { + phase, threadId: safeThreadId, executionId: safeExecutionId, + count: 1, totalMs: durationMs, maxMs: durationMs, + } }); + } + const key = JSON.stringify([phase, safeThreadId, safeExecutionId]); + const sample = this.samples.get(key); + if (sample) { + sample.count += 1; + sample.totalMs += durationMs; + sample.maxMs = Math.max(sample.maxMs, durationMs); + return; + } + if (this.samples.size === MAX_KEYS) { + this.overflowCount += 1; + return; + } + this.samples.set(key, { + phase, threadId: safeThreadId, executionId: safeExecutionId, + count: 1, totalMs: durationMs, maxMs: durationMs, + }); + } + + private isSlowOperation(phase: ServerWorkPhase, durationMs: number): boolean { + return durationMs >= REPORT_DELAY_MS + && (phase === "terminal-create" || phase === "thread-create" || phase === "agent-send"); + } + + /** Check a single probe interval; exposed to drive deterministic stall tests. */ + tick(now: number = NodePerfHooks.performance.now()): void { + const windowMs = now - this.lastTick; + this.lastTick = now; + const delayMs = Math.max(0, windowMs - INTERVAL_MS); + if (delayMs >= REPORT_DELAY_MS) { + this.emit({ kind: "server-work-stall", at: Date.now(), delayMs, windowMs, samples: [...this.samples.values()], eventApplyByType: this.eventApplyByType, narrativeCheckpoint: this.narrativeCheckpoint, narrativeCheckpointByStep: this.narrativeCheckpointByStep, overflowCount: this.overflowCount }); + } + this.samples.clear(); + this.eventApplyByType = emptyEventApplyTotals(); + this.narrativeCheckpoint = { count: 0, totalMs: 0, maxMs: 0 }; + this.narrativeCheckpointByStep = emptyNarrativeStepTotals(); + this.overflowCount = 0; + } +} + +/** Trace only the server loop; writer-worker delays are not server stalls. */ +export const serverWorkTrace = NodeWorkerThreads.isMainThread && process.env.MCODE_SERVER_WORK_TRACE === "1" + ? new ServerWorkTrace((report) => logger.warn("Server work trace", report)) + : null; + +serverWorkTrace?.start(); diff --git a/apps/server/src/features/agents/events/__tests__/provider-event-publication.test.ts b/apps/server/src/features/agents/events/__tests__/provider-event-publication.test.ts index 6a9107d97..087b3a7c7 100644 --- a/apps/server/src/features/agents/events/__tests__/provider-event-publication.test.ts +++ b/apps/server/src/features/agents/events/__tests__/provider-event-publication.test.ts @@ -49,6 +49,25 @@ describe("provider event publication ownership", () => { }); }); + it("publishes a committed terminal replay without changing a newer thread status", () => { + const deps = buildPublicationDeps(); + const event: AgentEvent = { + type: AgentEventType.TurnComplete, + threadId: "parent-thread", + turnExecutionId: "00000000-0000-4000-8000-000000000001", + publicationId: "1", + reason: "completed", + costUsd: null, + tokensIn: 1, + tokensOut: 1, + }; + + expect(publishParentProviderEvent(event, event, deps)).toBe(true); + expect(deps.publishAgentEvent).toHaveBeenCalledWith(event); + expect(deps.updateThreadStatus).not.toHaveBeenCalled(); + expect(deps.publishThreadStatus).not.toHaveBeenCalled(); + }); + it("publishes a parent error and updates its status", () => { const deps = buildPublicationDeps(); const parentEvent: AgentEvent = { diff --git a/apps/server/src/features/agents/events/persistence/hook-execution-repo.ts b/apps/server/src/features/agents/events/persistence/hook-execution-repo.ts index 1f10e39f7..05a90cadf 100644 --- a/apps/server/src/features/agents/events/persistence/hook-execution-repo.ts +++ b/apps/server/src/features/agents/events/persistence/hook-execution-repo.ts @@ -97,26 +97,26 @@ export class HookExecutionRepo { } /** Insert multiple hook execution records in a single transaction. */ - bulkCreate(inputs: CreateHookExecutionInput[]): void { + bulkCreate(inputs: CreateHookExecutionInput[], replaceExisting = false): void { if (inputs.length === 0) return; this.orm.transaction((tx) => { for (const item of inputs) { - tx.insert(hookExecutions) - .values({ - id: item.id ?? NodeCrypto.randomUUID(), - messageId: item.messageId, - hookName: item.hookName, - toolName: item.toolName, - phase: item.phase, - payload: item.payload, - durationMs: item.durationMs, - didBlock: item.didBlock ? 1 : 0, - startedAt: item.startedAt, - endedAt: item.endedAt, - sortOrder: item.sortOrder, - }) - .onConflictDoNothing() - .run(); + const { id, ...rest } = { + id: item.id ?? NodeCrypto.randomUUID(), + messageId: item.messageId, + hookName: item.hookName, + toolName: item.toolName, + phase: item.phase, + payload: item.payload, + durationMs: item.durationMs, + didBlock: item.didBlock ? 1 : 0, + startedAt: item.startedAt, + endedAt: item.endedAt, + sortOrder: item.sortOrder, + }; + const write = tx.insert(hookExecutions).values({ id, ...rest }); + if (replaceExisting) write.onConflictDoUpdate({ target: hookExecutions.id, set: rest }).run(); + else write.onConflictDoNothing().run(); } }); } diff --git a/apps/server/src/features/agents/events/provider-event-publication.ts b/apps/server/src/features/agents/events/provider-event-publication.ts index 82eb6d253..036b6e271 100644 --- a/apps/server/src/features/agents/events/provider-event-publication.ts +++ b/apps/server/src/features/agents/events/provider-event-publication.ts @@ -16,6 +16,9 @@ export function publishParentProviderEvent( if (!shouldPublishParentEvent(event)) return false; deps.publishAgentEvent(enrichedEvent); + // A semantic writer committed this status before releasing its terminal event. + // A late replay must not overwrite the status of a newer execution. + if (event.publicationId !== undefined) return true; if (event.type === "turnComplete") { deps.updateThreadStatus(event.threadId, "completed"); deps.publishThreadStatus({ threadId: event.threadId, status: "completed" }); diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts new file mode 100644 index 000000000..1f7aeab0e --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { AgentEventSchema, type AgentEvent } from "@mcode/contracts"; +import { CodexLiveEventReducer, type CodexPlanFeature } from "../codex-live-event-reducer.js"; +import { CodexLiveEventEffects } from "../codex-live-event-effects.js"; +import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; + +const execution = { threadId: "thread", turnId: "turn", executionId: "11111111-1111-4111-8111-111111111111" }; +const endedAt = "2026-09-25T12:00:00.000Z"; + +function event(type: AgentEvent["type"], fields: Record = {}): AgentEvent { + return AgentEventSchema().parse({ type, threadId: execution.threadId, turnExecutionId: execution.executionId, ...fields }); +} + +function setup(plan: CodexPlanFeature = "none") { + const reducer = new CodexLiveEventReducer(execution, plan); + const mapper = new CodexLiveEventEffects(execution, "user-message"); + const prepare = (type: AgentEvent["type"], fields: Record = {}) => { + const reduction = reducer.reduce(event(type, fields)); + if (reduction.kind !== "reduced") throw new Error(reduction.reason); + return mapper.prepare(reduction, endedAt); + }; + prepare("turnStarted"); + return { prepare, mapper }; +} + +describe("CodexLiveEventEffects", () => { + it("checkpoints only the result tool while keeping full recovery and terminal narrative consistent", () => { + const { prepare } = setup(); + prepare("textDelta", { delta: "Thought", isFinalResponse: false }); + prepare("toolUse", { toolCallId: "read", toolName: "Read", toolInput: { file_path: "file.txt" } }); + const result = prepare("toolResult", { toolCallId: "read", output: "contents", isError: false }); + expect(result.effects.narrative?.items).toMatchObject([{ kind: "toolCall", record: { id: "read", output_summary: "contents" } }]); + expect(result.effects.narrative?.items).toHaveLength(1); + expect(prepare("assistantMessageBoundary", { isFinalResponse: false }).effects.narrative).toBeUndefined(); + const terminal = prepare("turnComplete", { tokensIn: 0, tokensOut: 0, reason: "stop", costUsd: 0 }); + expect(terminal.terminal?.narrative).toHaveLength(2); + expect(result.effects.narrative?.items).toHaveLength(1); + expect(() => prepare("toolResult", { toolCallId: "read", output: "late", isError: false })) + .toThrow("post-terminal event needs the terminal lifecycle owner"); + }); + + it("moves unknown text into narrative and restarts its checkpoint sequence", () => { + const { prepare } = setup(); + expect(prepare("textDelta", { delta: "Looking at the files" }).effects.text).toEqual({ + kind: "append", inputs: [{ ...execution, sequence: 1, text: "Looking at the files" }], + }); + const boundary = prepare("assistantMessageBoundary", { isFinalResponse: false }); + expect(boundary.effects.text).toEqual({ kind: "reclassify", expectedText: "Looking at the files" }); + expect(boundary.effects.narrative?.items).toEqual([ + expect.objectContaining({ kind: "narrationSegment", record: expect.objectContaining({ text: "Looking at the files" }) }), + ]); + expect(prepare("textDelta", { delta: "Fixed", isFinalResponse: true }).effects.text).toEqual({ + kind: "append", inputs: [{ ...execution, sequence: 1, text: "Fixed" }], + }); + }); + + it("promotes narration while deleting its old recovery record", () => { + const { prepare } = setup(); + const thought = prepare("textDelta", { delta: "Answer", isFinalResponse: false }); + const id = thought.effects.narrative?.items[0]?.record.id; + expect(id).toBeDefined(); + const promoted = prepare("assistantMessageBoundary", { isFinalResponse: true }); + expect(promoted.effects.text).toEqual({ kind: "promote", input: { ...execution, sequence: 1, text: "Answer" } }); + expect(promoted.effects.narrative?.items).toEqual([]); + expect(promoted.effects.narrative?.discardedItemIds).toEqual([`narrationSegment:${id}`]); + expect(prepare("assistantMessageBoundary", { isFinalResponse: true }).effects.narrative).toBeUndefined(); + }); + + it("retains TaskCreate input until its result and keeps file intents separate", () => { + const { prepare } = setup(); + const use = prepare("toolUse", { toolCallId: "create", toolName: "TaskCreate", toolInput: { subject: "Fix login" } }); + expect(use.runtime).toContainEqual(expect.objectContaining({ kind: "tool-use" })); + expect(use.effects.narrative?.items[0]?.kind).toBe("toolCall"); + const result = prepare("toolResult", { toolCallId: "create", output: "Task #42 created", isError: false }); + expect(result.effects.taskIntents).toEqual([ + { kind: "append-task", task: { id: "42", content: "Fix login", status: "pending", group: "Tasks" } }, + ]); + }); + + it("carries system intents and leaves context projection explicit", () => { + const { prepare } = setup(); + const system = prepare("system", { subtype: "sdk_session_id:session-1" }); + expect(system.effects.systemIntents).toEqual([{ kind: "session-cursor", event: system.publication.event }]); + const context = prepare("contextEstimate", { tokensIn: 120, totalProcessedTokens: 120, contextWindow: 1000 }); + expect(context.runtime).toEqual([{ kind: "context-usage", tokensIn: 120, contextWindow: 1000 }]); + }); + + it("uses the same assigned assistant identity for live and terminal data", () => { + const { prepare } = setup(); + prepare("textDelta", { delta: "Answer", isFinalResponse: true }); + const message = prepare("message", { content: "Answer", tokens: null }); + const messageId = deriveTurnAssistantMessageId(execution.threadId, "user-message"); + expect(message.effects.message).toMatchObject({ precedingMessageId: "user-message", messageId, content: "Answer" }); + expect(message.publication.event).toMatchObject({ type: "message", messageId }); + const terminal = prepare("error", { error: "Provider disconnected" }); + expect(terminal.terminal).toMatchObject({ threadId: execution.threadId, executionId: execution.executionId, + endedAt, outcome: "errored", assistant: { messageId, content: "Answer" } }); + expect(terminal.publication.after).toBe("terminal"); + expect(terminal.runtime).toContainEqual({ kind: "turn-error", error: "Provider disconnected" }); + expect(structuredClone(terminal)).toEqual(terminal); + }); + + it("carries parsed plan questions with their completing text event", () => { + const { prepare } = setup("questions"); + const question = { id: "q1", category: "AUTH", question: "Which login?", options: [ + { id: "o1", title: "Passkey", description: "Use passkeys.", recommended: true }, + { id: "o2", title: "Password", description: "Use passwords." }, + ] }; + const text = `\`\`\`plan-questions\n${JSON.stringify([question])}\n\`\`\``; + const result = prepare("textDelta", { delta: text }); + expect(result.effects.planQuestions).toEqual([question]); + expect(result.effects.text.kind).toBe("append"); + expect(result.publication.after).toBe("writer"); + }); + + it("rejects another execution before advancing checkpoint state", () => { + const { mapper, prepare } = setup(); + const other = new CodexLiveEventReducer({ ...execution, turnId: "other-turn" }); + const reduction = other.reduce(event("turnStarted")); + if (reduction.kind !== "reduced") throw new Error(reduction.reason); + expect(() => mapper.prepare(reduction)).toThrow("another execution"); + expect(prepare("textDelta", { delta: "Answer" }).effects.text).toMatchObject({ inputs: [{ sequence: 1 }] }); + }); + + it("associates parsed plan output with the staged assistant body", () => { + const { prepare } = setup("output"); + const plan = { title: "Login plan", sections: [ + { id: "s1", title: "Implementation", level: 1, content: "Add passkey login." }, + ] }; + prepare("textDelta", { delta: `\`\`\`plan-output\n${JSON.stringify(plan)}\n\`\`\`` }); + const result = prepare("message", { content: "Provider prose", tokens: null }); + expect(result.effects.planOutput).toEqual({ title: "Login plan", + contentMd: "## Implementation\n\nAdd passkey login.", + sectionsJson: '[{"id":"s1","title":"Implementation","level":1}]', changeSummary: null }); + expect(result.effects.message?.content).toBe("Provider prose"); + expect(result.publication.event).toMatchObject({ messageId: result.effects.message?.messageId }); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts new file mode 100644 index 000000000..e6811e34d --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts @@ -0,0 +1,348 @@ +import { describe, expect, it } from "vitest"; +import { AgentEventSchema, type AgentEvent } from "@mcode/contracts"; +import { CodexEventMapper } from "../../../../../../../packages/providers/src/private/codex/codex-event-mapper.js"; +import { CodexLiveEventReducer } from "../codex-live-event-reducer.js"; + +const execution = { + threadId: "test-thread", + turnId: "test-turn", + executionId: "11111111-1111-4111-8111-111111111111", +}; + +function event(type: AgentEvent["type"], fields: Record = {}): AgentEvent { + return AgentEventSchema().parse({ type, threadId: execution.threadId, turnExecutionId: execution.executionId, ...fields }); +} + +function boundByProvider(mapped: AgentEvent): AgentEvent { + return { ...mapped, turnExecutionId: execution.executionId }; +} + +function reduceEvent(reducer: CodexLiveEventReducer, type: AgentEvent["type"], fields: Record = {}) { + const reduction = reducer.reduce(event(type, fields)); + if (reduction.kind !== "reduced") throw new Error(reduction.reason); + return reduction; +} + +describe("CodexLiveEventReducer", () => { + it("returns only changed recovery items and retains the complete terminal snapshot", () => { + const reducer = new CodexLiveEventReducer(execution); + reduceEvent(reducer, "turnStarted"); + reduceEvent(reducer, "toolUse", { toolCallId: "first", toolName: "Read", toolInput: { file_path: "one" } }); + const second = reduceEvent(reducer, "toolUse", { toolCallId: "second", toolName: "Read", toolInput: { file_path: "two" } }); + expect(second.writer).toContainEqual({ kind: "narrative-recovery", discardedItemIds: [], + items: [expect.objectContaining({ kind: "toolCall", record: expect.objectContaining({ id: "second" }) })] }); + const result = reduceEvent(reducer, "toolResult", { toolCallId: "first", output: "contents", isError: false }); + expect(result.writer).toContainEqual({ kind: "narrative-recovery", discardedItemIds: [], + items: [expect.objectContaining({ kind: "toolCall", record: expect.objectContaining({ id: "first", output_summary: "contents" }) })] }); + const boundary = reduceEvent(reducer, "assistantMessageBoundary", { isFinalResponse: false }); + expect(boundary.writer).toContainEqual({ kind: "narrative-recovery", items: [], discardedItemIds: [] }); + const terminal = reduceEvent(reducer, "turnComplete", { reason: "stop", costUsd: 0, tokensIn: 0, tokensOut: 0 }); + const projection = terminal.writer.find((intent) => intent.kind === "terminal-projection"); + expect(projection?.narrative.map((item) => item.record.id)).toEqual(["first", "second"]); + const lateHook = reduceEvent(reducer, "hookStarted", { hookName: "stop", hookType: "stop" }); + expect(lateHook.writer).toContainEqual({ kind: "narrative-recovery", discardedItemIds: [], + items: [expect.objectContaining({ kind: "hook" })] }); + }); + + it("keeps recovery output independent of buffered state and emits discard-only deltas", () => { + const reducer = new CodexLiveEventReducer(execution); + reduceEvent(reducer, "turnStarted"); + const thought = reduceEvent(reducer, "textDelta", { delta: "Answer", isFinalResponse: false }); + const item = thought.writer.find((intent) => intent.kind === "narrative-recovery")?.items[0]; + if (item?.kind !== "narrationSegment") throw new Error("Expected narration recovery"); + const id = item.record.id; + item.record.text = "Mutated caller copy"; + const extended = reduceEvent(reducer, "textDelta", { delta: " continues", isFinalResponse: false }); + expect(extended.writer).toContainEqual({ kind: "narrative-recovery", discardedItemIds: [], + items: [expect.objectContaining({ record: expect.objectContaining({ id, text: "Answer continues" }) })] }); + expect(item.record.text).toBe("Mutated caller copy"); + const promoted = reduceEvent(reducer, "assistantMessageBoundary", { isFinalResponse: true }); + expect(promoted.writer).toContainEqual({ kind: "narrative-recovery", items: [], discardedItemIds: [`narrationSegment:${id}`] }); + expect(promoted.writer).toContainEqual({ kind: "assistant-text-promote", text: "Answer continues" }); + const repeated = reduceEvent(reducer, "assistantMessageBoundary", { isFinalResponse: true }); + expect(repeated.writer).toContainEqual({ kind: "narrative-recovery", items: [], discardedItemIds: [] }); + }); + + it("carries parsed plan questions as data on the completing text event", () => { + const reducer = new CodexLiveEventReducer(execution, "questions"); + const question = { id: "q1", category: "AUTH", question: "Which login?", options: [ + { id: "o1", title: "Passkey", description: "Use passkeys.", recommended: true }, + { id: "o2", title: "Password", description: "Use passwords." }, + ] }; + const block = `\`\`\`plan-questions\n${JSON.stringify([question])}\n\`\`\``; + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + const first = reducer.reduce(event("textDelta", { delta: block.slice(0, 20) })); + expect(first.kind).toBe("reduced"); + if (first.kind !== "reduced") return; + expect(first.writer).not.toContainEqual(expect.objectContaining({ kind: "plan-questions" })); + const second = reducer.reduce(event("textDelta", { delta: block.slice(20) })); + expect(second.kind).toBe("reduced"); + if (second.kind !== "reduced") return; + expect(second.writer).toContainEqual({ kind: "plan-questions", questions: [question] }); + expect(second.publication.after).toBe("writer"); + expect(reducer.reduce(event("textDelta", { delta: block }))).toMatchObject({ + kind: "reduced", writer: expect.not.arrayContaining([{ kind: "plan-questions", questions: [question] }]), + }); + }); + + it("carries plan output data with the assistant body and bounds parser input", () => { + const reducer = new CodexLiveEventReducer(execution, "output"); + const plan = { title: "Login plan", sections: [ + { id: "s1", title: "Implementation", level: 1, content: "Add passkey login." }, + ] }; + const block = `\`\`\`plan-output\n${JSON.stringify(plan)}\n\`\`\``; + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + expect(reducer.reduce(event("textDelta", { delta: block })).kind).toBe("reduced"); + const message = reducer.reduce(event("message", { content: "Provider prose", tokens: null })); + expect(message.kind).toBe("reduced"); + if (message.kind !== "reduced") return; + expect(message.writer).toContainEqual({ kind: "plan-output", output: { + title: "Login plan", contentMd: "## Implementation\n\nAdd passkey login.", + sectionsJson: '[{"id":"s1","title":"Implementation","level":1}]', changeSummary: null, + } }); + expect(message.writer[0]).toMatchObject({ kind: "assistant-body", content: "Provider prose" }); + + const bounded = new CodexLiveEventReducer(execution, "questions"); + expect(bounded.reduce(event("turnStarted")).kind).toBe("reduced"); + expect(bounded.reduce(event("textDelta", { delta: "x".repeat(256 * 1024) })).kind).toBe("reduced"); + expect(bounded.reduce(event("textDelta", { delta: "x" }))).toEqual({ + kind: "unsupported", eventType: "textDelta", reason: "plan text exceeds the execution projection limit", + }); + }); + + it("reduces a Codex notification sequence through tool narration and final answer", () => { + const mapper = new CodexEventMapper(execution.threadId); + const reducer = new CodexLiveEventReducer(execution); + const reductions = [reducer.reduce(event("turnStarted"))]; + const notifications = [ + { method: "item/agentMessage/delta", params: { itemId: "thought", delta: "I will check." } }, + { method: "item/completed", params: { item: { type: "agentMessage", id: "thought" } } }, + { method: "item/started", params: { item: { type: "commandExecution", id: "cmd", command: "pwd" } } }, + { method: "item/completed", params: { item: { type: "commandExecution", id: "cmd", command: "pwd", output: "/repo", exitCode: 0 } } }, + { method: "item/agentMessage/delta", params: { itemId: "answer", delta: "Done." } }, + { method: "item/completed", params: { item: { type: "agentMessage", id: "answer" } } }, + { method: "turn/completed", params: { turn: { status: "completed" } } }, + ]; + for (const notification of notifications) { + for (const runtimeEvent of mapper.mapNotification({ jsonrpc: "2.0", ...notification })) { + reductions.push(reducer.reduce(boundByProvider(runtimeEvent.event))); + } + } + + expect(reductions.every((result) => result.kind === "reduced")).toBe(true); + const reduced = reductions.filter((result) => result.kind === "reduced"); + expect(reduced.map((result) => result.publication.event.type)).toEqual([ + "turnStarted", "textDelta", "assistantMessageBoundary", "toolUse", "toolResult", + "textDelta", "assistantMessageBoundary", "message", "turnComplete", + ]); + const toolUse = reduced.find((result) => result.publication.event.type === "toolUse"); + expect(toolUse?.writer).toContainEqual(expect.objectContaining({ kind: "tool-use" })); + const message = reduced.find((result) => result.publication.event.type === "message"); + expect(message?.writer).toContainEqual(expect.objectContaining({ + kind: "assistant-body", content: "Done.", attachments: [], + })); + const terminal = reduced.find((result) => result.publication.event.type === "turnComplete"); + expect(terminal?.writer).toContainEqual(expect.objectContaining({ + kind: "terminal-projection", source: "turnComplete", outcome: "completed", + assistant: expect.objectContaining({ content: "Done." }), + })); + expect(terminal?.publication.after).toBe("terminal"); + expect(structuredClone(reduced)).toEqual(reduced); + }); + + it("reclassifies unknown text as narration and carries attachments and late hooks", () => { + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + const delta = reducer.reduce(event("textDelta", { delta: "working" })); + expect(delta.kind).toBe("reduced"); + if (delta.kind !== "reduced") return; + expect(delta.writer).toContainEqual({ kind: "assistant-text-delta", delta: "working", classification: "unknown" }); + const boundary = reducer.reduce(event("assistantMessageBoundary", { isFinalResponse: false })); + expect(boundary.kind).toBe("reduced"); + if (boundary.kind !== "reduced") return; + expect(boundary.writer).toContainEqual({ kind: "assistant-text-reclassify", text: "working", classification: "narration" }); + expect(boundary.writer).toContainEqual(expect.objectContaining({ + kind: "narrative-recovery", + items: [expect.objectContaining({ kind: "narrationSegment", record: expect.objectContaining({ text: "working" }) })], + })); + + const attachment = { id: "image", name: "image.png", mimeType: "image/png", sizeBytes: 12 }; + expect(reducer.reduce(event("generatedAttachment", { attachment })).kind).toBe("reduced"); + const message = reducer.reduce(event("message", { content: "Answer", tokens: null })); + expect(message.kind).toBe("reduced"); + if (message.kind !== "reduced") return; + expect(message.writer).toContainEqual(expect.objectContaining({ kind: "assistant-body", attachments: [attachment] })); + + expect(reducer.reduce(event("turnComplete", { reason: "completed", costUsd: null, tokensIn: 2, tokensOut: 1 })).kind).toBe("reduced"); + const started = reducer.reduce(event("hookStarted", { hookName: "stop", hookType: "stop" })); + expect(started.kind).toBe("reduced"); + if (started.kind !== "reduced") return; + expect(started.writer).toContainEqual(expect.objectContaining({ kind: "hook-started", late: true })); + const completed = reducer.reduce(event("hookCompleted", { hookName: "stop", exitCode: 0, durationMs: 10, didBlock: false })); + expect(completed.kind).toBe("reduced"); + if (completed.kind !== "reduced") return; + expect(completed.writer).toContainEqual(expect.objectContaining({ kind: "hook-completed", late: true })); + expect(completed.publication.after).toBe("terminal"); + }); + + it("rejects unbound session events and reduces explicitly bound notices and cursors", () => { + const mapper = new CodexEventMapper(execution.threadId); + const reducer = new CodexLiveEventReducer(execution); + const session = reducer.reduce(mapper.sessionStartedEvent().event); + expect(session).toEqual({ + kind: "unsupported", eventType: "system", + reason: "event without execution identity needs the provider event routing owner", + }); + const boundSession = reducer.reduce(event("system", { subtype: "provider.session.started" })); + expect(boundSession).toMatchObject({ kind: "reduced", writer: [{ kind: "notice-session" }] }); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + + const notice = mapper.mapNotification({ method: "unknown/new-method", params: {} })[0]?.event; + expect(notice?.type).toBe("system"); + if (!notice) return; + expect(reducer.reduce(notice)).toEqual({ + kind: "unsupported", eventType: "system", + reason: "event without execution identity needs the provider event routing owner", + }); + const reduced = reducer.reduce(boundByProvider(notice)); + expect(reduced.kind).toBe("reduced"); + if (reduced.kind !== "reduced") return; + expect(reduced.writer.map(({ kind }) => kind)).toEqual(["system-notice"]); + expect(reduced.publication.after).toBe("writer"); + expect(structuredClone(reduced)).toEqual(reduced); + + const cursor = reducer.reduce(event("system", { subtype: "sdk_session_id:native-1" })); + expect(cursor.kind).toBe("reduced"); + if (cursor.kind !== "reduced") return; + expect(cursor.writer.map(({ kind }) => kind)).toEqual(["session-cursor"]); + }); + + it("records mapped context outside compaction and leaves an unsupported terminal unchanged", () => { + const mapper = new CodexEventMapper(execution.threadId, "native-main"); + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + mapper.prepareForTurn(); + mapper.mapNotification({ method: "turn/started", params: { threadId: "native-main", turn: { id: "native-turn" } } }); + const usage = { totalTokens: 120, inputTokens: 100, cachedInputTokens: 40, outputTokens: 20, reasoningOutputTokens: 5 }; + const mapped = mapper.mapNotification({ method: "thread/tokenUsage/updated", params: { + threadId: "native-main", turnId: "native-turn", + tokenUsage: { total: usage, last: usage, modelContextWindow: 200_000 }, + } })[0]?.event; + expect(mapped?.type).toBe("contextEstimate"); + if (!mapped) return; + expect(reducer.reduce(mapped)).toEqual({ + kind: "unsupported", eventType: "contextEstimate", + reason: "event without execution identity needs the provider event routing owner", + }); + const first = reducer.reduce(boundByProvider(mapped)); + expect(first.kind).toBe("reduced"); + if (first.kind !== "reduced") return; + expect(first.writer).toEqual([{ kind: "context-usage", tokensIn: 100, contextWindow: 200_000 }]); + + expect(reducer.reduce(event("compacting", { active: true }))).toMatchObject({ + kind: "reduced", writer: [{ kind: "compaction-started" }], + }); + const during = reducer.reduce(boundByProvider(mapped)); + expect(during).toMatchObject({ kind: "reduced", writer: [] }); + const completion = event("turnComplete", { reason: "end_turn", costUsd: null, tokensIn: 100, tokensOut: 20 }); + expect(reducer.reduce(completion)).toEqual({ + kind: "unsupported", eventType: "turnComplete", + reason: "turn completion during compaction needs the compaction terminal owner", + }); + expect(reducer.reduce(event("compactSummary", { summary: "Shortened context" }))).toMatchObject({ + kind: "reduced", writer: [{ kind: "compaction-summary", summary: "Shortened context" }], + }); + expect(reducer.reduce(event("compacting", { active: true }))).toMatchObject({ + kind: "reduced", writer: [{ kind: "compaction-started" }], + }); + expect(reducer.reduce(event("compacting", { active: false }))).toMatchObject({ + kind: "reduced", writer: [{ kind: "compaction-divider" }], + }); + expect(reducer.reduce(boundByProvider(mapped))).toMatchObject({ + kind: "reduced", writer: [{ kind: "context-usage", tokensIn: 100, contextWindow: 200_000 }], + }); + expect(reducer.reduce(completion)).toMatchObject({ + kind: "reduced", writer: [{ kind: "context-usage", tokensIn: 100 }, { kind: "terminal-projection" }, { kind: "feature-event" }], + }); + }); + + it("publishes mapper retry and status events and projects a failed turn", () => { + const mapper = new CodexEventMapper(execution.threadId, "native-main"); + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + mapper.prepareForTurn(); + mapper.mapNotification({ method: "turn/started", params: { threadId: "native-main", turn: { id: "native-turn" } } }); + const retry = mapper.mapNotification({ method: "error", params: { + threadId: "native-main", error: { message: "temporary" }, willRetry: true, + } })[0]?.event; + expect(retry?.type).toBe("apiRetry"); + if (!retry) return; + expect(reducer.reduce(boundByProvider(retry))).toMatchObject({ kind: "reduced", writer: [], publication: { after: "writer" } }); + const quota = AgentEventSchema().parse({ type: "quotaUpdate", threadId: execution.threadId, providerId: "codex", categories: [] }); + expect(reducer.reduce(quota)).toMatchObject({ kind: "unsupported", eventType: "quotaUpdate" }); + expect(reducer.reduce(boundByProvider(quota))).toMatchObject({ kind: "reduced", writer: [] }); + const status = mapper.mapNotification({ method: "mcpServer/startupStatus/updated", params: { + threadId: "native-main", name: "search", status: "failed", error: "offline", + } })[0]?.event; + expect(status?.type).toBe("mcpServerStartupStatus"); + if (!status) return; + expect(reducer.reduce(boundByProvider(status))).toMatchObject({ kind: "reduced", writer: [] }); + + const failure = mapper.mapNotification({ method: "turn/completed", params: { + threadId: "native-main", turn: { id: "native-turn", status: "failed", items: [], error: { message: "failed" } }, + } }).find(({ event: mapped }) => mapped.type === "error")?.event; + expect(failure?.type).toBe("error"); + if (!failure) return; + const terminal = reducer.reduce(boundByProvider(failure)); + expect(terminal).toMatchObject({ + kind: "reduced", writer: [{ kind: "turn-error", error: "failed" }, { kind: "terminal-projection", source: "error", outcome: "errored" }], + publication: { after: "terminal" }, + }); + expect(structuredClone(terminal)).toEqual(terminal); + }); + + it("keeps post-turn goal and status publications while rejecting unowned goal receipts", () => { + const mapper = new CodexEventMapper(execution.threadId, "native-main"); + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + expect(reducer.reduce(event("turnComplete", { reason: "end_turn", costUsd: null, tokensIn: 0, tokensOut: 0 })).kind).toBe("reduced"); + const goalEvents = mapper.mapNotification({ method: "thread/goal/updated", params: { + threadId: "native-main", turnId: "native-turn", + goal: { threadId: "native-main", objective: "Ship", status: "complete", tokenBudget: null, tokensUsed: 5, + timeUsedSeconds: 2, createdAt: 1, updatedAt: 2 }, + } }).map(({ event: mapped }) => mapped); + expect(goalEvents.map(({ type }) => type)).toEqual(["goalUpdated", "message", "goalCleared"]); + const [goalUpdated, receipt, goalCleared] = goalEvents; + if (!goalUpdated || !receipt || !goalCleared) return; + expect(reducer.reduce(goalUpdated)).toMatchObject({ kind: "unsupported", eventType: "goalUpdated" }); + expect(reducer.reduce(boundByProvider(goalUpdated))).toMatchObject({ kind: "reduced", writer: [], publication: { after: "terminal" } }); + expect(reducer.reduce(boundByProvider(receipt))).toEqual({ + kind: "unsupported", eventType: "message", reason: "post-turn goal receipt needs the goal message owner", + }); + expect(reducer.reduce(boundByProvider(goalCleared))).toMatchObject({ kind: "reduced", writer: [], publication: { after: "terminal" } }); + expect(reducer.reduce(event("ended", { turnExecutionId: execution.executionId }))).toMatchObject({ kind: "reduced" }); + }); + + it("rejects another execution and unowned events without changing the active execution", () => { + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + expect(reducer.reduce(event("textDelta", { delta: "wrong", turnExecutionId: "22222222-2222-4222-8222-222222222222" }))).toEqual({ + kind: "unsupported", eventType: "textDelta", reason: "different execution", + }); + for (const status of [ + event("modelFallback", { requestedModel: "a", actualModel: "b" }), + event("toolInputDelta", { partialJson: "{" }), + event("toolProgress", { toolCallId: "tool-1", toolName: "command_execution", elapsedSeconds: 1 }), + event("providerUnavailable", { providerId: "codex", reason: "disabled" }), + event("hookProgress", { hookName: "check", output: "running" }), + ]) { + expect(reducer.reduce(status)).toMatchObject({ + kind: "reduced", writer: [], publication: { event: status, after: "writer" }, + }); + } + const final = reducer.reduce(event("textDelta", { delta: "right", isFinalResponse: true })); + expect(final.kind).toBe("reduced"); + if (final.kind !== "reduced") return; + expect(final.writer).toContainEqual({ kind: "assistant-text-delta", delta: "right", classification: "final" }); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-file-evidence-coordinator.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-file-evidence-coordinator.test.ts new file mode 100644 index 000000000..b664ced5e --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-file-evidence-coordinator.test.ts @@ -0,0 +1,131 @@ +import "reflect-metadata"; +import * as NodeFSPromises from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { ProviderFileMutationStart } from "@mcode/contracts"; + +import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { RealGitExecutor } from "../../../projects/git/execution/real-git-executor.js"; +import { SnapshotService } from "../../../projects/diffs/snapshots/snapshot-service.js"; +import { TurnDiffRepo } from "../../turns/persistence/turn-diff-repo.js"; +import { TurnDiffService } from "../../turns/turn-diff-service.js"; +import { TurnFileTracker } from "../../turns/turn-file-tracker.js"; +import { + ExecutionFileEvidenceCoordinator, + prepareFrozenExecutionFileEvidence, +} from "../execution-file-evidence-coordinator.js"; + +const directories: string[] = []; +const databases: Database[] = []; +const input = { + threadId: "thread", turnId: "turn", executionId: "execution", + deliveryAttempt: 1, baselineRef: null, +}; + +async function directory(): Promise { + const path = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-worker-files-")); + directories.push(path); + return path; +} + +function tracker(): TurnFileTracker { + return new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, "win32"); +} + +function diffs(): TurnDiffService { + const db = openMemoryDatabase(); + databases.push(db); + return new TurnDiffService(new TurnDiffRepo(db)); +} + +function mutation(executionId = input.executionId, deliveryAttempt = input.deliveryAttempt): ProviderFileMutationStart { + return { threadId: input.threadId, turnExecutionId: executionId, deliveryAttempt, + toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; +} + +function toolUse(event: ProviderFileMutationStart) { + return { type: "toolUse" as const, threadId: event.threadId, + turnExecutionId: event.turnExecutionId, toolCallId: event.toolCallId, + toolName: event.toolName, toolInput: event.toolInput }; +} + +afterEach(async () => { + for (const db of databases.splice(0)) db.close(); + await Promise.all(directories.splice(0).map((path) => NodeFSPromises.rm(path, { recursive: true, force: true }))); +}); + +describe("ExecutionFileEvidenceCoordinator", () => { + it("transfers pre-edit evidence and settles only the sealed execution", async () => { + const cwd = await directory(); + const file = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(file, "before\n"); + const host = tracker(); + const worker = tracker(); + const native = diffs(); + const coordinator = new ExecutionFileEvidenceCoordinator(host, native); + const handoff = coordinator.begin({ ...input, cwd }); + expect(worker.beginTurnFromHandoff(structuredClone(handoff), { + threadId: input.threadId, executionId: input.executionId, cwd, + })).toBe(true); + const event = mutation(); + + expect(coordinator.capture(event)).toBe(true); + await NodeFSPromises.writeFile(file, "after\nextra\n"); + const captured = coordinator.take(toolUse(event), input.deliveryAttempt); + if (!captured) throw new Error("Expected pre-edit file evidence"); + expect(await worker.observeCapturedToolUse(toolUse(event), structuredClone(captured))).toBe(true); + await worker.observeToolResult(input.threadId, event.toolCallId); + const patch = "diff --git a/tracked.txt b/tracked.txt\nindex 1..2\n--- a/tracked.txt\n+++ b/tracked.txt\n@@ -1 +1,2 @@\n-before\n+after\n+extra\n"; + expect(native.push({ + turnId: input.turnId, turnExecutionId: input.executionId, + deliveryAttempt: input.deliveryAttempt, revision: 1, + state: "snapshot", nativeFidelity: "agent", patch, + })).toBe("accepted"); + const frozen = coordinator.seal({ ...input, outcome: "completed" }); + if (!frozen) throw new Error("Expected frozen file evidence"); + expect(() => structuredClone(frozen)).not.toThrow(); + expect(coordinator.seal({ ...input, outcome: "completed" })).toBe(frozen); + expect(coordinator.capture(event)).toBe(false); + + const prepared = await prepareFrozenExecutionFileEvidence( + structuredClone(frozen), worker, new SnapshotService(new RealGitExecutor()), + ); + expect(prepared).toMatchObject({ + threadId: input.threadId, turnId: input.turnId, executionId: input.executionId, + deliveryAttempt: input.deliveryAttempt, filesChanged: ["tracked.txt"], + fileEffects: { fileCount: 1, additions: 2, deletions: 1 }, + selectedTurnDiff: { thread_id: input.threadId, source: "native", patch, revision: 1 }, + }); + expect(() => structuredClone(prepared)).not.toThrow(); + expect(coordinator.retire(input.threadId, input.executionId, input.deliveryAttempt)).toBe(true); + expect(await host.finalEvidenceForExecution(handoff)).toBeNull(); + }); + + it("fences Stop, retries, and late terminal callbacks by execution and attempt", async () => { + const cwd = await directory(); + await NodeFSPromises.writeFile(NodePath.join(cwd, "tracked.txt"), "before\n"); + const coordinator = new ExecutionFileEvidenceCoordinator(tracker(), diffs()); + coordinator.begin({ ...input, cwd }); + expect(() => coordinator.begin({ ...input, deliveryAttempt: 0, cwd })).toThrow("positive delivery attempt"); + expect(coordinator.capture(mutation())).toBe(true); + expect(coordinator.fence(input.threadId, input.executionId, 2)).toBe(false); + expect(coordinator.fence(input.threadId, input.executionId, 1)).toBe(true); + expect(coordinator.take(toolUse(mutation()), 1)).toBeNull(); + expect(coordinator.seal({ ...input, deliveryAttempt: 2, outcome: "cancelled" })).toBeNull(); + const stopped = coordinator.seal({ ...input, outcome: "cancelled" }); + expect(stopped?.outcome).toBe("cancelled"); + expect(() => coordinator.seal({ ...input, outcome: "completed" })).toThrow("outcome changed"); + expect(() => coordinator.begin({ ...input, deliveryAttempt: 2, cwd })).toThrow("Retire sealed"); + expect(coordinator.retire(input.threadId, input.executionId, 1)).toBe(true); + + coordinator.begin({ ...input, deliveryAttempt: 2, cwd }); + expect(coordinator.capture(mutation(input.executionId, 1))).toBe(false); + expect(coordinator.capture(mutation(input.executionId, 2))).toBe(true); + expect(coordinator.seal({ ...input, outcome: "cancelled" })).toBeNull(); + expect(coordinator.retire(input.threadId, input.executionId, 1)).toBe(false); + expect(coordinator.retire(input.threadId, input.executionId, 2)).toBe(true); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-file-observation-handoff.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-file-observation-handoff.test.ts new file mode 100644 index 000000000..19f8546f0 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-file-observation-handoff.test.ts @@ -0,0 +1,116 @@ +import * as NodeFSPromises from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { ProviderFileMutationStart } from "@mcode/contracts"; + +import { TurnFileTracker } from "../../turns/turn-file-tracker.js"; +import { ExecutionFileObservationHandoff } from "../execution-file-observation-handoff.js"; + +const directories: string[] = []; +const platform = "win32" as const; + +async function directory(): Promise { + const path = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-worker-file-observation-")); + directories.push(path); + return path; +} + +function tracker(): TurnFileTracker { + return new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, platform); +} + +function mutation(executionId: string, toolCallId = "edit", deliveryAttempt = 1): ProviderFileMutationStart { + return { threadId: "thread", turnExecutionId: executionId, deliveryAttempt, + toolCallId, toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; +} + +function toolUse(event: ProviderFileMutationStart) { + return { type: "toolUse" as const, threadId: event.threadId, + turnExecutionId: event.turnExecutionId, toolCallId: event.toolCallId, + toolName: event.toolName, toolInput: event.toolInput }; +} + +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => NodeFSPromises.rm(path, { recursive: true, force: true }))); +}); + +describe("ExecutionFileObservationHandoff", () => { + it("transfers a pre-edit baseline after the provider has changed the file", async () => { + const cwd = await directory(); + const path = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(path, "before\n"); + const host = tracker(); + const worker = tracker(); + const expected = { threadId: "thread", executionId: "current", cwd }; + const handoff = host.beginExecutionTurn({ ...expected, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, handoff, 1); + const event = mutation(expected.executionId); + + expect(bridge.capture(event)).toBe(true); + await NodeFSPromises.writeFile(path, "after\nextra\n"); + expect(worker.beginTurnFromHandoff(structuredClone(handoff), expected)).toBe(true); + const captured = bridge.take(toolUse(event), 1); + if (!captured) throw new Error("Expected the pre-edit observation"); + expect(await worker.observeCapturedToolUse(toolUse(event), structuredClone(captured))).toBe(true); + await worker.observeToolResult("thread", "edit"); + expect(await worker.finalizeTurn("thread")).toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); + expect(bridge.take(toolUse(event), 1)).toBeNull(); + }); + + it("rejects an old child after retry even when its tool call ID collides", async () => { + const cwd = await directory(); + await NodeFSPromises.writeFile(NodePath.join(cwd, "tracked.txt"), "before\n"); + const host = tracker(); + const old = host.beginExecutionTurn({ threadId: "thread", executionId: "same", cwd, baselineRef: null }); + const oldBridge = new ExecutionFileObservationHandoff(host, old, 1); + expect(oldBridge.capture(mutation("same"))).toBe(true); + const current = host.beginExecutionTurn({ threadId: "thread", executionId: "same", cwd, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, current, 2); + + expect(oldBridge.take(toolUse(mutation("same")), 1)).toBeNull(); + expect(bridge.capture(mutation("same"))).toBe(false); + expect(bridge.capture(mutation("same", "edit", 2))).toBe(true); + expect(bridge.take(toolUse(mutation("same")), 1)).toBeNull(); + expect(bridge.take(toolUse(mutation("same", "edit", 2)), 2)).toMatchObject({ + executionId: "same", generation: current.generation, + }); + }); + + it("fences captures and pending evidence after Stop", async () => { + const cwd = await directory(); + const host = tracker(); + const handoff = host.beginExecutionTurn({ threadId: "thread", executionId: "current", cwd, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, handoff, 1); + const event = mutation("current"); + expect(bridge.capture(event)).toBe(true); + bridge.close(); + expect(bridge.take(toolUse(event), 1)).toBeNull(); + expect(bridge.capture(mutation("current", "later"))).toBe(false); + }); + + it("discards a colliding tool call without replacing its first pre-edit baseline", async () => { + const cwd = await directory(); + const host = tracker(); + const handoff = host.beginExecutionTurn({ threadId: "thread", executionId: "current", cwd, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, handoff, 1); + const event = mutation("current"); + expect(bridge.capture(event)).toBe(true); + expect(bridge.capture(event)).toBe(false); + expect(bridge.take(toolUse(event), 1)).toBeNull(); + }); + + it("bounds pending observations and rejects a mutation without an exact attempt", async () => { + const cwd = await directory(); + const host = tracker(); + const handoff = host.beginExecutionTurn({ threadId: "thread", executionId: "current", cwd, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, handoff, 1); + expect(bridge.capture({ ...mutation("current"), turnExecutionId: undefined })).toBe(false); + expect(bridge.capture({ ...mutation("current"), deliveryAttempt: undefined })).toBe(false); + for (let index = 0; index < 8; index += 1) { + expect(bridge.capture(mutation("current", `edit-${index}`))).toBe(true); + } + expect(bridge.capture(mutation("current", "overflow"))).toBe(false); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts new file mode 100644 index 000000000..349af331a --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from "vitest"; + +import { ExecutionMailboxOwner } from "../execution-mailbox-owner.js"; +import { ExecutionProviderEventOwnership } from "../execution-provider-event-ownership.js"; +import { ExecutionMailboxScheduler, type ExecutionMailboxCommand } from "../execution-mailbox-scheduler.js"; +import type { ExecutionIdentity, ExecutionWorkerPort, ExecutionWorkerReply, ExecutionWorkerRequest } from "../execution-mailbox-protocol.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution-worker-handler.js"; + +type Command = ExecutionMailboxCommand; + +class WorkerPort implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; + readonly requests: ExecutionWorkerRequest[] = []; + + postMessage(request: ExecutionWorkerRequest): void { this.requests.push(request); } + terminate(): void {} + + reply(index: number, result: ExecutionWorkerResult): void { + const request = this.requests[index]; + if (!request) throw new Error(`Missing request ${index}`); + this.onmessage?.(new MessageEvent("message", { data: { ...request, result } })); + } +} + +const execution: ExecutionIdentity = { threadId: "thread-1", turnId: "turn-1", executionId: "execution-1" }; +const parentTurn = { + thread: { id: execution.threadId, workspaceId: "workspace-1", providerId: "codex", createdAt: "2026-09-24T00:00:00.000Z" }, + turnId: execution.turnId, + executionId: execution.executionId, + permissionMode: "supervised" as const, + providerIdentities: [], + userMessage: { kind: "create" as const, messageId: "user-1", content: "Run", sequence: 1 }, +}; + +describe("ExecutionMailboxOwner", () => { + it("claims before durable start and admits Stop behind earlier commands", async () => { + const worker = new WorkerPort(); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 8, maxPendingBytes: 128_000, reservedControl: 2, reservedControlBytes: 16_000, + maxPerExecutionPending: 8, maxPerExecutionBytes: 128_000, + reservedPerExecutionControl: 2, reservedPerExecutionControlBytes: 16_000, + }, + createWorker: () => worker, + onWorkerLost: () => {}, + }); + const owner = new ExecutionMailboxOwner(scheduler); + const starting = owner.start({ execution, ownerEpoch: 1, providerId: "codex", parentTurn }); + expect(worker.requests[0]?.command.kind).toBe("start"); + expect(owner.current(execution.threadId)?.execution).toEqual(execution); + await expect(owner.start({ execution, ownerEpoch: 2, providerId: "codex", parentTurn })) + .rejects.toThrow("already has an execution owner"); + worker.reply(0, { kind: "committed", operationId: `${worker.requests[0]!.lease.leaseId}:1`, durableRevision: 1 }); + await starting; + + const checkpoint = owner.submit(execution, { kind: "checkpoint", phase: "running", nativeCursor: null }); + const stopped = owner.stop(execution, "stop-1"); + expect(worker.requests[1]?.command.kind).toBe("checkpoint"); + await expect(owner.submit(execution, { kind: "assistant-text", inputs: [] })) + .rejects.toThrow("stopping"); + expect(() => owner.submit({ ...execution, executionId: "stale" }, { kind: "checkpoint", phase: "running", nativeCursor: null })) + .toThrow("No matching execution owner"); + worker.reply(1, { kind: "committed", operationId: `${worker.requests[1]!.lease.leaseId}:2`, durableRevision: 1 }); + expect(worker.requests[2]).toMatchObject({ stopWatermark: 2, command: { kind: "stop", requestId: "stop-1" } }); + worker.reply(2, { kind: "committed", operationId: `${worker.requests[2]!.lease.leaseId}:3`, durableRevision: 1 }); + await checkpoint; + await stopped; + scheduler.shutdown(); + }); +}); + +describe("ExecutionProviderEventOwnership", () => { + it("submits an exact attempt to its mailbox and rejects retired or superseded attempts", async () => { + const worker = new WorkerPort(); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 8, maxPendingBytes: 128_000, reservedControl: 2, reservedControlBytes: 16_000, + maxPerExecutionPending: 8, maxPerExecutionBytes: 128_000, + reservedPerExecutionControl: 2, reservedPerExecutionControlBytes: 16_000, + }, + createWorker: () => worker, + onWorkerLost: () => {}, + }); + const owner = new ExecutionMailboxOwner(scheduler); + const ownership = new ExecutionProviderEventOwnership(owner); + const acknowledged: string[] = []; + let reportAcknowledgement: (() => void) | undefined; + const acknowledgementStarted = new Promise((resolve) => { reportAcknowledgement = resolve; }); + let finishAcknowledgement: (() => void) | undefined; + const acknowledgement = new Promise((resolve) => { finishAcknowledgement = resolve; }); + ownership.bindCommitted(async (owned, result) => { + acknowledged.push(`${owned.executionId}:${result.operationId}`); + reportAcknowledgement?.(); + await acknowledgement; + }); + const prepared: string[] = []; + ownership.bindCommandPreparation(async (owned, batch) => { + prepared.push(`${owned.executionId}:${batch.deliveryAttempt}`); + return { kind: "event", phase: batch.phase, nativeCursor: batch.nativeCursor ?? null, events: batch.events }; + }); + expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); + const start = owner.start({ execution, ownerEpoch: 1, providerId: "codex", parentTurn }); + await expect(ownership.bind(execution, 1)).rejects.toThrow("no matching execution owner"); + worker.reply(0, { kind: "committed", operationId: `${worker.requests[0]!.lease.leaseId}:1`, durableRevision: 1 }); + await start; + await ownership.bind(execution, 1); + const route = ownership.resolve(execution.executionId); + if (route.kind !== "worker") throw new Error("Expected worker route"); + const batch = { + threadId: execution.threadId, turnId: execution.turnId, executionId: execution.executionId, + batchId: "batch-1", deliveryAttempt: 1, phase: "running", events: [], + }; + const submitted = route.submit(batch); + await Promise.resolve(); + expect(worker.requests[1]?.command).toMatchObject({ kind: "event", phase: "running", events: [] }); + let nextAttemptBound = false; + const nextAttempt = ownership.bind(execution, 2).then(() => { nextAttemptBound = true; }); + expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); + await expect(route.submit(batch)).rejects.toThrow("retired execution attempt"); + worker.reply(1, { + kind: "committed", operationId: `${worker.requests[1]!.lease.leaseId}:2`, durableRevision: 1, + providerCommit: { + outcome: "committed", conversationRevision: 1, rosterRevision: 0, + acceptedThrough: 1, durableThrough: 1, eventCount: 0, + }, + }); + await acknowledgementStarted; + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(nextAttemptBound).toBe(false); + finishAcknowledgement?.(); + await expect(submitted).resolves.toMatchObject({ batchId: "batch-1", + deliveryAttempt: 1, commit: { outcome: "committed" } }); + expect(acknowledged).toEqual([`${execution.executionId}:${worker.requests[1]!.lease.leaseId}:2`]); + expect(prepared).toEqual([`${execution.executionId}:1`]); + + await nextAttempt; + expect(ownership.resolve(execution.executionId)).toMatchObject({ kind: "worker", deliveryAttempt: 2 }); + await ownership.retire(execution); + expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); + expect(ownership.resolve("legacy-execution", "claude")).toEqual({ kind: "legacy" }); + expect(worker.requests).toHaveLength(2); + scheduler.shutdown(); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts new file mode 100644 index 000000000..99f214cc3 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -0,0 +1,385 @@ +import { describe, expect, it } from "vitest"; + +import { + ExecutionMailboxScheduler, + type ExecutionMailboxLimits, + type ExecutionLostAssignment, +} from "../execution-mailbox-scheduler.js"; +import type { + ExecutionIdentity, + ExecutionWorkerPort, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "../execution-mailbox-protocol.js"; + +type Work = + | { readonly kind: "start" } + | { readonly kind: "event"; readonly sequence: number } + | { readonly kind: "checkpoint"; readonly revision: number } + | { readonly kind: "effect-result"; readonly effectId: string } + | { readonly kind: "stage-terminal" } + | { readonly kind: "finish-from-state" } + | { readonly kind: "finalize" }; +type Command = Work | { readonly kind: "stop"; readonly requestId: string }; +type Result = { readonly revision: number }; + +const LIMITS: ExecutionMailboxLimits = { + maxPending: 8, + maxPendingBytes: 800, + reservedControl: 2, + reservedControlBytes: 200, + maxPerExecutionPending: 5, + maxPerExecutionBytes: 500, + reservedPerExecutionControl: 2, + reservedPerExecutionControlBytes: 200, +}; + +class FakeWorker implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; + readonly requests: ExecutionWorkerRequest[] = []; + terminated = false; + + postMessage(request: ExecutionWorkerRequest): void { + this.requests.push(request); + } + + terminate(): void { + this.terminated = true; + } + + reply(request: ExecutionWorkerRequest, revision: number): void { + this.onmessage?.(new MessageEvent("message", { + data: { ...request, result: { revision } }, + })); + } + + crash(): void { + this.onerror?.(new ErrorEvent("error")); + } + + close(): void { + this.onclose?.(); + } +} + +function execution(threadId: string, executionId = `execution-${threadId}`): ExecutionIdentity { + return { threadId, turnId: `turn-${threadId}`, executionId }; +} + +function fixture(workerCount = 1, limits = LIMITS) { + const workers: FakeWorker[] = []; + const lost: ExecutionLostAssignment[][] = []; + const scheduler = new ExecutionMailboxScheduler({ + workerCount, + limits, + createWorker: () => { + const worker = new FakeWorker(); + workers.push(worker); + return worker; + }, + onWorkerLost: (affected) => lost.push([...affected]), + }); + return { scheduler, workers, lost }; +} + +function claimed(scheduler: ExecutionMailboxScheduler, identity: ExecutionIdentity, epoch = 1) { + const claim = scheduler.claim(identity, epoch); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + return claim.lease; +} + +function admitted( + scheduler: ExecutionMailboxScheduler, + identity: ExecutionIdentity, + lease: ReturnType, + command: Command, + byteLength = 100, +) { + const admission = scheduler.submit({ execution: identity, lease, command, byteLength }); + if (admission.kind !== "admitted") throw new Error(`Admission failed: ${admission.kind}`); + return admission; +} + +function request(worker: FakeWorker, index: number): ExecutionWorkerRequest { + const next = worker.requests[index]; + if (!next) throw new Error(`Missing worker request ${index}`); + return next; +} + +describe("ExecutionMailboxScheduler", () => { + it("admits worker state finalization after the Stop watermark", async () => { + const { scheduler, workers } = fixture(); + const identity = execution("stopped"); + const lease = claimed(scheduler, identity); + const stop = admitted(scheduler, identity, lease, { kind: "stop", requestId: "stop" }); + const finish = admitted(scheduler, identity, lease, { kind: "finish-from-state" }); + const worker = workers[0]; + if (!worker) throw new Error("Expected worker"); + expect(worker.requests).toHaveLength(1); + worker.reply(request(worker, 0), 1); + expect(request(worker, 1).command.kind).toBe("finish-from-state"); + worker.reply(request(worker, 1), 2); + await expect(stop.completion).resolves.toMatchObject({ kind: "reply" }); + await expect(finish.completion).resolves.toMatchObject({ kind: "reply" }); + scheduler.shutdown(); + }); + + it("keeps an execution on one worker and processes each mailbox in order", async () => { + const { scheduler, workers } = fixture(2); + const first = execution("first"); + const second = execution("second"); + const third = execution("third"); + const firstLease = claimed(scheduler, first); + const secondLease = claimed(scheduler, second); + const thirdLease = claimed(scheduler, third); + expect(firstLease.workerIndex).toBe(0); + expect(secondLease.workerIndex).toBe(1); + expect(thirdLease.workerIndex).toBe(0); + + const firstStart = admitted(scheduler, first, firstLease, { kind: "start" }); + const firstEvent = admitted(scheduler, first, firstLease, { kind: "event", sequence: 1 }); + const thirdStart = admitted(scheduler, third, thirdLease, { kind: "start" }); + const secondStart = admitted(scheduler, second, secondLease, { kind: "start" }); + expect(workers[0]?.requests).toHaveLength(1); + expect(workers[1]?.requests).toHaveLength(1); + + workers[0]?.reply(request(workers[0], 0), 1); + expect(request(workers[0]!, 1).execution).toEqual(first); + workers[0]?.reply(request(workers[0]!, 1), 2); + expect(request(workers[0]!, 2).execution).toEqual(third); + workers[0]?.reply(request(workers[0]!, 2), 1); + workers[1]?.reply(request(workers[1]!, 0), 1); + + expect([firstStart.ordinal, firstEvent.ordinal, thirdStart.ordinal]).toEqual([1, 2, 1]); + expect(await firstEvent.completion).toEqual({ kind: "reply", result: { revision: 2 } }); + expect(await thirdStart.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(await secondStart.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(scheduler.release(first, firstLease)).toBe(true); + expect(scheduler.depth().pending).toBe(0); + scheduler.shutdown(); + }); + + it("reserves count and byte credits for control commands within one execution", async () => { + const { scheduler, workers } = fixture(); + const identity = execution("busy"); + const lease = claimed(scheduler, identity); + const first = admitted(scheduler, identity, lease, { kind: "event", sequence: 1 }); + const second = admitted(scheduler, identity, lease, { kind: "event", sequence: 2 }); + const third = admitted(scheduler, identity, lease, { kind: "event", sequence: 3 }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "event", sequence: 4 }, byteLength: 100 })) + .toEqual({ kind: "overloaded" }); + const stop = admitted(scheduler, identity, lease, { kind: "stop", requestId: "stop-1" }); + expect(scheduler.depth()).toMatchObject({ pending: 4, pendingBytes: 400 }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "stop", requestId: "stop-2" }, byteLength: 100 })) + .toEqual({ kind: "stop-already-requested" }); + const checkpoint = admitted(scheduler, identity, lease, { kind: "checkpoint", revision: 1 }); + expect(scheduler.depth()).toMatchObject({ pending: 5, pendingBytes: 500 }); + + for (let index = 0; index < 5; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); + expect((await stop.completion).kind).toBe("reply"); + expect((await checkpoint.completion).kind).toBe("reply"); + expect((await first.completion).kind).toBe("reply"); + expect((await second.completion).kind).toBe("reply"); + expect((await third.completion).kind).toBe("reply"); + expect(workers[0]?.requests.map((item) => item.command.kind)).toEqual(["event", "event", "event", "stop", "checkpoint"]); + scheduler.shutdown(); + }); + + it("closes ordinary admission at Stop's watermark while lifecycle controls still settle", async () => { + const { scheduler, workers } = fixture(); + const identity = execution("stopping"); + const lease = claimed(scheduler, identity); + const start = admitted(scheduler, identity, lease, { kind: "start" }); + const event = admitted(scheduler, identity, lease, { kind: "event", sequence: 1 }); + const stop = admitted(scheduler, identity, lease, { kind: "stop", requestId: "stop-now" }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "event", sequence: 2 }, byteLength: 100 })) + .toEqual({ kind: "stopping" }); + const checkpoint = admitted(scheduler, identity, lease, { kind: "checkpoint", revision: 2 }); + const effect = admitted(scheduler, identity, lease, { kind: "effect-result", effectId: "file-1" }); + expect(request(workers[0]!, 0).ordinal).toBe(1); + workers[0]?.reply(request(workers[0]!, 0), 1); + workers[0]?.reply(request(workers[0]!, 1), 2); + expect(request(workers[0]!, 2)).toMatchObject({ + ordinal: 3, + stopWatermark: 2, + command: { kind: "stop", requestId: "stop-now" }, + }); + workers[0]?.reply(request(workers[0]!, 2), 3); + expect(await stop.completion).toEqual({ kind: "reply", result: { revision: 3 } }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "event", sequence: 3 }, byteLength: 100 })) + .toEqual({ kind: "stopping" }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "stop", requestId: "again" }, byteLength: 100 })) + .toEqual({ kind: "stop-already-requested" }); + const staged = admitted(scheduler, identity, lease, { kind: "stage-terminal" }); + const finalize = admitted(scheduler, identity, lease, { kind: "finalize" }); + for (let index = 3; index < 7; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); + expect(await start.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(await event.completion).toEqual({ kind: "reply", result: { revision: 2 } }); + expect((await checkpoint.completion).kind).toBe("reply"); + expect((await effect.completion).kind).toBe("reply"); + expect((await staged.completion).kind).toBe("reply"); + expect((await finalize.completion).kind).toBe("reply"); + expect(scheduler.release(identity, lease)).toBe(true); + scheduler.shutdown(); + }); + + it("keeps a global byte reserve when several executions fill normal capacity", async () => { + const limits: ExecutionMailboxLimits = { + ...LIMITS, + maxPending: 10, + maxPendingBytes: 500, + reservedControl: 1, + reservedControlBytes: 100, + maxPerExecutionBytes: 500, + }; + const { scheduler } = fixture(1, limits); + const first = execution("first"); + const second = execution("second"); + const third = execution("third"); + const firstLease = claimed(scheduler, first); + const secondLease = claimed(scheduler, second); + const thirdLease = claimed(scheduler, third); + admitted(scheduler, first, firstLease, { kind: "event", sequence: 1 }, 200); + admitted(scheduler, second, secondLease, { kind: "event", sequence: 1 }, 200); + expect(scheduler.submit({ execution: third, lease: thirdLease, command: { kind: "start" }, byteLength: 100 })) + .toEqual({ kind: "overloaded" }); + admitted(scheduler, first, firstLease, { kind: "stop", requestId: "reserved" }, 100); + expect(scheduler.depth()).toMatchObject({ pending: 3, pendingBytes: 500 }); + expect(scheduler.submit({ execution: second, lease: secondLease, command: { kind: "stop", requestId: "full" }, byteLength: 100 })) + .toEqual({ kind: "overloaded" }); + scheduler.shutdown(); + }); + + it("lets another execution progress while Stop remains behind its own earlier events", async () => { + const { scheduler, workers } = fixture(); + const chatty = execution("chatty"); + const quiet = execution("quiet"); + const chattyLease = claimed(scheduler, chatty); + const quietLease = claimed(scheduler, quiet); + admitted(scheduler, chatty, chattyLease, { kind: "event", sequence: 1 }); + admitted(scheduler, chatty, chattyLease, { kind: "event", sequence: 2 }); + const stop = admitted(scheduler, chatty, chattyLease, { kind: "stop", requestId: "stop-chatty" }); + const quietEvent = admitted(scheduler, quiet, quietLease, { kind: "event", sequence: 1 }); + for (let index = 0; index < 4; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); + expect(workers[0]?.requests.map((item) => `${item.execution.threadId}:${item.command.kind}`)) + .toEqual(["chatty:event", "chatty:event", "quiet:event", "chatty:stop"]); + expect((await quietEvent.completion).kind).toBe("reply"); + expect((await stop.completion).kind).toBe("reply"); + scheduler.shutdown(); + }); + + it("revokes a crashed worker without replay and fences late replies from its generation", async () => { + const { scheduler, workers, lost } = fixture(); + const oldExecution = execution("same-thread", "old-execution"); + const oldLease = claimed(scheduler, oldExecution); + const first = admitted(scheduler, oldExecution, oldLease, { kind: "event", sequence: 1 }); + const queued = admitted(scheduler, oldExecution, oldLease, { kind: "event", sequence: 2 }); + const oldRequest = request(workers[0]!, 0); + workers[0]?.crash(); + expect(await first.completion).toEqual({ kind: "worker-lost" }); + expect(await queued.completion).toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[{ execution: oldExecution, lease: oldLease }]]); + expect(workers[0]?.terminated).toBe(true); + expect(scheduler.claim(execution("unavailable"), 2)).toEqual({ kind: "worker-unavailable" }); + expect(scheduler.submit({ execution: oldExecution, lease: oldLease, command: { kind: "stop", requestId: "stale" }, byteLength: 100 })) + .toEqual({ kind: "stale-execution" }); + + expect(scheduler.replaceWorker(0)).toBe(false); + expect(scheduler.claim(oldExecution, 2)).toEqual({ kind: "thread-busy" }); + const recovery = { kind: "conflict" as const, operationId: `${oldLease.leaseId}:worker-lost`, + recoveryState: "not-started" as const }; + expect(scheduler.reconcileLost(oldExecution, oldLease, { ...recovery, operationId: "wrong" })).toBe(false); + expect(scheduler.reconcileLost(oldExecution, { ...oldLease, ownerEpoch: 2 }, recovery)).toBe(false); + expect(scheduler.reconcileLost(oldExecution, oldLease, recovery)).toBe(true); + expect(scheduler.reconcileLost(oldExecution, oldLease, recovery)).toBe(false); + expect(scheduler.replaceWorker(0)).toBe(true); + expect(workers[1]?.requests).toHaveLength(0); + + const newExecution = oldExecution; + const newLease = claimed(scheduler, newExecution, 2); + expect(newLease.workerGeneration).toBeGreaterThan(oldLease.workerGeneration); + expect(scheduler.submit({ execution: newExecution, lease: oldLease, command: { kind: "start" }, byteLength: 100 })) + .toEqual({ kind: "stale-execution" }); + const newEvent = admitted(scheduler, newExecution, newLease, { kind: "event", sequence: 1 }); + let newEventSettled = false; + void newEvent.completion.then(() => { newEventSettled = true; }); + workers[0]?.reply(oldRequest, 99); + await Promise.resolve(); + expect(newEventSettled).toBe(false); + expect(scheduler.depth().pending).toBe(1); + expect(workers[1]?.terminated).toBe(false); + workers[1]?.reply(request(workers[1]!, 0), 1); + expect(await newEvent.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(scheduler.release(oldExecution, oldLease)).toBe(false); + scheduler.shutdown(); + }); + + it("revokes a worker whose reply claims a different execution", async () => { + const { scheduler, workers, lost } = fixture(); + const identity = execution("target"); + const lease = claimed(scheduler, identity); + const admission = admitted(scheduler, identity, lease, { kind: "start" }); + const sent = request(workers[0]!, 0); + workers[0]?.onmessage?.(new MessageEvent("message", { + data: { ...sent, execution: execution("other"), result: { revision: 1 } }, + })); + expect(await admission.completion).toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[{ execution: identity, lease }]]); + scheduler.shutdown(); + }); + + it("keeps a different worker usable after one worker crashes", async () => { + const { scheduler, workers, lost } = fixture(2); + const failed = execution("failed"); + const healthy = execution("healthy"); + const alsoFailed = execution("also-failed"); + const failedLease = claimed(scheduler, failed); + const healthyLease = claimed(scheduler, healthy); + const alsoFailedLease = claimed(scheduler, alsoFailed); + const failedEvent = admitted(scheduler, failed, failedLease, { kind: "event", sequence: 1 }); + const healthyEvent = admitted(scheduler, healthy, healthyLease, { kind: "event", sequence: 1 }); + const alsoFailedEvent = admitted(scheduler, alsoFailed, alsoFailedLease, { kind: "event", sequence: 1 }); + workers[0]?.crash(); + workers[1]?.reply(request(workers[1]!, 0), 1); + expect(await failedEvent.completion).toEqual({ kind: "worker-lost" }); + expect(await alsoFailedEvent.completion).toEqual({ kind: "worker-lost" }); + expect(await healthyEvent.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(lost).toEqual([[ + { execution: failed, lease: failedLease }, + { execution: alsoFailed, lease: alsoFailedLease }, + ]]); + const another = execution("another"); + expect(claimed(scheduler, another).workerIndex).toBe(1); + scheduler.shutdown(); + }); + + it("revokes retained commands when a worker exits without an error", async () => { + const { scheduler, workers, lost } = fixture(); + const identity = execution("closed"); + const lease = claimed(scheduler, identity); + const event = admitted(scheduler, identity, lease, { kind: "event", sequence: 1 }); + workers[0]?.close(); + expect(await event.completion).toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[{ execution: identity, lease }]]); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 1 }); + expect(scheduler.reconcileLost(identity, lease, { + kind: "conflict", operationId: `${lease.leaseId}:worker-lost`, recoveryState: "not-started", + })).toBe(true); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); + scheduler.shutdown(); + }); + + it("rejects a second active execution on the same thread and settles shutdown", async () => { + const { scheduler } = fixture(); + const identity = execution("one", "first"); + const lease = claimed(scheduler, identity); + expect(scheduler.claim(execution("one", "second"), 2)).toEqual({ kind: "thread-busy" }); + const admission = admitted(scheduler, identity, lease, { kind: "start" }); + expect(scheduler.release(identity, lease)).toBe(false); + scheduler.shutdown(); + expect(await admission.completion).toEqual({ kind: "shutdown" }); + expect(scheduler.depth()).toMatchObject({ pending: 0, pendingBytes: 0, activeExecutions: 0 }); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts new file mode 100644 index 000000000..fb76f1d62 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts @@ -0,0 +1,161 @@ +import type { ProviderEventDraft } from "@mcode/providers"; +import { describe, expect, it } from "vitest"; + +import type { + DataOnlyParentTurnFinishInput, + DataOnlyParentTurnStartInput, +} from "../../canonical/canonical-parent-turn-write.js"; +import { + ExecutionMailboxScheduler, + type ExecutionMailboxLimits, + type ExecutionLostAssignment, +} from "../execution-mailbox-scheduler.js"; +import type { ExecutionIdentity, ExecutionLease } from "../execution-mailbox-protocol.js"; +import { ExecutionThreadWorkerPort } from "../execution-worker-port.js"; +import type { + ExecutionSemanticOperation, + ExecutionSemanticWriter, + ExecutionWorkCommand, + ExecutionWorkerResult, + ExecutionWriteReceipt, +} from "../execution-worker-handler.js"; + +const EXECUTION: ExecutionIdentity = { + threadId: "real-worker-thread", + turnId: "real-worker-turn", + executionId: "00000000-0000-4000-8000-000000000002", +}; + +const START_INPUT: DataOnlyParentTurnStartInput = { + thread: { id: EXECUTION.threadId, workspaceId: "fixture-workspace", providerId: "codex", createdAt: "2026-09-24T12:00:00.000Z" }, + turnId: EXECUTION.turnId, + executionId: EXECUTION.executionId, + permissionMode: "full", + providerIdentities: [], + userMessage: { kind: "create", content: "Test", sequence: 1 }, +}; + +const FINISH_INPUT: DataOnlyParentTurnFinishInput = { + threadId: EXECUTION.threadId, + turnId: EXECUTION.turnId, + executionId: EXECUTION.executionId, + providerId: "codex", + providerIdentities: [], + outcome: "cancelled", + projection: { message: null, narrative: [] }, +}; + +const LIMITS: ExecutionMailboxLimits = { + maxPending: 8, + maxPendingBytes: 8_000, + reservedControl: 4, + reservedControlBytes: 4_000, + maxPerExecutionPending: 6, + maxPerExecutionBytes: 6_000, + reservedPerExecutionControl: 3, + reservedPerExecutionControlBytes: 3_000, +}; + +class AcknowledgingWriter implements ExecutionSemanticWriter { + readonly operations: ExecutionSemanticOperation[] = []; + + async transact(operation: ExecutionSemanticOperation): Promise { + this.operations.push(operation); + return { kind: "committed", operationId: operation.operationId, durableRevision: this.operations.length }; + } +} + +function fixture(writer: ExecutionSemanticWriter, workerUrl?: URL) { + const ports: ExecutionThreadWorkerPort[] = []; + const lost: ExecutionLostAssignment[][] = []; + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: LIMITS, + createWorker: () => { + const port = new ExecutionThreadWorkerPort(writer, workerUrl); + ports.push(port); + return port; + }, + onWorkerLost: (executions) => lost.push([...executions]), + }); + const claim = scheduler.claim(EXECUTION, 1); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + return { scheduler, port: ports[0]!, lost, lease: claim.lease }; +} + +function submit( + scheduler: ExecutionMailboxScheduler, + lease: ExecutionLease, + command: Parameters[0]["command"], +) { + const admission = scheduler.submit({ execution: EXECUTION, lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Admission failed: ${admission.kind}`); + return admission.completion; +} + +function eventDraft(): ProviderEventDraft { + return { + eventId: "real-worker-event-1", + routing: { ...EXECUTION, itemId: "real-worker-item-1" }, + sourceProviderId: "codex", + sourceIdentities: [], + sourceSequence: 1, + payload: { type: "turn.started", startedAt: "2026-09-24T12:00:00.000Z" }, + }; +} + +describe("ExecutionThreadWorkerPort", () => { + it("starts and closes the real Bun Worker", async () => { + const port = new ExecutionThreadWorkerPort(new AcknowledgingWriter()); + try { + await expect(port.whenReady()).resolves.toBe(true); + await expect(port.close()).resolves.toBe(true); + } finally { + port.terminate(); + } + }, 10_000); + + it("runs a complete synthetic execution through a real Worker and writer RPC", async () => { + const writer = new AcknowledgingWriter(); + const { scheduler, port, lease } = fixture(writer); + try { + await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 1 } }); + await expect(port.whenReady()).resolves.toBe(true); + await expect(submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 2 } }); + await expect(submit(scheduler, lease, { kind: "stop", requestId: "stop-real-worker" })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 3 } }); + await expect(submit(scheduler, lease, { kind: "checkpoint", phase: "stopping", nativeCursor: null })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 4 } }); + await expect(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled", input: FINISH_INPUT })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 5 } }); + await expect(submit(scheduler, lease, { kind: "release" })) + .resolves.toEqual({ kind: "reply", result: { kind: "released" } }); + expect(scheduler.release(EXECUTION, lease)).toBe(true); + expect(writer.operations.map((operation) => operation.mutation.kind)) + .toEqual(["begin", "append-events", "stop-requested", "checkpoint", "finish"]); + expect(writer.operations[2]?.mutation).toEqual({ + kind: "stop-requested", requestId: "stop-real-worker", lastAdmittedOrdinal: 2, + }); + } finally { + scheduler.shutdown(); + } + }, 10_000); + + it("treats a clean but unexpected Worker exit as ownership loss", async () => { + const exitWorkerUrl = new URL("./fixtures/exit.worker.ts", import.meta.url); + const { scheduler, port, lost, lease } = fixture(new AcknowledgingWriter(), exitWorkerUrl); + try { + await expect(port.whenReady()).resolves.toBe(true); + await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT })) + .resolves.toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[{ execution: EXECUTION, lease }]]); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 1 }); + expect(scheduler.claim(EXECUTION, 2)).toEqual({ kind: "thread-busy" }); + expect(scheduler.replaceWorker(0)).toBe(false); + } finally { + scheduler.shutdown(); + } + }, 10_000); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-file-evidence.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-file-evidence.test.ts new file mode 100644 index 000000000..b034deda9 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-file-evidence.test.ts @@ -0,0 +1,137 @@ +import * as NodeFSPromises from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { ProviderFileMutationStart } from "@mcode/contracts"; + +import { TurnFileTracker } from "../../turns/turn-file-tracker.js"; +import { ExecutionFileEvidenceCoordinator } from "../execution-file-evidence-coordinator.js"; +import { ExecutionWorkerFileEvidence } from "../execution-worker-file-evidence.js"; + +const directories: string[] = []; +const execution = { threadId: "thread", turnId: "turn", executionId: "execution" }; + +async function directory(): Promise { + const path = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-worker-file-settle-")); + directories.push(path); + return path; +} + +function coordinator() { + const captureTracker = new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, process.platform); + return new ExecutionFileEvidenceCoordinator(captureTracker, { + begin: () => {}, + takeFinalizationEvidence: () => null, + clearExecution: () => {}, + }); +} + +function mutation(deliveryAttempt = 1): ProviderFileMutationStart { + return { + threadId: execution.threadId, turnExecutionId: execution.executionId, + deliveryAttempt, toolCallId: "edit", toolName: "Edit", + toolInput: { file_path: "tracked.txt" }, + }; +} + +function toolUse(event: ProviderFileMutationStart) { + return { + type: "toolUse" as const, threadId: event.threadId, + turnExecutionId: event.turnExecutionId, toolCallId: event.toolCallId, + toolName: event.toolName, toolInput: event.toolInput, + }; +} + +function toolResult() { + return { + type: "toolResult" as const, threadId: execution.threadId, + turnExecutionId: execution.executionId, toolCallId: "edit", + output: "done", isError: false, + }; +} + +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => NodeFSPromises.rm(path, { recursive: true, force: true }))); +}); + +describe("ExecutionWorkerFileEvidence", () => { + it("settles a pre-edit capture on the worker after the tool result", async () => { + const cwd = await directory(); + const file = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(file, "before\n"); + const host = coordinator(); + const worker = new ExecutionWorkerFileEvidence(); + const handoff = host.begin({ ...execution, deliveryAttempt: 1, cwd, baselineRef: null }); + expect(worker.begin({ execution, deliveryAttempt: 1, cwd, handoff: structuredClone(handoff) })).toBe(true); + const event = mutation(); + expect(host.capture(event)).toBe(true); + await NodeFSPromises.writeFile(file, "after\nextra\n"); + const captured = host.take(toolUse(event), 1); + if (!captured) throw new Error("Expected captured pre-edit evidence"); + expect(await worker.observeToolUse({ execution, deliveryAttempt: 1, event: toolUse(event), + captured: structuredClone(captured) })).toBe(true); + expect(await worker.observeToolResult({ execution, deliveryAttempt: 1, event: toolResult() })) + .toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); + const frozen = host.seal({ ...execution, deliveryAttempt: 1, outcome: "completed" }); + if (!frozen) throw new Error("Expected frozen terminal evidence"); + + const prepared = await worker.settle(structuredClone(frozen)); + expect(prepared).toMatchObject({ + threadId: execution.threadId, turnId: execution.turnId, + executionId: execution.executionId, deliveryAttempt: 1, + filesChanged: ["tracked.txt"], fileEffects: { fileCount: 1, additions: 2, deletions: 1 }, + }); + expect(await worker.settle(structuredClone(frozen))).toBe(prepared); + expect(() => structuredClone(prepared)).not.toThrow(); + expect(worker.retire(execution, 1)).toBe(true); + expect(worker.retire(execution, 1)).toBe(false); + }); + + it("rejects a different root or attempt before it can change file evidence", async () => { + const cwd = await directory(); + const otherCwd = await directory(); + await NodeFSPromises.writeFile(NodePath.join(cwd, "tracked.txt"), "before\n"); + const host = coordinator(); + const worker = new ExecutionWorkerFileEvidence(); + const handoff = host.begin({ ...execution, deliveryAttempt: 2, cwd, baselineRef: null }); + expect(worker.begin({ execution, deliveryAttempt: 2, cwd: otherCwd, handoff })).toBe(false); + expect(worker.begin({ execution, deliveryAttempt: 2, cwd, handoff })).toBe(true); + const event = mutation(2); + expect(host.capture(event)).toBe(true); + const captured = host.take(toolUse(event), 2); + if (!captured) throw new Error("Expected captured pre-edit evidence"); + expect(await worker.observeToolUse({ execution, deliveryAttempt: 1, event: toolUse(event), captured })).toBe(false); + expect(await worker.observeToolResult({ execution, deliveryAttempt: 1, event: toolResult() })).toBeNull(); + const frozen = host.seal({ ...execution, deliveryAttempt: 2, outcome: "cancelled" }); + if (!frozen) throw new Error("Expected frozen terminal evidence"); + expect(await worker.settle({ ...frozen, deliveryAttempt: 1 })).toBeNull(); + expect(worker.retire(execution, 1)).toBe(false); + expect(worker.retire(execution, 2)).toBe(true); + }); + + it("settles an open file tool when Stop arrives before its result", async () => { + const cwd = await directory(); + const file = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(file, "before\n"); + const host = coordinator(); + const worker = new ExecutionWorkerFileEvidence(); + const handoff = host.begin({ ...execution, deliveryAttempt: 1, cwd, baselineRef: null }); + expect(worker.begin({ execution, deliveryAttempt: 1, cwd, handoff })).toBe(true); + const event = mutation(); + expect(host.capture(event)).toBe(true); + const captured = host.take(toolUse(event), 1); + if (!captured) throw new Error("Expected captured pre-edit evidence"); + expect(await worker.observeToolUse({ execution, deliveryAttempt: 1, event: toolUse(event), captured })) + .toBe(true); + await NodeFSPromises.writeFile(file, "after\n"); + + const frozen = host.seal({ ...execution, deliveryAttempt: 1, outcome: "cancelled" }); + if (!frozen) throw new Error("Expected frozen terminal evidence"); + expect(await worker.settle(frozen)).toMatchObject({ + fileEffects: { fileCount: 1, additions: 1, deletions: 1 }, + }); + expect(await worker.observeToolResult({ execution, deliveryAttempt: 1, event: toolResult() })) + .toBeNull(); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts new file mode 100644 index 000000000..d5686ac86 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -0,0 +1,418 @@ +import type { ProviderEventDraft } from "@mcode/providers"; +import { describe, expect, it } from "vitest"; + +import type { + DataOnlyParentTurnFinishInput, + DataOnlyParentTurnStartInput, +} from "../../canonical/canonical-parent-turn-write.js"; +import type { ParentNarrativeRecoveryCommit } from "../../turns/parent-turn-durability.js"; +import { + ExecutionMailboxScheduler, + type ExecutionMailboxCommand, + type ExecutionMailboxLimits, + type ExecutionLostAssignment, +} from "../execution-mailbox-scheduler.js"; +import type { + ExecutionIdentity, + ExecutionLease, + ExecutionWorkerPort, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "../execution-mailbox-protocol.js"; +import { + ExecutionWorkerHandler, + type ExecutionLivePublicationIntent, + type ExecutionSemanticOperation, + type ExecutionSemanticWriter, + type ExecutionWorkCommand, + type ExecutionWorkerResult, + type ExecutionWriteReceipt, +} from "../execution-worker-handler.js"; + +type Command = ExecutionMailboxCommand; + +const EXECUTION: ExecutionIdentity = { + threadId: "fixture-thread", + turnId: "fixture-turn", + executionId: "00000000-0000-4000-8000-000000000001", +}; + +const START_INPUT: DataOnlyParentTurnStartInput = { + thread: { id: EXECUTION.threadId, workspaceId: "fixture-workspace", providerId: "codex", createdAt: "2026-09-24T12:00:00.000Z" }, + turnId: EXECUTION.turnId, + executionId: EXECUTION.executionId, + permissionMode: "full", + providerIdentities: [], + userMessage: { kind: "create", content: "Test", sequence: 1 }, +}; + +const FINISH_INPUT: DataOnlyParentTurnFinishInput = { + threadId: EXECUTION.threadId, + turnId: EXECUTION.turnId, + executionId: EXECUTION.executionId, + providerId: "codex", + providerIdentities: [], + outcome: "cancelled", + projection: { message: null, narrative: [] }, +}; + +const NARRATIVE_INPUT: ParentNarrativeRecoveryCommit = { + executionId: EXECUTION.executionId, + items: [{ + kind: "narrationSegment", + record: { + id: "thought-1", message_id: "message-1", text: "Working", + started_at: "2026-09-24T12:00:00.000Z", ended_at: null, sort_order: 1, + }, + }], +}; + +const PUBLICATION = { + after: "writer", + event: { type: "turnStarted", threadId: EXECUTION.threadId }, +} satisfies ExecutionLivePublicationIntent; + +const TEXT_INPUT = { + ...EXECUTION, + sequence: 1, + text: "Hello", +}; + +const LIMITS: ExecutionMailboxLimits = { + maxPending: 12, + maxPendingBytes: 12_000, + reservedControl: 6, + reservedControlBytes: 6_000, + maxPerExecutionPending: 8, + maxPerExecutionBytes: 8_000, + reservedPerExecutionControl: 5, + reservedPerExecutionControlBytes: 5_000, +}; + +class RecordingWriter implements ExecutionSemanticWriter { + readonly operations: ExecutionSemanticOperation[] = []; + beforeCommit: ((operation: ExecutionSemanticOperation) => Promise) | undefined; + conflictKind: ExecutionSemanticOperation["mutation"]["kind"] | undefined; + + async transact(operation: ExecutionSemanticOperation): Promise { + this.operations.push(operation); + await this.beforeCommit?.(operation); + if (operation.mutation.kind === this.conflictKind) { + return { kind: "conflict", operationId: operation.operationId }; + } + return { kind: "committed", operationId: operation.operationId, durableRevision: this.operations.length }; + } +} + +class InlineWorker implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; + readonly requests: ExecutionWorkerRequest[] = []; + terminated = false; + + constructor(private readonly handler: ExecutionWorkerHandler) {} + + postMessage(request: ExecutionWorkerRequest): void { + this.requests.push(request); + void this.handler.handle(request) + .then((reply) => this.onmessage?.(new MessageEvent("message", { data: reply }))) + .catch(() => this.onerror?.(new ErrorEvent("error"))); + } + + terminate(): void { + this.terminated = true; + } +} + +function fixture(writer = new RecordingWriter()) { + const workers: InlineWorker[] = []; + const handlers: ExecutionWorkerHandler[] = []; + const lost: ExecutionLostAssignment[][] = []; + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: LIMITS, + createWorker: () => { + const handler = new ExecutionWorkerHandler(writer); + const worker = new InlineWorker(handler); + handlers.push(handler); + workers.push(worker); + return worker; + }, + onWorkerLost: (executions) => lost.push([...executions]), + }); + const claim = scheduler.claim(EXECUTION, 1); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + return { scheduler, worker: workers[0]!, handler: handlers[0]!, writer, lost, lease: claim.lease }; +} + +function submit( + scheduler: ExecutionMailboxScheduler, + lease: ExecutionLease, + command: Command, +) { + const admission = scheduler.submit({ execution: EXECUTION, lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Admission failed: ${admission.kind}`); + return admission; +} + +function eventDraft(): ProviderEventDraft { + return { + eventId: "fixture-event-1", + routing: { ...EXECUTION, itemId: "fixture-item-1" }, + sourceProviderId: "codex", + sourceIdentities: [], + sourceSequence: 1, + payload: { type: "turn.started", startedAt: "2026-09-24T12:00:00.000Z" }, + }; +} + +async function committed(admission: ReturnType, revision: number): Promise { + await expect(admission.completion).resolves.toMatchObject({ + kind: "reply", + result: { kind: "committed", durableRevision: revision }, + }); +} + +describe("ExecutionWorkerHandler through its scheduler", () => { + it("commits a canonical draft and its parent effects in one operation", async () => { + const { scheduler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await committed(submit(scheduler, lease, { + kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()], + parentLive: { text: { kind: "append", inputs: [TEXT_INPUT] }, narrative: NARRATIVE_INPUT }, + livePublication: [PUBLICATION], + }), 2); + expect(writer.operations[1]).toMatchObject({ + mutation: { + kind: "append-events", events: [eventDraft()], + parentLive: { text: { kind: "append", inputs: [TEXT_INPUT] }, narrative: NARRATIVE_INPUT }, + }, + livePublication: [PUBLICATION], + }); + expect(writer.operations).toHaveLength(2); + scheduler.shutdown(); + }); + + it("rejects a compound event with mismatched parent routing or publication count", async () => { + const { scheduler, handler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + for (const command of [ + { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()], + parentLive: { text: { kind: "append", inputs: [{ ...TEXT_INPUT, executionId: "other" }] } }, livePublication: [PUBLICATION] }, + { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()], + parentLive: { text: { kind: "unchanged" } }, livePublication: [] }, + ] as const) { + await expect(handler.handle({ + requestId: 2, execution: EXECUTION, lease, ordinal: 2, command, + })).resolves.toMatchObject({ + result: { kind: "rejected", reason: "invalid-text-routing" }, + }); + } + expect(writer.operations).toHaveLength(1); + scheduler.shutdown(); + }); + + it("commits one compound live event and forwards its publication receipt", async () => { + class PublicationWriter extends RecordingWriter { + override async transact(operation: ExecutionSemanticOperation): Promise { + const receipt = await super.transact(operation); + return receipt.kind === "committed" && operation.livePublication + ? { ...receipt, livePublication: operation.livePublication.map((intent) => ({ + ...intent, publicationId: `${operation.operationId}:0`, + })) } + : receipt; + } + } + + const { scheduler, writer, lease } = fixture(new PublicationWriter()); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + const admission = submit(scheduler, lease, { + kind: "live-event", text: { kind: "append", inputs: [TEXT_INPUT] }, + narrative: NARRATIVE_INPUT, publication: PUBLICATION, + }); + await expect(admission.completion).resolves.toMatchObject({ + kind: "reply", + result: { + kind: "committed", operationId: `${lease.leaseId}:2`, durableRevision: 2, + livePublication: [{ ...PUBLICATION, publicationId: `${lease.leaseId}:2:0` }], + }, + }); + expect(writer.operations).toHaveLength(2); + expect(writer.operations[1]).toEqual({ + operationId: `${lease.leaseId}:2`, execution: EXECUTION, lease, ordinal: 2, + mutation: { + kind: "live-event", text: { kind: "append", inputs: [TEXT_INPUT] }, narrative: NARRATIVE_INPUT, + }, + livePublication: [PUBLICATION], + }); + scheduler.shutdown(); + }); + + it("rejects live-event text or narrative for a different execution before writing", async () => { + const { scheduler, handler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + + for (const command of [ + { kind: "live-event", text: { kind: "append", inputs: [{ ...TEXT_INPUT, turnId: "other-turn" }] }, publication: PUBLICATION }, + { kind: "live-event", text: { kind: "promote", input: { ...TEXT_INPUT, executionId: "other-execution" } }, publication: PUBLICATION }, + ] as const) { + await expect(handler.handle({ + requestId: 2, execution: EXECUTION, lease, ordinal: 2, command, + })).resolves.toMatchObject({ + result: { kind: "rejected", reason: "invalid-text-routing" }, + }); + } + await expect(handler.handle({ + requestId: 2, execution: EXECUTION, lease, ordinal: 2, + command: { + kind: "live-event", text: { kind: "unchanged" }, + narrative: { ...NARRATIVE_INPUT, executionId: "other-execution" }, publication: PUBLICATION, + }, + })).resolves.toMatchObject({ + result: { kind: "rejected", reason: "invalid-narrative-routing" }, + }); + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual(["begin"]); + scheduler.shutdown(); + }); + + it("maps a running narrative delta to one fenced semantic operation", async () => { + const { scheduler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await committed(submit(scheduler, lease, { kind: "narrative-delta", input: NARRATIVE_INPUT }), 2); + + expect(writer.operations[1]).toMatchObject({ + operationId: `${lease.leaseId}:2`, execution: EXECUTION, lease, ordinal: 2, + mutation: { kind: "narrative-delta", input: NARRATIVE_INPUT }, + }); + scheduler.shutdown(); + }); + + it("rejects a narrative delta for another execution before it reaches the writer", async () => { + const { scheduler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await expect(submit(scheduler, lease, { + kind: "narrative-delta", input: { ...NARRATIVE_INPUT, executionId: "another-execution" }, + }).completion).resolves.toEqual({ + kind: "reply", result: { kind: "rejected", reason: "invalid-narrative-routing" }, + }); + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual(["begin"]); + scheduler.shutdown(); + }); + + it("rejects a narrative delta once the execution is stopping", async () => { + const { scheduler, handler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await committed(submit(scheduler, lease, { kind: "stop", requestId: "stop-1" }), 2); + const reply = await handler.handle({ + requestId: 3, execution: EXECUTION, lease, ordinal: 3, + command: { kind: "narrative-delta", input: NARRATIVE_INPUT }, + }); + expect(reply.result).toEqual({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual(["begin", "stop-requested"]); + scheduler.shutdown(); + }); + + it("runs one synthetic turn from start through Stop, checkpoint, and finalization", async () => { + const { scheduler, worker, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await committed(submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }), 2); + const stop = submit(scheduler, lease, { kind: "stop", requestId: "stop-1" }); + expect(scheduler.submit({ + execution: EXECUTION, + lease, + command: { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }, + byteLength: 1_000, + })).toEqual({ kind: "stopping" }); + await committed(stop, 3); + await committed(submit(scheduler, lease, { kind: "checkpoint", phase: "stopping", nativeCursor: "cursor-1" }), 4); + await committed(submit(scheduler, lease, { kind: "effect-result", effectId: "file-1", settled: true }), 5); + await committed(submit(scheduler, lease, { kind: "provider-outcome", outcome: "cancelled" }), 6); + await committed(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled", input: FINISH_INPUT }), 7); + await expect(submit(scheduler, lease, { kind: "release" }).completion).resolves.toEqual({ + kind: "reply", + result: { kind: "released" }, + }); + expect(scheduler.release(EXECUTION, lease)).toBe(true); + + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual([ + "begin", "append-events", "stop-requested", "checkpoint", "effect-result", "provider-outcome", "finish", + ]); + expect(writer.operations[2]?.mutation).toEqual({ + kind: "stop-requested", requestId: "stop-1", lastAdmittedOrdinal: 2, + }); + expect(worker.requests.map((request) => request.ordinal)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); + expect(worker.requests.every((request) => request.execution.executionId === EXECUTION.executionId + && request.lease.leaseId === lease.leaseId)).toBe(true); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); + scheduler.shutdown(); + }); + + it("does not acknowledge a worker command until its writer receipt arrives", async () => { + const writer = new RecordingWriter(); + let releaseWrite: (() => void) | undefined; + writer.beforeCommit = () => new Promise((resolve) => { releaseWrite = resolve; }); + const { scheduler, worker, lease } = fixture(writer); + const start = submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }); + const event = submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }); + await Promise.resolve(); + expect(worker.requests).toHaveLength(1); + expect(scheduler.depth()).toMatchObject({ pending: 2 }); + releaseWrite?.(); + await committed(start, 1); + expect(worker.requests).toHaveLength(2); + releaseWrite?.(); + await committed(event, 2); + scheduler.shutdown(); + }); + + it("rejects a start whose transaction input names another execution", async () => { + const { scheduler, writer, lease } = fixture(); + const input = { ...START_INPUT, executionId: "another-execution" }; + await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input }).completion) + .resolves.toEqual({ kind: "reply", result: { kind: "rejected", reason: "invalid-transition" } }); + expect(writer.operations).toEqual([]); + scheduler.shutdown(); + }); + + it("rejects finalization when its projected outcome disagrees with the command", async () => { + const { scheduler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + const input = { ...FINISH_INPUT, outcome: "completed" as const }; + await expect(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled", input }).completion) + .resolves.toEqual({ kind: "reply", result: { kind: "rejected", reason: "invalid-transition" } }); + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual(["begin"]); + scheduler.shutdown(); + }); + + it("rejects a writer conflict without reporting a durable command", async () => { + const writer = new RecordingWriter(); + writer.conflictKind = "append-events"; + const { scheduler, lease } = fixture(writer); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await expect(submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }).completion).resolves.toEqual({ + kind: "reply", + result: { kind: "rejected", reason: "writer-conflict" }, + }); + await expect(submit(scheduler, lease, { kind: "checkpoint", phase: "running", nativeCursor: null }).completion) + .resolves.toEqual({ kind: "reply", result: { kind: "rejected", reason: "out-of-order" } }); + scheduler.shutdown(); + }); + + it("revokes the execution when the writer fails instead of acknowledging its event", async () => { + const writer = new RecordingWriter(); + writer.beforeCommit = async (operation) => { + if (operation.mutation.kind === "append-events") throw new Error("database unavailable"); + }; + const { scheduler, worker, lost, lease } = fixture(writer); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await expect(submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }).completion) + .resolves.toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[{ execution: EXECUTION, lease }]]); + expect(worker.terminated).toBe(true); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 1 }); + expect(scheduler.claim(EXECUTION, 2)).toEqual({ kind: "thread-busy" }); + expect(scheduler.replaceWorker(0)).toBe(false); + scheduler.shutdown(); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts new file mode 100644 index 000000000..cb799595c --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts @@ -0,0 +1,302 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { CanonicalAgentWriterClient } from "../../canonical/canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../../canonical/canonical-execution-writer-port.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; +import type { ExecutionWorkerPort, ExecutionWorkerReply, ExecutionWorkerRequest } from "../execution-mailbox-protocol.js"; +import type { ExecutionLostAssignment, ExecutionMailboxCommand } from "../execution-mailbox-scheduler.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution-worker-handler.js"; +import { ExecutionThreadWorkerPort } from "../execution-worker-port.js"; +import { ExecutionWorkerLossCoordinator } from "../execution-worker-loss-coordinator.js"; + +const NOW = "2026-09-24T10:00:00.000Z"; +const execution = { + threadId: "lost-worker-thread", turnId: "lost-worker-turn", + executionId: "00000000-0000-4000-8000-000000000188", +} as const; +type Command = ExecutionMailboxCommand; + +const limits = { + maxPending: 8, maxPendingBytes: 8_000, reservedControl: 2, reservedControlBytes: 2_000, + maxPerExecutionPending: 6, maxPerExecutionBytes: 6_000, + reservedPerExecutionControl: 2, reservedPerExecutionControlBytes: 2_000, +}; + +class CrashingPort implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; + private readonly inner: ExecutionThreadWorkerPort; + + constructor(writer: CanonicalExecutionWriterPort) { + this.inner = new ExecutionThreadWorkerPort(writer); + this.inner.onmessage = (event) => this.onmessage?.(event); + this.inner.onerror = (event) => this.onerror?.(event); + this.inner.onclose = () => this.onclose?.(); + } + + postMessage(request: ExecutionWorkerRequest): void { + this.inner.postMessage(request); + } + + terminate(): void { + this.inner.terminate(); + } + + crash(): void { + this.inner.terminate(); + this.onclose?.(); + } +} + +describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { + let directory: string; + let db: Database; + let writer: CanonicalAgentWriterClient; + let coordinator: ExecutionWorkerLossCoordinator; + let workers: CrashingPort[]; + let published: string[]; + let recoveryIncidentIds: string[]; + let recoveredAssignments: ExecutionLostAssignment[]; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-lost-worker-host-")); + db = openDatabase({ dbPath: NodePath.join(directory, "app.sqlite") }); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("lost-worker-workspace", "Workspace", directory, NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(execution.threadId, "lost-worker-workspace", "Thread", "main", "codex", NOW, NOW); + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + published = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + recoveryIncidentIds = []; + recoveredAssignments = []; + workers = []; + coordinator = new ExecutionWorkerLossCoordinator({ + interruptWorkerLoss: (input) => { + recoveryIncidentIds.push(input.recoveryIncidentId); + return port.interruptWorkerLoss(input); + }, + }, { + workerCount: 1, + limits, + createWorker: () => { + const worker = new CrashingPort(port); + workers.push(worker); + return worker; + }, + }, (assignment) => recoveredAssignments.push(assignment)); + }); + + afterEach(async () => { + coordinator.scheduler.shutdown(); + await writer.close(); + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + async function start(coordinator: ExecutionWorkerLossCoordinator, identity = execution) { + const claim = coordinator.scheduler.claim(identity, 1); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + const command: ExecutionWorkCommand = { + kind: "start", providerId: "codex", + input: { + thread: { id: identity.threadId, workspaceId: "lost-worker-workspace", providerId: "codex", createdAt: NOW }, + turnId: identity.turnId, executionId: identity.executionId, + permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: `${identity.threadId}-user`, content: "Question", sequence: 1 }, + }, + }; + const admitted = coordinator.scheduler.submit({ execution: identity, lease: claim.lease, command, byteLength: 1_000 }); + if (admitted.kind !== "admitted") throw new Error(`Start not admitted: ${admitted.kind}`); + expect(await admitted.completion).toMatchObject({ kind: "reply", result: { kind: "committed" } }); + return claim.lease; + } + + it("carries committed start, tool narrative, live publication, and finish through both workers", async () => { + const claim = coordinator.scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + const send = async (command: ExecutionWorkCommand) => { + const admitted = coordinator.scheduler.submit({ execution, lease: claim.lease, command, + byteLength: Buffer.byteLength(JSON.stringify(command), "utf8") }); + if (admitted.kind !== "admitted") throw new Error(`Command not admitted: ${admitted.kind}`); + const completion = await admitted.completion; + if (completion.kind !== "reply" || completion.result.kind !== "committed") { + throw new Error(`Command did not commit: ${JSON.stringify(completion)}`); + } + return completion.result; + }; + const started = { type: "turnStarted" as const, threadId: execution.threadId, + turnExecutionId: execution.executionId }; + const begin = await send({ + kind: "start", providerId: "codex", + input: { + thread: { id: execution.threadId, workspaceId: "lost-worker-workspace", providerId: "codex", createdAt: NOW }, + turnId: execution.turnId, executionId: execution.executionId, + permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: `${execution.threadId}-user`, content: "Question", sequence: 1 }, + }, + livePublication: [{ after: "writer", event: started }], + }); + expect(begin.livePublication).toEqual([{ + publicationId: "1", after: "writer", event: started, + }]); + expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(execution.executionId, begin.operationId)) + .toMatchObject({ receipt_json: expect.stringContaining('"publicationId":"1"') }); + + const tool = { kind: "toolCall" as const, sequence: 2, sortOrder: 0, record: { + id: "transport-tool", message_id: "", parent_tool_call_id: null, + tool_name: "Read", input_summary: "CONTEXT.md", output_summary: "read", + status: "completed" as const, started_at: NOW, completed_at: NOW, sort_order: 0, + } }; + await send({ kind: "narrative-delta", input: { executionId: execution.executionId, items: [tool] } }); + expect(db.prepare("SELECT id FROM canonical_agent_items WHERE id = ?").get("toolCall:transport-tool")) + .toEqual({ id: "toolCall:transport-tool" }); + + const toolUse = { type: "toolUse" as const, threadId: execution.threadId, + turnExecutionId: execution.executionId, toolCallId: "transport-tool", toolName: "Read", + toolInput: { path: "CONTEXT.md" } }; + const event = await send({ kind: "event", phase: "running", nativeCursor: null, events: [{ + eventId: `${execution.executionId}:transport-tool`, + routing: { ...execution, itemId: "transport-tool" }, + sourceProviderId: "codex", sourceIdentities: [], sourceSequence: 1, + payload: { type: "item.recorded", item: { + id: "transport-tool", threadId: execution.threadId, turnId: execution.turnId, + kind: "tool-call", providerIdentities: [], + payload: { projection: "toolCall", toolName: "Read", path: "CONTEXT.md" }, + createdAt: NOW, updatedAt: NOW, + } }, + }], livePublication: [{ after: "writer", event: toolUse }] }); + expect(event.livePublication).toEqual([{ + publicationId: "2", after: "writer", event: toolUse, + }]); + expect(published).toContain(`${execution.executionId}:transport-tool`); + expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(execution.executionId, event.operationId)) + .toMatchObject({ receipt_json: expect.stringContaining('"publicationId":"2"') }); + + await send({ kind: "provider-outcome", outcome: "completed" }); + await send({ kind: "stage-terminal", input: { + threadId: execution.threadId, executionId: execution.executionId, + outcome: "completed", endedAt: NOW, + assistant: { content: "Done", model: null, attachments: [] }, narrative: [tool], + } }); + const ended = { type: "ended" as const, threadId: execution.threadId, + turnExecutionId: execution.executionId, outcome: "completed" as const }; + const finish = await send({ kind: "finalize", outcome: "completed", input: { + ...execution, providerId: "codex", providerIdentities: [], outcome: "completed", + projection: { kind: "writer-staged" }, + }, livePublication: [{ after: "terminal", event: ended }] }); + expect(finish.livePublication).toEqual([{ + publicationId: "3", after: "terminal", event: ended, + }]); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: "completed" }); + expect(new MessageRepo(db).listIncludingInternal(execution.threadId)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Done", outcome: "completed", is_internal: false, + })); + expect(db.prepare("SELECT id, status FROM tool_call_records WHERE id = ?").get("transport-tool")) + .toEqual({ id: "transport-tool", status: "completed" }); + expect(published.indexOf(`${execution.executionId}:transport-tool`)) + .toBeLessThan(published.indexOf(`${execution.executionId}:turn.completed`)); + + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: "completed" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'assistant' AND is_internal = 0") + .get(execution.threadId)).toEqual({ count: 1 }); + }); + + it("interrupts a crashed worker before replacing its slot", async () => { + const lease = await start(coordinator); + new ParentAssistantTextCheckpointService(db).appendChunk([{ ...execution, sequence: 1, text: "Partial answer" }]); + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(recoveredAssignments).toEqual([{ execution, lease }]); + expect(workers).toHaveLength(2); + expect(db.prepare("SELECT terminal_outcome, recovery_incident_id FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toMatchObject({ terminal_outcome: "interrupted", + recovery_incident_id: expect.stringMatching(/^worker-loss:[0-9a-f]{64}$/) }); + expect(new MessageRepo(db).listIncludingInternal(execution.threadId)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Partial answer", outcome: "interrupted", is_internal: false, + })); + expect(published).toContain(`${execution.executionId}:recovery-interrupted`); + expect(coordinator.scheduler.submit({ execution, lease, command: { + kind: "checkpoint", phase: "running", nativeCursor: null, + }, byteLength: 100 })).toEqual({ kind: "stale-execution" }); + expect(coordinator.scheduler.claim({ ...execution, executionId: "new-execution" }, 2).kind).toBe("claimed"); + }); + + it("keeps the slot fenced after a failed writer receipt until explicit retry", async () => { + const lease = await start(coordinator); + db.run(`CREATE TRIGGER fail_loss_receipt BEFORE INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:worker-lost' BEGIN SELECT RAISE(ABORT, 'loss receipt unavailable'); END`); + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toMatchObject({ kind: "failed", workerIndex: 0, + unresolved: [{ execution, lease }] }); + expect(workers).toHaveLength(1); + expect(coordinator.scheduler.claim(execution, 2)).toEqual({ kind: "thread-busy" }); + expect(coordinator.scheduler.claim({ threadId: "other", turnId: "other", executionId: "other" }, 2)) + .toEqual({ kind: "worker-unavailable" }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: null }); + db.run("DROP TRIGGER fail_loss_receipt"); + expect(await coordinator.retryFailed(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(workers).toHaveLength(2); + expect(recoveryIncidentIds).toHaveLength(2); + expect(recoveryIncidentIds[1]).toBe(recoveryIncidentIds[0]); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: "interrupted" }); + }); + + it("keeps a shared slot fenced until every lost task has writer evidence", async () => { + const second = { + threadId: "second-lost-thread", turnId: "second-lost-turn", + executionId: "00000000-0000-4000-8000-000000000189", + }; + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(second.threadId, "lost-worker-workspace", "Second", "main", "codex", NOW, NOW); + await start(coordinator); + await start(coordinator, second); + db.run(`CREATE TRIGGER fail_second_loss BEFORE INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:worker-lost' AND NEW.execution_id = '${second.executionId}' + BEGIN SELECT RAISE(ABORT, 'second loss receipt unavailable'); END`); + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toMatchObject({ kind: "failed", workerIndex: 0, + unresolved: [{ execution: second }] }); + expect(workers).toHaveLength(1); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: "interrupted" }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(second.executionId)).toEqual({ terminal_outcome: null }); + expect(coordinator.scheduler.claim(second, 2)).toEqual({ kind: "thread-busy" }); + db.run("DROP TRIGGER fail_second_loss"); + expect(await coordinator.retryFailed(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(workers).toHaveLength(2); + expect(recoveryIncidentIds).toHaveLength(3); + expect(recoveryIncidentIds[2]).toBe(recoveryIncidentIds[1]); + expect(recoveryIncidentIds[0]).not.toBe(recoveryIncidentIds[1]); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(second.executionId)).toEqual({ terminal_outcome: "interrupted" }); + const incidents = db.prepare("SELECT recovery_incident_id FROM canonical_agent_ingest_checkpoints WHERE execution_id IN (?, ?)") + .all(execution.executionId, second.executionId); + expect(new Set(incidents.map((row) => JSON.stringify(row))).size).toBe(2); + }); + + it("replaces an idle crashed slot using the callback's slot index", async () => { + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(workers).toHaveLength(2); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-parent-events.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-parent-events.test.ts new file mode 100644 index 000000000..e3da0eb25 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-parent-events.test.ts @@ -0,0 +1,297 @@ +import { AgentEventSchema, ProviderRuntimeEventSchema, type AgentEvent } from "@mcode/contracts"; +import type { ProviderEventDraft } from "@mcode/providers"; +import * as NodeFSPromises from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeOS from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { ExecutionWorkerHandler, type ExecutionSemanticOperation, type ExecutionSemanticWriter, + type ExecutionWorkCommand, type ExecutionWriteReceipt } from "../execution-worker-handler.js"; +import { TurnFileTracker } from "../../turns/turn-file-tracker.js"; + +const execution = { threadId: "thread", turnId: "turn", executionId: "11111111-1111-4111-8111-111111111111" }; +const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "lease-1" }; +const now = "2026-09-25T12:00:00.000Z"; +const directories: string[] = []; + +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => NodeFSPromises.rm(path, { recursive: true, force: true }))); +}); + +async function fileFixture() { + const cwd = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-handler-files-")); + directories.push(cwd); + const tracker = new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, process.platform); + const handoff = tracker.beginExecutionTurn({ ...execution, cwd, baselineRef: null }); + return { cwd, tracker, handoff }; +} + +function eventDraft(sequence: number, type: AgentEvent["type"], fields: Record = {}): ProviderEventDraft { + const event = AgentEventSchema().parse({ type, threadId: execution.threadId, turnExecutionId: execution.executionId, ...fields }); + const itemId = `item-${sequence}`; + return { eventId: `event-${sequence}`, routing: { ...execution, itemId }, sourceProviderId: "codex", sourceIdentities: [], + sourceSequence: sequence, payload: { type: "item.recorded", item: { + id: itemId, threadId: execution.threadId, turnId: execution.turnId, kind: "system", providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent: { event } }, createdAt: now, updatedAt: now, + } } }; +} + +function start(): Extract { + return { kind: "start", providerId: "codex", parentLive: { planFeature: "none", precedingMessageId: "user" }, + input: { thread: { id: execution.threadId, workspaceId: "workspace", providerId: "codex", createdAt: now }, + turnId: execution.turnId, executionId: execution.executionId, permissionMode: "full", providerIdentities: [], + userMessage: { kind: "create", messageId: "user", content: "Task", sequence: 1 } } }; +} + +class Writer implements ExecutionSemanticWriter { + readonly operations: ExecutionSemanticOperation[] = []; + fail = false; + throwFailure = false; + async transact(operation: ExecutionSemanticOperation): Promise { + this.operations.push(operation); + if (this.throwFailure) throw new Error("writer disconnected"); + return this.fail ? { kind: "conflict", operationId: operation.operationId } + : { kind: "committed", operationId: operation.operationId, durableRevision: this.operations.length }; + } +} + +function fixture() { + const writer = new Writer(); + const handler = new ExecutionWorkerHandler(writer); + const send = (ordinal: number, command: ExecutionWorkCommand) => handler.handle({ requestId: ordinal, execution, lease, ordinal, command }); + const event = (ordinal: number, draft: ProviderEventDraft) => send(ordinal, { + kind: "event", phase: "running", nativeCursor: null, events: [draft], + }); + const stop = (ordinal: number) => handler.handle({ requestId: ordinal, execution, lease, ordinal, + stopWatermark: ordinal - 1, command: { kind: "stop", requestId: "user-stop" } }); + return { writer, send, event, stop }; +} + +describe("execution worker parent event ownership", () => { + it.each(["codex", "claude", "cursor"] as const)("preserves %s partial text and narrative when Stop finishes from worker state", async (providerId) => { + const { writer, send, event, stop } = fixture(); + const command = start(); + await send(1, { ...command, providerId, input: { ...command.input, thread: { ...command.input.thread, providerId } } }); + const draft = (sequence: number, type: AgentEvent["type"], fields: Record = {}) => ({ + ...eventDraft(sequence, type, fields), sourceProviderId: providerId, + }); + await event(2, draft(1, "turnStarted")); + await event(3, draft(2, "textDelta", { delta: "Checking files", isFinalResponse: false })); + await event(4, draft(3, "textDelta", { delta: "Partial unclassified answer" })); + expect((await stop(5)).result.kind).toBe("committed"); + const reply = await send(6, { kind: "finish-from-state", outcome: "cancelled", + input: { ...execution, providerId, providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" } } }); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations).toHaveLength(6); + expect(writer.operations[5]).toMatchObject({ mutation: { kind: "finish-live-event", outcome: "cancelled", + projection: { assistant: { content: "Partial unclassified answer" }, narrative: [ + expect.objectContaining({ kind: "narrationSegment", record: expect.objectContaining({ text: "Checking files" }) }), + ] } }, livePublication: [{ after: "terminal", event: { type: "ended", outcome: "interrupted" } }] }); + expect(writer.operations[5]?.mutation).not.toHaveProperty("providerEvent"); + expect((await send(7, { kind: "release" })).result).toEqual({ kind: "released" }); + }); + + it("finishes a setup error before turnStarted without losing the admitted execution", async () => { + const { writer, send } = fixture(); + await send(1, start()); + const input = { ...execution, providerId: "codex", providerIdentities: [], outcome: "errored", projection: { kind: "writer-staged" } } as const; + expect((await send(2, { kind: "finish-from-state", outcome: "errored", input })).result.kind).toBe("rejected"); + const reply = await send(2, { kind: "finish-from-state", outcome: "errored", input: { ...input, error: "Provider executable missing" } }); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations).toHaveLength(2); + expect(writer.operations[1]).toMatchObject({ mutation: { kind: "finish-live-event", outcome: "errored", + projection: { assistant: { content: "" }, narrative: [] } }, + livePublication: [{ event: { type: "error", error: "Provider executable missing" } }], + }); + }); + + it("settles captured edits on the worker before its single terminal writer call", async () => { + const { cwd, tracker, handoff } = await fileFixture(); + const { writer, send } = fixture(); + const file = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(file, "before\n"); + await send(1, start()); + expect((await send(2, { kind: "begin-files", cwd, handoff, deliveryAttempt: 1 })).result.kind).toBe("committed"); + const command = (draft: ProviderEventDraft): Extract => ({ + kind: "event", phase: "running", nativeCursor: null, deliveryAttempt: 1, events: [draft], + }); + await send(3, command(eventDraft(1, "turnStarted"))); + const toolInput = { file_path: "tracked.txt" }; + const captured = tracker.captureToolUseObservation(execution.threadId, "edit", "Edit", toolInput); + if (!captured) throw new Error("Expected captured file evidence"); + await NodeFSPromises.writeFile(file, "after\nextra\n"); + const use = eventDraft(2, "toolUse", { toolCallId: "edit", toolName: "Edit", toolInput }); + expect((await send(4, { ...command(use), capturedFileObservation: captured })).result.kind).toBe("committed"); + expect((await send(5, command(eventDraft(3, "toolResult", { toolCallId: "edit", output: "Done", isError: false })))).result.kind).toBe("committed"); + const terminal = eventDraft(4, "error", { error: "Disconnected" }); + const reply = await send(6, { ...command(terminal), + terminalInput: { ...execution, providerId: "codex", providerIdentities: [], outcome: "errored", projection: { kind: "writer-staged" } }, + frozenFileEvidence: { handoff, turnId: execution.turnId, deliveryAttempt: 1, outcome: "errored", nativeDiff: null } }); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations).toHaveLength(6); + expect(writer.operations[5]?.mutation).toMatchObject({ kind: "finish-live-event", input: { + error: "Disconnected", fileEvidence: { filesChanged: ["tracked.txt"], fileEffects: { fileCount: 1, additions: 2, deletions: 1 } }, + } }); + expect((await send(7, { kind: "release" })).result).toEqual({ kind: "released" }); + }); + + it("fences attempts and poisons a terminal whose file evidence is missing", async () => { + const { cwd, handoff } = await fileFixture(); + const { writer, send } = fixture(); + await send(1, start()); + await send(2, { kind: "begin-files", cwd, handoff, deliveryAttempt: 1 }); + const started = { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft(1, "turnStarted")] } satisfies ExecutionWorkCommand; + expect((await send(3, { ...started, deliveryAttempt: 2 })).result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect((await send(3, { ...started, deliveryAttempt: 1 })).result.kind).toBe("committed"); + const terminal = { ...started, events: [eventDraft(2, "error", { error: "Disconnected" })], deliveryAttempt: 1, + terminalInput: { ...execution, providerId: "codex", providerIdentities: [], outcome: "errored", projection: { kind: "writer-staged" } }, + } satisfies ExecutionWorkCommand; + expect((await send(4, terminal)).result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect((await send(4, terminal)).result).toMatchObject({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations).toHaveLength(3); + }); + + it.each(["claude", "cursor"] as const)("prepares %s parent text, message features, and terminal writes", async (providerId) => { + const { writer, send, event } = fixture(); + const command = start(); + await send(1, { ...command, providerId, input: { ...command.input, thread: { ...command.input.thread, providerId } } }); + const draft = (sequence: number, type: AgentEvent["type"], fields: Record = {}) => ({ + ...eventDraft(sequence, type, fields), sourceProviderId: providerId, + }); + await event(2, draft(1, "turnStarted")); + await event(3, draft(2, "textDelta", { delta: "Answer", isFinalResponse: true })); + expect(writer.operations[2]?.mutation).toMatchObject({ parentLive: { + text: { inputs: [{ sequence: 1, text: "Answer" }] }, + } }); + const message = await event(4, draft(3, "message", { content: "Answer", tokens: null })); + expect(message.result).toMatchObject({ kind: "committed", parentEvent: { + runtime: expect.arrayContaining([{ kind: "assistant-message-feature", providerId, + event: expect.objectContaining({ type: "message", content: "Answer" }) }]), + } }); + const terminal = draft(4, "error", { error: "Disconnected" }); + const reply = await send(5, { kind: "event", phase: "running", nativeCursor: null, events: [terminal], + terminalInput: { ...execution, providerId, providerIdentities: [], outcome: "errored", projection: { kind: "writer-staged" } } }); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations[4]?.mutation).toMatchObject({ kind: "finish-live-event", input: { providerId }, providerEvent: { events: [terminal] } }); + }); + + it.each(["claude", "cursor"] as const)("rejects unsupported %s events and another provider's drafts", async (providerId) => { + const { writer, send, event } = fixture(); + const command = start(); + await send(1, { ...command, providerId, input: { ...command.input, thread: { ...command.input.thread, providerId } } }); + expect((await event(2, eventDraft(1, "turnStarted"))).result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect((await event(2, { ...eventDraft(2, "textDelta", { delta: "Before start" }), sourceProviderId: providerId })).result) + .toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect(writer.operations).toHaveLength(1); + }); + + it("derives text writes inside the worker and commits them with their canonical draft", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + const started = await event(2, eventDraft(1, "turnStarted")); + expect(started.result).toMatchObject({ kind: "committed", parentEvent: { runtime: [{ kind: "turn-started" }] } }); + const draft = eventDraft(2, "textDelta", { delta: "Answer", isFinalResponse: true }); + const reply = await event(3, draft); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations[2]).toMatchObject({ + mutation: { kind: "append-events", events: [draft], parentLive: { + text: { kind: "append", inputs: [{ ...execution, sequence: 1, text: "Answer" }] }, + } }, livePublication: [{ after: "writer", event: { type: "textDelta", delta: "Answer" } }], + }); + }); + + it("rejects caller parent effects when the worker owns the reducer", async () => { + const { writer, send } = fixture(); + await send(1, start()); + const reply = await send(2, { kind: "event", phase: "running", nativeCursor: null, + events: [eventDraft(1, "turnStarted")], parentLive: { text: { kind: "unchanged" } } }); + expect(reply.result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect(writer.operations).toHaveLength(1); + }); + + it("rejects malformed parent identity before writing or advancing the reducer", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + const stale = eventDraft(1, "turnStarted"); + if (stale.payload.type !== "item.recorded") throw new Error("Expected runtime item"); + const reply = await event(2, { ...stale, payload: { ...stale.payload, item: { ...stale.payload.item, turnId: "other" } } }); + expect(reply.result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect(writer.operations).toHaveLength(1); + }); + + it("poisons prepared reducer state when its write conflicts", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + await event(2, eventDraft(1, "turnStarted")); + writer.fail = true; + const draft = eventDraft(2, "textDelta", { delta: "Answer" }); + expect((await event(3, draft)).result).toMatchObject({ kind: "rejected", reason: "writer-conflict" }); + writer.fail = false; + expect((await event(3, draft)).result).toMatchObject({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations).toHaveLength(3); + }); + + it("poisons prepared reducer state when the writer throws", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + writer.throwFailure = true; + await expect(event(2, eventDraft(1, "turnStarted"))).rejects.toThrow("writer disconnected"); + writer.throwFailure = false; + expect((await event(2, eventDraft(1, "turnStarted"))).result).toMatchObject({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations).toHaveLength(2); + }); + + it("requires a matching start message identity and supported provider for live ownership", async () => { + for (const command of [ + { ...start(), parentLive: { planFeature: "none", precedingMessageId: "other-user" } }, + { ...start(), providerId: "claude" }, + ] satisfies ExecutionWorkCommand[]) { + const { writer, send } = fixture(); + expect((await send(1, command)).result).toMatchObject({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations).toHaveLength(0); + } + }); + + it("keeps child runtime evidence on the writer path without changing parent text", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + await event(2, eventDraft(1, "turnStarted")); + const child = eventDraft(2, "textDelta", { delta: "Child answer" }); + if (child.payload.type !== "item.recorded") throw new Error("Expected runtime item"); + const runtime = ProviderRuntimeEventSchema().parse(child.payload.item.payload.runtimeEvent); + const reply = await event(3, { ...child, payload: { ...child.payload, item: { ...child.payload.item, + payload: { ...child.payload.item.payload, runtimeEvent: { ...runtime, + extension: { providerId: "codex", kind: "codex-collaboration", + child: { nativeThreadId: "child-thread", parentCollaborationItemId: "spawn-1" } } } }, + } } }); + expect(reply.result).toMatchObject({ kind: "committed" }); + expect(reply.result).not.toHaveProperty("parentEvent"); + expect(writer.operations[2]?.mutation).not.toHaveProperty("parentLive"); + await event(4, eventDraft(3, "textDelta", { delta: "Parent answer" })); + expect(writer.operations[3]?.mutation).toMatchObject({ parentLive: { + text: { inputs: [{ sequence: 1, text: "Parent answer" }] }, + } }); + }); + + it("requires terminal evidence then commits the terminal draft with finish in one operation", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + await event(2, eventDraft(1, "turnStarted")); + await event(3, eventDraft(2, "textDelta", { delta: "Partial answer", isFinalResponse: true })); + const terminal = eventDraft(3, "error", { error: "Disconnected" }); + expect((await event(4, terminal)).result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect(writer.operations).toHaveLength(3); + const reply = await send(4, { kind: "event", phase: "running", nativeCursor: null, events: [terminal], + terminalInput: { ...execution, providerId: "codex", providerIdentities: [], outcome: "errored", + projection: { kind: "writer-staged" } } }); + expect(reply.result).toMatchObject({ kind: "committed", parentEvent: { + terminal: { outcome: "errored", assistant: { content: "Partial answer" } }, + } }); + expect(writer.operations).toHaveLength(4); + expect(writer.operations[3]).toMatchObject({ mutation: { kind: "finish-live-event", + providerEvent: { events: [terminal] }, outcome: "errored", + projection: { assistant: { content: "Partial answer" } } }, + livePublication: [{ after: "terminal", event: { type: "error", error: "Disconnected" } }], + }); + expect((await send(5, { kind: "release" })).result).toEqual({ kind: "released" }); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/fixtures/exit.worker.ts b/apps/server/src/features/agents/execution/__tests__/fixtures/exit.worker.ts new file mode 100644 index 000000000..b7ffa9bc9 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/fixtures/exit.worker.ts @@ -0,0 +1,2 @@ +globalThis.onmessage = (): void => globalThis.close(); +globalThis.postMessage({ kind: "ready" }); diff --git a/apps/server/src/features/agents/execution/__tests__/provider-live-event-effects.test.ts b/apps/server/src/features/agents/execution/__tests__/provider-live-event-effects.test.ts new file mode 100644 index 000000000..63e584e20 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/provider-live-event-effects.test.ts @@ -0,0 +1,90 @@ +import { AgentEventSchema, type AgentEvent } from "@mcode/contracts"; +import { describe, expect, it } from "vitest"; + +import { OtherProviderLiveEventEffects } from "../provider-live-event-effects.js"; + +const execution = { + threadId: "provider-thread", + turnId: "provider-turn", + executionId: "11111111-1111-4111-8111-111111111111", +}; + +function event(type: AgentEvent["type"], fields: Record = {}): AgentEvent { + return AgentEventSchema().parse({ type, threadId: execution.threadId, + turnExecutionId: execution.executionId, ...fields }); +} + +describe("OtherProviderLiveEventEffects", () => { + it("prepares Claude text, assistant message, cursor, and terminal writes for one execution", () => { + const owner = new OtherProviderLiveEventEffects("claude", execution, "user-message"); + const start = owner.prepare(event("turnStarted")); + expect(start).toMatchObject({ kind: "prepared", providerId: "claude", execution, + runtime: [{ kind: "turn-started" }] }); + + const cursor = owner.prepare(event("system", { subtype: "sdk_session_id:claude-session" })); + expect(cursor).toMatchObject({ kind: "prepared", effects: { + systemIntents: [{ kind: "session-cursor", event: { subtype: "sdk_session_id:claude-session" } }], + } }); + + const delta = owner.prepare(event("textDelta", { delta: "Answer" })); + expect(delta).toMatchObject({ kind: "prepared", effects: { + text: { kind: "append", inputs: [{ ...execution, sequence: 1, text: "Answer" }] }, + } }); + expect(owner.prepare(event("assistantMessageBoundary", { isFinalResponse: true }))).toMatchObject({ + kind: "prepared", publication: { after: "writer" }, + }); + const message = owner.prepare(event("message", { content: "Answer", tokens: null })); + expect(message).toMatchObject({ kind: "prepared", effects: { + message: { content: "Answer", precedingMessageId: "user-message" }, + }, runtime: [{ kind: "assistant-message-feature", providerId: "claude", + event: { type: "message", content: "Answer" } }] }); + const terminal = owner.prepare(event("turnComplete", { + reason: "completed", costUsd: null, tokensIn: 4, tokensOut: 2, + }), "2026-09-25T00:00:00.000Z"); + expect(terminal).toMatchObject({ kind: "prepared", publication: { after: "terminal" }, + terminal: { threadId: execution.threadId, executionId: execution.executionId, + outcome: "completed", endedAt: "2026-09-25T00:00:00.000Z", + assistant: { content: "Answer" } } }); + expect(structuredClone(terminal)).toEqual(terminal); + expect(owner.prepare(event("hookStarted", { hookName: "stop", hookType: "stop" }))) + .toMatchObject({ kind: "prepared", publication: { after: "terminal" }, + runtime: [{ kind: "hook-started", late: true }] }); + expect(owner.prepare(event("message", { content: "Goal achieved in 2s.", tokens: null }))) + .toMatchObject({ kind: "unsupported", reason: expect.stringContaining("goal receipt") }); + }); + + it("prepares Cursor unknown text and its final message without a boundary", () => { + const owner = new OtherProviderLiveEventEffects("cursor", execution, "user-message"); + expect(owner.prepare(event("turnStarted"))).toMatchObject({ kind: "prepared" }); + expect(owner.prepare(event("textDelta", { delta: "Working" }))).toMatchObject({ + kind: "prepared", effects: { text: { kind: "append" } }, + }); + expect(owner.prepare(event("message", { content: "Done", tokens: null }))).toMatchObject({ + kind: "prepared", effects: { message: { content: "Done" } }, + runtime: [{ kind: "assistant-message-feature", providerId: "cursor" }], + }); + expect(owner.prepare(event("turnComplete", { + reason: "completed", costUsd: null, tokensIn: 0, tokensOut: 0, + }))).toMatchObject({ kind: "prepared", terminal: { outcome: "completed", + assistant: { content: "Done" } } }); + }); + + it("rejects an unbound or stale event without advancing the owner", () => { + const owner = new OtherProviderLiveEventEffects("claude", execution, "user-message"); + expect(owner.prepare(event("turnStarted", { turnExecutionId: undefined }))).toMatchObject({ + kind: "unsupported", reason: expect.stringContaining("without execution identity"), + }); + expect(owner.prepare(event("turnStarted", { + turnExecutionId: "22222222-2222-4222-8222-222222222222", + }))).toMatchObject({ + kind: "unsupported", reason: "different execution", + }); + expect(owner.prepare(event("turnStarted"))).toMatchObject({ kind: "prepared" }); + expect(owner.prepare(event("turnComplete", { + reason: "completed", costUsd: null, tokensIn: 0, tokensOut: 0, providerId: "codex", + }))).toMatchObject({ kind: "unsupported", reason: "different provider" }); + expect(owner.prepare(event("turnComplete", { + reason: "completed", costUsd: null, tokensIn: 0, tokensOut: 0, providerId: "claude", + }))).toMatchObject({ kind: "prepared", terminal: { outcome: "completed" } }); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/worker-owned-turn-runtime.test.ts b/apps/server/src/features/agents/execution/__tests__/worker-owned-turn-runtime.test.ts new file mode 100644 index 000000000..67720c9f2 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/worker-owned-turn-runtime.test.ts @@ -0,0 +1,73 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; +import { WorkerOwnedTurnRuntime } from "../worker-owned-turn-runtime.js"; + +const NOW = "2026-09-25T10:00:00.000Z"; +const identities = Array.from({ length: 5 }, (_, index) => ({ + threadId: `failure-isolation-thread-${index}`, + turnId: `failure-isolation-turn-${index}`, + executionId: `00000000-0000-4000-8000-0000000002${String(index).padStart(2, "0")}`, +})); + +let directory: string | undefined; +let database: ReturnType | undefined; +let runtime: WorkerOwnedTurnRuntime | undefined; + +afterEach(async () => { + await runtime?.close(); + database?.close(true); + if (directory) NodeFS.rmSync(directory, { recursive: true, force: true }); + runtime = undefined; + database = undefined; + directory = undefined; +}); + +it("contains a rejected writer operation to its execution while a slot peer keeps running", async () => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-rejected-execution-")); + const dbPath = NodePath.join(directory, "app.sqlite"); + database = openDatabase({ dbPath }); + database.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("failure-isolation-workspace", "Workspace", directory, NOW, NOW); + const insertThread = database.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)"); + for (const identity of identities) { + insertThread.run(identity.threadId, "failure-isolation-workspace", "Thread", "main", "codex", NOW, NOW); + } + runtime = new WorkerOwnedTurnRuntime(dbPath, new AgentEventPublicationRegistry()); + await runtime.whenReady(); + for (const identity of identities) { + await runtime.owner.start({ execution: identity, ownerEpoch: 1, providerId: "codex", parentTurn: { + thread: { id: identity.threadId, workspaceId: "failure-isolation-workspace", providerId: "codex", createdAt: NOW }, + turnId: identity.turnId, executionId: identity.executionId, permissionMode: "supervised", + providerIdentities: [], userMessage: { kind: "create", messageId: `${identity.threadId}-user`, content: "Question", sequence: 1 }, + } }); + } + const failed = identities[0]!; + const peer = identities[4]!; + expect(runtime.owner.current(failed.threadId)?.lease.workerIndex) + .toBe(runtime.owner.current(peer.threadId)?.lease.workerIndex); + database.run(`CREATE TRIGGER fail_one_append BEFORE INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:append-events' AND NEW.execution_id = '${failed.executionId}' + BEGIN SELECT RAISE(ABORT, 'injected append failure'); END`); + await expect(runtime.owner.submit(failed, { kind: "event", phase: "running", nativeCursor: null, events: [{ + eventId: `${failed.executionId}:event:1`, routing: { ...failed, itemId: "failed-tool" }, + sourceProviderId: "codex", sourceIdentities: [], sourceSequence: 1, + payload: { type: "item.recorded", item: { id: "failed-tool", threadId: failed.threadId, turnId: failed.turnId, + kind: "tool-call", providerIdentities: [], payload: { projection: "toolCall", toolName: "Read", path: "CONTEXT.md" }, + createdAt: NOW, updatedAt: NOW } }, + }] })).rejects.toThrow("writer-failure"); + await runtime.recoverRejected(failed); + expect(runtime.owner.current(failed.threadId)).toBeUndefined(); + expect(database.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(failed.executionId)).toEqual({ terminal_outcome: "interrupted" }); + expect(database.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(peer.executionId)).toEqual({ terminal_outcome: null }); + await expect(runtime.owner.submit(peer, { kind: "checkpoint", phase: "running", nativeCursor: null })) + .resolves.toMatchObject({ kind: "committed" }); + expect(runtime.owner.isStarted(peer)).toBe(true); +}); diff --git a/apps/server/src/features/agents/execution/codex-live-event-effects.ts b/apps/server/src/features/agents/execution/codex-live-event-effects.ts new file mode 100644 index 000000000..472661f62 --- /dev/null +++ b/apps/server/src/features/agents/execution/codex-live-event-effects.ts @@ -0,0 +1,153 @@ +import type { DataOnlyParentTerminalProjectionInput } from "../canonical/canonical-parent-turn-write.js"; +import type { CodexSystemWriterIntent } from "../canonical/canonical-codex-system-error-projection.js"; +import { taskToolWriteIntents, type TaskToolCall } from "../tasks/task-tool-intent-reducer.js"; +import { deriveTurnAssistantMessageId } from "../turns/turn-assistant-message-id.js"; +import type { ParentAssistantTextCheckpointInput } from "../turns/parent-assistant-text-checkpoint-service.js"; +import type { CodexLiveReduction, CodexLiveWriterIntent } from "./codex-live-event-reducer.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import type { ExecutionLivePublicationIntent, ParentLiveEffects } from "./execution-worker-handler.js"; + +type ReducedEvent = Extract; +type TerminalIntent = Extract; + +/** Effects that still require an execution runtime or a canonical projection owner. */ +export type CodexLiveRuntimeIntent = Extract | { readonly kind: "feature-event"; readonly feature: "goal-refresh"; readonly event: ReducedEvent["publication"]["event"] }; + +/** Cloneable parent writes and the remaining work required before publication. */ +export interface PreparedCodexLiveEvent { + readonly effects: ParentLiveEffects; + readonly publication: ExecutionLivePublicationIntent; + readonly runtime: readonly CodexLiveRuntimeIntent[]; + readonly terminal?: DataOnlyParentTerminalProjectionInput; +} + +/** + * Maps one execution's reduced events to writer data. Advance only in mailbox order; + * discard this mapper and its reducer if a prepared operation fails to commit. + */ +export class CodexLiveEventEffects { + private sequence = 0; + private readonly calls = new Map(); + private assignedMessageId: string | undefined; + + constructor(private readonly execution: ExecutionIdentity, private readonly precedingMessageId: string) {} + + /** Prepare every parent write while retaining intents owned by other execution collaborators. */ + prepare(reduction: ReducedEvent, endedAt = new Date().toISOString()): PreparedCodexLiveEvent { + this.requireExecution(reduction); + let effects: ParentLiveEffects = { text: { kind: "unchanged" } }; + const runtime: CodexLiveRuntimeIntent[] = []; + let terminal: DataOnlyParentTerminalProjectionInput | undefined; + for (const intent of reduction.writer) { + if (intent.kind === "terminal-projection") terminal = this.terminalProjection(intent, endedAt); + else effects = this.applyIntent(intent, effects, runtime); + } + const event = reduction.publication.event; + const publication = event.type === "message" && effects.message + ? { ...reduction.publication, event: { ...event, messageId: effects.message.messageId } } + : reduction.publication; + return structuredClone({ effects, publication, runtime, ...(terminal ? { terminal } : {}) }); + } + + private requireExecution(reduction: ReducedEvent): void { + const actual = reduction.execution; + const event = reduction.publication.event; + if (actual.threadId !== this.execution.threadId || actual.turnId !== this.execution.turnId + || actual.executionId !== this.execution.executionId || event.threadId !== actual.threadId + || event.turnExecutionId !== actual.executionId) throw new Error("Codex effects belong to another execution"); + } + + private applyIntent( + intent: Exclude, + effects: ParentLiveEffects, + runtime: CodexLiveRuntimeIntent[], + ): ParentLiveEffects { + switch (intent.kind) { + case "assistant-text-delta": + return intent.delta ? { ...effects, text: { kind: "append", inputs: [this.checkpoint(intent.delta)] } } : effects; + case "assistant-text-promote": + return { ...effects, text: { kind: "promote", input: this.checkpoint(intent.text) } }; + case "assistant-text-reclassify": + this.sequence = 0; + return { ...effects, text: { kind: "reclassify", expectedText: intent.text } }; + default: return this.applyNonTextIntent(intent, effects, runtime); + } + } + + private applyNonTextIntent( + intent: Exclude, + effects: ParentLiveEffects, + runtime: CodexLiveRuntimeIntent[], + ): ParentLiveEffects { + switch (intent.kind) { + case "assistant-body": return this.messageEffects(intent, effects); + case "narrative-recovery": return this.narrativeEffects(intent, effects); + case "feature-event": return this.featureEffects(intent, effects, runtime); + case "plan-questions": return { ...effects, planQuestions: intent.questions }; + case "plan-output": return { ...effects, planOutput: intent.output }; + case "notice-session": + case "system-notice": + case "session-cursor": return this.systemEffects(intent, effects); + default: return this.runtimeEffects(intent, effects, runtime); + } + } + + private checkpoint(text: string): ParentAssistantTextCheckpointInput { + return { ...this.execution, sequence: ++this.sequence, text }; + } + + private messageEffects(intent: Extract, effects: ParentLiveEffects): ParentLiveEffects { + this.assignedMessageId ??= deriveTurnAssistantMessageId(this.execution.threadId, this.precedingMessageId); + return { ...effects, message: { precedingMessageId: this.precedingMessageId, messageId: this.assignedMessageId, + content: intent.content, model: intent.model, attachments: intent.attachments } }; + } + + private narrativeEffects(intent: Extract, effects: ParentLiveEffects): ParentLiveEffects { + if (intent.items.length === 0 && intent.discardedItemIds.length === 0) return effects; + return { ...effects, narrative: { executionId: this.execution.executionId, + items: intent.items, discardedItemIds: intent.discardedItemIds } }; + } + + private featureEffects( + intent: Extract, + effects: ParentLiveEffects, + runtime: CodexLiveRuntimeIntent[], + ): ParentLiveEffects { + if (intent.feature === "goal-refresh") runtime.push({ ...intent, feature: "goal-refresh" }); + // The reducer emits parsed plans and assistant bodies as their own intents. + if (intent.feature !== "task-tool") return effects; + const event = intent.event; + const bufferedCalls = [...this.calls.values()]; + const taskIntents = event.type === "toolUse" + ? taskToolWriteIntents({ kind: "tool-use", ...event, bufferedCalls }) + : event.type === "toolResult" + ? taskToolWriteIntents({ kind: "tool-result", ...event, bufferedCalls }) : []; + return taskIntents.length ? { ...effects, taskIntents } : effects; + } + + private systemEffects(intent: CodexSystemWriterIntent, effects: ParentLiveEffects): ParentLiveEffects { + return { ...effects, systemIntents: [...effects.systemIntents ?? [], intent] }; + } + + private runtimeEffects(intent: CodexLiveRuntimeIntent, effects: ParentLiveEffects, runtime: CodexLiveRuntimeIntent[]): ParentLiveEffects { + if (intent.kind === "tool-use") { + const event = intent.event; + this.calls.set(event.toolCallId, { toolCallId: event.toolCallId, toolName: event.toolName, + parentToolCallId: event.parentToolCallId, _rawToolInput: event.toolInput }); + } + runtime.push(intent); + return effects; + } + + private terminalProjection(intent: TerminalIntent, endedAt: string): DataOnlyParentTerminalProjectionInput { + return { threadId: this.execution.threadId, executionId: this.execution.executionId, + outcome: intent.outcome, endedAt, + assistant: { content: intent.assistant.content, model: intent.assistant.model, + attachments: intent.assistant.attachments, ...(this.assignedMessageId ? { messageId: this.assignedMessageId } : {}) }, + narrative: intent.narrative }; + } +} diff --git a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts new file mode 100644 index 000000000..d58e12b0c --- /dev/null +++ b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts @@ -0,0 +1,476 @@ +import type { AgentEvent, ParentNarrativeRecoveryItem, PlanQuestion, StoredAttachment, TurnOutcome } from "@mcode/contracts"; +import { NarrativeTurnState, type NarrativeTurnStateEffect } from "../conversation/narrative/narrative-turn-state.js"; +import { AssistantExecutionState, type AssistantMaterializationInput } from "../turns/assistant-execution-state.js"; +import { NarrativeRecoveryDelta, type PreparedNarrativeRecoveryDelta } from "../turns/narrative-recovery-delta.js"; +import { PlanExecutionState, type PlanPersistenceReady } from "../planning/plan-execution-state.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; + +type ToolUseEvent = Extract; +type ToolResultEvent = Extract; +type MessageEvent = Extract; +type TextDeltaEvent = Extract; +type ContextEvent = Extract; +type SystemEvent = Extract; +const MAX_PLAN_TEXT_BYTES = 256 * 1024; + +/** Which plan parser, if any, was armed when this execution began. */ +export type CodexPlanFeature = "none" | "questions" | "output"; + +/** A runtime termination carries its actual failure or cancellation outcome. */ +export type SyntheticTerminalInput = + | { readonly outcome: "errored"; readonly error: string } + | { readonly outcome: "cancelled" | "interrupted" }; + +const BEFORE_START_EVENTS = new Set([ + "system", "quotaUpdate", "goalUpdated", "goalCleared", "mcpServerStartupStatus", +]); +const AFTER_COMPLETION_EVENTS = new Set([ + "ended", "hookStarted", "hookProgress", "hookCompleted", ...BEFORE_START_EVENTS, +]); +const TRANSIENT_STATUS_EVENTS = new Set([ + "apiRetry", "rateLimited", "quotaUpdate", "providerUnavailable", "modelFallback", + "toolInputDelta", "toolProgress", "hookProgress", "goalUpdated", "goalCleared", + "mcpServerStartupStatus", +]); + +// A new contract variant must receive an owner decision before the reducer accepts it. +const UNSUPPORTED_FEATURE_REASON = { + turnStarted: null, + message: null, + generatedAttachment: null, + toolUse: null, + toolResult: null, + turnComplete: null, + error: null, + ended: null, + system: null, + compacting: null, + compactSummary: null, + modelFallback: null, + textDelta: null, + toolInputDelta: null, + toolProgress: null, + contextEstimate: null, + quotaUpdate: null, + providerUnavailable: null, + rateLimited: null, + apiRetry: null, + hookStarted: null, + hookProgress: null, + hookCompleted: null, + assistantMessageBoundary: null, + goalUpdated: null, + goalCleared: null, + mcpServerStartupStatus: null, +} satisfies Record; + +/** Data to commit for one provider event before releasing its publication. */ +export type CodexLiveWriterIntent = + | { readonly kind: "turn-started" } + | { readonly kind: "assistant-text-delta"; readonly delta: string; readonly classification: "final" | "unknown" } + | { readonly kind: "assistant-text-reclassify"; readonly text: string; readonly classification: "narration" } + | { readonly kind: "assistant-text-promote"; readonly text: string } + | { readonly kind: "assistant-body"; readonly content: string; readonly model: string | null; readonly attachments: StoredAttachment[]; readonly tokens: number | null } + | { readonly kind: "generated-attachment"; readonly attachment: StoredAttachment } + | { readonly kind: "tool-use"; readonly event: ToolUseEvent } + | { readonly kind: "tool-result"; readonly event: ToolResultEvent } + | { readonly kind: "hook-started"; readonly hookId: string; readonly late: boolean } + | { readonly kind: "hook-completed"; readonly hookId: string; readonly late: boolean; readonly exitCode: number; readonly durationMs: number; readonly didBlock: boolean } + | { readonly kind: "narrative-recovery"; readonly items: readonly ParentNarrativeRecoveryItem[]; readonly discardedItemIds: readonly string[] } + | { readonly kind: "narrative-effect"; readonly effect: NarrativeTurnStateEffect } + | { readonly kind: "feature-event"; readonly feature: "plan-text" | "assistant-message" | "task-tool" | "goal-refresh"; readonly event: AgentEvent } + | { readonly kind: "plan-questions"; readonly questions: readonly PlanQuestion[] } + | { readonly kind: "plan-output"; readonly output: PlanPersistenceReady } + | { readonly kind: "context-usage"; readonly tokensIn: number; readonly contextWindow?: number } + | { readonly kind: "compaction-started" } + | { readonly kind: "compaction-divider" } + | { readonly kind: "compaction-summary"; readonly summary: string } + | { readonly kind: "notice-session"; readonly event: SystemEvent } + | { readonly kind: "system-notice"; readonly event: SystemEvent } + | { readonly kind: "session-cursor"; readonly event: SystemEvent } + | { readonly kind: "turn-error"; readonly error: string } + | { readonly kind: "terminal-projection"; readonly source: "turnComplete" | "error" | "ended"; readonly outcome: TurnOutcome; readonly assistant: AssistantMaterializationInput; readonly narrative: ParentNarrativeRecoveryItem[] } + | { readonly kind: "turn-ended" }; + +/** A publication is released only after the writer accepts every intent for the event. */ +export interface CodexLivePublicationIntent { + readonly event: AgentEvent; + readonly after: "writer" | "terminal"; +} + +/** Unsupported means no reducer state was changed and no publication may be emitted. */ +export type CodexLiveReduction = + | { readonly kind: "reduced"; readonly execution: ExecutionIdentity; readonly writer: CodexLiveWriterIntent[]; readonly publication: CodexLivePublicationIntent } + | { readonly kind: "unsupported"; readonly eventType: AgentEvent["type"]; readonly reason: string }; + +/** + * Reduces one Codex execution's live AgentEvents without database or transport calls. + * The caller must supply an exact turnExecutionId from provider turn binding; + * session-level events without that proof belong to a separate owner. + * Preparation advances narrative checkpoints before the writer commits. + * The caller must discard this instance if preparation or its writer operation fails, because + * the next event may only observe state whose preceding intents were committed. + */ +export class CodexLiveEventReducer { + private readonly narrative: NarrativeTurnState; + private readonly narrativeDelta = new NarrativeRecoveryDelta(); + private readonly assistant = new AssistantExecutionState(); + private phase: "awaiting-start" | "active" | "completed" | "ended" = "awaiting-start"; + private unknownText = ""; + private knownFinalText = false; + private compacting = false; + private readonly plan: PlanExecutionState | null; + private planTextBytes = 0; + private planQuestionsResolved = false; + + constructor(readonly execution: ExecutionIdentity, readonly planFeature: CodexPlanFeature = "none") { + this.narrative = new NarrativeTurnState(execution); + this.plan = planFeature === "none" ? null : new PlanExecutionState(); + if (planFeature === "questions") this.plan?.beginQuestionGeneration(); + if (planFeature === "output") this.plan?.beginOutputGeneration(); + } + + reduce(input: AgentEvent): CodexLiveReduction { + const rejection = this.identityRejection(input) ?? UNSUPPORTED_FEATURE_REASON[input.type] + ?? this.phaseRejection(input) ?? this.textRejection(input) ?? this.planTextRejection(input); + if (rejection) return this.unsupported(input, rejection); + let event: AgentEvent; + try { + event = structuredClone(input); + } catch { + return this.unsupported(input, "event is not cloneable"); + } + + const writer = this.apply(event); + if (!writer) return this.unsupported(input, "reducer dispatch owner has no handler for this event"); + for (const effect of this.narrative.takeEffects()) writer.push({ kind: "narrative-effect", effect }); + return structuredClone({ + kind: "reduced", + execution: this.execution, + writer, + publication: { + event: this.publicationEvent(event), + after: this.publicationBarrier(event), + }, + }); + } + + /** Preserve partial text and narrative when the runtime must finish without a provider terminal event. */ + finishFromState(input: SyntheticTerminalInput): CodexLiveReduction { + const event: AgentEvent = input.outcome === "errored" + ? { type: "error", threadId: this.execution.threadId, turnExecutionId: this.execution.executionId, error: input.error } + : { type: "ended", threadId: this.execution.threadId, turnExecutionId: this.execution.executionId, outcome: "interrupted" }; + if (this.phase === "completed" || this.phase === "ended") return this.unsupported(event, "execution already terminal"); + this.phase = "completed"; + const writer: CodexLiveWriterIntent[] = [{ kind: "terminal-projection", + source: input.outcome === "errored" ? "error" : "ended", outcome: input.outcome, + assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(this.execution.threadId) }]; + if (input.outcome === "errored") writer.push({ kind: "turn-error", error: input.error }); + else writer.push({ kind: "turn-ended" }); + return structuredClone({ kind: "reduced", execution: this.execution, writer, + publication: { event, after: "terminal" } }); + } + + private identityRejection(event: AgentEvent): string | undefined { + if (event.threadId !== this.execution.threadId) return "different thread"; + if (!event.turnExecutionId) return "event without execution identity needs the provider event routing owner"; + if (event.turnExecutionId !== this.execution.executionId) return "different execution"; + return undefined; + } + + private phaseRejection(event: AgentEvent): string | undefined { + if (this.phase === "ended") return "execution already ended"; + if (this.phase === "awaiting-start" && event.type !== "turnStarted" && !BEFORE_START_EVENTS.has(event.type)) { + return "turn has not started"; + } + if (this.phase !== "awaiting-start" && event.type === "turnStarted") return "turn already started"; + if (this.phase === "completed") return this.completedPhaseRejection(event); + if (this.compacting && event.type === "turnComplete") return "turn completion during compaction needs the compaction terminal owner"; + return undefined; + } + + private completedPhaseRejection(event: AgentEvent): string | undefined { + if (AFTER_COMPLETION_EVENTS.has(event.type)) return undefined; + return event.type === "message" ? "post-turn goal receipt needs the goal message owner" + : "post-terminal event needs the terminal lifecycle owner"; + } + + private textRejection(event: AgentEvent): string | undefined { + switch (event.type) { + case "assistantMessageBoundary": + return !event.isFinalResponse && this.knownFinalText + ? "mixed final and unknown assistant text" : undefined; + case "textDelta": return this.deltaRejection(event); + case "turnComplete": + case "ended": + case "error": + return this.unknownText ? "assistant text still lacks a boundary" : undefined; + default: return undefined; + } + } + + private deltaRejection(event: TextDeltaEvent): string | undefined { + if (event.isFinalResponse === false && this.knownFinalText) return "mixed final and nonfinal assistant text"; + if (event.isFinalResponse !== undefined && this.unknownText) return "mixed classified and unknown assistant text"; + return undefined; + } + + private planTextRejection(event: AgentEvent): string | undefined { + if (!this.plan || this.planQuestionsResolved || event.type !== "textDelta") return undefined; + return this.planTextBytes + Buffer.byteLength(event.delta, "utf8") > MAX_PLAN_TEXT_BYTES + ? "plan text exceeds the execution projection limit" : undefined; + } + + private publicationEvent(event: AgentEvent): AgentEvent { + return event.type === "ended" && event.outcome === "cancelled" + ? { ...event, outcome: "interrupted" } + : event; + } + + private publicationBarrier(event: AgentEvent): "writer" | "terminal" { + return this.phase === "completed" || (event.type === "ended" && event.outcome !== undefined) + ? "terminal" : "writer"; + } + + private apply(event: AgentEvent): CodexLiveWriterIntent[] | undefined { + return this.applyNarrative(event) ?? this.applyLifecycle(event); + } + + private applyNarrative(event: AgentEvent): CodexLiveWriterIntent[] | undefined { + switch (event.type) { + case "textDelta": return this.textDelta(event); + case "assistantMessageBoundary": return this.boundary(event); + case "message": return this.message(event); + case "generatedAttachment": return this.attachment(event); + case "toolUse": return this.toolUse(event); + case "toolResult": return this.toolResult(event); + case "hookStarted": return this.hookStarted(event); + case "hookCompleted": return this.hookCompleted(event); + default: return undefined; + } + } + + private applyLifecycle(event: AgentEvent): CodexLiveWriterIntent[] | undefined { + switch (event.type) { + case "turnStarted": return this.start(event); + case "turnComplete": return this.turnComplete(event); + case "error": return this.error(event); + case "ended": return this.ended(event); + case "contextEstimate": return this.contextEstimate(event); + case "compacting": return this.compactingEvent(event); + case "compactSummary": return this.compactSummary(event); + default: return this.applyStatus(event); + } + } + + private applyStatus(event: AgentEvent): CodexLiveWriterIntent[] | undefined { + if (event.type === "system") return this.system(event); + // The legacy turn application only publishes these transient statuses. + return TRANSIENT_STATUS_EVENTS.has(event.type) ? [] : undefined; + } + + private start(event: Extract): CodexLiveWriterIntent[] { + this.narrative.beginTurn(event.threadId); + this.narrative.resetTurnCounters(event.threadId); + this.phase = "active"; + return [{ kind: "turn-started" }]; + } + + private textDelta(event: TextDeltaEvent): CodexLiveWriterIntent[] { + const writer: CodexLiveWriterIntent[] = [{ kind: "feature-event", feature: "plan-text", event }]; + if (this.plan && !this.planQuestionsResolved) { + this.planTextBytes += Buffer.byteLength(event.delta, "utf8"); + const ready = this.plan.feedText(event.delta); + if (ready) { + this.planQuestionsResolved = true; + writer.push({ kind: "plan-questions", questions: ready.questions }); + } + } + if (event.isFinalResponse === false) { + this.narrative.openOrExtendThought(event.threadId, event.delta); + writer.push(this.recovery()); + } else { + this.assistant.appendStreamingText(event.delta); + if (event.isFinalResponse === undefined) this.unknownText += event.delta; + else this.knownFinalText = true; + writer.push({ kind: "assistant-text-delta", delta: event.delta, classification: event.isFinalResponse ? "final" : "unknown" }); + } + return writer; + } + + private boundary(event: Extract): CodexLiveWriterIntent[] { + const writer: CodexLiveWriterIntent[] = []; + if (event.isFinalResponse) { + const text = this.narrative.takeOpenThought(event.threadId); + if (text) { + this.assistant.appendStreamingText(text); + writer.push({ kind: "assistant-text-promote", text }); + } + this.unknownText = ""; + } else { + if (this.unknownText) { + const staged = this.narrative.stageNarrationSegment(event.threadId, this.unknownText); + if (staged) this.narrative.applyStagedNarrationSegment(event.threadId, staged); + writer.push({ kind: "assistant-text-reclassify", text: this.unknownText, classification: "narration" }); + this.assistant.resetStreamingText(); + this.unknownText = ""; + } + this.narrative.closeOpenThought(event.threadId); + } + this.knownFinalText = false; + writer.push(this.recovery()); + return writer; + } + + private message(event: MessageEvent): CodexLiveWriterIntent[] { + this.assistant.bufferBody(event.content, event.model ?? null, event.attachments ?? []); + const body = this.assistant.materializationInput(event.model ?? null); + this.assistant.resetStreamingText(); + this.narrative.clearAgentStackOnMessage(event.threadId); + this.knownFinalText = false; + this.unknownText = ""; + const writer: CodexLiveWriterIntent[] = [ + { kind: "assistant-body", content: body.content, model: body.model, attachments: body.attachments, tokens: event.tokens }, + { kind: "feature-event", feature: "assistant-message", event }, + ]; + if (this.plan && this.planFeature === "output") { + const output = this.plan.consumeAssistantMessage(event.content); + if (output) writer.push({ kind: "plan-output", output }); + } + return writer; + } + + private attachment(event: Extract): CodexLiveWriterIntent[] { + this.assistant.bufferAttachments([event.attachment]); + return [{ kind: "generated-attachment", attachment: event.attachment }]; + } + + private toolUse(event: ToolUseEvent): CodexLiveWriterIntent[] { + this.narrative.closeOpenThought(event.threadId); + const parentToolCallId = this.narrative.bufferToolCall(event.threadId, event); + const attributed = { ...event, parentToolCallId }; + return [ + { kind: "tool-use", event: attributed }, + { kind: "feature-event", feature: "task-tool", event: attributed }, + this.recovery(), + ]; + } + + private toolResult(event: ToolResultEvent): CodexLiveWriterIntent[] { + this.narrative.updateBufferedToolCallOutput(event.threadId, event.toolCallId, event.output, event.isError, + event.toolInput, { + exitCode: event.exitCode, + outputTruncated: event.outputTruncated, + outputTotalBytes: event.outputTotalBytes, + outputArtifactPath: event.outputArtifactPath, + }, event.subagentPresentation); + return [ + { kind: "tool-result", event }, + { kind: "feature-event", feature: "task-tool", event }, + this.recoveryIntent(this.narrativeDelta.prepareToolUpdate(this.narrative.toolRecoveryItem(event.threadId, event.toolCallId))), + ]; + } + + private hookStarted(event: Extract): CodexLiveWriterIntent[] { + const late = this.phase === "completed"; + if (!late) this.narrative.closeOpenThought(event.threadId); + const hookId = this.narrative.openHook(event.threadId, { + hookName: event.hookName, + toolName: late ? null : event.toolName ?? null, + phase: late ? "stop" : event.hookType, + payload: JSON.stringify({ hookType: late ? "stop" : event.hookType, toolName: late ? null : event.toolName ?? null }), + sortOrder: this.narrative.nextSortOrder(event.threadId), + }); + return [{ kind: "hook-started", hookId, late }, this.recovery()]; + } + + private hookCompleted(event: Extract): CodexLiveWriterIntent[] { + const open = this.narrative.peekOpenHook(event.threadId, event.hookName); + if (!open) return []; + this.narrative.pushClosedHook(event.threadId, { + ...open, + messageId: "", + durationMs: event.durationMs, + didBlock: event.didBlock, + endedAt: new Date().toISOString(), + }); + this.narrative.removeOpenHook(event.threadId, event.hookName); + return [{ kind: "hook-completed", hookId: open.id, late: this.phase === "completed", + exitCode: event.exitCode, durationMs: event.durationMs, didBlock: event.didBlock }, this.recovery()]; + } + + private turnComplete(event: Extract): CodexLiveWriterIntent[] { + this.phase = "completed"; + return [ + ...this.contextUsage(event), + { kind: "terminal-projection", source: "turnComplete", outcome: "completed", + assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(event.threadId) }, + { kind: "feature-event", feature: "goal-refresh", event }, + ]; + } + + private error(event: Extract): CodexLiveWriterIntent[] { + this.phase = "completed"; + return [ + { kind: "turn-error", error: event.error }, + { kind: "terminal-projection", source: "error", outcome: "errored", + assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(event.threadId) }, + ]; + } + + private contextEstimate(event: Extract): CodexLiveWriterIntent[] { + return event.totalProcessedTokens === undefined ? [] : this.contextUsage(event); + } + + private contextUsage(event: ContextEvent): CodexLiveWriterIntent[] { + return event.tokensIn > 0 && !this.compacting + ? [{ kind: "context-usage", tokensIn: event.tokensIn, + ...(event.contextWindow !== undefined ? { contextWindow: event.contextWindow } : {}) }] + : []; + } + + private compactingEvent(event: Extract): CodexLiveWriterIntent[] { + this.compacting = event.active; + return [{ kind: event.active ? "compaction-started" : "compaction-divider" }]; + } + + private compactSummary(event: Extract): CodexLiveWriterIntent[] { + this.compacting = false; + return [{ kind: "compaction-summary", summary: event.summary }]; + } + + private system(event: SystemEvent): CodexLiveWriterIntent[] { + const writer: CodexLiveWriterIntent[] = []; + if (event.subtype === "provider.session.started") writer.push({ kind: "notice-session", event }); + if (event.subtype.startsWith("provider.notice.") && event.message) writer.push({ kind: "system-notice", event }); + if (event.subtype.startsWith("sdk_session_id:") || event.subtype === "sdk_session_invalidated") { + writer.push({ kind: "session-cursor", event }); + } + return writer; + } + + private ended(event: Extract): CodexLiveWriterIntent[] { + const writer: CodexLiveWriterIntent[] = []; + if (event.outcome && this.phase !== "completed") { + writer.push({ kind: "terminal-projection", source: "ended", + outcome: event.outcome === "cancelled" ? "interrupted" : event.outcome, + assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(event.threadId) }); + } + writer.push({ kind: "turn-ended" }); + this.phase = "ended"; + return writer; + } + + private recovery(): CodexLiveWriterIntent { + return this.recoveryIntent(this.narrativeDelta.prepare(this.narrative.recoverySnapshot(this.execution.threadId))); + } + + private recoveryIntent(delta: PreparedNarrativeRecoveryDelta | null): CodexLiveWriterIntent { + delta?.acknowledge(); + return { kind: "narrative-recovery", items: delta?.items ?? [], discardedItemIds: delta?.discardedItemIds ?? [] }; + } + + private unsupported(event: AgentEvent, reason: string): CodexLiveReduction { + return { kind: "unsupported", eventType: event.type, reason }; + } +} diff --git a/apps/server/src/features/agents/execution/execution-file-evidence-coordinator.ts b/apps/server/src/features/agents/execution/execution-file-evidence-coordinator.ts new file mode 100644 index 000000000..7ac0a5afe --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-file-evidence-coordinator.ts @@ -0,0 +1,152 @@ +import type { AgentEvent, ProviderFileMutationStart, TurnOutcome } from "@mcode/contracts"; + +import type { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; +import type { PreparedTurnDiffEvidence, TurnDiffService } from "../turns/turn-diff-service.js"; +import { + prepareExecutionFileEvidence, + type PreparedExecutionFileEvidence, +} from "../turns/turn-execution-file-evidence.js"; +import type { + CapturedToolUseObservation, + FileTurnHandoff, + TurnFileTracker, +} from "../turns/turn-file-tracker.js"; +import { ExecutionFileObservationHandoff } from "./execution-file-observation-handoff.js"; + +type ToolUse = Extract; +type DiffHandoff = Pick; + +/** Cloneable terminal file data retained until the writer acknowledges finalization. */ +export interface FrozenExecutionFileEvidence { + readonly handoff: FileTurnHandoff; + readonly turnId: string; + readonly deliveryAttempt: number; + readonly outcome: TurnOutcome; + readonly nativeDiff: PreparedTurnDiffEvidence | null; +} + +interface ActiveFileExecution { + readonly handoff: FileTurnHandoff; + readonly turnId: string; + readonly deliveryAttempt: number; + readonly observations: ExecutionFileObservationHandoff; + frozen?: FrozenExecutionFileEvidence; +} + +/** Captures pre-edit evidence on the provider callback and seals one attempt for worker settlement. */ +export class ExecutionFileEvidenceCoordinator { + private readonly active = new Map(); + + constructor( + private readonly captureTracker: TurnFileTracker, + private readonly diffs: DiffHandoff, + ) {} + + /** Open the exact execution only after its durable start has been acknowledged. */ + begin(input: { + readonly threadId: string; + readonly turnId: string; + readonly executionId: string; + readonly deliveryAttempt: number; + readonly cwd: string; + readonly baselineRef: string | null; + }): FileTurnHandoff { + if (!input.turnId || !Number.isSafeInteger(input.deliveryAttempt) || input.deliveryAttempt < 1) { + throw new Error("File evidence requires a turn ID and positive delivery attempt"); + } + const previous = this.active.get(input.threadId); + if (previous?.frozen) throw new Error("Retire sealed file evidence before opening another attempt"); + const handoff = this.captureTracker.beginExecutionTurn(input); + const observations = new ExecutionFileObservationHandoff( + this.captureTracker, handoff, input.deliveryAttempt, + ); + previous?.observations.close(); + if (previous) this.captureTracker.clearTurn(input.threadId, previous.handoff.generation); + this.diffs.begin({ threadId: input.threadId, turnId: input.turnId, + turnExecutionId: input.executionId, deliveryAttempt: input.deliveryAttempt }); + this.active.set(input.threadId, { + handoff, turnId: input.turnId, deliveryAttempt: input.deliveryAttempt, observations, + }); + return handoff; + } + + /** Run synchronously before the provider is allowed to mutate a named file. */ + capture(event: ProviderFileMutationStart): boolean { + return this.active.get(event.threadId)?.observations.capture(event) ?? false; + } + + /** Transfer the original pre-edit observation with its matching admitted tool event. */ + take(event: ToolUse, deliveryAttempt: number): CapturedToolUseObservation | null { + return this.active.get(event.threadId)?.observations.take(event, deliveryAttempt) ?? null; + } + + /** Reject later callbacks at the Stop or retry cut while retaining settled evidence. */ + fence(threadId: string, executionId: string, deliveryAttempt: number): boolean { + const active = this.match(threadId, executionId, deliveryAttempt); + if (!active) return false; + active.observations.close(); + return true; + } + + /** Freeze native evidence before asynchronous file settlement or terminal publication. */ + seal(input: { + readonly threadId: string; + readonly turnId: string; + readonly executionId: string; + readonly deliveryAttempt: number; + readonly outcome: TurnOutcome; + }): FrozenExecutionFileEvidence | null { + const active = this.match(input.threadId, input.executionId, input.deliveryAttempt); + if (!active || active.turnId !== input.turnId) return null; + if (active.frozen) { + if (active.frozen.outcome !== input.outcome) throw new Error("File evidence outcome changed after sealing"); + return active.frozen; + } + active.observations.close(); + const nativeDiff = this.diffs.takeFinalizationEvidence(input.threadId, input.executionId); + if (nativeDiff && (nativeDiff.turnId !== input.turnId + || nativeDiff.turnExecutionId !== input.executionId + || nativeDiff.deliveryAttempt !== input.deliveryAttempt)) { + throw new Error("Native diff does not belong to the sealed execution attempt"); + } + active.frozen = Object.freeze({ + handoff: active.handoff, turnId: input.turnId, + deliveryAttempt: input.deliveryAttempt, outcome: input.outcome, nativeDiff, + }); + return active.frozen; + } + + /** Release the capture tracker only after the terminal writer receipt or explicit abandonment. */ + retire(threadId: string, executionId: string, deliveryAttempt: number): boolean { + const active = this.match(threadId, executionId, deliveryAttempt); + if (!active) return false; + active.observations.close(); + this.active.delete(threadId); + this.diffs.clearExecution(threadId, executionId); + this.captureTracker.clearTurn(threadId, active.handoff.generation); + return true; + } + + private match(threadId: string, executionId: string, deliveryAttempt: number): ActiveFileExecution | null { + const active = this.active.get(threadId); + return active?.handoff.executionId === executionId && active.deliveryAttempt === deliveryAttempt + ? active : null; + } +} + +/** Settle a frozen attempt on its execution worker before the terminal writer operation. */ +export function prepareFrozenExecutionFileEvidence( + frozen: FrozenExecutionFileEvidence, + tracker: TurnFileTracker, + snapshots: SnapshotService, +): Promise { + return prepareExecutionFileEvidence({ + handoff: frozen.handoff, + turnId: frozen.turnId, + deliveryAttempt: frozen.deliveryAttempt, + outcome: frozen.outcome, + nativeDiff: frozen.nativeDiff, + tracker, + snapshots, + }); +} diff --git a/apps/server/src/features/agents/execution/execution-file-observation-handoff.ts b/apps/server/src/features/agents/execution/execution-file-observation-handoff.ts new file mode 100644 index 000000000..2e99e5b81 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-file-observation-handoff.ts @@ -0,0 +1,94 @@ +import type { AgentEvent, ProviderFileMutationStart } from "@mcode/contracts"; + +import type { + CapturedToolUseObservation, + FileTurnHandoff, + TurnFileTracker, +} from "../turns/turn-file-tracker.js"; + +const MAX_PENDING = 8; +const MAX_PENDING_BYTES = 4 * 1_048_576; +const MAX_CAPTURE_BYTES = 1_310_720; +const MAX_SEEN = 1_024; + +type PendingObservation = { readonly captured: CapturedToolUseObservation; readonly bytes: number }; +type ToolUse = Extract; + +/** Keeps synchronous pre-edit evidence for one exact execution until its tool event is admitted. */ +export class ExecutionFileObservationHandoff { + private readonly pending = new Map(); + private readonly seen = new Set(); + private pendingBytes = 0; + private closed = false; + + constructor( + private readonly tracker: TurnFileTracker, + private readonly handoff: FileTurnHandoff, + private readonly deliveryAttempt: number, + ) { + if (!Number.isSafeInteger(deliveryAttempt) || deliveryAttempt < 1) { + throw new Error("A positive file observation delivery attempt is required"); + } + } + + /** Capture before returning to the provider notification loop; no asynchronous read precedes this call. */ + capture(event: ProviderFileMutationStart): boolean { + if (!this.matches(event)) return false; + const key = event.toolCallId; + if (this.seen.has(key)) { + this.forget(key); + return false; + } + if (this.seen.size >= MAX_SEEN) return false; + this.seen.add(key); + if (this.pending.size >= MAX_PENDING) return false; + const captured = this.tracker.captureToolUseObservation( + event.threadId, key, event.toolName, event.toolInput, + ); + if (!captured || !this.matchesCapture(captured)) return false; + const bytes = Buffer.byteLength(JSON.stringify(captured), "utf8"); + if (bytes > MAX_CAPTURE_BYTES || this.pendingBytes + bytes > MAX_PENDING_BYTES) return false; + this.pending.set(key, { captured, bytes }); + this.pendingBytes += bytes; + return true; + } + + /** The caller must supply the tool event's source attempt, not its current scheduler attempt. */ + take(event: ToolUse, deliveryAttempt: number): CapturedToolUseObservation | null { + if (!this.matches({ ...event, deliveryAttempt })) return null; + const pending = this.pending.get(event.toolCallId); + if (!pending) return null; + this.forget(event.toolCallId); + return pending.captured; + } + + /** Fence Stop, retry, worker loss, or terminal handling before any later provider callback. */ + close(): void { + this.closed = true; + this.pending.clear(); + this.seen.clear(); + this.pendingBytes = 0; + } + + private matches(event: { readonly threadId: string; readonly turnExecutionId?: string; + readonly deliveryAttempt?: number }): boolean { + return !this.closed && event.threadId === this.handoff.threadId + && event.turnExecutionId === this.handoff.executionId + && event.deliveryAttempt === this.deliveryAttempt + && this.tracker.getCurrentTurnId(this.handoff.threadId) === String(this.handoff.generation); + } + + private matchesCapture(captured: CapturedToolUseObservation): boolean { + return captured.executionId === this.handoff.executionId + && captured.generation === this.handoff.generation + && captured.generationToken === this.handoff.generationToken + && captured.canonicalRoot === this.handoff.canonicalRoot; + } + + private forget(key: string): void { + const pending = this.pending.get(key); + if (!pending) return; + this.pending.delete(key); + this.pendingBytes -= pending.bytes; + } +} diff --git a/apps/server/src/features/agents/execution/execution-mailbox-owner.ts b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts new file mode 100644 index 000000000..5ca45bbc8 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts @@ -0,0 +1,130 @@ +import type { DataOnlyParentTurnStartInput } from "../canonical/canonical-parent-turn-write.js"; +import type { ExecutionParentStartContext } from "./provider-execution-event-state.js"; +import { sameExecution } from "./execution-mailbox-protocol.js"; +import type { ExecutionIdentity, ExecutionLease, ExecutionMailboxCompletion } from "./execution-mailbox-protocol.js"; +import type { ExecutionMailboxScheduler, ExecutionRecoveryReceipt } from "./execution-mailbox-scheduler.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; + +type Scheduler = ExecutionMailboxScheduler; + +interface OwnedExecution { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + started: boolean; +} + +/** Claims one mailbox before a parent turn can be written or sent to a provider. */ +export class ExecutionMailboxOwner { + private readonly active = new Map(); + + constructor(private readonly scheduler: Scheduler) {} + + /** Start the exact execution on its assigned worker and wait for its durable receipt. */ + async start(input: { + readonly execution: ExecutionIdentity; + readonly ownerEpoch: number; + readonly providerId: string; + readonly parentTurn: DataOnlyParentTurnStartInput; + readonly parentLive?: ExecutionParentStartContext; + readonly publishParentStart?: boolean; + }): Promise { + if (this.active.has(input.execution.threadId)) throw new Error("Thread already has an execution owner"); + const claim = this.scheduler.claim(input.execution, input.ownerEpoch); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const owner = { execution: input.execution, lease: claim.lease, started: false }; + this.active.set(input.execution.threadId, owner); + try { + const result = await this.submitOwned(owner, { + kind: "start", providerId: input.providerId, input: input.parentTurn, + ...(input.parentLive ? { parentLive: input.parentLive } : {}), + ...(input.publishParentStart ? { publishParentStart: true } : {}), + }); + if (result.kind !== "committed") throw new Error(`Execution start was not committed: ${input.execution.executionId}`); + owner.started = true; + } catch (error) { + // A lost worker is reconciled by its coordinator. A rejected start did not create worker state. + if (this.scheduler.release(owner.execution, owner.lease)) this.active.delete(input.execution.threadId); + throw error; + } + } + + /** Admit one ordered command only for the current thread execution and lease. */ + submit(execution: ExecutionIdentity, command: ExecutionWorkCommand): Promise { + return this.submitOwned(this.requireOwner(execution), command); + } + + /** Put Stop behind every already admitted event before awaiting provider teardown. */ + stop(execution: ExecutionIdentity, requestId: string): Promise { + const owner = this.requireOwner(execution); + return this.submitOwned(owner, { kind: "stop", requestId }); + } + + /** Release the slot only after finalization and its release command have settled. */ + async release(execution: ExecutionIdentity): Promise { + const owner = this.requireOwner(execution); + const result = await this.submitOwned(owner, { kind: "release" }); + if (result.kind !== "released" || !this.scheduler.release(owner.execution, owner.lease)) { + throw new Error(`Execution release was not acknowledged: ${execution.executionId}`); + } + this.active.delete(execution.threadId); + } + + /** Release one rejected execution only after the writer has durably settled that exact lease. */ + async releaseRecovered(execution: ExecutionIdentity, recovery: ExecutionRecoveryReceipt): Promise { + const owner = this.requireOwner(execution); + if (recovery.operationId !== `${owner.lease.leaseId}:worker-lost` + || recovery.kind === "conflict" && recovery.recoveryState !== "already-terminal") { + throw new Error("Recovered release has no matching durable evidence"); + } + const result = await this.submitOwned(owner, { kind: "release", recovery }); + if (result.kind !== "released" || !this.scheduler.release(owner.execution, owner.lease)) { + throw new Error(`Recovered execution release was not acknowledged: ${execution.executionId}`); + } + this.active.delete(execution.threadId); + } + + /** Forget a worker-lost execution only after the recovery coordinator released its lease. */ + forgetRecovered(execution: ExecutionIdentity, lease: ExecutionLease): void { + const owner = this.active.get(execution.threadId); + if (owner && sameExecution(owner.execution, execution) && owner.lease.leaseId === lease.leaseId) { + this.active.delete(execution.threadId); + } + } + + /** Current exact owner, for provider ingress and Stop routing. */ + current(threadId: string): OwnedExecution | undefined { + return this.active.get(threadId); + } + + /** Whether this owner received its durable start receipt. */ + isStarted(execution: ExecutionIdentity): boolean { + const owner = this.active.get(execution.threadId); + return owner?.started === true && sameExecution(owner.execution, execution); + } + + private requireOwner(execution: ExecutionIdentity): OwnedExecution { + const owner = this.active.get(execution.threadId); + if (!owner || !sameExecution(owner.execution, execution)) { + throw new Error(`No matching execution owner: ${execution.executionId}`); + } + return owner; + } + + private async submitOwned( + owner: OwnedExecution, + command: ExecutionWorkCommand | { readonly kind: "stop"; readonly requestId: string }, + ): Promise { + const byteLength = Buffer.byteLength(JSON.stringify({ execution: owner.execution, lease: owner.lease, command }), "utf8"); + const admitted = this.scheduler.submit({ ...owner, command, byteLength }); + if (admitted.kind !== "admitted") throw new Error(`Execution command admission failed: ${admitted.kind}`); + return requireReply(await admitted.completion); + } +} + +function requireReply(completion: ExecutionMailboxCompletion): ExecutionWorkerResult { + if (completion.kind !== "reply") throw new Error(`Execution worker ${completion.kind}`); + if (completion.result.kind === "rejected") { + throw new Error(`Execution worker rejected command: ${completion.result.reason}`); + } + return completion.result; +} diff --git a/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts b/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts new file mode 100644 index 000000000..4a96e9157 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts @@ -0,0 +1,63 @@ +/** The exact turn attempt that owns every command and reply in a mailbox. */ +export interface ExecutionIdentity { + readonly threadId: string; + readonly turnId: string; + readonly executionId: string; +} + +/** A durable owner epoch plus the local worker incarnation that may use it. */ +export interface ExecutionLease { + readonly ownerEpoch: number; + readonly workerIndex: number; + readonly workerGeneration: number; + readonly leaseId: string; +} + +/** One ordered, data-only command sent to the worker that owns an execution. */ +export interface ExecutionWorkerRequest { + readonly requestId: number; + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly ordinal: number; + /** Last admitted ordinal before Stop. Present only on a Stop request. */ + readonly stopWatermark?: number; + readonly command: Command; +} + +/** A reply must echo the complete identity and lease before it can be accepted. */ +export interface ExecutionWorkerReply { + readonly requestId: number; + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly ordinal: number; + readonly result: Result; +} + +/** A worker reply is transport completion; the result defines durable success or failure. */ +export type ExecutionMailboxCompletion = + | { readonly kind: "reply"; readonly result: Result } + | { readonly kind: "worker-lost" } + | { readonly kind: "shutdown" }; + +/** The worker port used by the fixed execution mailbox scheduler. */ +export interface ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null; + onerror: ((event: ErrorEvent) => void) | null; + onclose: (() => void) | null; + postMessage(request: ExecutionWorkerRequest): void; + terminate(): void; +} + +/** Identity fields carried by inbound provider routing, where the turn may be unresolved. */ +export type RoutedExecutionIdentity = Omit & { readonly turnId?: string | undefined }; + +/** Two executions match only when every durable identity field agrees. */ +export function sameExecution(left: RoutedExecutionIdentity, right: ExecutionIdentity): boolean { + return left.threadId === right.threadId && left.turnId === right.turnId && left.executionId === right.executionId; +} + +/** Two leases match only when the epoch, worker incarnation, and lease id all agree. */ +export function sameLease(left: ExecutionLease, right: ExecutionLease): boolean { + return left.ownerEpoch === right.ownerEpoch && left.workerIndex === right.workerIndex + && left.workerGeneration === right.workerGeneration && left.leaseId === right.leaseId; +} diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts new file mode 100644 index 000000000..d18a5b3b7 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -0,0 +1,449 @@ +import * as NodeCrypto from "node:crypto"; + +import { sameExecution, sameLease } from "./execution-mailbox-protocol.js"; +import type { + ExecutionIdentity, + ExecutionLease, + ExecutionMailboxCompletion, + ExecutionWorkerPort, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "./execution-mailbox-protocol.js"; + +/** The caller supplies typed start, event, checkpoint, outcome, and finalization work. */ +export type ExecutionMailboxCommand = + | Work + | { readonly kind: "stop"; readonly requestId: string }; + +/** Lifecycle results that must be allowed to settle after Stop is admitted. */ +export const EXECUTION_CONTROL_KINDS = [ + "checkpoint", "effect-result", "provider-outcome", "stage-terminal", "finalize", "release", + "finish-from-state", "finish-live-event", "post-terminal-event", +] as const; +/** A command kind that must still settle after Stop begins draining the mailbox. */ +export type ExecutionControlKind = typeof EXECUTION_CONTROL_KINDS[number]; +const CONTROL_KINDS: ReadonlySet = new Set(EXECUTION_CONTROL_KINDS); + +interface Pending { + readonly request: ExecutionWorkerRequest; + readonly byteLength: number; + readonly control: boolean; + readonly resolve: (completion: ExecutionMailboxCompletion) => void; +} + +interface Assignment { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly slot: Slot; + nextOrdinal: number; + pendingCount: number; + pendingBytes: number; + normalCount: number; + normalBytes: number; + stopping: boolean; + revoked: boolean; +} + +interface Slot { + readonly index: number; + generation: number; + worker: ExecutionWorkerPort | undefined; + inFlight: Pending | undefined; + readonly queues: Map[]>; + readonly readyThreads: string[]; + activeCount: number; +} + +/** Capacity includes posted work until its reply arrives. Control has a reserve. */ +export interface ExecutionMailboxLimits { + readonly maxPending: number; + readonly maxPendingBytes: number; + readonly reservedControl: number; + readonly reservedControlBytes: number; + readonly maxPerExecutionPending: number; + readonly maxPerExecutionBytes: number; + readonly reservedPerExecutionControl: number; + readonly reservedPerExecutionControlBytes: number; +} + +/** Exact ownership that a host must revoke before replacing a lost worker. */ +export interface ExecutionLostAssignment { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; +} + +/** Writer evidence that a lost worker's execution can release its thread and slot. */ +export type ExecutionRecoveryReceipt = + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number } + | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState?: "not-started" | "already-terminal" }; + +/** The host supplies durable owner epochs and a worker factory for each fixed slot. */ +export interface ExecutionMailboxOptions { + readonly workerCount: number; + readonly limits: ExecutionMailboxLimits; + readonly createWorker: (workerIndex: number) => ExecutionWorkerPort, Result>; + readonly onWorkerLost: (assignments: readonly ExecutionLostAssignment[], workerIndex: number) => void; +} + +/** The result of binding a turn attempt to a worker slot before its start is sent. */ +export type ExecutionClaim = + | { readonly kind: "claimed"; readonly lease: ExecutionLease } + | { readonly kind: "thread-busy" | "worker-unavailable" | "shutdown" }; + +/** The result of admitting one command; an admitted command resolves through its completion promise. */ +export type ExecutionAdmission = + | { readonly kind: "admitted"; readonly ordinal: number; readonly completion: Promise> } + | { readonly kind: "stale-execution" | "overloaded" | "invalid-size" | "stop-already-requested" | "stopping" | "shutdown" }; + +type AdmissionDecision = + | { readonly kind: "accept"; readonly control: boolean; readonly stop: boolean } + | { readonly kind: "invalid-size" | "overloaded" | "stop-already-requested" | "stopping" }; + +/** Bounded state visible to diagnostics without exposing command payloads. */ +export interface ExecutionMailboxDepth { + readonly pending: number; + readonly pendingBytes: number; + readonly activeExecutions: number; + readonly workers: readonly { readonly index: number; readonly queued: number; readonly inFlight: boolean }[]; +} + +/** + * Holds an execution on one fixed worker for its lifetime. This scheduler only + * admits and transports commands; a worker reply carries the writer's durable + * receipt and must be interpreted by the caller before publishing success. + */ +export class ExecutionMailboxScheduler { + private readonly slots: Slot, Result>[]; + private readonly byThread = new Map, Result>>(); + private readonly limits: ExecutionMailboxLimits; + private readonly createWorker: ExecutionMailboxOptions["createWorker"]; + private readonly onWorkerLost: ExecutionMailboxOptions["onWorkerLost"]; + private pendingCount = 0; + private pendingBytes = 0; + private normalCount = 0; + private normalBytes = 0; + private nextRequestId = 1; + private nextSlotIndex = 0; + private stopped = false; + + constructor(options: ExecutionMailboxOptions) { + validateOptions(options); + this.limits = options.limits; + this.createWorker = options.createWorker; + this.onWorkerLost = options.onWorkerLost; + this.slots = Array.from({ length: options.workerCount }, (_, index) => ({ + index, + generation: 0, + worker: undefined, + inFlight: undefined, + queues: new Map(), + readyThreads: [], + activeCount: 0, + })); + try { + for (const slot of this.slots) this.startWorker(slot); + } catch (error) { + for (const slot of this.slots) slot.worker?.terminate(); + throw error; + } + } + + /** Bind one turn attempt and its durable owner epoch before submitting start. */ + claim(execution: ExecutionIdentity, ownerEpoch: number): ExecutionClaim { + if (this.stopped) return { kind: "shutdown" }; + if (!validIdentity(execution) || !positiveInteger(ownerEpoch)) throw new Error("Invalid execution identity or owner epoch"); + if (this.byThread.has(execution.threadId)) return { kind: "thread-busy" }; + const slot = this.chooseSlot(); + if (!slot) return { kind: "worker-unavailable" }; + const lease: ExecutionLease = { + ownerEpoch, + workerIndex: slot.index, + workerGeneration: slot.generation, + leaseId: NodeCrypto.randomUUID(), + }; + this.byThread.set(execution.threadId, { + execution: { ...execution }, + lease, + slot, + nextOrdinal: 1, + pendingCount: 0, + pendingBytes: 0, + normalCount: 0, + normalBytes: 0, + stopping: false, + revoked: false, + }); + slot.activeCount += 1; + return { kind: "claimed", lease }; + } + + /** Admit in FIFO order. byteLength must cover the entire cloneable request. */ + submit(input: { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly command: ExecutionMailboxCommand; + readonly byteLength: number; + }): ExecutionAdmission { + if (this.stopped) return { kind: "shutdown" }; + const assignment = this.currentAssignment(input.execution, input.lease); + if (!assignment || assignment.revoked) return { kind: "stale-execution" }; + const decision = this.admissionDecision(assignment, input.command, input.byteLength); + if (decision.kind !== "accept") return { kind: decision.kind }; + const ordinal = assignment.nextOrdinal++; + const request: ExecutionWorkerRequest> = { + requestId: this.nextRequestId++, + execution: assignment.execution, + lease: assignment.lease, + ordinal, + ...(decision.stop ? { stopWatermark: ordinal - 1 } : {}), + command: input.command, + }; + let resolveCompletion: (completion: ExecutionMailboxCompletion) => void = () => {}; + const completion = new Promise>((resolve) => { + resolveCompletion = resolve; + }); + const pending: Pending, Result> = { + request, + byteLength: input.byteLength, + control: decision.control, + resolve: resolveCompletion, + }; + this.retain(assignment, pending); + if (decision.stop) assignment.stopping = true; + this.enqueue(assignment.slot, pending); + this.dispatch(assignment.slot); + return { kind: "admitted", ordinal, completion }; + } + + /** Release only after the worker has acknowledged every command for the execution. */ + release(execution: ExecutionIdentity, lease: ExecutionLease): boolean { + const assignment = this.currentAssignment(execution, lease); + if (!assignment || assignment.revoked || assignment.pendingCount !== 0) return false; + this.byThread.delete(execution.threadId); + assignment.slot.activeCount -= 1; + return true; + } + + /** Release a crashed worker's ownership only after its durable interruption is acknowledged. */ + reconcileLost(execution: ExecutionIdentity, lease: ExecutionLease, receipt: ExecutionRecoveryReceipt): boolean { + const assignment = this.currentAssignment(execution, lease); + if (!assignment?.revoked || assignment.pendingCount !== 0 + || !validRecoveryReceipt(lease, receipt)) return false; + this.byThread.delete(execution.threadId); + assignment.slot.activeCount -= 1; + return true; + } + + /** Replace a failed slot only after its revoked executions have been reconciled. */ + replaceWorker(index: number): boolean { + const slot = this.slots[index]; + if (this.stopped || !slot || slot.worker || slot.activeCount !== 0) return false; + this.startWorker(slot); + return true; + } + + /** Stop admission and settle retained work without replaying external effects. */ + shutdown(): void { + if (this.stopped) return; + this.stopped = true; + for (const slot of this.slots) { + slot.worker?.terminate(); + slot.worker = undefined; + this.settleSlot(slot, "shutdown"); + } + this.byThread.clear(); + } + + /** Return only counts, so telemetry cannot log provider or user content. */ + depth(): ExecutionMailboxDepth { + return { + pending: this.pendingCount, + pendingBytes: this.pendingBytes, + activeExecutions: this.byThread.size, + workers: this.slots.map((slot) => ({ + index: slot.index, + queued: [...slot.queues.values()].reduce((sum, queue) => sum + queue.length, 0), + inFlight: slot.inFlight !== undefined, + })), + }; + } + + private chooseSlot(): Slot, Result> | undefined { + const available = this.slots.filter((slot) => slot.worker !== undefined); + if (available.length === 0) return undefined; + const least = Math.min(...available.map((slot) => slot.activeCount)); + for (let offset = 0; offset < this.slots.length; offset += 1) { + const index = (this.nextSlotIndex + offset) % this.slots.length; + const slot = this.slots[index]; + if (slot?.worker && slot.activeCount === least) { + this.nextSlotIndex = (index + 1) % this.slots.length; + return slot; + } + } + return undefined; + } + + private currentAssignment(execution: ExecutionIdentity, lease: ExecutionLease): Assignment, Result> | undefined { + const assignment = this.byThread.get(execution.threadId); + if (!assignment || !sameExecution(assignment.execution, execution) || !sameLease(assignment.lease, lease)) return undefined; + return assignment; + } + + private admissionDecision( + assignment: Assignment, Result>, + command: ExecutionMailboxCommand, + bytes: number, + ): AdmissionDecision { + if (!positiveInteger(bytes)) return { kind: "invalid-size" }; + const stop = command.kind === "stop"; + const control = stop || CONTROL_KINDS.has(command.kind); + if (stop && assignment.stopping) return { kind: "stop-already-requested" }; + if (assignment.stopping && !control) return { kind: "stopping" }; + if (!this.hasCapacity(assignment, bytes, control)) return { kind: "overloaded" }; + return { kind: "accept", control, stop }; + } + + private hasCapacity(assignment: Assignment, Result>, bytes: number, control: boolean): boolean { + const limits = this.limits; + if (this.pendingCount + 1 > limits.maxPending || this.pendingBytes + bytes > limits.maxPendingBytes) return false; + if (assignment.pendingCount + 1 > limits.maxPerExecutionPending + || assignment.pendingBytes + bytes > limits.maxPerExecutionBytes) return false; + if (control) return true; + return this.normalCount + 1 <= limits.maxPending - limits.reservedControl + && this.normalBytes + bytes <= limits.maxPendingBytes - limits.reservedControlBytes + && assignment.normalCount + 1 <= limits.maxPerExecutionPending - limits.reservedPerExecutionControl + && assignment.normalBytes + bytes <= limits.maxPerExecutionBytes - limits.reservedPerExecutionControlBytes; + } + + private retain(assignment: Assignment, Result>, pending: Pending, Result>): void { + this.pendingCount += 1; + this.pendingBytes += pending.byteLength; + assignment.pendingCount += 1; + assignment.pendingBytes += pending.byteLength; + if (pending.control) return; + this.normalCount += 1; + this.normalBytes += pending.byteLength; + assignment.normalCount += 1; + assignment.normalBytes += pending.byteLength; + } + + private settle(pending: Pending, Result>, completion: ExecutionMailboxCompletion): void { + const assignment = this.byThread.get(pending.request.execution.threadId); + this.pendingCount -= 1; + this.pendingBytes -= pending.byteLength; + if (assignment && sameLease(assignment.lease, pending.request.lease)) { + assignment.pendingCount -= 1; + assignment.pendingBytes -= pending.byteLength; + if (!pending.control) { + assignment.normalCount -= 1; + assignment.normalBytes -= pending.byteLength; + } + } + if (!pending.control) { + this.normalCount -= 1; + this.normalBytes -= pending.byteLength; + } + pending.resolve(completion); + } + + private enqueue(slot: Slot, Result>, pending: Pending, Result>): void { + const threadId = pending.request.execution.threadId; + const queue = slot.queues.get(threadId); + if (queue) queue.push(pending); + else { + slot.queues.set(threadId, [pending]); + slot.readyThreads.push(threadId); + } + } + + private dispatch(slot: Slot, Result>): void { + if (this.stopped || !slot.worker || slot.inFlight) return; + const threadId = slot.readyThreads.shift(); + if (!threadId) return; + const queue = slot.queues.get(threadId); + const next = queue?.shift(); + if (!queue || !next) throw new Error("Mailbox ready list lost its queue"); + if (queue.length > 0) slot.readyThreads.push(threadId); + else slot.queues.delete(threadId); + slot.inFlight = next; + try { + slot.worker.postMessage(next.request); + } catch { + this.workerLost(slot, slot.generation); + } + } + + private startWorker(slot: Slot, Result>): void { + if (this.stopped) return; + const generation = ++slot.generation; + const worker = this.createWorker(slot.index); + worker.onmessage = (event) => this.receive(slot, generation, event.data); + worker.onerror = () => this.workerLost(slot, generation); + worker.onclose = () => this.workerLost(slot, generation); + slot.worker = worker; + } + + private receive(slot: Slot, Result>, generation: number, reply: ExecutionWorkerReply): void { + if (this.stopped || slot.generation !== generation) return; + const pending = slot.inFlight; + if (!pending || !matchesReply(pending.request, reply)) { + this.workerLost(slot, generation); + return; + } + slot.inFlight = undefined; + this.settle(pending, { kind: "reply", result: reply.result }); + this.dispatch(slot); + } + + private workerLost(slot: Slot, Result>, generation: number): void { + if (this.stopped || slot.generation !== generation) return; + const worker = slot.worker; + slot.worker = undefined; + slot.generation += 1; + worker?.terminate(); + const revokedAssignments = [...this.byThread.values()].filter((assignment) => assignment.slot === slot); + for (const assignment of revokedAssignments) assignment.revoked = true; + const revoked = revokedAssignments.map((assignment) => ({ execution: assignment.execution, lease: assignment.lease })); + this.settleSlot(slot, "worker-lost"); + this.onWorkerLost(revoked, slot.index); + } + + private settleSlot(slot: Slot, Result>, kind: "worker-lost" | "shutdown"): void { + const pending = [slot.inFlight, ...slot.queues.values()].flat().filter((item) => item !== undefined); + slot.inFlight = undefined; + slot.queues.clear(); + slot.readyThreads.length = 0; + for (const item of pending) this.settle(item, { kind }); + } +} + +function validRecoveryReceipt(lease: ExecutionLease, receipt: ExecutionRecoveryReceipt): boolean { + return receipt.operationId === `${lease.leaseId}:worker-lost` + && (receipt.kind === "conflict" || Number.isSafeInteger(receipt.durableRevision)); +} + +function matchesReply(request: ExecutionWorkerRequest, reply: ExecutionWorkerReply): boolean { + return request.requestId === reply.requestId && request.ordinal === reply.ordinal + && sameExecution(request.execution, reply.execution) && sameLease(request.lease, reply.lease); +} + +function validIdentity(execution: ExecutionIdentity): boolean { + return execution.threadId.length > 0 && execution.turnId.length > 0 && execution.executionId.length > 0; +} + +function positiveInteger(value: number): boolean { + return Number.isSafeInteger(value) && value > 0; +} + +function validateOptions(options: ExecutionMailboxOptions): void { + const limits = options.limits; + const values = [options.workerCount, limits.maxPending, limits.maxPendingBytes, limits.maxPerExecutionPending, + limits.maxPerExecutionBytes, limits.reservedControl, limits.reservedControlBytes, + limits.reservedPerExecutionControl, limits.reservedPerExecutionControlBytes]; + if (values.some((value) => !positiveInteger(value))) throw new Error("Mailbox limits must be positive safe integers"); + if (limits.reservedControl >= limits.maxPending || limits.reservedControlBytes >= limits.maxPendingBytes + || limits.reservedPerExecutionControl >= limits.maxPerExecutionPending + || limits.reservedPerExecutionControlBytes >= limits.maxPerExecutionBytes) { + throw new Error("Mailbox control reserves must leave normal capacity"); + } +} diff --git a/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts b/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts new file mode 100644 index 000000000..a3557a43b --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts @@ -0,0 +1,170 @@ +import { sameExecution } from "./execution-mailbox-protocol.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import type { ExecutionMailboxOwner } from "./execution-mailbox-owner.js"; +import type { ProviderEventCommitReceipt } from "@mcode/providers"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; +import type { ProviderEventOwnership, ProviderEventOwnershipRoute, WorkerOwnedProviderEventBatch, WorkerOwnedProviderEventResult } from "../../providers/composition/provider-host-ports.js"; + +interface ActiveRoute { + readonly kind: "active"; + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly pending: Set>; +} + +interface SwitchingRoute { + readonly kind: "switching"; + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly drained: Promise; +} + +type Route = ActiveRoute | SwitchingRoute; + +/** + * Binds provider batches to the execution mailbox after its durable start. + * Unknown and retired executions fail closed while this worker route is active. + */ +export class ExecutionProviderEventOwnership implements ProviderEventOwnership { + private readonly routes = new Map(); + private readonly workerProviders = new Set(["codex"]); + private onCommitted: ((execution: ExecutionIdentity, result: Extract) => Promise) | undefined; + private prepareCommand: ((execution: ExecutionIdentity, batch: WorkerOwnedProviderEventBatch) => Promise>) | undefined; + + constructor(private readonly owner: Pick) {} + + /** Run execution lifecycle effects after the writer acknowledgement and before the provider receives its batch receipt. */ + bindCommitted(onCommitted: (execution: ExecutionIdentity, result: Extract) => Promise): void { + this.onCommitted = onCommitted; + } + + /** Prepare exact file and terminal inputs before admitting the provider batch to its mailbox. */ + bindCommandPreparation(prepare: (execution: ExecutionIdentity, batch: WorkerOwnedProviderEventBatch) => Promise>): void { + this.prepareCommand = prepare; + } + + /** Admit one provider attempt only while the exact mailbox still owns the thread. */ + async bind(execution: ExecutionIdentity, deliveryAttempt: number): Promise { + this.requireOwner(execution, deliveryAttempt); + const existing = this.routes.get(execution.executionId); + if (existing?.kind === "switching") { + await existing.drained; + return this.bind(execution, deliveryAttempt); + } + if (existing) this.requireReplacement(existing, execution, deliveryAttempt); + if (existing?.deliveryAttempt === deliveryAttempt) return; + if (existing) await this.drainPrevious(existing, execution, deliveryAttempt); + this.routes.set(execution.executionId, { + kind: "active", execution, deliveryAttempt, pending: new Set(), + }); + } + + private requireOwner(execution: ExecutionIdentity, deliveryAttempt: number): void { + if (!Number.isSafeInteger(deliveryAttempt) || deliveryAttempt < 1 + || !this.owner.isStarted(execution)) { + throw new Error("Provider attempt has no matching execution owner"); + } + } + + private requireReplacement(existing: ActiveRoute, execution: ExecutionIdentity, deliveryAttempt: number): void { + if (!sameExecution(existing.execution, execution) || deliveryAttempt < existing.deliveryAttempt) { + throw new Error("Provider attempt cannot replace its current owner"); + } + } + + private async drainPrevious(existing: ActiveRoute, execution: ExecutionIdentity, deliveryAttempt: number): Promise { + const switching: SwitchingRoute = { + kind: "switching", execution, deliveryAttempt, + drained: Promise.allSettled(existing.pending).then(() => undefined), + }; + this.routes.set(execution.executionId, switching); + await switching.drained; + if (this.routes.get(execution.executionId) !== switching || !this.owner.isStarted(execution)) { + throw new Error("Provider attempt lost its execution owner while draining"); + } + } + + /** Keep the execution rejected after its terminal fence releases ownership. */ + async retire(execution: ExecutionIdentity): Promise { + const current = this.routes.get(execution.executionId); + if (current && sameExecution(current.execution, execution)) { + this.routes.delete(execution.executionId); + if (current.kind === "switching") await current.drained; + else await Promise.allSettled(current.pending); + } + } + + /** Select the exact mailbox, then check again before asynchronous admission. */ + resolve(executionId: string, sourceProviderId?: string): ProviderEventOwnershipRoute { + const route = this.routes.get(executionId); + if (route?.kind !== "active" || !this.owner.isStarted(route.execution)) { + return { kind: sourceProviderId && !this.workerProviders.has(sourceProviderId) ? "legacy" : "rejected" }; + } + return { + kind: "worker", + ...route.execution, + deliveryAttempt: route.deliveryAttempt, + submit: (batch) => this.submit(route, batch), + }; + } + + private async submit(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): Promise { + if (!this.isCurrent(route, batch)) { + throw new Error("Provider batch belongs to a retired execution attempt"); + } + const pending = this.commitAndNotify(route, batch); + route.pending.add(pending); + try { + return await pending; + } finally { + route.pending.delete(pending); + } + } + + private async commitAndNotify(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): Promise { + const result = await this.submitPrepared(route, batch); + if (result.kind !== "committed" || !result.providerCommit) { + throw new Error("Execution worker did not commit provider batch"); + } + await this.notifyCommitted(route.execution, result); + const commit: ProviderEventCommitReceipt = { + ...result.providerCommit, + outcome: result.providerCommit.outcome === "terminal-outcome-confirmed" + ? "duplicate" : result.providerCommit.outcome, + }; + if (commit.outcome !== "committed" && result.providerEvents?.length) { + throw new Error("Noncommitted provider batch returned projected events"); + } + return { + batchId: batch.batchId, + deliveryAttempt: batch.deliveryAttempt, + commit, + providerEvents: result.providerEvents ?? [], + }; + } + + private async submitPrepared(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): Promise { + const command = this.prepareCommand + ? await this.prepareCommand(route.execution, batch) + : { kind: "event" as const, phase: batch.phase, nativeCursor: batch.nativeCursor ?? null, events: batch.events }; + if (!this.isCurrent(route, batch) || command.kind !== "event" + || command.events !== batch.events || command.phase !== batch.phase) { + throw new Error("Prepared provider event lost its execution owner"); + } + return await this.owner.submit(route.execution, command); + } + + private isCurrent(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): boolean { + return this.routes.get(route.execution.executionId) === route + && this.owner.isStarted(route.execution) + && batch.deliveryAttempt === route.deliveryAttempt + && sameExecution(batch, route.execution); + } + + private async notifyCommitted( + execution: ExecutionIdentity, + result: Extract, + ): Promise { + await this.onCommitted?.(execution, result); + } +} diff --git a/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts b/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts new file mode 100644 index 000000000..a0e3ad04b --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts @@ -0,0 +1,155 @@ +import "reflect-metadata"; +import type { AgentEvent, TurnFileEffectSummary } from "@mcode/contracts"; +import { hostRuntime } from "@mcode/shared/node/host-runtime"; + +import { RealGitExecutor } from "../../projects/git/execution/real-git-executor.js"; +import { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; +import { TurnFileTracker, type CapturedToolUseObservation, type FileTurnHandoff } from "../turns/turn-file-tracker.js"; +import type { PreparedExecutionFileEvidence } from "../turns/turn-execution-file-evidence.js"; +import { sameExecution } from "./execution-mailbox-protocol.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import { + prepareFrozenExecutionFileEvidence, + type FrozenExecutionFileEvidence, +} from "./execution-file-evidence-coordinator.js"; + +type ToolUse = Extract; +type ToolResult = Extract; +const MAX_OPEN_TOOL_CALLS = 64; + +interface ActiveFileExecution { + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly handoff: FileTurnHandoff; + readonly openToolCalls: Set; + sealed: boolean; + settlement?: Promise; +} + +/** Runs file attribution and terminal settlement on the execution worker, with no database access. */ +export class ExecutionWorkerFileEvidence { + private readonly active = new Map(); + private readonly tracker: TurnFileTracker; + + constructor(private readonly snapshots = new SnapshotService(new RealGitExecutor())) { + this.tracker = new TurnFileTracker( + (cwd, ref, path) => snapshots.getFileAtRef(cwd, ref, path), + () => {}, + hostRuntime.platform, + ); + } + + /** Install the main-loop handoff only against the scheduler's independent cwd and execution. */ + begin(input: { + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly cwd: string; + readonly handoff: FileTurnHandoff; + }): boolean { + if (!Number.isSafeInteger(input.deliveryAttempt) || input.deliveryAttempt < 1) return false; + const { execution, handoff } = input; + const current = this.active.get(execution.threadId); + if (current) return this.matches(current, execution, input.deliveryAttempt) + && current.handoff.generationToken === handoff.generationToken; + if (!this.tracker.beginTurnFromHandoff(handoff, { + threadId: execution.threadId, executionId: execution.executionId, cwd: input.cwd, + })) return false; + this.active.set(execution.threadId, { + execution, deliveryAttempt: input.deliveryAttempt, handoff, + openToolCalls: new Set(), sealed: false, + }); + return true; + } + + /** Attach the captured pre-edit state before processing the matching tool event. */ + async observeToolUse(input: { + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly event: ToolUse; + readonly captured: CapturedToolUseObservation | null; + }): Promise { + const active = this.activeFor(input.execution, input.deliveryAttempt); + if (!active || active.sealed || !input.captured || !this.matchesEvent(active, input.event) + || active.openToolCalls.has(input.event.toolCallId) + || active.openToolCalls.size >= MAX_OPEN_TOOL_CALLS) return false; + const observed = await this.tracker.observeCapturedToolUse(input.event, input.captured); + if (observed) active.openToolCalls.add(input.event.toolCallId); + return observed; + } + + /** Recompute the tracked net effect after a previously captured tool finishes. */ + async observeToolResult(input: { + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly event: ToolResult; + }): Promise { + const active = this.activeFor(input.execution, input.deliveryAttempt); + if (!active || active.sealed || !this.matchesEvent(active, input.event) + || !active.openToolCalls.has(input.event.toolCallId)) return null; + await this.tracker.observeToolResult(active.execution.threadId, input.event.toolCallId); + active.openToolCalls.delete(input.event.toolCallId); + return this.tracker.finalizeTurn(active.execution.threadId, active.handoff.generation); + } + + /** Apply a baseline captured after start to this attempt's tracker generation only. */ + async setBaselineRef(execution: ExecutionIdentity, deliveryAttempt: number, ref: string): Promise { + const active = this.activeFor(execution, deliveryAttempt); + if (!active || active.sealed) return false; + await this.tracker.setBaselineRef(execution.threadId, active.handoff.generation, ref); + return true; + } + + /** Freeze exact attempt evidence before the one terminal writer operation. */ + settle(frozen: FrozenExecutionFileEvidence): Promise { + const active = this.active.get(frozen.handoff.threadId); + if (!active || !this.matches(active, { + threadId: frozen.handoff.threadId, turnId: frozen.turnId, + executionId: frozen.handoff.executionId, + }, frozen.deliveryAttempt) + || active.handoff.generation !== frozen.handoff.generation + || active.handoff.generationToken !== frozen.handoff.generationToken) return Promise.resolve(null); + if (active.settlement) return active.settlement; + active.sealed = true; + const settlement = this.settleActive(active, frozen); + active.settlement = settlement; + void settlement.catch(() => { + if (active.settlement === settlement) active.settlement = undefined; + }); + return settlement; + } + + /** Clear a settled generation only after its final writer receipt. */ + retire(execution: ExecutionIdentity, deliveryAttempt: number): boolean { + const active = this.activeFor(execution, deliveryAttempt); + if (!active) return false; + this.active.delete(execution.threadId); + this.tracker.clearTurn(execution.threadId, active.handoff.generation); + return true; + } + + private activeFor(execution: ExecutionIdentity, deliveryAttempt: number): ActiveFileExecution | null { + const active = this.active.get(execution.threadId); + return active && this.matches(active, execution, deliveryAttempt) ? active : null; + } + + private matches(active: ActiveFileExecution, execution: ExecutionIdentity, deliveryAttempt: number): boolean { + return sameExecution(active.execution, execution) + && active.deliveryAttempt === deliveryAttempt; + } + + private matchesEvent(active: ActiveFileExecution, event: ToolUse | ToolResult): boolean { + return event.threadId === active.execution.threadId + && event.turnExecutionId === active.execution.executionId; + } + + private async settleActive( + active: ActiveFileExecution, + frozen: FrozenExecutionFileEvidence, + ): Promise { + for (const toolCallId of active.openToolCalls) { + await this.tracker.observeToolResult(active.execution.threadId, toolCallId); + } + active.openToolCalls.clear(); + return prepareFrozenExecutionFileEvidence(frozen, this.tracker, this.snapshots); + } +} diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts new file mode 100644 index 000000000..d4c007c59 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -0,0 +1,806 @@ +import type { AgentEvent, ParentNarrativeRecoveryItem, PlanQuestion, PlanRecord, TurnFileEffectSummary, TurnOutcome } from "@mcode/contracts"; +import type { ProviderEventDraft } from "@mcode/providers"; + +import type { + DataOnlyParentLiveMessageInput, + DataOnlyParentTerminalProjectionInput, + DataOnlyParentTurnFinishInput, + DataOnlyParentTurnStartInput, +} from "../canonical/canonical-parent-turn-write.js"; +import type { CodexSystemWriterIntent } from "../canonical/canonical-codex-system-error-projection.js"; +import type { CanonicalAgentCommitResult } from "../canonical/canonical-agent-boundary.js"; +import type { ProviderEventIngressEvent } from "../../providers/composition/provider-event-ingress.js"; +import type { + ParentAssistantTextCheckpointInput, + ParentAssistantTextCheckpointResult, +} from "../turns/parent-assistant-text-checkpoint-service.js"; +import type { ParentNarrativeRecoveryCommit } from "../turns/parent-turn-durability.js"; +import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; +import type { PlanPersistenceReady } from "../planning/plan-execution-state.js"; +import { sameExecution, sameLease } from "./execution-mailbox-protocol.js"; +import type { + ExecutionIdentity, + ExecutionLease, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "./execution-mailbox-protocol.js"; +import type { ExecutionMailboxCommand, ExecutionRecoveryReceipt } from "./execution-mailbox-scheduler.js"; +import { ProviderExecutionEventState, type ExecutionParentStartContext, type PreparedProviderLiveEvent } from "./provider-execution-event-state.js"; +import { ExecutionWorkerFileEvidence } from "./execution-worker-file-evidence.js"; +import type { FrozenExecutionFileEvidence } from "./execution-file-evidence-coordinator.js"; +import type { CapturedToolUseObservation, FileTurnHandoff } from "../turns/turn-file-tracker.js"; +import type { SyntheticTerminalInput } from "./codex-live-event-reducer.js"; + +/** Data that an execution worker may receive without a server dependency container. */ +export type ExecutionWorkCommand = + | { readonly kind: "start"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput; readonly parentLive?: ExecutionParentStartContext; readonly publishParentStart?: boolean; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } + | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } + | { readonly kind: "begin-files"; readonly cwd: string; readonly handoff: FileTurnHandoff; readonly deliveryAttempt: number } + | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly parentLive?: ParentLiveEffects; readonly terminalInput?: DataOnlyParentTurnFinishInput; readonly deliveryAttempt?: number; readonly capturedFileObservation?: CapturedToolUseObservation | null; readonly frozenFileEvidence?: FrozenExecutionFileEvidence; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } + | { readonly kind: "assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } + | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } + | ({ readonly kind: "live-event"; readonly publication: ExecutionLivePublicationIntent } & ParentLiveEffects) + | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } + | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } + | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } + | { readonly kind: "stage-terminal"; readonly input: DataOnlyParentTerminalProjectionInput } + | { readonly kind: "finalize"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } + | { readonly kind: "finish-live-event"; readonly outcome: TurnOutcome; readonly projection: DataOnlyParentTerminalProjectionInput; readonly input: DataOnlyParentTurnFinishInput; readonly providerEvent?: TerminalProviderEventInput; readonly frozenFileEvidence?: FrozenExecutionFileEvidence; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } + | { readonly kind: "finish-from-state"; readonly outcome: SyntheticTerminalInput["outcome"]; readonly input: DataOnlyParentTurnFinishInput; readonly frozenFileEvidence?: FrozenExecutionFileEvidence } + | ({ readonly kind: "post-terminal-event"; readonly publication: ExecutionLivePublicationIntent } & PostTerminalEffects) + | { readonly kind: "release"; readonly recovery?: ExecutionRecoveryReceipt }; + +/** Parent effects prepared from the same provider event as its canonical draft. */ +export interface ParentLiveEffects { + readonly text: + | { readonly kind: "unchanged" } + | { readonly kind: "append"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } + | { readonly kind: "reclassify"; readonly expectedText: string } + | { readonly kind: "promote"; readonly input: ParentAssistantTextCheckpointInput }; + readonly narrative?: ParentNarrativeRecoveryCommit; + readonly taskIntents?: readonly TaskToolWriteIntent[]; + readonly systemIntents?: readonly CodexSystemWriterIntent[]; + readonly message?: DataOnlyParentLiveMessageInput; + readonly planQuestions?: readonly PlanQuestion[]; + readonly planOutput?: PlanPersistenceReady; +} + +/** The only effects admitted after an execution's terminal projection has committed. */ +export interface PostTerminalEffects { + readonly hooks?: readonly Extract[]; + readonly systemIntents?: readonly CodexSystemWriterIntent[]; + readonly providerEvent?: TerminalProviderEventInput; +} + +/** Provider evidence committed with a terminal projection; synthetic Stop completion omits it. */ +export interface TerminalProviderEventInput { + readonly phase: string; + readonly nativeCursor: unknown | null; + readonly events: readonly ProviderEventDraft[]; +} + +/** One complete semantic mutation. The single writer decides its SQL transaction. */ +export interface ExecutionSemanticOperation { + readonly operationId: string; + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly ordinal: number; + /** Live Codex events to release with this operation's durable receipt. */ + readonly livePublication?: readonly ExecutionLivePublicationIntent[]; + readonly mutation: + | { readonly kind: "begin"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } + | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } + | { readonly kind: "append-events"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly parentLive?: ParentLiveEffects } + | { readonly kind: "append-assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } + | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } + | ({ readonly kind: "live-event" } & ParentLiveEffects) + | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } + | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } + | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } + | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } + | { readonly kind: "stage-terminal"; readonly input: DataOnlyParentTerminalProjectionInput } + | { readonly kind: "worker-lost"; readonly reason: string; readonly recoveryIncidentId: string } + | { readonly kind: "finish"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput } + | { readonly kind: "finish-live-event"; readonly outcome: TurnOutcome; readonly projection: DataOnlyParentTerminalProjectionInput; readonly input: DataOnlyParentTurnFinishInput; readonly providerEvent?: TerminalProviderEventInput } + | ({ readonly kind: "post-terminal-event" } & PostTerminalEffects); +} + +/** A live event has one durability barrier and stays inert until its semantic effects commit. */ +export interface ExecutionLivePublicationIntent { + readonly event: AgentEvent; + readonly after: "writer" | "terminal"; +} + +/** Stable identity lets the transport deduplicate a replayed publication receipt. */ +export interface ExecutionLivePublicationReceipt extends ExecutionLivePublicationIntent { + readonly publicationId: string; +} + +/** Plan questions may be pushed only after their text event's durable writer receipt. */ +export interface ExecutionPlanQuestionsReceipt { + readonly publicationId: string; + readonly threadId: string; + readonly questions: readonly PlanQuestion[]; +} + +/** Provider-facing receipt values retained with the execution operation. */ +export type ExecutionProviderCommitReceipt = Pick< + CanonicalAgentCommitResult, + "outcome" | "conversationRevision" | "rosterRevision" | "acceptedThrough" | "durableThrough" +> & { readonly eventCount: number }; + +/** A committed runtime event after writer-local provider interpretation. */ +export type ProjectedCommittedProviderEvent = Omit< + ProviderEventIngressEvent, + "sourceKind" | "canonicalReceipt" | "runtimeExtension" +> & { + readonly sourceKind: "canonical-commit"; + readonly canonicalReceipt: NonNullable; +}; + +/** Writer-confirmed terminal projection surfaced after its durable finish. */ +export interface ExecutionTerminalPersistenceReceipt { + readonly messageId: string | null; + readonly outcome: TurnOutcome; + readonly toolCallCount: number; + readonly filesChanged: readonly string[]; + readonly fileEffects?: TurnFileEffectSummary; +} + +/** A writer reply is valid only after the semantic operation commits durably. */ +export type ExecutionWriteReceipt = + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[]; readonly planQuestions?: ExecutionPlanQuestionsReceipt; readonly planOutput?: PlanRecord; readonly terminalPersistence?: ExecutionTerminalPersistenceReceipt } + | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState?: "not-started" | "already-terminal" }; + +/** + * Implemented by one acknowledged writer, not by a worker-local SQLite connection. + * It must fence the durable lease and enforce terminal prerequisites in the + * same transaction that records each semantic operation. + */ +export interface ExecutionSemanticWriter { + transact(operation: ExecutionSemanticOperation): Promise; +} + +/** Execution runtime effects returned only after their associated parent write commits. */ +export type ExecutionParentEventResult = Pick; + +/** A command result that never calls an uncommitted mutation successful. */ +export type ExecutionWorkerResult = + | (Extract & { readonly parentEvent?: ExecutionParentEventResult }) + | { readonly kind: "released" } + | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | "invalid-stop-watermark" | "writer-conflict" | "writer-failure" }; + +type WorkerCommand = ExecutionMailboxCommand; + +interface ExecutionState { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly providerId: string; + readonly parentEvents?: ProviderExecutionEventState; + nextOrdinal: number; + phase: "running" | "stopping" | "finalized" | "poisoned"; + durableRevision: number; + fileAttempt?: number; +} + +interface PreparedExecutionRequest { + readonly request: ExecutionWorkerRequest; + readonly parentEvent?: ExecutionParentEventResult; +} + +/** + * Applies one worker command at a time per slot. This module has no database + * handle, provider process, or publication callback. It only advances state + * after the single writer acknowledges the semantic operation. + */ +export class ExecutionWorkerHandler { + private readonly states = new Map(); + + constructor(private readonly writer: ExecutionSemanticWriter, private readonly files = new ExecutionWorkerFileEvidence()) {} + + /** Whether every execution assigned to this worker has released its state. */ + get isIdle(): boolean { return this.states.size === 0; } + + /** Process one mailbox command and echo its exact execution and lease. */ + async handle(request: ExecutionWorkerRequest): Promise> { + const result = await this.apply(request); + return { + requestId: request.requestId, + execution: request.execution, + lease: request.lease, + ordinal: request.ordinal, + result, + }; + } + + private async apply(request: ExecutionWorkerRequest): Promise { + const state = this.states.get(request.execution.threadId); + if (request.command.kind === "start" || request.command.kind === "resume") { + return await this.begin(request, state); + } + if (!state) return { kind: "rejected", reason: "no-execution" }; + return await this.applyToState(request, state); + } + + private async applyToState( + request: ExecutionWorkerRequest, + state: ExecutionState, + ): Promise { + const rejection = commandRejection(request, state); + if (rejection) return { kind: "rejected", reason: rejection }; + if (request.command.kind === "release") return this.release(request, state); + if (state.phase === "poisoned" || state.phase === "finalized" + && request.command.kind !== "event" && request.command.kind !== "post-terminal-event") { + return { kind: "rejected", reason: "invalid-transition" }; + } + const prepared = await this.prepareOwnedRequest(request, state); + if (!prepared) return { kind: "rejected", reason: "invalid-event-routing" }; + const mutation = mutationFor(prepared.request, state); + if (!mutation) { + if (prepared.parentEvent) state.phase = "poisoned"; + return { kind: "rejected", reason: invalidReason(request) }; + } + return await this.commitRequest(prepared, state, mutation); + } + + private async prepareOwnedRequest(request: ExecutionWorkerRequest, state: ExecutionState): Promise { + try { + const prepared = this.prepareRequest(request, state); + if (!prepared) return undefined; + const withFiles = await this.prepareFileEffects(prepared, state); + if (!withFiles && prepared.parentEvent) state.phase = "poisoned"; + return withFiles; + } catch (error) { + if (state.parentEvents) state.phase = "poisoned"; + throw error; + } + } + + private async commitRequest( + prepared: PreparedExecutionRequest, + state: ExecutionState, + mutation: ExecutionSemanticOperation["mutation"], + ): Promise { + const { request } = prepared; + let receipt: ExecutionWriteReceipt; + try { + receipt = await this.writer.transact(operationFor(request, mutation)); + } catch (error) { + if (state.parentEvents) state.phase = "poisoned"; + throw error; + } + if (!isDurableReceipt(receipt, request, state.durableRevision)) { + if (state.parentEvents) state.phase = "poisoned"; + return { kind: "rejected", reason: "writer-conflict" }; + } + state.nextOrdinal += 1; + state.durableRevision = receipt.durableRevision; + if (request.command.kind === "stop") state.phase = "stopping"; + if (request.command.kind === "finalize" || request.command.kind === "finish-live-event") { + state.phase = "finalized"; + if (state.fileAttempt !== undefined) this.files.retire(state.execution, state.fileAttempt); + } + return prepared.parentEvent ? { ...receipt, parentEvent: prepared.parentEvent } : receipt; + } + + private prepareRequest(request: ExecutionWorkerRequest, state: ExecutionState): PreparedExecutionRequest | undefined { + const command = request.command; + if (command.kind === "finish-from-state") return this.prepareSyntheticFinish(request, command, state); + if (!state.parentEvents || command.kind !== "event") return { request }; + if (state.phase === "finalized") return this.preparePostTerminalEvent(request, command, state); + if (!validOwnedEventCommand(command, state)) return undefined; + const reduction = state.parentEvents.prepare(command.events, command.terminalInput !== undefined); + if (reduction.kind === "rejected") return undefined; + if (reduction.kind === "writer-owned") return { request }; + const { effects, ...parentEvent } = reduction.prepared; + if (parentEvent.publication.after === "terminal") { + const terminal = prepareTerminalEvent(command, parentEvent, state.providerId); + if (!terminal) { state.phase = "poisoned"; return undefined; } + return { request: { ...request, command: terminal }, parentEvent }; + } + return { request: { ...request, command: { ...command, + parentLive: effects, livePublication: [parentEvent.publication] } }, parentEvent }; + } + + + private prepareSyntheticFinish( + request: ExecutionWorkerRequest, + command: Extract, + state: ExecutionState, + ): PreparedExecutionRequest | undefined { + const terminalInput = syntheticTerminalInput(command, state); + if (!terminalInput || !state.parentEvents) return undefined; + const reduction = state.parentEvents.finishFromState(terminalInput); + if (reduction.kind !== "parent" || !reduction.prepared.terminal) return undefined; + const { effects: _effects, ...parentEvent } = reduction.prepared; + const projection = reduction.prepared.terminal; + const finish: Extract = { + kind: "finish-live-event", outcome: command.outcome, projection, + input: { ...command.input, projection: { kind: "writer-staged", + ...(projection.assistant.messageId ? { messageId: projection.assistant.messageId } : {}) } }, + ...(command.frozenFileEvidence ? { frozenFileEvidence: command.frozenFileEvidence } : {}), + livePublication: [parentEvent.publication], + }; + return { request: { ...request, command: finish }, parentEvent }; + } + + private preparePostTerminalEvent( + request: ExecutionWorkerRequest, + command: Extract, + state: ExecutionState, + ): PreparedExecutionRequest | undefined { + if (!validLateOwnedEventCommand(command, state) || !state.parentEvents) return undefined; + const reduction = state.parentEvents.prepare(command.events, true); + if (reduction.kind !== "parent" || reduction.prepared.terminal) return undefined; + const { effects, ...prepared } = reduction.prepared; + const parentEvent = { ...prepared, publication: { ...prepared.publication, after: "terminal" as const } }; + const hooks = effects.narrative?.items.filter((item) => item.kind === "hook"); + return { parentEvent, request: { ...request, command: { + kind: "post-terminal-event", publication: parentEvent.publication, + ...(hooks?.length ? { hooks } : {}), ...(effects.systemIntents ? { systemIntents: effects.systemIntents } : {}), + providerEvent: { phase: command.phase, nativeCursor: command.nativeCursor, events: command.events }, + } } }; + } + + private async prepareFileEffects(prepared: PreparedExecutionRequest, state: ExecutionState): Promise { + const command = prepared.request.command; + if (command.kind === "begin-files") { + return this.beginFileEffects(command, state) ? prepared : undefined; + } + if (command.kind === "finish-live-event") { + if (state.fileAttempt !== undefined) return this.prepareTerminalFiles(prepared, command, state); + return command.frozenFileEvidence ? undefined : prepared; + } + if (command.kind !== "event" || state.fileAttempt === undefined) return prepared; + return await this.observeEventFiles(prepared, command, state) ? prepared : undefined; + } + + private beginFileEffects(command: Extract, state: ExecutionState): boolean { + if (!state.parentEvents || state.nextOrdinal !== 2 || !this.files.begin({ + execution: state.execution, deliveryAttempt: command.deliveryAttempt, cwd: command.cwd, handoff: command.handoff, + })) return false; + state.fileAttempt = command.deliveryAttempt; + return true; + } + + private async observeEventFiles( + prepared: PreparedExecutionRequest, + command: Extract, + state: ExecutionState, + ): Promise { + if (state.fileAttempt === undefined) return false; + const event = prepared.parentEvent?.publication.event; + if (event?.type === "toolUse" && command.capturedFileObservation) { + return this.files.observeToolUse({ execution: state.execution, deliveryAttempt: state.fileAttempt, + event, captured: command.capturedFileObservation }); + } + if (event?.type === "toolResult") await this.files.observeToolResult({ + execution: state.execution, deliveryAttempt: state.fileAttempt, event, + }); + return true; + } + + private async prepareTerminalFiles( + prepared: PreparedExecutionRequest, + command: Extract, + state: ExecutionState, + ): Promise { + const frozen = command.frozenFileEvidence; + if (!frozen || frozen.outcome !== command.outcome || frozen.deliveryAttempt !== state.fileAttempt) return undefined; + const evidence = await this.files.settle(frozen); + if (!evidence) return undefined; + return { ...prepared, request: { ...prepared.request, command: { ...command, + input: { ...command.input, deliveryAttempt: state.fileAttempt, fileEvidence: evidence } } } }; + } + + private async begin( + request: ExecutionWorkerRequest, + existing: ExecutionState | undefined, + ): Promise { + const rejection = beginRejection(request, existing); + if (rejection) return { kind: "rejected", reason: rejection }; + const command = request.command; + if (command.kind !== "start" && command.kind !== "resume") throw new Error("Invalid begin command escaped validation"); + const parent = parentReducerState(command, request.execution); + const started = this.admissionStart(command, parent); + const receipt = await this.writer.transact(operationFor( + withParentStartPublication(request, started), beginMutation(command), + )); + if (!isDurableReceipt(receipt, request, 0)) { + return { kind: "rejected", reason: "writer-conflict" }; + } + this.states.set(request.execution.threadId, { + execution: request.execution, + lease: request.lease, + providerId: command.providerId, + ...parent, + nextOrdinal: 2, + phase: "running", + durableRevision: receipt.durableRevision, + }); + return started ? { ...receipt, parentEvent: started } : receipt; + } + + private admissionStart( + command: Extract, + parent: Pick, + ): PreparedProviderLiveEvent | undefined { + if (command.kind !== "start" || !command.publishParentStart) return undefined; + const started = parent.parentEvents?.startFromAdmission(); + if (started?.kind !== "parent" || started.prepared.publication.event.type !== "turnStarted") { + throw new Error("Worker-owned parent start could not be prepared"); + } + return started.prepared; + } + + private release(request: ExecutionWorkerRequest, state: ExecutionState): ExecutionWorkerResult { + if (state.phase !== "finalized" && !validRecoveryRelease(request.command, state.lease)) { + return { kind: "rejected", reason: "invalid-transition" }; + } + if (state.fileAttempt !== undefined) this.files.retire(state.execution, state.fileAttempt); + this.states.delete(request.execution.threadId); + return { kind: "released" }; + } +} + +function beginRejection( + request: ExecutionWorkerRequest, + existing: ExecutionState | undefined, +): "invalid-transition" | "out-of-order" | null { + if (existing) return "invalid-transition"; + if (request.ordinal !== 1) return "out-of-order"; + const command = request.command; + if (command.kind !== "start" && command.kind !== "resume") return "invalid-transition"; + return command.kind === "start" && !validStartInput(command, request.execution) + ? "invalid-transition" : null; +} + +function beginMutation(command: Extract): ExecutionSemanticOperation["mutation"] { + return command.kind === "start" + ? { kind: "begin", providerId: command.providerId, input: command.input } + : { kind: "resume", providerId: command.providerId, checkpointId: command.checkpointId }; +} + +function withParentStartPublication( + request: ExecutionWorkerRequest, + started: PreparedProviderLiveEvent | undefined, +): ExecutionWorkerRequest { + return started && request.command.kind === "start" + ? { ...request, command: { ...request.command, livePublication: [started.publication] } } + : request; +} + +function mutationFor( + request: ExecutionWorkerRequest, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + const command = request.command; + switch (command.kind) { + case "post-terminal-event": { + if (state.phase !== "finalized") return undefined; + const { publication: _publication, ...mutation } = command; + return mutation; + } + case "event": + return eventMutation(command, request.execution, state); + case "narrative-delta": + return narrativeMutation(command, request.execution, state); + case "live-event": + return liveEventMutation(command, request.execution, state); + case "stop": + return stopMutation(command, request, state); + case "finalize": + case "finish-live-event": + return finishMutation(command, request.execution, state.providerId); + default: return auxiliaryMutation(command, request.execution, state); + } +} + +function auxiliaryMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (command.kind === "start" || command.kind === "resume" || command.kind === "release" || command.kind === "finish-from-state") return undefined; + // File handoff is the first command after durable start, before a provider cursor exists. + if (command.kind === "begin-files") return { kind: "checkpoint", phase: "running", nativeCursor: null }; + return metadataOrTextMutation(command, execution, state); +} + +function syntheticTerminalInput( + command: Extract, + state: ExecutionState, +): SyntheticTerminalInput | undefined { + if (state.phase !== "running" && state.phase !== "stopping") return undefined; + if (command.input.outcome !== command.outcome || command.input.providerId !== state.providerId + || !sameExecution(command.input, state.execution)) return undefined; + switch (command.outcome) { + case "cancelled": return { outcome: "cancelled" }; + case "interrupted": return { outcome: "interrupted" }; + case "errored": return errorTerminalInput(command.input.error); + } +} + +function errorTerminalInput(error: unknown): Extract | undefined { + return typeof error === "string" && error.length > 0 ? { outcome: "errored", error } : undefined; +} + +function validOwnedEventCommand(command: Extract, state: ExecutionState): boolean { + return command.parentLive === undefined && command.livePublication === undefined && state.phase === "running" + && (state.fileAttempt === undefined || command.deliveryAttempt === state.fileAttempt); +} + +function validLateOwnedEventCommand(command: Extract, state: ExecutionState): boolean { + return command.parentLive === undefined && command.livePublication === undefined + && (state.fileAttempt === undefined || command.deliveryAttempt === state.fileAttempt); +} + +function metadataOrTextMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (command.kind !== "assistant-text") return command; + return state.phase === "running" && validTextRouting(command.inputs, execution) + ? { kind: "append-assistant-text", inputs: command.inputs } : undefined; +} + +function eventMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (state.phase !== "running" || !validEventRouting(command.events, execution) + || (command.parentLive !== undefined && (!validLiveEventRouting(command.parentLive, execution) + || command.livePublication?.length !== 1))) return undefined; + return { + kind: "append-events", phase: command.phase, nativeCursor: command.nativeCursor, events: command.events, + ...(command.parentLive ? { parentLive: command.parentLive } : {}), + }; +} + +function narrativeMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (state.phase !== "running" || command.input?.executionId !== execution.executionId) return undefined; + return { kind: "narrative-delta", input: command.input }; +} + +function liveEventMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (state.phase !== "running" || !validLiveEventRouting(command, execution)) return undefined; + return { + kind: "live-event", text: command.text, + ...(command.narrative ? { narrative: command.narrative } : {}), + ...(command.taskIntents ? { taskIntents: command.taskIntents } : {}), + ...(command.systemIntents ? { systemIntents: command.systemIntents } : {}), + ...(command.message ? { message: command.message } : {}), + ...(command.planQuestions ? { planQuestions: command.planQuestions } : {}), + ...(command.planOutput ? { planOutput: command.planOutput } : {}), + }; +} + +function stopMutation( + command: Extract, + request: ExecutionWorkerRequest, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (state.phase !== "running" || request.stopWatermark !== request.ordinal - 1) return undefined; + return { kind: "stop-requested", requestId: command.requestId, lastAdmittedOrdinal: request.stopWatermark }; +} + +function finishMutation( + command: Extract, + execution: ExecutionIdentity, + providerId: string, +): ExecutionSemanticOperation["mutation"] | undefined { + if (!validFinishInput(command, execution, providerId)) return undefined; + if (command.kind === "finish-live-event") { + if (command.projection.threadId !== execution.threadId || command.projection.executionId !== execution.executionId + || command.projection.outcome !== command.outcome) return undefined; + if (command.providerEvent && !validEventRouting(command.providerEvent.events, execution)) return undefined; + return { kind: "finish-live-event", outcome: command.outcome, projection: command.projection, input: command.input, + ...(command.providerEvent ? { providerEvent: command.providerEvent } : {}) }; + } + return { kind: "finish", outcome: command.outcome, input: command.input }; +} + +function commandRejection( + request: ExecutionWorkerRequest, + state: ExecutionState, +): Extract["reason"] | null { + if (!sameExecution(state.execution, request.execution) || !sameLease(state.lease, request.lease)) return "stale-execution"; + if (validRecoveryRelease(request.command, state.lease)) return null; + if (request.ordinal !== state.nextOrdinal) return "out-of-order"; + return null; +} + +function validRecoveryRelease(command: WorkerCommand, lease: ExecutionLease): boolean { + if (command.kind !== "release" || !command.recovery + || command.recovery.operationId !== `${lease.leaseId}:worker-lost`) return false; + return command.recovery.kind === "committed" && Number.isSafeInteger(command.recovery.durableRevision) + || command.recovery.kind === "conflict" && command.recovery.recoveryState === "already-terminal"; +} + +function invalidReason(request: ExecutionWorkerRequest): Extract["reason"] { + if (request.command.kind === "stop" && request.stopWatermark !== request.ordinal - 1) return "invalid-stop-watermark"; + if (request.command.kind === "live-event") { + return invalidLiveEventReason(request.command, request.execution) ?? "invalid-transition"; + } + return invalidRoutingReason(request.command, request.execution) ?? "invalid-transition"; +} + +function invalidRoutingReason( + command: Exclude, + execution: ExecutionIdentity, +): "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | null { + switch (command.kind) { + case "event": + return invalidEventRoutingReason(command, execution); + case "assistant-text": + return validTextRouting(command.inputs, execution) ? null : "invalid-text-routing"; + case "narrative-delta": + return command.input?.executionId === execution.executionId ? null : "invalid-narrative-routing"; + default: + return null; + } +} + +function invalidLiveEventReason( + command: Extract, + execution: ExecutionIdentity, +): "invalid-text-routing" | "invalid-narrative-routing" | null { + if (validLiveEventRouting(command, execution)) return null; + return command.narrative !== undefined && command.narrative?.executionId !== execution.executionId + ? "invalid-narrative-routing" : "invalid-text-routing"; +} + +function validEventRouting(events: readonly ProviderEventDraft[], execution: ExecutionIdentity): boolean { + return events.length > 0 && events.every((event) => sameExecution(event.routing, execution)); +} + +function validTextRouting(inputs: readonly ParentAssistantTextCheckpointInput[], execution: ExecutionIdentity): boolean { + return Array.isArray(inputs) && inputs.length > 0 && inputs.every((input) => input + && sameExecution(input, execution)); +} + +function validLiveEventRouting( + command: ParentLiveEffects, + execution: ExecutionIdentity, +): boolean { + if (command.narrative !== undefined && command.narrative?.executionId !== execution.executionId) return false; + if (!validPlanMessageRouting(command)) return false; + switch (command.text?.kind) { + case "unchanged": + case "reclassify": + return true; + case "append": + return validTextRouting(command.text.inputs, execution); + case "promote": + return validTextRouting([command.text.input], execution); + default: + return false; + } +} + +function validPlanMessageRouting(command: ParentLiveEffects): boolean { + return command.planOutput === undefined || command.message !== undefined; +} + +function validStartInput( + command: Extract, + execution: ExecutionIdentity, +): boolean { + return command.providerId === command.input.thread.providerId + && command.input.thread.id === execution.threadId + && command.input.turnId === execution.turnId + && command.input.executionId === execution.executionId + && validParentStartContext(command); +} + +function invalidEventRoutingReason( + command: Extract, + execution: ExecutionIdentity, +): "invalid-event-routing" | "invalid-text-routing" | null { + if (!validEventRouting(command.events, execution)) return "invalid-event-routing"; + return command.parentLive !== undefined && (!validLiveEventRouting(command.parentLive, execution) + || command.livePublication?.length !== 1) ? "invalid-text-routing" : null; +} + +function parentReducerState( + command: Extract, + execution: ExecutionIdentity, +): Pick { + if (command.kind !== "start" || !command.parentLive) return {}; + switch (command.providerId) { + case "codex": return { parentEvents: new ProviderExecutionEventState("codex", execution, command.parentLive) }; + case "claude": return { parentEvents: new ProviderExecutionEventState("claude", execution, command.parentLive) }; + case "cursor": return { parentEvents: new ProviderExecutionEventState("cursor", execution, command.parentLive) }; + default: throw new Error("Unsupported execution parent provider"); + } +} + +function validParentStartContext(command: Extract): boolean { + const context = command.parentLive; + if (!context) return true; + return supportedParentProvider(command.providerId) && typeof context.precedingMessageId === "string" + && context.precedingMessageId.length > 0 && context.precedingMessageId === command.input.userMessage.messageId + && (context.planFeature === "none" || context.planFeature === "questions" || context.planFeature === "output"); +} + +function supportedParentProvider(providerId: string): boolean { + return providerId === "codex" || providerId === "claude" || providerId === "cursor"; +} + +function prepareTerminalEvent( + command: Extract, + parentEvent: ExecutionParentEventResult, + providerId: string, +): Extract | undefined { + const projection = parentEvent.terminal; + const input = command.terminalInput; + if (!projection || !input || input.outcome !== projection.outcome || input.providerId !== providerId + || input.threadId !== projection.threadId || input.executionId !== projection.executionId) return undefined; + return { kind: "finish-live-event", outcome: projection.outcome, projection, + input: { ...input, ...(parentEvent.publication.event.type === "error" ? { error: parentEvent.publication.event.error } : {}), + projection: { kind: "writer-staged", + ...(projection.assistant.messageId ? { messageId: projection.assistant.messageId } : {}) } }, + providerEvent: { phase: command.phase, nativeCursor: command.nativeCursor, events: command.events }, + ...(command.frozenFileEvidence ? { frozenFileEvidence: command.frozenFileEvidence } : {}), + livePublication: [parentEvent.publication] }; +} + +function validFinishInput( + command: Extract, + execution: ExecutionIdentity, + providerId: string, +): boolean { + return command.outcome === command.input.outcome + && command.input.providerId === providerId + && sameExecution(command.input, execution); +} + +function operationFor( + request: ExecutionWorkerRequest, + mutation: ExecutionSemanticOperation["mutation"], +): ExecutionSemanticOperation { + const livePublication = livePublicationFor(request.command); + return { + operationId: operationId(request), + execution: request.execution, + lease: request.lease, + ordinal: request.ordinal, + mutation, + ...(livePublication ? { livePublication } : {}), + }; +} + +function livePublicationFor(command: WorkerCommand): readonly ExecutionLivePublicationIntent[] | undefined { + switch (command.kind) { + case "start": + case "event": + case "finish-live-event": + case "finalize": return command.livePublication; + case "live-event": return [command.publication]; + case "post-terminal-event": return [command.publication]; + default: return undefined; + } +} + +function operationId(request: ExecutionWorkerRequest): string { + return `${request.lease.leaseId}:${request.ordinal}`; +} + +function isDurableReceipt( + receipt: ExecutionWriteReceipt, + request: ExecutionWorkerRequest, + previousRevision: number, +): receipt is Extract { + return receipt.kind === "committed" && receipt.operationId === operationId(request) + && Number.isSafeInteger(receipt.durableRevision) && receipt.durableRevision >= previousRevision; +} diff --git a/apps/server/src/features/agents/execution/execution-worker-loss-coordinator.ts b/apps/server/src/features/agents/execution/execution-worker-loss-coordinator.ts new file mode 100644 index 000000000..86ebdfe6b --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-worker-loss-coordinator.ts @@ -0,0 +1,122 @@ +import * as NodeCrypto from "node:crypto"; + +import type { CanonicalExecutionWriterPort } from "../canonical/canonical-execution-writer-port.js"; +import { + ExecutionMailboxScheduler, + type ExecutionLostAssignment, + type ExecutionMailboxOptions, + type ExecutionRecoveryReceipt, +} from "./execution-mailbox-scheduler.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult, ExecutionWriteReceipt } from "./execution-worker-handler.js"; + +const WORKER_LOSS_REASON = "The provider could not prove that this execution was still active after its worker exited."; + +type Scheduler = ExecutionMailboxScheduler; +type SchedulerOptions = Omit, "onWorkerLost">; + +/** Reconciliation result for a worker slot. A failure leaves that slot unavailable. */ +export type WorkerLossResolution = + | { readonly kind: "recovered"; readonly workerIndex: number } + | { readonly kind: "failed"; readonly workerIndex: number; readonly error: Error; readonly unresolved: readonly ExecutionLostAssignment[] }; + +interface LostSlot { + readonly workerIndex: number; + readonly unresolved: ExecutionLostAssignment[]; + task: Promise | null; + result: WorkerLossResolution | null; +} + +/** Owns the durable handoff between a crashed execution worker and its replacement. */ +export class ExecutionWorkerLossCoordinator { + readonly scheduler: Scheduler; + private readonly slots = new Map(); + + constructor( + private readonly writer: Pick, + options: SchedulerOptions, + private readonly onRecovered?: (assignment: ExecutionLostAssignment) => void, + ) { + this.scheduler = new ExecutionMailboxScheduler({ + ...options, + onWorkerLost: (assignments, workerIndex) => this.workerLost(assignments, workerIndex), + }); + } + + /** Wait for the current recovery attempt without starting another attempt. */ + waitForRecovery(workerIndex: number): Promise { + const slot = this.slots.get(workerIndex); + if (!slot?.task) throw new Error(`No lost execution worker at slot ${workerIndex}`); + return slot.task; + } + + /** Retry a failed reconciliation explicitly; no provider command is replayed. */ + retryFailed(workerIndex: number): Promise { + const slot = this.slots.get(workerIndex); + if (!slot || slot.result?.kind !== "failed") { + throw new Error(`No failed execution recovery at slot ${workerIndex}`); + } + this.schedule(slot); + if (!slot.task) throw new Error(`No pending execution recovery at slot ${workerIndex}`); + return slot.task; + } + + private workerLost(assignments: readonly ExecutionLostAssignment[], workerIndex: number): void { + const slot: LostSlot = { + workerIndex, + unresolved: [...assignments], + task: null, + result: null, + }; + this.slots.set(workerIndex, slot); + this.schedule(slot); + } + + private schedule(slot: LostSlot): void { + slot.result = null; + slot.task = Promise.resolve().then(() => this.reconcile(slot)).then((result) => { + slot.result = result; + return result; + }); + } + + private async reconcile(slot: LostSlot): Promise { + try { + while (slot.unresolved.length > 0) { + const assignment = slot.unresolved[0]; + if (!assignment) break; + const receipt = await this.writer.interruptWorkerLoss({ + ...assignment, + reason: WORKER_LOSS_REASON, + recoveryIncidentId: workerLossIncidentId(assignment), + }); + const evidence = recoveryEvidence(receipt); + if (!evidence || !this.scheduler.reconcileLost(assignment.execution, assignment.lease, evidence)) { + throw new Error(`Lost execution has no matching durable recovery evidence: ${assignment.execution.executionId}`); + } + this.onRecovered?.(assignment); + slot.unresolved.shift(); + } + if (!this.scheduler.replaceWorker(slot.workerIndex)) { + throw new Error(`Lost execution worker slot ${slot.workerIndex} could not be replaced`); + } + return { kind: "recovered", workerIndex: slot.workerIndex }; + } catch (error) { + return { kind: "failed", workerIndex: slot.workerIndex, + error: error instanceof Error ? error : new Error(String(error)), + unresolved: [...slot.unresolved] }; + } + } +} + +function recoveryEvidence(receipt: ExecutionWriteReceipt): ExecutionRecoveryReceipt | null { + if (receipt.kind === "committed") return receipt; + if (!receipt.recoveryState) return null; + return { ...receipt, recoveryState: receipt.recoveryState }; +} + +/** Stable incident identity for a writer-backed interruption of one exact lease. */ +export function workerLossIncidentId(assignment: ExecutionLostAssignment): string { + const hash = NodeCrypto.createHash("sha256") + .update(JSON.stringify([assignment.execution, assignment.lease])).digest("hex"); + return `worker-loss:${hash}`; +} diff --git a/apps/server/src/features/agents/execution/execution-worker-port.ts b/apps/server/src/features/agents/execution/execution-worker-port.ts new file mode 100644 index 000000000..acf7702d8 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-worker-port.ts @@ -0,0 +1,210 @@ +import { sameExecution, sameLease } from "./execution-mailbox-protocol.js"; +import type { + ExecutionWorkerPort, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "./execution-mailbox-protocol.js"; +import type { ExecutionMailboxCommand } from "./execution-mailbox-scheduler.js"; +import type { + ExecutionSemanticOperation, + ExecutionSemanticWriter, + ExecutionWorkCommand, + ExecutionWorkerResult, + ExecutionWriteReceipt, +} from "./execution-worker-handler.js"; + +type Command = ExecutionMailboxCommand; +type Request = ExecutionWorkerRequest; +type Reply = ExecutionWorkerReply; + +/** Only data crosses the Worker boundary; the single writer remains on the host. */ +export type ExecutionWorkerInbound = + | { readonly kind: "command"; readonly request: Request } + | { readonly kind: "writer-receipt"; readonly rpcId: number; readonly receipt: ExecutionWriteReceipt } + | { readonly kind: "writer-failure"; readonly rpcId: number } + | { readonly kind: "close" }; + +/** The Worker requests one durable transaction and returns its acknowledged result. */ +export type ExecutionWorkerOutbound = + | { readonly kind: "ready" } + | { readonly kind: "writer-request"; readonly rpcId: number; readonly operation: ExecutionSemanticOperation } + | { readonly kind: "command-reply"; readonly reply: Reply } + | { readonly kind: "fatal" } + | { readonly kind: "closed" }; + +/** A real Bun Worker port for one fixed scheduler slot. */ +export class ExecutionThreadWorkerPort implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; + + private readonly worker: Worker; + private readonly readyPromise: Promise; + private resolveReady: (ready: boolean) => void = () => {}; + private closePromise: Promise | undefined; + private resolveClose: ((graceful: boolean) => void) | undefined; + private closeTimer: ReturnType | undefined; + private state: "starting" | "ready" | "closing" | "closed" = "starting"; + private queuedBeforeReady: Request | undefined; + private inFlight: Request | undefined; + private writerRpcPending = false; + private writerCalled = false; + + constructor( + private readonly writer: ExecutionSemanticWriter, + workerUrl: URL = defaultWorkerUrl(), + ) { + this.readyPromise = new Promise((resolve) => { this.resolveReady = resolve; }); + const workerOptions = { type: "module", smol: true } satisfies WorkerOptions & { smol: boolean }; + this.worker = new Worker(workerUrl, workerOptions); + this.worker.onmessage = (event: MessageEvent) => this.receive(event.data); + this.worker.onerror = (event) => this.fail(event); + this.worker.onmessageerror = () => this.fail(new ErrorEvent("error")); + this.worker.addEventListener("close", () => this.closed()); + } + + /** Startup resolves false if the Worker exits before it can accept commands. */ + whenReady(): Promise { + return this.readyPromise; + } + + postMessage(request: Request): void { + if (this.state === "closed" || this.state === "closing") throw new Error("Execution worker is closed"); + if (this.inFlight) throw new Error("Execution worker already has an in-flight command"); + this.inFlight = request; + this.writerCalled = false; + if (this.state === "starting") this.queuedBeforeReady = request; + else this.send({ kind: "command", request }); + } + + /** Ask the Worker to close, then force termination if it does not respond. */ + close(): Promise { + if (this.state === "closed") return Promise.resolve(true); + if (this.closePromise) return this.closePromise; + this.state = "closing"; + this.closePromise = new Promise((resolve) => { this.resolveClose = resolve; }); + this.closeTimer = setTimeout(() => this.terminate(), 2_000); + this.send({ kind: "close" }); + return this.closePromise; + } + + terminate(): void { + if (this.state === "closed") return; + this.state = "closed"; + this.worker.terminate(); + this.finishClose(false); + } + + private receive(message: ExecutionWorkerOutbound): void { + if (this.state === "closed") return; + switch (message.kind) { + case "ready": + this.ready(); + break; + case "writer-request": + void this.write(message.rpcId, message.operation); + break; + case "command-reply": + this.reply(message.reply); + break; + case "fatal": + this.fail(new ErrorEvent("error")); + break; + case "closed": + this.closed(); + break; + default: { + const exhaustive: never = message; + throw new Error(`Unknown execution worker message: ${String(exhaustive)}`); + } + } + } + + private ready(): void { + if (this.state !== "starting") return; + this.state = "ready"; + this.resolveReady(true); + const request = this.queuedBeforeReady; + this.queuedBeforeReady = undefined; + if (request) this.send({ kind: "command", request }); + } + + private async write(rpcId: number, operation: ExecutionSemanticOperation): Promise { + if (this.state !== "ready" || this.writerRpcPending || this.writerCalled || !matchesOperation(this.inFlight, operation)) { + this.fail(new ErrorEvent("error")); + return; + } + this.writerRpcPending = true; + this.writerCalled = true; + try { + const receipt = await this.writer.transact(operation); + if (this.state === "ready") this.send({ kind: "writer-receipt", rpcId, receipt }); + } catch { + if (this.state === "ready") this.send({ kind: "writer-failure", rpcId }); + } finally { + this.writerRpcPending = false; + } + } + + private reply(reply: Reply): void { + if (this.state !== "ready" || !matchesReply(this.inFlight, reply)) { + this.fail(new ErrorEvent("error")); + return; + } + this.inFlight = undefined; + this.onmessage?.(new MessageEvent("message", { data: reply })); + } + + private send(message: ExecutionWorkerInbound): void { + try { + this.worker.postMessage(message); + } catch { + this.fail(new ErrorEvent("error")); + } + } + + private fail(event: ErrorEvent): void { + if (this.state === "closed") return; + try { + this.onerror?.(event); + } finally { + this.terminate(); + } + } + + private closed(): void { + if (this.state === "closed") return; + const graceful = this.state === "closing"; + this.state = "closed"; + this.worker.terminate(); + this.finishClose(graceful); + this.onclose?.(); + } + + private finishClose(graceful: boolean): void { + if (this.closeTimer) clearTimeout(this.closeTimer); + this.closeTimer = undefined; + this.resolveReady(false); + this.resolveClose?.(graceful); + this.resolveClose = undefined; + } +} + +function defaultWorkerUrl(): URL { + const workerFile = import.meta.url.endsWith(".cjs") ? "./execution.worker.cjs" : "./execution.worker.ts"; + return new URL(workerFile, import.meta.url); +} + +function matchesOperation(request: Request | undefined, operation: ExecutionSemanticOperation): boolean { + return request !== undefined && request.ordinal === operation.ordinal + && sameExecution(request.execution, operation.execution) + && sameLease(request.lease, operation.lease) + && operation.operationId === `${request.lease.leaseId}:${request.ordinal}`; +} + +function matchesReply(request: Request | undefined, reply: Reply): boolean { + return request !== undefined && request.requestId === reply.requestId + && request.ordinal === reply.ordinal + && sameExecution(request.execution, reply.execution) + && sameLease(request.lease, reply.lease); +} diff --git a/apps/server/src/features/agents/execution/execution.worker.ts b/apps/server/src/features/agents/execution/execution.worker.ts new file mode 100644 index 000000000..eb69c3dcf --- /dev/null +++ b/apps/server/src/features/agents/execution/execution.worker.ts @@ -0,0 +1,128 @@ +import { gc } from "bun"; +import { + ExecutionWorkerHandler, + type ExecutionSemanticOperation, + type ExecutionSemanticWriter, + type ExecutionWriteReceipt, +} from "./execution-worker-handler.js"; +import type { ExecutionWorkerInbound, ExecutionWorkerOutbound } from "./execution-worker-port.js"; + +interface PendingWrite { + readonly rpcId: number; + readonly operationId: string; + readonly resolve: (receipt: ExecutionWriteReceipt) => void; + readonly reject: () => void; +} + +class ExecutionWriterRpcFailure extends Error {} + +let nextRpcId = 1; +let pendingWrite: PendingWrite | undefined; +let commandActive = false; +let closed = false; + +const writer: ExecutionSemanticWriter = { + transact: (operation: ExecutionSemanticOperation): Promise => { + if (closed || pendingWrite) return Promise.reject(new Error("Execution writer RPC unavailable")); + const rpcId = nextRpcId++; + return new Promise((resolve, reject) => { + pendingWrite = { + rpcId, + operationId: operation.operationId, + resolve, + reject: () => reject(new ExecutionWriterRpcFailure("Execution writer RPC failed")), + }; + post({ kind: "writer-request", rpcId, operation }); + }); + }, +}; + +const handler = new ExecutionWorkerHandler(writer); + +globalThis.onmessage = (event: MessageEvent): void => { + const message = event.data; + switch (message.kind) { + case "command": + handleCommand(message.request); + break; + case "writer-receipt": + finishWrite(message.rpcId, message.receipt); + break; + case "writer-failure": + failWrite(message.rpcId); + break; + case "close": + closeWorker(); + break; + default: { + const exhaustive: never = message; + throw new Error(`Unknown execution worker input: ${String(exhaustive)}`); + } + } +}; + +post({ kind: "ready" }); + +function handleCommand(request: Extract["request"]): void { + if (closed || commandActive) { + failWorker(); + return; + } + commandActive = true; + void handler.handle(request) + .then((reply) => { + if (!closed) { + post({ kind: "command-reply", reply }); + if (reply.result.kind === "released" && handler.isIdle) gc(true); + } + }) + .catch((error: unknown) => { + if (error instanceof ExecutionWriterRpcFailure) { + post({ kind: "command-reply", reply: { requestId: request.requestId, + execution: request.execution, lease: request.lease, ordinal: request.ordinal, + result: { kind: "rejected", reason: "writer-failure" } } }); + } else { + failWorker(); + } + }) + .finally(() => { commandActive = false; }); +} + +function finishWrite(rpcId: number, receipt: ExecutionWriteReceipt): void { + const pending = pendingWrite; + if (!pending || pending.rpcId !== rpcId || pending.operationId !== receipt.operationId) { + failWorker(); + return; + } + pendingWrite = undefined; + pending.resolve(receipt); +} + +function failWrite(rpcId: number): void { + const pending = pendingWrite; + if (!pending || pending.rpcId !== rpcId) { + failWorker(); + return; + } + pendingWrite = undefined; + pending.reject(); +} + +function failWorker(): void { + if (closed) return; + post({ kind: "fatal" }); + closeWorker(); +} + +function closeWorker(): void { + if (closed) return; + closed = true; + pendingWrite?.reject(); + pendingWrite = undefined; + post({ kind: "closed" }); + globalThis.close(); +} + +function post(message: ExecutionWorkerOutbound): void { + globalThis.postMessage(message); +} diff --git a/apps/server/src/features/agents/execution/provider-execution-event-state.ts b/apps/server/src/features/agents/execution/provider-execution-event-state.ts new file mode 100644 index 000000000..94f50221d --- /dev/null +++ b/apps/server/src/features/agents/execution/provider-execution-event-state.ts @@ -0,0 +1,149 @@ +import { ProviderRuntimeEventSchema, type AgentEvent, type ProviderRuntimeEvent } from "@mcode/contracts"; +import type { ProviderEventDraft } from "@mcode/providers"; +import { processProviderEventWorkerTask } from "../../providers/composition/provider-event-worker-protocol.js"; +import { CodexLiveEventReducer, type CodexPlanFeature, type SyntheticTerminalInput } from "./codex-live-event-reducer.js"; +import { CodexLiveEventEffects, type PreparedCodexLiveEvent } from "./codex-live-event-effects.js"; +import { sameExecution } from "./execution-mailbox-protocol.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import { OtherProviderLiveEventEffects, type OtherProviderRuntimeIntent } from "./provider-live-event-effects.js"; + +/** Providers whose parent event effects have an execution worker implementation. */ +export type ExecutionLiveProviderId = "codex" | "claude" | "cursor"; + +/** Immutable reducer setup supplied with the same durable parent start. */ +export interface ExecutionParentStartContext { + readonly planFeature: CodexPlanFeature; + readonly precedingMessageId: string; +} + +/** A provider preparation retains all runtime work alongside its writer data. */ +export type PreparedProviderLiveEvent = Omit & { + readonly runtime: readonly OtherProviderRuntimeIntent[]; +}; + +/** Parent effects are prepared only for ordinary, explicitly routed provider events. */ +export type PreparedExecutionParentEvent = + | { readonly kind: "writer-owned" } + | { readonly kind: "parent"; readonly prepared: PreparedProviderLiveEvent } + | { readonly kind: "rejected" }; + +/** Owns the volatile parent reducer; the execution must be poisoned after a failed write. */ +export class ProviderExecutionEventState { + private readonly parent: ParentReducer; + + constructor(private readonly providerId: ExecutionLiveProviderId, private readonly execution: ExecutionIdentity, context: ExecutionParentStartContext) { + this.parent = createParentReducer(providerId, execution, context); + } + + /** Validate the whole draft before any reducer state changes. Child evidence stays writer-owned. */ + prepare(drafts: readonly ProviderEventDraft[], allowTerminal = false): PreparedExecutionParentEvent { + const parents: AgentEvent[] = []; + for (const draft of drafts) { + const result = this.parentEvent(draft); + if (result.kind === "rejected") return result; + if (result.kind === "parent") parents.push(result.event); + } + if (parents.length === 0) return { kind: "writer-owned" }; + if (parents.length !== 1 || drafts.length !== 1) return { kind: "rejected" }; + const event = parents[0]; + if (!event) return { kind: "rejected" }; + return this.reduceParent(event, allowTerminal); + } + + /** Synthesize terminal projection from this execution's buffers without host-side reconstruction. */ + finishFromState(input: SyntheticTerminalInput): PreparedExecutionParentEvent { + switch (this.parent.providerId) { + case "codex": { + const reduction = this.parent.reducer.finishFromState(input); + return reduction.kind === "reduced" + ? { kind: "parent", prepared: this.parent.effects.prepare(reduction) } : { kind: "rejected" }; + } + case "claude": return preparedOtherResult(this.parent.effects.finishFromState(input)); + case "cursor": return preparedOtherResult(this.parent.effects.finishFromState(input)); + } + } + + /** Publish the admission-owned start before the provider can stream its first event. */ + startFromAdmission(): PreparedExecutionParentEvent { + return this.reduceParent({ type: "turnStarted", threadId: this.execution.threadId, + turnExecutionId: this.execution.executionId }, false); + } + + private reduceParent(event: AgentEvent, allowTerminal: boolean): PreparedExecutionParentEvent { + if (!allowTerminal && isTerminalEvent(event)) return { kind: "rejected" }; + switch (this.parent.providerId) { + case "codex": { + const reduction = this.parent.reducer.reduce(event); + return reduction.kind === "reduced" + ? { kind: "parent", prepared: this.parent.effects.prepare(reduction) } : { kind: "rejected" }; + } + case "claude": return prepareOtherParent(this.parent.effects, event); + case "cursor": return prepareOtherParent(this.parent.effects, event); + } + } + + private parentEvent(draft: ProviderEventDraft): + | { kind: "parent"; event: AgentEvent } + | { kind: "writer-owned" } + | { kind: "rejected" } { + if (draft.sourceProviderId !== this.providerId || !this.matchesExecution(draft)) { + return { kind: "rejected" }; + } + if (draft.payload.type !== "item.recorded") return { kind: "writer-owned" }; + const item = draft.payload.item; + if (item.payload.projection !== "providerRuntimeEvent") return { kind: "writer-owned" }; + const runtime = this.validatedRuntimeEvent(draft, item); + if (!runtime) return { kind: "rejected" }; + if (hasWriterOwnedExtension(runtime)) return { kind: "writer-owned" }; + return { kind: "parent", event: runtime.event }; + } + + private validatedRuntimeEvent( + draft: ProviderEventDraft, + item: Extract["item"], + ): ProviderRuntimeEvent | undefined { + const runtime = ProviderRuntimeEventSchema().safeParse(item.payload.runtimeEvent); + if (!runtime.success || item.threadId !== draft.routing.threadId || item.turnId !== draft.routing.turnId + || item.id !== draft.routing.itemId) return undefined; + const parsed = processProviderEventWorkerTask({ kind: "provider-runtime", providerId: this.providerId, runtimeEvent: runtime.data }); + if (parsed.status === "rejected") return undefined; + if (parsed.status !== "accepted" || parsed.event.event.threadId !== this.execution.threadId + || parsed.event.event.turnExecutionId !== this.execution.executionId) return undefined; + return { ...runtime.data, event: parsed.event.event }; + } + + private matchesExecution(draft: ProviderEventDraft): boolean { + return sameExecution(draft.routing, this.execution); + } +} + +type ParentReducer = + | { readonly providerId: "codex"; readonly reducer: CodexLiveEventReducer; readonly effects: CodexLiveEventEffects } + | { readonly providerId: "claude" | "cursor"; readonly effects: OtherProviderLiveEventEffects }; + +function createParentReducer(providerId: ExecutionLiveProviderId, execution: ExecutionIdentity, context: ExecutionParentStartContext): ParentReducer { + switch (providerId) { + case "codex": return { providerId, reducer: new CodexLiveEventReducer(execution, context.planFeature), + effects: new CodexLiveEventEffects(execution, context.precedingMessageId) }; + case "claude": return { providerId, effects: new OtherProviderLiveEventEffects("claude", execution, context.precedingMessageId, context.planFeature) }; + case "cursor": return { providerId, effects: new OtherProviderLiveEventEffects("cursor", execution, context.precedingMessageId, context.planFeature) }; + } +} + +function prepareOtherParent(effects: OtherProviderLiveEventEffects, event: AgentEvent): PreparedExecutionParentEvent { + return preparedOtherResult(effects.prepare(event)); +} + +function preparedOtherResult(prepared: ReturnType): PreparedExecutionParentEvent { + return prepared.kind === "prepared" ? { kind: "parent", prepared } : { kind: "rejected" }; +} + +function isTerminalEvent(event: AgentEvent): boolean { + return event.type === "turnComplete" || event.type === "error" + || event.type === "ended" && event.outcome !== undefined; +} + +function hasWriterOwnedExtension(runtime: ProviderRuntimeEvent): boolean { + const extension = runtime.extension; + return Boolean(extension?.child || extension?.collaboration || extension?.continuation); +} diff --git a/apps/server/src/features/agents/execution/provider-live-event-effects.ts b/apps/server/src/features/agents/execution/provider-live-event-effects.ts new file mode 100644 index 000000000..d98017aef --- /dev/null +++ b/apps/server/src/features/agents/execution/provider-live-event-effects.ts @@ -0,0 +1,100 @@ +import type { AgentEvent } from "@mcode/contracts"; + +import type { DataOnlyParentTerminalProjectionInput } from "../canonical/canonical-parent-turn-write.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import { CodexLiveEventEffects, type CodexLiveRuntimeIntent } from "./codex-live-event-effects.js"; +import { CodexLiveEventReducer, type CodexPlanFeature, type CodexLiveReduction, type SyntheticTerminalInput } from "./codex-live-event-reducer.js"; +import type { ExecutionLivePublicationIntent, ParentLiveEffects } from "./execution-worker-handler.js"; + +/** Providers whose AgentEvents use the shared parent turn projection. */ +export type OtherLiveProviderId = "claude" | "cursor"; + +/** Runtime work retained after the event and its parent effects are durable. */ +export type OtherProviderRuntimeIntent = CodexLiveRuntimeIntent | { + readonly kind: "assistant-message-feature"; + readonly providerId: OtherLiveProviderId; + readonly event: Extract; +}; + +/** One execution-bound provider event prepared for a single semantic write. */ +export type OtherProviderLivePreparation = + | { + readonly kind: "prepared"; + readonly providerId: OtherLiveProviderId; + readonly execution: ExecutionIdentity; + readonly effects: ParentLiveEffects; + readonly publication: ExecutionLivePublicationIntent; + readonly runtime: readonly OtherProviderRuntimeIntent[]; + readonly terminal?: DataOnlyParentTerminalProjectionInput; + } + | { + readonly kind: "unsupported"; + readonly providerId: OtherLiveProviderId; + readonly execution: ExecutionIdentity; + readonly eventType: AgentEvent["type"]; + readonly reason: string; + }; + +/** + * Prepares Claude or Cursor parent writes from their normalized AgentEvents. + * Keep one instance per execution and discard it after any failed writer commit. + */ +export class OtherProviderLiveEventEffects { + private readonly reducer: CodexLiveEventReducer; + private readonly effects: CodexLiveEventEffects; + + constructor( + readonly providerId: OtherLiveProviderId, + readonly execution: ExecutionIdentity, + precedingMessageId: string, + planFeature: CodexPlanFeature = "none", + ) { + this.reducer = new CodexLiveEventReducer(execution, planFeature); + this.effects = new CodexLiveEventEffects(execution, precedingMessageId); + } + + /** Reduce exactly one event; unsupported input never yields a publication. */ + prepare(event: AgentEvent, endedAt?: string): OtherProviderLivePreparation { + if (event.type === "turnComplete" && event.providerId && event.providerId !== this.providerId) { + return { + kind: "unsupported", + providerId: this.providerId, + execution: this.execution, + eventType: event.type, + reason: "different provider", + }; + } + return this.prepareReduction(this.reducer.reduce(event), endedAt); + } + + /** Prepare an interrupted execution from worker-owned buffers, including unclassified partial text. */ + finishFromState(input: SyntheticTerminalInput, endedAt?: string): OtherProviderLivePreparation { + return this.prepareReduction(this.reducer.finishFromState(input), endedAt); + } + + private prepareReduction(reduction: CodexLiveReduction, endedAt?: string): OtherProviderLivePreparation { + if (reduction.kind === "unsupported") { + return { + kind: "unsupported", + providerId: this.providerId, + execution: this.execution, + eventType: reduction.eventType, + reason: reduction.reason, + }; + } + const prepared = this.effects.prepare(reduction, endedAt); + const messageFeature = reduction.writer.find((intent) => intent.kind === "feature-event" + && intent.feature === "assistant-message"); + const runtime: OtherProviderRuntimeIntent[] = [...prepared.runtime]; + if (messageFeature?.kind === "feature-event" && messageFeature.event.type === "message") { + runtime.push({ kind: "assistant-message-feature", providerId: this.providerId, event: messageFeature.event }); + } + return structuredClone({ + kind: "prepared", + providerId: this.providerId, + execution: this.execution, + ...prepared, + runtime, + }); + } +} diff --git a/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts b/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts new file mode 100644 index 000000000..32d4e3cc0 --- /dev/null +++ b/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts @@ -0,0 +1,114 @@ +import { broadcast } from "../../../application/transport/push.js"; +import { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; +import { publishCanonicalAgentEvents } from "../canonical/canonical-agent-boundary.js"; +import { CanonicalExecutionWriterPort } from "../canonical/canonical-execution-writer-port.js"; +import { ExecutionLivePublicationRelease } from "../canonical/execution-live-publication-release.js"; +import { ExecutionPlanQuestionRelease } from "../canonical/execution-plan-question-release.js"; +import { AgentEventPublicationRegistry } from "../orchestration/agent-event-publication-registry.js"; +import { ExecutionMailboxOwner } from "./execution-mailbox-owner.js"; +import { ExecutionMailboxScheduler, type ExecutionMailboxLimits } from "./execution-mailbox-scheduler.js"; +import { ExecutionProviderEventOwnership } from "./execution-provider-event-ownership.js"; +import { ExecutionThreadWorkerPort } from "./execution-worker-port.js"; +import type { ExecutionIdentity, ExecutionLease } from "./execution-mailbox-protocol.js"; +import { ExecutionWorkerLossCoordinator, workerLossIncidentId } from "./execution-worker-loss-coordinator.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; + +const EXECUTION_WORKER_COUNT = 4; +const EXECUTION_MAILBOX_LIMITS: ExecutionMailboxLimits = { + maxPending: 256, + maxPendingBytes: 16 * 1024 * 1024, + reservedControl: 32, + reservedControlBytes: 2 * 1024 * 1024, + maxPerExecutionPending: 64, + maxPerExecutionBytes: 4 * 1024 * 1024, + reservedPerExecutionControl: 8, + reservedPerExecutionControlBytes: 512 * 1024, +}; + +/** Owns the fixed execution pool and sole semantic writer for this server process. */ +export class WorkerOwnedTurnRuntime { + readonly writer: CanonicalAgentWriterClient; + readonly writerPort: CanonicalExecutionWriterPort; + readonly workerLoss: ExecutionWorkerLossCoordinator; + readonly scheduler: ExecutionMailboxScheduler; + readonly owner: ExecutionMailboxOwner; + readonly providerEvents: ExecutionProviderEventOwnership; + private readonly initialWorkers: ExecutionThreadWorkerPort[] = []; + private readonly rejectedRecoveries = new Map>(); + private onRecovered: ((execution: ExecutionIdentity) => void) | undefined; + + constructor(dbPath: string, publication: AgentEventPublicationRegistry) { + this.writer = new CanonicalAgentWriterClient(dbPath); + this.writerPort = new CanonicalExecutionWriterPort( + this.writer, + publishCanonicalAgentEvents, + new ExecutionLivePublicationRelease(publication), + new ExecutionPlanQuestionRelease((threadId, questions) => { + broadcast("plan.questions", { threadId, questions: [...questions] }); + }), + ); + this.workerLoss = new ExecutionWorkerLossCoordinator(this.writerPort, { + workerCount: EXECUTION_WORKER_COUNT, + limits: EXECUTION_MAILBOX_LIMITS, + createWorker: () => { + const worker = new ExecutionThreadWorkerPort(this.writerPort); + if (this.initialWorkers.length < EXECUTION_WORKER_COUNT) this.initialWorkers.push(worker); + return worker; + }, + }, (assignment) => { + this.owner.forgetRecovered(assignment.execution, assignment.lease); + this.onRecovered?.(assignment.execution); + }); + this.scheduler = this.workerLoss.scheduler; + this.owner = new ExecutionMailboxOwner(this.scheduler); + this.providerEvents = new ExecutionProviderEventOwnership(this.owner); + } + + /** Notify the runtime controller only after writer-backed worker-loss recovery releases the lease. */ + bindRecovered(onRecovered: (execution: ExecutionIdentity) => void): void { + this.onRecovered = onRecovered; + } + + /** Interrupt and release only the rejected execution; other assignments keep their worker. */ + async recoverRejected(execution: ExecutionIdentity): Promise { + const existing = this.rejectedRecoveries.get(execution.executionId); + if (existing) return existing; + const owned = this.owner.current(execution.threadId); + if (!owned || owned.execution.executionId !== execution.executionId) return; + const task = this.recoverRejectedOwned(execution, owned.lease); + this.rejectedRecoveries.set(execution.executionId, task); + try { + await task; + } finally { + this.rejectedRecoveries.delete(execution.executionId); + } + } + + private async recoverRejectedOwned(execution: ExecutionIdentity, lease: ExecutionLease): Promise { + const receipt = await this.writerPort.interruptWorkerLoss({ + execution, lease, + reason: "The provider event could not be durably applied to this execution.", + recoveryIncidentId: workerLossIncidentId({ execution, lease }), + }); + if (receipt.kind !== "committed" && receipt.recoveryState !== "already-terminal") { + throw new Error("Rejected execution has no durable recovery evidence"); + } + await this.owner.releaseRecovered(execution, receipt); + this.onRecovered?.(execution); + } + + /** Wait for the writer and every initial slot before accepting a provider turn. */ + async whenReady(): Promise { + await this.writer.whenReady(); + const ready = await Promise.all(this.initialWorkers.map((worker) => worker.whenReady())); + if (ready.length !== EXECUTION_WORKER_COUNT || ready.some((value) => !value)) { + throw new Error("Execution worker pool did not start"); + } + } + + /** Stop mailbox admission before closing its durable writer. */ + async close(): Promise { + this.scheduler.shutdown(); + await this.writer.close(); + } +} diff --git a/apps/server/src/features/agents/index.ts b/apps/server/src/features/agents/index.ts index f0e5b315b..eebd966e8 100644 --- a/apps/server/src/features/agents/index.ts +++ b/apps/server/src/features/agents/index.ts @@ -16,9 +16,6 @@ export { SubagentLifecycleService } from "./collaboration/subagent-lifecycle-ser /** Full canonical agent system boundary used by the server composition roots. */ export { CanonicalAgentBoundary } from "./canonical/canonical-agent-boundary.js"; -/** Temporary compatibility façade for callers migrating to the named boundary. */ -export { CanonicalAgentEventSink } from "./canonical/canonical-agent-event-sink.js"; - /** Publishes canonical agent events for the server composition roots. */ export { publishCanonicalAgentEvents } from "./canonical/canonical-agent-boundary.js"; diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-existing-worktree.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-existing-worktree.test.ts index abb2fe070..ec2048b00 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-existing-worktree.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-existing-worktree.test.ts @@ -10,7 +10,7 @@ import { WorkspaceRepo } from "../../../projects/persistence/workspace-repo.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { createAgentServiceForTest, goalLifecycleForAgentServiceTest } from "./agent-service-test-harness.js"; import { WorkspaceEnvironmentService } from "../../../projects/environment/workspace-environment-service.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import type { GitService } from "../../../projects/index.js"; import type { ThreadService } from "../../../thread-control/index.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; @@ -128,7 +128,7 @@ function createAgentServiceHarness(automaticSetup?: {} as never, undefined, undefined, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), resolvedAutomaticSetup as never, undefined, undefined, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts index 27e5ca42c..86817107c 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts @@ -3,7 +3,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import type { ApprovalReviewSupport, Thread, IProviderRegistry } from "@mcode/contracts"; import { supportsInternalThreadControl } from "../../turns/turn-admission-dispatch-coordinator.js"; import { createAgentServiceForTest } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { ProviderAvailabilityService } from "../../../providers/availability/provider-availability-service.js"; @@ -222,7 +222,7 @@ function buildService({ undefined, threadControlMcp as never, undefined, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), ); return { svc, @@ -340,6 +340,8 @@ describe("AgentService.sendMessage — admission gates", () => { sourceTurnId: "source-turn", sourceProviderId: "claude", }, + expect.any(String), + undefined, ); expect(threadRepo.updateStatus).toHaveBeenCalledWith(THREAD_ID, "active"); expect(providerStub.sendTurn).toHaveBeenCalledTimes(1); diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-goal-command.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-goal-command.test.ts index d4836d0cc..9e84ccb08 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-goal-command.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-goal-command.test.ts @@ -19,7 +19,7 @@ import { waitForAgentServiceIngressForTest, wrapProviderEmitterForRuntimeEvents, } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { GoalLifecycleService } from "../../goals/goal-lifecycle-service.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; @@ -176,7 +176,7 @@ function buildService(db: Database) { undefined, undefined, undefined, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), undefined, undefined, undefined, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts index 1557663c3..a657e1878 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts @@ -26,14 +26,14 @@ import { streamAgentReliabilityTextForTest, waitForAgentServiceIngressForTest, } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { TaskPersistenceService } from "../../tasks/task-persistence-service.js"; import { ParentAssistantTextCheckpointService, PARENT_ASSISTANT_TEXT_RETAINED_LIMITS, } from "../../turns/parent-assistant-text-checkpoint-service.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { broadcast } from "../../../../application/transport/push.js"; import { isTurnScopedEvent } from "../../turns/turn-runtime.js"; @@ -145,7 +145,7 @@ function makeThread(overrides: Partial = {}): Thread { interface Built { service: AgentService; providerEmitter: NodeEvents.EventEmitter; - canonicalSink: CanonicalAgentEventSink; + canonicalSink: CanonicalAgentBoundary; db: Database; messageRepo: MessageRepo; thoughtBulk: ReturnType; @@ -160,7 +160,7 @@ interface Built { function build(options: { db?: Database; - canonicalSink?: CanonicalAgentEventSink; + canonicalSink?: CanonicalAgentBoundary; messageRepo?: MessageRepo; parentAssistantTextCheckpoints?: ParentAssistantTextCheckpointService; onProviderEvent?: (event: AgentEvent) => void; @@ -270,7 +270,7 @@ function build(options: { transaction: vi.fn((fn: Function) => fn), prepare: vi.fn(() => ({ run: vi.fn() })), } as unknown as Database); - const canonicalSink = options.canonicalSink ?? createCanonicalAgentEventSinkStub(db); + const canonicalSink = options.canonicalSink ?? createCanonicalAgentBoundaryStub(db); const parentAssistantTextCheckpoints = options.parentAssistantTextCheckpoints ?? new ParentAssistantTextCheckpointService(db); @@ -372,7 +372,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const { providerEmitter, service } = build({ db, @@ -442,7 +442,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const { providerEmitter, service } = build({ db, @@ -501,7 +501,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; let sqliteAvailable = false; const appendChunk = ParentAssistantTextCheckpointService.prototype.appendChunk; @@ -569,7 +569,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const restoreChunks = ParentAssistantTextCheckpointService.prototype.restoreChunks; const restoreChunksSpy = vi.spyOn(ParentAssistantTextCheckpointService.prototype, "restoreChunks") @@ -635,7 +635,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const restoreChunksSpy = vi.spyOn(ParentAssistantTextCheckpointService.prototype, "restoreChunks") .mockImplementation(() => { throw Object.assign(new Error("database locked"), { code: "SQLITE_BUSY" }); @@ -697,7 +697,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const restoreChunksSpy = vi.spyOn(ParentAssistantTextCheckpointService.prototype, "restoreChunks") .mockImplementation(() => { throw Object.assign(new Error("database locked"), { code: "SQLITE_BUSY" }); @@ -767,7 +767,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const journalService = new ParentAssistantTextCheckpointService( db, undefined, @@ -851,7 +851,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const appendSpy = vi.spyOn(ParentAssistantTextCheckpointService.prototype, "appendChunk") .mockImplementation(() => { @@ -920,7 +920,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const { providerEmitter, service, thoughtBulk, narrativeStore } = build({ db, @@ -990,7 +990,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const observed: Array<{ type: string; persisted: string | undefined }> = []; const { providerEmitter, service } = build({ db, @@ -1047,7 +1047,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const messageRepo = new SqliteMessageRepo(db); const { service } = build({ @@ -1599,7 +1599,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const { service, providerEmitter, hookBulk, narrativeStore } = build({ db, @@ -1930,7 +1930,7 @@ describe("AgentService narrative persistence", () => { "INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "codex", now, now); const published = vi.fn(); - const canonicalSink = new CanonicalAgentEventSink(db, published); + const canonicalSink = new CanonicalAgentBoundary(db, published); const { providerEmitter, service, narrativeStore } = build({ db, canonicalSink }); const executionId = narrativeExecutionId(service); const messages = new SqliteMessageRepo(db); @@ -2093,7 +2093,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-nested", "Parent", "main", "codex", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const { providerEmitter, service } = build({ db, canonicalSink }); const executionId = narrativeExecutionId(service); const messages = new SqliteMessageRepo(db); @@ -2352,7 +2352,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-failure", "Parent", "main", "codex", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const { providerEmitter, service } = build({ db, canonicalSink }); const executionId = narrativeExecutionId(service); const messages = new SqliteMessageRepo(db); @@ -2432,7 +2432,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-action", "Parent", "main", "codex", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const { providerEmitter, service } = build({ db, canonicalSink }); const executionId = narrativeExecutionId(service); const messages = new SqliteMessageRepo(db); diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-plan-marker.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-plan-marker.test.ts index 9deb40fab..ac20a5e45 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-plan-marker.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-plan-marker.test.ts @@ -10,7 +10,7 @@ import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { PlanQuestionAnswersRepo } from "../../planning/persistence/plan-question-answers-repo.js"; import { TurnSnapshotRepo } from "../../turns/persistence/turn-snapshot-repo.js"; import { createAgentServiceForTest } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { PlanQuestionService } from "../../planning/plan-question-service.js"; import { PlanTurnService } from "../../planning/plan-turn-service.js"; @@ -120,7 +120,7 @@ function buildService(db: Database) { undefined, undefined, undefined, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), ); const plans = new PlanTurnService( threadRepo, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-session-invalidated.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-session-invalidated.test.ts index 37793d64e..27213272d 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-session-invalidated.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-session-invalidated.test.ts @@ -19,7 +19,7 @@ import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-rep import { TurnSnapshotRepo } from "../../turns/persistence/turn-snapshot-repo.js"; import { AgentService } from "../agent-service.js"; import { createAgentServiceForTest, startAgentServiceIngressForTest, wrapProviderEmitterForRuntimeEvents } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import type { GitService } from "../../../projects/index.js"; @@ -147,7 +147,7 @@ describe("AgentService clears sdk_session_id on session invalidation", () => { assertUsable: vi.fn(), } as unknown as ProviderAvailabilityService; providerEventIngress = new ProviderEventIngress(); - const canonicalSink = createCanonicalAgentEventSinkStub(db); + const canonicalSink = createCanonicalAgentBoundaryStub(db); Object.assign(canonicalSink, { recordNativeCursor: vi.fn(() => true) }); svc = createAgentServiceForTest( diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-transient-retry.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-transient-retry.test.ts index f79ab4cec..fb83c7ffe 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-transient-retry.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-transient-retry.test.ts @@ -10,7 +10,7 @@ import { wrapProviderEmitterForRuntimeEvents, } from "./agent-service-test-harness.js"; import { publishParentProviderEvent } from "../../events/provider-event-publication.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { ThreadControlMutationReservationService } from "../../../thread-control/index.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; @@ -219,7 +219,7 @@ function buildService(): { undefined, threadControlMcp as never, mutationReservations, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), ); return { diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts index 2ccd1fdea..1a65b16ef 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts @@ -40,8 +40,8 @@ import { waitForAgentServiceIngressForTest, wrapProviderEmitterForRuntimeEvents, } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { broadcast } from "../../../../application/transport/push.js"; @@ -340,7 +340,7 @@ function buildService( undefined, undefined, mutationReservations, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), undefined, undefined, ); @@ -530,7 +530,7 @@ describe("AgentService turn cleanup", () => { it("marks a replayed queued plan answer after projecting its persisted user message", async () => { const { service, messageRepo, planQuestionAnswersRepo } = buildService(); - vi.mocked(messageRepo.findByIdInThread).mockReturnValue({ id: "queued-plan-answer", sequence: 1 } as never); + vi.mocked(messageRepo.findByIdInThread).mockReturnValue({ id: "queued-plan-answer", role: "user", sequence: 1 } as never); await service.dispatchQueuedAutomaticTurn({ threadId: THREAD_ID, @@ -584,6 +584,9 @@ describe("AgentService turn cleanup", () => { undefined, undefined, undefined, + undefined, + expect.any(String), + undefined, ); }); @@ -1022,6 +1025,9 @@ describe("AgentService turn cleanup", () => { undefined, undefined, bundle, + undefined, + expect.any(String), + undefined, ); expect((providerEmitter as any).sendTurn).toHaveBeenCalledWith( expect.objectContaining({ @@ -1454,7 +1460,7 @@ describe("AgentService Ended finalization", () => { stopSession: ReturnType; interruptChildTurn: ReturnType; }; - let canonicalSink: CanonicalAgentEventSink; + let canonicalSink: CanonicalAgentBoundary; let canonicalEvents: CanonicalAgentEventEnvelope[]; let service: AgentService; let pendingPlanOutputs: Map; @@ -1520,7 +1526,7 @@ describe("AgentService Ended finalization", () => { listAnsweredForThread: vi.fn(() => []), } as unknown as PlanQuestionAnswersRepo; - canonicalSink = new CanonicalAgentEventSink(db, (events) => { + canonicalSink = new CanonicalAgentBoundary(db, (events) => { canonicalEvents.push(...events); }); pendingPlanOutputs = new Map(); diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-started.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-started.test.ts index 4b32248a3..9232da236 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-started.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-started.test.ts @@ -17,7 +17,7 @@ import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-rep import { TurnSnapshotRepo } from "../../turns/persistence/turn-snapshot-repo.js"; import { AgentService } from "../agent-service.js"; import { createAgentServiceForTest, startAgentServiceIngressForTest, wrapProviderEmitterForRuntimeEvents } from "./agent-service-test-harness.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import type { GitService } from "../../../projects/index.js"; @@ -43,7 +43,7 @@ describe("AgentService.sendMessage emits TurnStarted", () => { let toolCallRecordRepo: ToolCallRecordRepo; let turnSnapshotRepo: TurnSnapshotRepo; let svc: AgentService; - let canonicalSink: CanonicalAgentEventSink; + let canonicalSink: CanonicalAgentBoundary; let providerStub: NodeEvents.EventEmitter & Partial & { sendTurn: ReturnType; }; @@ -123,7 +123,7 @@ describe("AgentService.sendMessage emits TurnStarted", () => { assertUsable: vi.fn(), } as unknown as ProviderAvailabilityService; - canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); svc = createAgentServiceForTest( threadRepo, workspaceRepo, diff --git a/apps/server/src/features/agents/orchestration/__tests__/start-agent-orchestration.test.ts b/apps/server/src/features/agents/orchestration/__tests__/start-agent-orchestration.test.ts index dc1cd36f8..90740477b 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/start-agent-orchestration.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/start-agent-orchestration.test.ts @@ -115,4 +115,21 @@ describe("agent orchestration", () => { }); expect(orchestration.pullRequestCompletionEffect.schedule).toHaveBeenCalledWith("thread-1"); }); + + it("releases a committed terminal receipt even when the legacy runtime suppression is set", () => { + const orchestration = buildOrchestration(); + orchestration.runtime.shouldSuppressTurnComplete.mockReturnValue(true); + const legacy: AgentEvent = { type: AgentEventType.TurnComplete, threadId: "thread-1", + reason: "completed", costUsd: null, tokensIn: 1, tokensOut: 1 }; + orchestration.publish(legacy); + expect(orchestration.publishedEvents).toEqual([]); + + const committed: AgentEvent = { ...legacy, + turnExecutionId: "00000000-0000-4000-8000-000000000001", publicationId: "1" }; + orchestration.publish(committed); + expect(orchestration.publishedEvents).toEqual([committed]); + expect(orchestration.threadRepo.updateStatus).not.toHaveBeenCalled(); + expect(orchestration.publishThreadStatus).not.toHaveBeenCalled(); + expect(orchestration.pullRequestCompletionEffect.schedule).toHaveBeenCalledWith("thread-1"); + }); }); diff --git a/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts b/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts index 65d5e9ef0..e7c74be7b 100644 --- a/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts +++ b/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts @@ -5,6 +5,7 @@ import type { NarrativeStore } from "../conversation/narrative/narrative-store.j import { publishParentProviderEvent } from "../events/provider-event-publication.js"; import { publishAgentPermissionEvents } from "../permissions/permission-publication.js"; import { sanitizePublicToolInput } from "../tools/input/public-tool-input.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; /** The runtime read surface needed to prepare renderer-facing provider events. */ export interface AgentEventPublicationRuntime { @@ -42,6 +43,14 @@ export class AgentEventPublicationService { /** Publish one provider event after its synchronous persistence application completed. */ publish(event: AgentEvent): void { + if (serverWorkTrace) { + serverWorkTrace.measure("publication", event.threadId, event.turnExecutionId, () => this.publishCore(event)); + return; + } + this.publishCore(event); + } + + private publishCore(event: AgentEvent): void { if (this.shouldSuppress(event)) return; const enriched = this.sanitize(this.enrich(event)); const published = publishParentProviderEvent(event, enriched, { @@ -53,6 +62,8 @@ export class AgentEventPublicationService { } private shouldSuppress(event: AgentEvent): boolean { + // A stable publication already passed the writer's semantic and terminal barriers. + if (event.publicationId) return false; if (event.type === AgentEventType.Ended) return this.dependencies.runtime.shouldSuppressTurnEnded(event.threadId); if (event.type === AgentEventType.TurnComplete) return this.dependencies.runtime.shouldSuppressTurnComplete(event.threadId); return event.type === AgentEventType.Error diff --git a/apps/server/src/features/agents/orchestration/turn-runtime-controller.ts b/apps/server/src/features/agents/orchestration/turn-runtime-controller.ts index cb4b8682e..b6c6edb17 100644 --- a/apps/server/src/features/agents/orchestration/turn-runtime-controller.ts +++ b/apps/server/src/features/agents/orchestration/turn-runtime-controller.ts @@ -5,9 +5,11 @@ * Extracted from apps/desktop/src/main/app-state.ts. */ +import * as NodeCrypto from "node:crypto"; import { injectable, inject, delay } from "tsyringe"; import { logger } from "@mcode/shared"; -import { AgentEventType, isSessionEvictable } from "@mcode/contracts"; +import { AgentEventType, ProviderRuntimeEventSchema, isSessionEvictable } from "@mcode/contracts"; +import type { CodexProviderBoundary } from "@mcode/providers"; import type { Thread, IProviderRegistry, @@ -17,6 +19,7 @@ import type { ProviderId, TurnRuntimeSnapshot, AgentStopResult, + ProviderRuntimeEvent, } from "@mcode/contracts"; import { TURN_FILE_EFFECTS, TurnFileEffects } from "../turns/turn-file-effects.js"; import type { WorkspaceEnvironmentAutomaticSetupDispatch } from "../../projects/environment/workspace-environment-service.js"; @@ -31,6 +34,13 @@ import { TurnErrorPolicy } from "../turns/turn-error-policy.js"; import { TurnRuntimeRegistry } from "../turns/turn-runtime.js"; import type { TurnOutcome } from "../turns/turn-outcome.js"; import { ProviderEventIngress } from "../../providers/composition/provider-event-ingress.js"; +import { ExecutionFileEvidenceCoordinator } from "../execution/execution-file-evidence-coordinator.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution/execution-worker-handler.js"; +import { WorkerOwnedTurnRuntime } from "../execution/worker-owned-turn-runtime.js"; +import type { WorkerOwnedProviderEventBatch } from "../../providers/composition/provider-host-ports.js"; +import { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; +import type { DataOnlyParentTurnFinishInput } from "../canonical/canonical-parent-turn-write.js"; import { TurnEventPipeline, type FinalizeTurnCommand, @@ -95,6 +105,21 @@ type PreparedStop = { runtime: TurnRuntimeSnapshot; }; +interface WorkerOwnedTurn { + readonly execution: ExecutionIdentity; + readonly prepared: PreparedTurnDispatch; + readonly provider: CodexProviderBoundary; + deliveryAttempt: number; + terminalCommitted: boolean; + persistedPublished: boolean; + terminalOutcome?: TurnOutcome; + releaseStarted: boolean; + deliveryFailed?: boolean; + deliveryFailureHandling?: boolean; + stopInProgress?: boolean; + stopRequested?: boolean; +} + /** Read-only runtime state available to server-owned diagnostics and recovery infrastructure. */ export interface AgentRuntimeAccess { /** Return the number of active agent sessions. */ @@ -184,6 +209,7 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE private readonly turnGenerations = new Map(); /** Single ordered pipeline for validated provider envelopes and terminal materialization. */ private readonly turnEventPipeline: TurnEventPipeline; + private readonly workerTurns = new Map(); constructor( @inject(TURN_RUNTIME_PERSISTENCE) private readonly runtimePersistence: TurnRuntimePersistence, @@ -220,9 +246,19 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE @inject(delay(() => ProviderTurnEventApplication)) private readonly eventApplication: ProviderTurnEventApplication, @inject(TurnDiffService) turnDiffs: TurnDiffService, + @inject("WorkerOwnedTurnRuntime", { isOptional: true }) + private readonly workerRuntime?: WorkerOwnedTurnRuntime, + @inject("ExecutionFileEvidenceCoordinator", { isOptional: true }) + private readonly workerFiles?: ExecutionFileEvidenceCoordinator, + @inject("WorkerTurnSnapshotService", { isOptional: true }) + private readonly snapshots?: SnapshotService, ) { this.turnDiffs = turnDiffs; - this.turnEventPipeline = new TurnEventPipeline(this, eventApplication, turnDiffs, providerEventIngress); + this.turnEventPipeline = new TurnEventPipeline(this, eventApplication, turnDiffs, providerEventIngress, + workerRuntime && workerFiles ? { + ownsFileMutation: (event) => Boolean(workerRuntime.owner.current(event.threadId)?.started), + capture: (event) => workerFiles.capture(event), + } : undefined); runtimeCommands.bind({ sendMessage: (command) => this.sendMessage(command), runtimeSnapshots: () => this.runtimeSnapshots(), @@ -237,6 +273,9 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE }); continuation.bind((executionId) => eventApplication.continueWithoutSaving(executionId)); reliability.bind((threadId) => eventApplication.streamReliabilityAssistantText(threadId)); + workerRuntime?.providerEvents.bindCommandPreparation((execution, batch) => this.prepareWorkerEvent(execution, batch)); + workerRuntime?.providerEvents.bindCommitted((execution, result) => this.applyWorkerReceipt(execution, result)); + workerRuntime?.bindRecovered((execution) => this.handleWorkerRecovery(execution)); this.eventPublication.registerPipelineStart(() => this.initializeProviderEvents()); } @@ -488,6 +527,12 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE /** Terminalize an admitted turn when durable admission cannot complete. */ private async abortTurnAdmission(lease: TurnRuntimeLease): Promise { + if (await this.finishUnsentWorkerTurn(lease.threadId, lease.turnExecutionId)) { + this.turnRuntime.terminalize(lease.threadId, lease.turnExecutionId, "errored"); + this.disarmTurnRetryWindow(lease.threadId); + this.trackSessionEnded(lease.threadId, lease.turnExecutionId); + return; + } if (!this.turnRuntime.terminalize(lease.threadId, lease.turnExecutionId, "errored")) return; await (this.finalizeTerminalTurn(lease.threadId, "errored", "turn admission failure") ?? Promise.resolve()); this.disarmTurnRetryWindow(lease.threadId); @@ -514,6 +559,8 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE await this.activatePreparedCommandEffect(prepared); await this.startPreparedProviderDispatch(prepared); } catch (error) { + logger.error("Prepared turn dispatch failed", { threadId: prepared.lease.threadId, + error: error instanceof Error ? error.message : String(error) }); await this.failPreparedTurnDispatch(prepared, error); } } @@ -521,6 +568,10 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE /** Initialize pipeline state after the parent-turn transaction has committed. */ private async prepareRuntimeDispatch(prepared: PreparedTurnDispatch): Promise { const { lease, request } = prepared; + if (prepared.workerOwned) { + await this.prepareWorkerRuntimeDispatch(prepared); + return; + } this.eventApplication.beginPreparedTurn(lease.threadId, lease.turnExecutionId, request.turnId); this.turnAdmissions.markDispatchActive(lease.threadId); this.emitProviderEvent(prepared.provider, { @@ -533,6 +584,255 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE this.eventApplication.recordContextSeed(lease.threadId, prepared.contextSeed, prepared.contextWindow ?? undefined); } + /** Bind file evidence and the exact provider route after the parent start receipt. */ + private async prepareWorkerRuntimeDispatch(prepared: PreparedTurnDispatch): Promise { + const worker = this.requireWorkerRuntime(); + const files = this.requireWorkerFiles(); + const snapshots = this.snapshots; + if (!snapshots) throw new Error("Worker-owned turn requires snapshot service"); + const execution = this.executionFor(prepared); + const provider = this.codexProvider(prepared.provider); + let baselineRef: string | null = null; + try { + baselineRef = await snapshots.captureRef(prepared.cwd); + } catch (error) { + logger.warn("Could not capture initial turn snapshot", { + threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error), + }); + } + if (!this.ownsTurnAdmission(prepared.lease)) throw new Error("Worker turn lost admission during file preparation"); + const handoff = files.begin({ ...execution, deliveryAttempt: 1, cwd: prepared.cwd, baselineRef }); + const began = await worker.owner.submit(execution, { + kind: "begin-files", cwd: prepared.cwd, handoff, deliveryAttempt: 1, + }); + if (began.kind !== "committed") throw new Error("Worker file handoff was not committed"); + this.markProviderTurnActive(execution.threadId); + await worker.providerEvents.bind(execution, 1); + this.workerTurns.set(execution.threadId, { + execution, prepared, provider, deliveryAttempt: 1, + terminalCommitted: false, persistedPublished: false, releaseStarted: false, + }); + provider.setCanonicalTurnDeliveryFailureHandler((routing, error) => this.handleWorkerDeliveryFailure(routing, error)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + this.turnAdmissions.markDispatchActive(execution.threadId); + } + + private executionFor(prepared: PreparedTurnDispatch): ExecutionIdentity { + return { + threadId: prepared.lease.threadId, + turnId: prepared.request.turnId, + executionId: prepared.lease.turnExecutionId, + }; + } + + private codexProvider(provider: IAgentProvider): CodexProviderBoundary { + if (provider.id !== "codex" || !("setCanonicalTurnEventDeliveryEnabled" in provider) + || !("fenceCanonicalTurnEvents" in provider)) { + throw new Error("Codex provider has no worker-owned event route"); + } + return provider as CodexProviderBoundary; + } + + private requireWorkerRuntime(): WorkerOwnedTurnRuntime { + if (!this.workerRuntime) throw new Error("Worker-owned turn runtime is unavailable"); + return this.workerRuntime; + } + + private requireWorkerFiles(): ExecutionFileEvidenceCoordinator { + if (!this.workerFiles) throw new Error("Worker-owned file capture is unavailable"); + return this.workerFiles; + } + + /** Attach host-captured evidence before the exact provider batch enters its mailbox. */ + private async prepareWorkerEvent( + execution: ExecutionIdentity, + batch: WorkerOwnedProviderEventBatch, + ): Promise> { + const active = this.requireActiveWorkerTurn(execution, batch.deliveryAttempt); + const event = this.parentRuntimeEvent(batch); + const fileObservation = event?.type === "toolUse" + ? this.requireWorkerFiles().take(event, batch.deliveryAttempt) : null; + const outcome = event ? workerTerminalOutcome(event) : null; + const terminal = outcome ? this.prepareWorkerTerminal(active, outcome) : null; + return { + kind: "event", phase: batch.phase, nativeCursor: batch.nativeCursor ?? null, + events: batch.events, deliveryAttempt: batch.deliveryAttempt, + ...(fileObservation ? { capturedFileObservation: fileObservation } : {}), + ...(terminal ? { terminalInput: terminal.input, frozenFileEvidence: terminal.files } : {}), + }; + } + + private requireActiveWorkerTurn(execution: ExecutionIdentity, attempt: number): WorkerOwnedTurn { + const active = this.workerTurns.get(execution.threadId); + if (!active || active.execution.executionId !== execution.executionId + || active.deliveryAttempt !== attempt) { + throw new Error("Provider event has no matching worker-owned turn"); + } + return active; + } + + private parentRuntimeEvent(batch: WorkerOwnedProviderEventBatch): AgentEvent | null { + if (batch.events.length !== 1) return null; + const draft = batch.events[0]; + if (!draft || draft.payload.type !== "item.recorded") return null; + const payload = draft.payload.item.payload; + if (payload.projection !== "providerRuntimeEvent") return null; + const parsed = ProviderRuntimeEventSchema().safeParse(payload.runtimeEvent); + if (!parsed.success) throw new Error("Worker event has an invalid provider runtime payload"); + if (writerOwnedProviderExtension(parsed.data.extension)) return null; + const event = parsed.data.event; + if (!matchesWorkerEvent(event, batch)) { + throw new Error("Worker event belongs to a different execution"); + } + return event; + } + + private prepareWorkerTerminal(active: WorkerOwnedTurn, outcome: TurnOutcome): { + input: DataOnlyParentTurnFinishInput; + files: NonNullable>; + } { + const files = this.requireWorkerFiles().seal({ ...active.execution, + deliveryAttempt: active.deliveryAttempt, outcome }); + if (!files) throw new Error("Terminal event lost its exact file evidence generation"); + return { + files, + input: { + ...active.execution, + providerId: active.prepared.providerId, + providerIdentities: active.prepared.parentStartInput.providerIdentities, + outcome, + deliveryAttempt: active.deliveryAttempt, + projection: { kind: "writer-staged" }, + }, + }; + } + + /** Update volatile runtime state only after the semantic writer has acknowledged the event. */ + private async applyWorkerReceipt( + execution: ExecutionIdentity, + result: Extract, + ): Promise { + const active = this.workerTurns.get(execution.threadId); + if (!active || active.execution.executionId !== execution.executionId) return; + const event = result.parentEvent?.publication.event; + if (!event) return; + this.applyWorkerFeatureReceipt(active, result); + this.publishWorkerPersistence(active, result); + this.applyWorkerTerminalReceipt(active, event); + if (event.type === "ended") { + // The callback still belongs to the route's pending set. Release on the next task. + setTimeout(() => { void this.releaseWorkerTurn(active).catch((error: unknown) => { + logger.error("Worker-owned terminal release failed", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); }, 0); + } + } + + private applyWorkerFeatureReceipt( + active: WorkerOwnedTurn, + result: Extract, + ): void { + const event = result.parentEvent?.publication.event; + if (event?.type === "message" && active.prepared.providerId === "codex") { + this.featureEffects.onAssistantMessage("codex", event); + } + for (const intent of result.parentEvent?.runtime ?? []) { + if (intent.kind === "assistant-message-feature") { + this.featureEffects.onAssistantMessage(intent.providerId, intent.event); + } + } + } + + private applyWorkerTerminalReceipt(active: WorkerOwnedTurn, event: AgentEvent): void { + if (event.type === "turnStarted") this.markProviderTurnActive(event.threadId); + const outcome = workerTerminalOutcome(event); + if (!outcome) return; + active.terminalCommitted = true; + active.terminalOutcome = outcome; + this.turnRuntime.terminalize(active.execution.threadId, active.execution.executionId, outcome); + } + + private publishWorkerPersistence( + active: WorkerOwnedTurn | undefined, + result: Extract, + ): void { + if (!active) return; + const persisted = result.terminalPersistence; + if (!persisted || active.persistedPublished) return; + active.persistedPublished = true; + broadcast("thread.status", { threadId: active.execution.threadId, + status: persisted.outcome === "cancelled" ? "interrupted" : persisted.outcome }); + broadcast("turn.persisted", { + threadId: active.execution.threadId, + turnId: active.execution.turnId, + messageId: persisted.messageId, + toolCallCount: persisted.toolCallCount, + filesChanged: [...persisted.filesChanged], + outcome: persisted.outcome, + executionId: active.execution.executionId, + ...(persisted.fileEffects ? { fileEffects: persisted.fileEffects } : {}), + }); + } + + private async releaseWorkerTurn(active: WorkerOwnedTurn): Promise { + if (active.releaseStarted || active.stopInProgress || !active.terminalCommitted) return; + active.releaseStarted = true; + const { execution, provider, deliveryAttempt } = active; + try { + const routing = { ...execution, deliveryAttempt }; + if (active.deliveryFailed) { + await provider.fenceCanonicalTurnEvents(routing).catch(() => {}); + await provider.retireCanonicalTurnEvents(routing).catch(() => {}); + } else { + await provider.fenceCanonicalTurnEvents(routing); + await provider.retireCanonicalTurnEvents(routing); + } + await this.requireWorkerRuntime().providerEvents.retire(execution); + await this.requireWorkerRuntime().owner.release(execution); + this.requireWorkerFiles().retire(execution.threadId, execution.executionId, deliveryAttempt); + if (this.workerTurns.get(execution.threadId) === active) this.workerTurns.delete(execution.threadId); + this.trackSessionEnded(execution.threadId, execution.executionId); + if (active.terminalOutcome === "completed") this.featureEffects.refreshAfterTurn(execution.threadId); + this.disarmTurnRetryWindow(execution.threadId); + this.clearTurnEndedState(execution.threadId); + } catch (error) { + active.releaseStarted = false; + await this.requireWorkerRuntime().recoverRejected(execution); + throw error; + } + } + + private handleWorkerRecovery(execution: ExecutionIdentity): void { + if (this.turnRuntime.snapshot(execution.threadId)?.turnExecutionId !== execution.executionId) return; + const active = this.workerTurns.get(execution.threadId); + if (active && active.execution.executionId !== execution.executionId) return; + if (active) this.workerTurns.delete(execution.threadId); + if (active) this.recoverWorkerProvider(active); + this.turnRuntime.terminalize(execution.threadId, execution.executionId, "interrupted"); + this.trackSessionEnded(execution.threadId, execution.executionId); + this.disarmTurnRetryWindow(execution.threadId); + this.clearTurnEndedState(execution.threadId); + } + + private recoverWorkerProvider(active: WorkerOwnedTurn): void { + const { execution } = active; + void active.provider.fenceCanonicalTurnEvents({ ...execution, + deliveryAttempt: active.deliveryAttempt }).catch((error: unknown) => { + logger.warn("Recovered worker provider fence failed", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); + void this.requireWorkerRuntime().providerEvents.retire(execution).catch((error: unknown) => { + logger.warn("Recovered worker provider route cleanup failed", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); + this.requireWorkerFiles().retire(execution.threadId, execution.executionId, active.deliveryAttempt); + void Promise.resolve(active.provider.stopSession(active.prepared.request.sessionId)).catch((error: unknown) => { + logger.warn("Recovered worker provider stop failed", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); + } + /** Activate command-specific state only after the runtime can still dispatch. */ private async activatePreparedCommandEffect(prepared: PreparedTurnDispatch): Promise { if (!this.ownsTurnAdmission(prepared.lease)) { @@ -618,7 +918,9 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE "activeTurn", () => { dispatch.dispatchStarted = true; - this.turnDiffs.begin({ ...dispatch.turnRequest, deliveryAttempt: dispatch.turnRequest.deliveryAttempt ?? 1 }); + if (!this.workerTurns.has(threadId)) { + this.turnDiffs.begin({ ...dispatch.turnRequest, deliveryAttempt: dispatch.turnRequest.deliveryAttempt ?? 1 }); + } return dispatch.resolvedProvider.sendTurn(dispatch.turnRequest); }, ); @@ -644,10 +946,69 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE error: unknown, ): Promise { if (!this.mutationReservations.owns(threadId, dispatch.mutationReservationToken, "activeTurn")) return; + const workerTurn = this.workerTurns.get(threadId); + if (workerTurn) { + try { + await this.failWorkerDispatch(workerTurn, error); + } catch (failure) { + await this.requireWorkerRuntime().recoverRejected(workerTurn.execution); + throw failure; + } + return; + } if (await this.runTransientTurnRetry(threadId, error)) return; await this.giveUpTransientTurnRetry(threadId, error); } + private async failWorkerDispatch(active: WorkerOwnedTurn, error: unknown): Promise { + const { execution, deliveryAttempt, provider } = active; + this.requireWorkerFiles().fence(execution.threadId, execution.executionId, deliveryAttempt); + if (active.deliveryFailed) { + await provider.fenceCanonicalTurnEvents({ ...execution, deliveryAttempt }).catch(() => {}); + } else { + await provider.fenceCanonicalTurnEvents({ ...execution, deliveryAttempt }); + } + if (active.terminalCommitted) return this.releaseWorkerTurn(active); + const frozen = this.requireWorkerFiles().seal({ ...execution, deliveryAttempt, outcome: "errored" }); + if (!frozen) throw new Error("Failed worker dispatch lost its file evidence generation"); + const result = await this.requireWorkerRuntime().owner.submit(execution, { + kind: "finish-from-state", outcome: "errored", frozenFileEvidence: frozen, + input: { ...execution, providerId: active.prepared.providerId, + providerIdentities: active.prepared.parentStartInput.providerIdentities, + outcome: "errored", error: error instanceof Error ? error.message : String(error), + deliveryAttempt, projection: { kind: "writer-staged" } }, + }); + if (result.kind !== "committed") throw new Error("Failed worker dispatch terminal did not commit"); + this.publishWorkerPersistence(active, result); + active.terminalCommitted = true; + active.terminalOutcome = "errored"; + this.turnRuntime.terminalize(execution.threadId, execution.executionId, "errored"); + await this.releaseWorkerTurn(active); + } + + private async handleWorkerDeliveryFailure( + routing: { threadId: string; turnId: string; executionId: string; deliveryAttempt: number }, + error: Error, + ): Promise { + const active = this.workerTurns.get(routing.threadId); + if (!active || active.execution.turnId !== routing.turnId + || active.execution.executionId !== routing.executionId + || active.deliveryAttempt !== routing.deliveryAttempt || active.deliveryFailureHandling) return; + active.deliveryFailed = true; + active.deliveryFailureHandling = true; + logger.error("Worker-owned canonical delivery failed", { threadId: routing.threadId, + executionId: routing.executionId, deliveryAttempt: routing.deliveryAttempt, error: error.message }); + try { + await this.failWorkerDispatch(active, error); + } catch (failure) { + logger.error("Worker-owned canonical delivery recovery failed", { threadId: routing.threadId, + executionId: routing.executionId, error: failure instanceof Error ? failure.message : String(failure) }); + await this.requireWorkerRuntime().recoverRejected(active.execution); + } finally { + active.deliveryFailureHandling = false; + } + } + /** Terminalize setup failures while preserving an explicit user-stop winner. */ private async failPreparedTurnDispatch(prepared: PreparedTurnDispatch, error: unknown): Promise { const runtime = this.turnRuntime.snapshot(prepared.lease.threadId); @@ -659,6 +1020,7 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE this.releaseTurnAdmission(prepared.lease); return; } + if (prepared.workerOwned) return this.failWorkerPreparedDispatch(prepared, error); if (this.turnRuntime.terminalize(prepared.lease.threadId, prepared.lease.turnExecutionId, "errored")) { await (this.finalizeTerminalTurn(prepared.lease.threadId, "errored", "send setup failure") ?? Promise.resolve()); this.disarmTurnRetryWindow(prepared.lease.threadId); @@ -669,6 +1031,79 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE throw error; } + private async failWorkerPreparedDispatch(prepared: PreparedTurnDispatch, error: unknown): Promise { + if (this.turnRetryDispatchByThread.get(prepared.lease.threadId)?.dispatchStarted) throw error; + await this.finishUnsentWorkerTurn(prepared.lease.threadId, prepared.lease.turnExecutionId); + this.turnRuntime.terminalize(prepared.lease.threadId, prepared.lease.turnExecutionId, "errored"); + this.disarmTurnRetryWindow(prepared.lease.threadId); + this.trackSessionEnded(prepared.lease.threadId, prepared.lease.turnExecutionId); + this.releaseTurnAdmission(prepared.lease); + await this.turnAdmissions.rollbackCommandEffect(prepared.commandEffect); + throw error; + } + + /** Finish a committed parent start that never reached provider dispatch. */ + private async finishUnsentWorkerTurn(threadId: string, executionId: string, outcome: "errored" | "cancelled" = "errored"): Promise { + const worker = this.workerRuntime; + const owned = worker?.owner.current(threadId); + if (!worker || !owned?.started || owned.execution.executionId !== executionId) return false; + const active = this.workerTurns.get(threadId); + const execution = owned.execution; + const result = await this.submitUnsentWorkerFinish(worker, execution, active, outcome); + if (!result) return true; + if (result.kind !== "committed") throw new Error("Unsent worker turn did not finish durably"); + this.publishWorkerPersistence(active, result); + await worker.providerEvents.retire(execution); + await worker.owner.release(execution); + if (active) this.requireWorkerFiles().retire(execution.threadId, execution.executionId, active.deliveryAttempt); + this.workerTurns.delete(execution.threadId); + return true; + } + + private async submitUnsentWorkerFinish( + worker: WorkerOwnedTurnRuntime, + execution: ExecutionIdentity, + active: WorkerOwnedTurn | undefined, + outcome: "errored" | "cancelled", + ): Promise { + try { + return await worker.owner.submit(execution, this.unsentFinishCommand(execution, active, outcome)); + } catch (error) { + await worker.recoverRejected(execution); + logger.error("Unsent worker turn recovered after rejected finish", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + return null; + } + } + + private unsentFinishCommand( + execution: ExecutionIdentity, + active: WorkerOwnedTurn | undefined, + outcome: "errored" | "cancelled", + ): Extract { + const frozen = this.unsentFileEvidence(execution, active, outcome); + const providerId = active?.prepared.providerId + ?? this.runtimePersistence.load(execution.threadId)?.provider ?? "codex"; + return { + kind: "finish-from-state", outcome, + input: { ...execution, providerId, + providerIdentities: active?.prepared.parentStartInput.providerIdentities ?? [], + outcome, projection: { kind: "writer-staged" }, + ...(outcome === "errored" ? { error: "Provider turn failed before dispatch" } : {}), + ...(active ? { deliveryAttempt: active.deliveryAttempt } : {}) }, + ...(frozen ? { frozenFileEvidence: frozen } : {}), + }; + } + + private unsentFileEvidence( + execution: ExecutionIdentity, + active: WorkerOwnedTurn | undefined, + outcome: "errored" | "cancelled", + ): ReturnType { + return active ? this.requireWorkerFiles().seal({ ...execution, + deliveryAttempt: active.deliveryAttempt, outcome }) : null; + } + /** Start a prepared first-turn command and return its authoritative admission outcome. */ private async sendInitialMessageAndSnapshot( command: SendMessageCommand, @@ -715,6 +1150,10 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE */ private async stopSessionInternal(threadId: string): Promise { const prepared = this.prepareStop(threadId); + const owned = this.workerRuntime?.owner.current(threadId); + if (owned?.started && owned.execution.executionId === prepared.runtime.turnExecutionId) { + return this.stopWorkerSession(prepared); + } if (!isRunningRuntime(prepared.runtime)) return this.alreadyTerminal(prepared); this.finishStopCheckpoint(prepared); void this.stopProviderForTurn(prepared).catch((error: unknown) => { @@ -727,6 +1166,106 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE return this.finalizeStoppedTurn(prepared); } + private async stopWorkerSession(prepared: PreparedStop): Promise { + const executionId = prepared.runtime.turnExecutionId!; + const active = this.workerTurns.get(prepared.threadId); + if (!active || !this.turnRetryDispatchByThread.get(prepared.threadId)?.dispatchStarted) { + await this.finishUnsentWorkerTurn(prepared.threadId, executionId, "cancelled"); + this.turnRuntime.terminalize(prepared.threadId, executionId, "cancelled"); + this.trackSessionEnded(prepared.threadId, executionId); + this.disarmTurnRetryWindow(prepared.threadId); + return this.cancelledWorkerStop(prepared); + } + if (!isRunningRuntime(prepared.runtime)) { + await this.releaseWorkerTurn(active); + return this.alreadyTerminal(prepared); + } + active.stopInProgress = true; + this.requireWorkerFiles().fence(prepared.threadId, executionId, active.deliveryAttempt); + const drained = active.provider.fenceCanonicalTurnEvents({ ...active.execution, + deliveryAttempt: active.deliveryAttempt }, { discardQueued: true }); + void this.stopWorkerProvider(prepared, active.provider); + try { + return await this.completeWorkerStop(prepared, active, drained); + } catch (error) { + await this.requireWorkerRuntime().recoverRejected(active.execution); + throw error; + } finally { + active.stopInProgress = false; + } + } + + private async completeWorkerStop( + prepared: PreparedStop, + active: WorkerOwnedTurn, + drained: Promise, + ): Promise { + await drained; + if (active.terminalCommitted) { + active.stopInProgress = false; + await this.releaseWorkerTurn(active); + return this.alreadyTerminal(prepared); + } + if (!active.stopRequested) { + const stopped = await this.requireWorkerRuntime().owner.stop(active.execution, NodeCrypto.randomUUID()); + if (stopped.kind !== "committed") throw new Error("Worker stop request did not commit"); + active.stopRequested = true; + } + const frozen = this.requireWorkerFiles().seal({ ...active.execution, + deliveryAttempt: active.deliveryAttempt, outcome: "cancelled" }); + if (!frozen) throw new Error("Worker stop lost its file evidence generation"); + await this.finishWorkerStop(active, frozen); + return this.cancelledWorkerStop(prepared); + } + + private async finishWorkerStop( + active: WorkerOwnedTurn, + frozen: NonNullable>, + ): Promise { + const result = await this.requireWorkerRuntime().owner.submit(active.execution, { + kind: "finish-from-state", outcome: "cancelled", frozenFileEvidence: frozen, + input: { ...active.execution, providerId: active.prepared.providerId, + providerIdentities: active.prepared.parentStartInput.providerIdentities, + outcome: "cancelled", deliveryAttempt: active.deliveryAttempt, + projection: { kind: "writer-staged" } }, + }); + if (result.kind !== "committed") throw new Error("Worker stop terminal did not commit"); + this.publishWorkerPersistence(active, result); + active.terminalCommitted = true; + active.terminalOutcome = "cancelled"; + this.turnRuntime.terminalize(active.execution.threadId, active.execution.executionId, "cancelled"); + active.stopInProgress = false; + await this.releaseWorkerTurn(active); + } + + private cancelledWorkerStop(prepared: PreparedStop): AgentStopResult { + return { threadId: prepared.threadId, turnExecutionId: prepared.runtime.turnExecutionId, + snapshot: this.turnRuntime.snapshot(prepared.threadId) ?? prepared.runtime, + status: "cancelled", dispatchState: prepared.dispatchState }; + } + + private async stopWorkerProvider(prepared: PreparedStop, provider: CodexProviderBoundary): Promise { + void Promise.resolve(this.featureEffects.stopDescendants(prepared.threadId)).catch((error: unknown) => { + logger.warn("Worker-owned descendant stop failed", { threadId: prepared.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); + const stop = Promise.resolve(provider.stopSession(prepared.sessionId)); + let timer: ReturnType | undefined; + try { + const timedOut = await Promise.race([ + stop.then(() => false), + new Promise((resolve) => { timer = setTimeout(() => resolve(true), PROVIDER_STOP_SETTLE_TIMEOUT_MS); }), + ]); + if (timedOut) this.evictUnresponsiveSession(prepared); + } catch (error) { + this.evictUnresponsiveSession(prepared); + logger.warn("Worker-owned provider stop failed", { threadId: prepared.threadId, + error: error instanceof Error ? error.message : String(error) }); + } finally { + clearTimeout(timer); + } + } + private prepareStop(threadId: string): PreparedStop { const reservationToken = this.activeMutationReservations.get(threadId); const runtime = this.turnRuntime.snapshot(threadId) ?? idleRuntime(threadId); @@ -1512,3 +2051,18 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE this.activeMutationReservations.clear(); } } + +function workerTerminalOutcome(event: AgentEvent): TurnOutcome | null { + if (event.type === "turnComplete") return "completed"; + if (event.type === "error") return "errored"; + if (event.type !== "ended" || event.outcome === undefined) return null; + return event.outcome === "cancelled" ? "interrupted" : event.outcome; +} + +function writerOwnedProviderExtension(extension: ProviderRuntimeEvent["extension"]): boolean { + return Boolean(extension?.child || extension?.collaboration || extension?.continuation); +} + +function matchesWorkerEvent(event: AgentEvent, batch: WorkerOwnedProviderEventBatch): boolean { + return event.threadId === batch.threadId && event.turnExecutionId === batch.executionId; +} diff --git a/apps/server/src/features/agents/planning/__tests__/plan-execution-state.test.ts b/apps/server/src/features/agents/planning/__tests__/plan-execution-state.test.ts new file mode 100644 index 000000000..79397e0d9 --- /dev/null +++ b/apps/server/src/features/agents/planning/__tests__/plan-execution-state.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; + +import { PlanExecutionState } from "../plan-execution-state.js"; + +const question = { + id: "q1", + category: "AUTH", + question: "Which login?", + options: [ + { id: "o1", title: "Passkey", description: "Use passkeys.", recommended: true }, + { id: "o2", title: "Password", description: "Use passwords." }, + ], +}; + +const structuredPlan = { + title: "Login plan", + changeSummary: "Use passkeys", + sections: [{ id: "s1", title: "Implementation", level: 1, content: "Add passkey login." }], +}; + +describe("PlanExecutionState", () => { + it("publishes one plain question outcome from chunked text", () => { + const state = new PlanExecutionState(); + state.beginQuestionGeneration(); + const block = `\`\`\`plan-questions\n${JSON.stringify([question])}\n\`\`\``; + + expect(state.feedText(block.slice(0, 23))).toBeNull(); + const ready = state.feedText(block.slice(23)); + expect(ready).toEqual({ questions: [question] }); + expect(structuredClone(ready)).toEqual(ready); + expect(state.feedText(block)).toBeNull(); + }); + + it("uses the parsed plan at the assistant message boundary", () => { + const state = new PlanExecutionState(); + state.beginOutputGeneration(); + const block = `\`\`\`plan-output\n${JSON.stringify(structuredPlan)}\n\`\`\``; + + expect(state.needsAssistantMaterialization()).toBe(true); + expect(state.feedText(block.slice(0, 19))).toBeNull(); + expect(state.feedText(block.slice(19))).toBeNull(); + expect(state.consumeAssistantMessage("provider prose")).toEqual({ + title: "Login plan", + contentMd: "## Implementation\n\nAdd passkey login.", + sectionsJson: '[{"id":"s1","title":"Implementation","level":1}]', + changeSummary: "Use passkeys", + }); + expect(state.needsAssistantMaterialization()).toBe(false); + expect(state.consumeAssistantMessage("# Another\n## Section")).toBeNull(); + }); + + it("uses assistant markdown when no structured block completes", () => { + const state = new PlanExecutionState(); + state.beginOutputGeneration(); + + expect(state.consumeAssistantMessage("# Fallback\n## Step\nDo it.")).toEqual({ + title: "Fallback", + contentMd: "# Fallback\n## Step\nDo it.", + sectionsJson: '[{"id":"s1","title":"Step","level":2}]', + changeSummary: null, + }); + expect(state.needsAssistantMaterialization()).toBe(false); + }); + + it("lets native exit markdown replace a pending streamed plan", () => { + const state = new PlanExecutionState(); + state.beginOutputGeneration(); + state.feedText(`\`\`\`plan-output\n${JSON.stringify(structuredPlan)}\n\`\`\``); + state.handleNativeExit("# Native plan\n## Deploy\nShip it."); + + expect(state.consumeAssistantMessage("ignored")).toEqual({ + title: "Native plan", + contentMd: "# Native plan\n## Deploy\nShip it.", + sectionsJson: '[{"id":"s1","title":"Deploy","level":2}]', + changeSummary: null, + }); + state.markPlanPersisted(); + state.handleNativeExit("# Later\n## Ignored"); + expect(state.consumeAssistantMessage("ignored")).toBeNull(); + }); +}); diff --git a/apps/server/src/features/agents/planning/__tests__/plan-turn-service-output.test.ts b/apps/server/src/features/agents/planning/__tests__/plan-turn-service-output.test.ts new file mode 100644 index 000000000..3d25a9c58 --- /dev/null +++ b/apps/server/src/features/agents/planning/__tests__/plan-turn-service-output.test.ts @@ -0,0 +1,68 @@ +import "reflect-metadata"; +import { AgentEventType, type AgentEvent } from "@mcode/contracts"; +import { describe, expect, it, vi } from "vitest"; + +import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { WorkspaceRepo } from "../../../projects/persistence/workspace-repo.js"; +import { ProviderRegistry } from "../../../providers/composition/provider-registry.js"; +import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { AgentRuntimeCommandPort, AgentTurnCommandPort } from "../../orchestration/agent-turn-command-port.js"; +import { PlanQuestionAnswersRepo } from "../persistence/plan-question-answers-repo.js"; +import { PlanRepo } from "../persistence/plan-repo.js"; +import { PlanQuestionService } from "../plan-question-service.js"; +import { PlanTurnService } from "../plan-turn-service.js"; + +vi.mock("../../../../application/transport/push.js", () => ({ broadcast: vi.fn() })); +import { broadcast } from "../../../../application/transport/push.js"; + +describe("PlanTurnService output", () => { + it("publishes parsed questions and persists the plan at its assistant message", () => { + const db = openMemoryDatabase(); + const workspace = new WorkspaceRepo(db).create("plans", process.cwd(), false); + const threads = new ThreadRepo(db); + const thread = threads.create(workspace.id, "plan", "direct", "main", false, "codex"); + const messages = new MessageRepo(db); + const plans = new PlanRepo(db); + const questions = new PlanQuestionService(messages, new PlanQuestionAnswersRepo(db)); + const service = new PlanTurnService( + threads, + new ProviderRegistry([]), + questions, + plans, + new AgentTurnCommandPort(new AgentRuntimeCommandPort()), + ); + const question = { + id: "q1", category: "AUTH", question: "Which login?", + options: [ + { id: "o1", title: "Passkey", description: "Use passkeys." }, + { id: "o2", title: "Password", description: "Use passwords." }, + ], + }; + + service.beginQuestionGeneration(thread.id); + service.onTextDelta(thread.id, `\`\`\`plan-questions\n${JSON.stringify([question])}\n\`\`\``); + expect(broadcast).toHaveBeenCalledWith("plan.questions", { threadId: thread.id, questions: [question] }); + + service.beginOutputGeneration(thread.id); + const output = { title: "Login plan", sections: [{ id: "s1", title: "Implement", level: 1, content: "Add passkeys." }] }; + const block = `\`\`\`plan-output\n${JSON.stringify(output)}\n\`\`\``; + service.onTextDelta(thread.id, block.slice(0, 24)); + service.onTextDelta(thread.id, block.slice(24)); + const assistant = messages.create(thread.id, "assistant", "Plan response", 1); + const event: Extract = { + type: AgentEventType.Message, threadId: thread.id, messageId: assistant.id, content: assistant.content, tokens: null, + }; + + expect(service.needsAssistantMaterialization(event)).toBe(true); + service.persistAssistantMessage(event); + expect(plans.getLatestForThread(thread.id)).toMatchObject({ + messageId: assistant.id, + title: "Login plan", + contentMd: "## Implement\n\nAdd passkeys.", + sectionsJson: [{ id: "s1", title: "Implement", level: 1 }], + }); + expect(service.needsAssistantMaterialization(event)).toBe(false); + db.close(true); + }); +}); diff --git a/apps/server/src/features/agents/planning/persistence/plan-repo.ts b/apps/server/src/features/agents/planning/persistence/plan-repo.ts index d6b9e20bc..8004a6936 100644 --- a/apps/server/src/features/agents/planning/persistence/plan-repo.ts +++ b/apps/server/src/features/agents/planning/persistence/plan-repo.ts @@ -111,6 +111,12 @@ export class PlanRepo { return row ? this.toRecord(row) : null; } + /** Find the plan bound to one durable assistant message. */ + getByMessageId(messageId: string): PlanRecord | null { + const row = this.orm.select().from(plans).where(eq(plans.messageId, messageId)).get(); + return row ? this.toRecord(row) : null; + } + private toRecord(row: Row): PlanRecord { return { id: row.id, diff --git a/apps/server/src/features/agents/planning/plan-execution-state.ts b/apps/server/src/features/agents/planning/plan-execution-state.ts new file mode 100644 index 000000000..cc03f8939 --- /dev/null +++ b/apps/server/src/features/agents/planning/plan-execution-state.ts @@ -0,0 +1,115 @@ +import type { PlanOutput, PlanQuestion } from "@mcode/contracts"; + +import { PlanOutputParser } from "./plan-output-parser.js"; +import { PlanQuestionParser } from "./plan-question-parser.js"; + +/** Parsed questions ready for the service to publish. */ +export interface PlanQuestionsReady { + questions: PlanQuestion[]; +} + +/** Plan data ready for the service to persist against an assistant message. */ +export interface PlanPersistenceReady { + title: string; + contentMd: string; + sectionsJson: string; + changeSummary: string | null; +} + +/** Parsing and output decisions for one plan execution, without database or transport dependencies. */ +export class PlanExecutionState { + private questionParser: PlanQuestionParser | undefined; + private outputParser: PlanOutputParser | undefined; + private pendingOutput: PlanOutput | undefined; + private pendingExitMarkdown: string | undefined; + private captured = false; + + beginQuestionGeneration(): void { + this.questionParser = new PlanQuestionParser(); + } + + beginOutputGeneration(): void { + this.outputParser = new PlanOutputParser(); + this.captured = false; + } + + feedText(delta: string): PlanQuestionsReady | null { + const questions = this.questionParser?.feed(delta); + if (questions) this.questionParser = undefined; + const output = this.outputParser?.feed(delta); + if (output) { + this.outputParser = undefined; + this.pendingOutput = output; + } + return questions ? { questions } : null; + } + + handleNativeExit(markdown: string): void { + if (this.captured) return; + this.outputParser = undefined; + this.pendingOutput = undefined; + this.pendingExitMarkdown = markdown; + } + + needsAssistantMaterialization(): boolean { + return this.pendingOutput !== undefined + || this.pendingExitMarkdown !== undefined + || this.outputParser !== undefined; + } + + consumeAssistantMessage(content: string): PlanPersistenceReady | null { + const output = this.pendingOutput; + if (output) { + this.pendingOutput = undefined; + this.outputParser = undefined; + this.pendingExitMarkdown = undefined; + const contentMd = output.sections.map((section) => ( + `${"#".repeat(section.level + 1)} ${section.title}\n\n${section.content}` + )).join("\n\n"); + const sectionsJson = JSON.stringify(output.sections.map((section) => ({ + id: section.id, + title: section.title, + level: section.level, + }))); + return { title: output.title, contentMd, sectionsJson, changeSummary: output.changeSummary ?? null }; + } + const markdown = this.pendingExitMarkdown; + if (markdown) { + this.pendingExitMarkdown = undefined; + this.outputParser = undefined; + return extractMarkdown(markdown); + } + if (!this.outputParser || !content) return null; + this.outputParser = undefined; + return extractMarkdown(content); + } + + hasPersistedPlan(): boolean { + return this.captured; + } + + markPlanPersisted(): void { + this.captured = true; + } +} + +function extractMarkdown(content: string): PlanPersistenceReady | null { + let title: string | null = null; + let nextId = 0; + const sections: Array<{ id: string; title: string; level: number }> = []; + for (const line of content.split("\n")) { + const match = /^(#{1,3})\s+(.+)/.exec(line); + if (!match) continue; + const level = match[1].length; + const heading = match[2].trim(); + if (!title) { + title = heading; + continue; + } + nextId += 1; + sections.push({ id: `s${nextId}`, title: heading, level }); + } + return title && sections.length > 0 + ? { title, contentMd: content, sectionsJson: JSON.stringify(sections), changeSummary: null } + : null; +} diff --git a/apps/server/src/features/agents/planning/plan-turn-service.ts b/apps/server/src/features/agents/planning/plan-turn-service.ts index 8505f5a31..e60162a30 100644 --- a/apps/server/src/features/agents/planning/plan-turn-service.ts +++ b/apps/server/src/features/agents/planning/plan-turn-service.ts @@ -5,7 +5,6 @@ import type { ContextWindowMode, IProviderRegistry, PermissionMode, - PlanOutput, ProviderId, ReasoningLevel, } from "@mcode/contracts"; @@ -15,8 +14,7 @@ import { AGENT_TURN_COMMAND_PORT, type AgentTurnCommandPort, } from "../orchestration/agent-turn-command-port.js"; -import { PlanOutputParser } from "./plan-output-parser.js"; -import { PlanQuestionParser } from "./plan-question-parser.js"; +import { PlanExecutionState, type PlanPersistenceReady } from "./plan-execution-state.js"; import { PlanQuestionService, type PlanAnswerInput } from "./plan-question-service.js"; import { PlanRepo } from "./persistence/plan-repo.js"; @@ -29,11 +27,7 @@ type ClaudePlanAnswerModeProvider = { /** Owns plan-question turns and durable plan-output materialization. */ @injectable() export class PlanTurnService { - private readonly questionParsers = new Map(); - private readonly outputParsers = new Map(); - private readonly pendingOutputs = new Map(); - private readonly pendingExitMarkdown = new Map(); - private readonly capturedThreads = new Set(); + private readonly executionByThread = new Map(); constructor( @inject(ThreadRepo) private readonly threadRepo: ThreadRepo, @@ -45,13 +39,12 @@ export class PlanTurnService { /** Start parsing one plan-question generation turn. */ beginQuestionGeneration(threadId: string): void { - this.questionParsers.set(threadId, new PlanQuestionParser()); + this.execution(threadId).beginQuestionGeneration(); } /** Start parsing one structured plan-output turn and arm its native provider mode. */ beginOutputGeneration(threadId: string): void { - this.outputParsers.set(threadId, new PlanOutputParser()); - this.capturedThreads.delete(threadId); + this.execution(threadId).beginOutputGeneration(); this.armNativeOutputMode(threadId); } @@ -94,65 +87,27 @@ ${userMessage}`; /** Consume one visible text delta while a plan turn is active. */ onTextDelta(threadId: string, delta: string): void { - const questions = this.questionParsers.get(threadId)?.feed(delta); - if (questions) { - this.questionParsers.delete(threadId); - broadcast("plan.questions", { threadId, questions }); - } - const output = this.outputParsers.get(threadId)?.feed(delta); - if (output) { - this.outputParsers.delete(threadId); - this.pendingOutputs.set(threadId, output); - } + const ready = this.executionByThread.get(threadId)?.feedText(delta); + if (ready) broadcast("plan.questions", { threadId, questions: ready.questions }); } /** Capture native plan markdown until its assistant message receives a durable identity. */ handleExitPlanMode(threadId: string, planMarkdown: string): void { - if (this.capturedThreads.has(threadId)) return; - this.outputParsers.delete(threadId); - this.pendingOutputs.delete(threadId); - this.pendingExitMarkdown.set(threadId, planMarkdown); + this.execution(threadId).handleNativeExit(planMarkdown); } /** Return whether a message needs early durable materialization for a plan record. */ needsAssistantMaterialization(event: PlanMessage): boolean { if (!event.messageId) return false; - return this.pendingOutputs.has(event.threadId) - || this.pendingExitMarkdown.has(event.threadId) - || this.outputParsers.has(event.threadId); + return this.executionByThread.get(event.threadId)?.needsAssistantMaterialization() ?? false; } /** Persist the one plan record that an assistant message can materialize. */ persistAssistantMessage(event: PlanMessage): void { if (!event.messageId) return; - const output = this.pendingOutputs.get(event.threadId); - if (output) { - this.pendingOutputs.delete(event.threadId); - this.outputParsers.delete(event.threadId); - this.pendingExitMarkdown.delete(event.threadId); - const content = output.sections.map((section) => ( - `${"#".repeat(section.level + 1)} ${section.title}\n\n${section.content}` - )).join("\n\n"); - const sections = JSON.stringify(output.sections.map((section) => ({ - id: section.id, - title: section.title, - level: section.level, - }))); - this.persistPlan(event.threadId, event.messageId, output.title, content, sections, output.changeSummary ?? null); - return; - } - const markdown = this.pendingExitMarkdown.get(event.threadId); - if (markdown) { - this.pendingExitMarkdown.delete(event.threadId); - this.outputParsers.delete(event.threadId); - const extracted = this.extractMarkdown(markdown); - if (extracted) this.persistPlan(event.threadId, event.messageId, extracted.title, extracted.contentMd, extracted.sectionsJson, null); - return; - } - if (!this.outputParsers.has(event.threadId) || !event.content) return; - this.outputParsers.delete(event.threadId); - const extracted = this.extractMarkdown(event.content); - if (extracted) this.persistPlan(event.threadId, event.messageId, extracted.title, extracted.contentMd, extracted.sectionsJson, null); + const execution = this.executionByThread.get(event.threadId); + const ready = execution?.consumeAssistantMessage(event.content); + if (execution && ready) this.persistPlan(event.threadId, event.messageId, ready, execution); } /** Submit answers and dispatch the complete answer turn through the command facade. */ @@ -190,11 +145,7 @@ ${userMessage}`; /** Clear volatile plan state once a turn reaches its terminal lifecycle. */ clearTurn(threadId: string): void { - this.questionParsers.delete(threadId); - this.outputParsers.delete(threadId); - this.pendingOutputs.delete(threadId); - this.pendingExitMarkdown.delete(threadId); - this.capturedThreads.delete(threadId); + this.executionByThread.delete(threadId); } private armNativeOutputMode(threadId: string): void { @@ -207,15 +158,13 @@ ${userMessage}`; private persistPlan( threadId: string, messageId: string, - title: string, - contentMd: string, - sectionsJson: string, - changeSummary: string | null, + ready: PlanPersistenceReady, + execution: PlanExecutionState, ): void { - if (this.capturedThreads.has(threadId)) return; + if (execution.hasPersistedPlan()) return; try { - const plan = this.planRepo.create(threadId, messageId, title, contentMd, sectionsJson, changeSummary); - this.capturedThreads.add(threadId); + const plan = this.planRepo.create(threadId, messageId, ready.title, ready.contentMd, ready.sectionsJson, ready.changeSummary); + execution.markPlanPersisted(); broadcast("plan.generated", { threadId, plan }); } catch (error) { logger.error("Failed to persist plan output", { @@ -225,24 +174,9 @@ ${userMessage}`; } } - private extractMarkdown(content: string): { title: string; contentMd: string; sectionsJson: string } | null { - let title: string | null = null; - let nextId = 0; - const sections: Array<{ id: string; title: string; level: number }> = []; - for (const line of content.split("\n")) { - const match = /^(#{1,3})\s+(.+)/.exec(line); - if (!match) continue; - const level = match[1].length; - const heading = match[2].trim(); - if (!title) { - title = heading; - continue; - } - nextId += 1; - sections.push({ id: `s${nextId}`, title: heading, level }); - } - return title && sections.length > 0 - ? { title, contentMd: content, sectionsJson: JSON.stringify(sections) } - : null; + private execution(threadId: string): PlanExecutionState { + const state = this.executionByThread.get(threadId) ?? new PlanExecutionState(); + this.executionByThread.set(threadId, state); + return state; } } diff --git a/apps/server/src/features/agents/recovery/__tests__/turn-recovery-service.test.ts b/apps/server/src/features/agents/recovery/__tests__/turn-recovery-service.test.ts index 33cf7bfd8..604f93050 100644 --- a/apps/server/src/features/agents/recovery/__tests__/turn-recovery-service.test.ts +++ b/apps/server/src/features/agents/recovery/__tests__/turn-recovery-service.test.ts @@ -9,9 +9,9 @@ import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/datab import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; import { - CanonicalAgentEventSink, + CanonicalAgentBoundary, type CanonicalAgentEventPublisher, -} from "../../canonical/canonical-agent-event-sink.js"; +} from "../../canonical/canonical-agent-boundary.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; @@ -29,7 +29,7 @@ const EXECUTION_ID = "00000000-0000-4000-8000-000000000015"; describe("TurnRecoveryService", () => { let db: Database; - let sink: CanonicalAgentEventSink; + let sink: CanonicalAgentBoundary; let threadRepo: ThreadRepo; let messageRepo: MessageRepo; let defaultCheckpoints: ParentAssistantTextCheckpointService; @@ -45,7 +45,7 @@ describe("TurnRecoveryService", () => { "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "workspace-recovery", "Recovery", "main", "codex", "active", NOW, NOW); published = vi.fn(); - sink = new CanonicalAgentEventSink(db, published); + sink = new CanonicalAgentBoundary(db, published); threadRepo = new ThreadRepo(db); messageRepo = new MessageRepo(db); defaultCheckpoints = new ParentAssistantTextCheckpointService(db); diff --git a/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts b/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts new file mode 100644 index 000000000..26a157fbf --- /dev/null +++ b/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts @@ -0,0 +1,237 @@ +import "reflect-metadata"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { WorkspaceRepo } from "../../../projects/persistence/workspace-repo.js"; +import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; +import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; +import { ThoughtSegmentRepo } from "../../conversation/narrative/persistence/thought-segment-repo.js"; +import { HookExecutionRepo } from "../../events/persistence/hook-execution-repo.js"; +import { TaskRepo } from "../../orchestration/persistence/task-repo.js"; +import type { ExecutionIdentity } from "../../execution/execution-mailbox-protocol.js"; +import { TaskPersistenceService } from "../task-persistence-service.js"; +import { + TaskToolIntentReducer, + type TaskToolCommand, + type TaskToolWriteIntent, +} from "../task-tool-intent-reducer.js"; + +describe("TaskToolIntentReducer", () => { + let db: Database; + let tasks: TaskRepo; + let narrative: NarrativeStore; + let service: TaskPersistenceService; + let liveThread: string; + let intentThread: string; + let execution: ExecutionIdentity; + let reducer: TaskToolIntentReducer; + + beforeEach(() => { + db = openMemoryDatabase(); + const workspace = new WorkspaceRepo(db).create("task-tool-reducer", `${process.cwd()}#task-tool-reducer`, false); + const threads = new ThreadRepo(db); + liveThread = threads.create(workspace.id, "live", "direct", "main").id; + intentThread = threads.create(workspace.id, "intents", "direct", "main").id; + tasks = new TaskRepo(db); + narrative = new NarrativeStore( + new MessageRepo(db), + new ToolCallRecordRepo(db), + new ThoughtSegmentRepo(db), + new HookExecutionRepo(db), + db, + ); + narrative.beginTurn(liveThread); + service = new TaskPersistenceService(tasks, narrative); + execution = { threadId: liveThread, turnId: "turn-1", executionId: "execution-1" }; + reducer = new TaskToolIntentReducer(execution); + }); + + afterEach(() => db.close()); + + function applyToIntentThread(intents: readonly TaskToolWriteIntent[]): void { + for (const intent of intents) { + switch (intent.kind) { + case "upsert-group": tasks.upsertGroup(intentThread, intent.group, intent.tasks); break; + case "append-task": tasks.appendTask(intentThread, intent.task); break; + case "update-task": tasks.updateTask(intentThread, intent.id, intent.patch, intent.group); break; + case "remove-task": tasks.removeTask(intentThread, intent.id, intent.group); break; + } + } + } + + function compare(command: TaskToolCommand, applyLive: () => void): TaskToolWriteIntent[] { + const reduction = reducer.reduce(execution, command); + expect(reduction.kind).toBe("intents"); + if (reduction.kind !== "intents") throw new Error("Expected task intents"); + expect(structuredClone(reduction)).toEqual(reduction); + applyToIntentThread(reduction.intents); + applyLive(); + expect(tasks.get(liveThread)).toEqual(tasks.get(intentThread)); + return reduction.intents; + } + + function toolUse( + toolCallId: string, + toolName: string, + toolInput: Record, + parentToolCallId?: string, + ): TaskToolWriteIntent[] { + const attributedParent = narrative.bufferToolCall(liveThread, { + toolCallId, toolName, toolInput, parentToolCallId, + }); + return compare({ + kind: "tool-use", toolName, toolInput, + parentToolCallId: attributedParent, + bufferedCalls: narrative.getBufferedToolCalls(liveThread), + }, () => service.onToolUse(liveThread, { toolName, toolInput, parentToolCallId: attributedParent })); + } + + function toolResult(toolCallId: string, output: string, isError = false): TaskToolWriteIntent[] { + return compare({ + kind: "tool-result", toolCallId, output, isError, + bufferedCalls: narrative.getBufferedToolCalls(liveThread), + }, () => service.onToolResult(liveThread, toolCallId, output, isError)); + } + + it("matches live rows for grouped TodoWrite, update_plan, and status normalization", () => { + expect(toolUse("todos-main", "TodoWrite", { + todos: [ + { content: "main pending", status: "unknown" }, + { content: "main active", status: "in-progress" }, + { content: " ", status: "completed" }, + ], + })).toEqual([{ + kind: "upsert-group", group: "Tasks", + tasks: [ + { content: "main pending", status: "pending", group: "Tasks" }, + { content: "main active", status: "in_progress", group: "Tasks" }, + ], + }]); + + toolUse("agent", "Agent", { description: " Build sub feature " }); + expect(toolUse("todos-sub", "TodoWrite", { + todos: [{ content: "sub cancelled", status: "canceled" }], + }, "agent")).toEqual([{ + kind: "upsert-group", group: "Build sub feature", + tasks: [{ content: "sub cancelled", status: "cancelled", group: "Build sub feature" }], + }]); + narrative.clearAgentStackOnMessage(liveThread); + expect(toolUse("plan", "update_plan", { + tasks: ["first step", { title: "second step", status: "inProgress" }, { description: " " }], + })).toEqual([{ + kind: "upsert-group", group: "Tasks", + tasks: [ + { content: "first step", status: "pending", group: "Tasks" }, + { content: "second step", status: "in_progress", group: "Tasks" }, + ], + }]); + expect(tasks.get(liveThread)).toEqual([ + { content: "sub cancelled", status: "cancelled", group: "Build sub feature" }, + { content: "first step", status: "pending", group: "Tasks" }, + { content: "second step", status: "in_progress", group: "Tasks" }, + ]); + }); + + it("matches live rows for TaskCreate result IDs, updates, and deletion", () => { + toolUse("agent", "Agent", { prompt: " Inspect files " }); + expect(toolUse("create", "TaskCreate", { + subject: " Write tests ", description: " Cover edge cases ", activeForm: " Writing tests ", + }, "agent")).toEqual([]); + expect(toolResult("create", "Created task #27 successfully")).toEqual([{ + kind: "append-task", + task: { + id: "27", content: "Write tests - Cover edge cases", status: "pending", + activeForm: "Writing tests", group: "Inspect files", + }, + }]); + expect(toolUse("update", "TaskUpdate", { + taskId: 27, status: "inProgress", subject: " Test updates ", activeForm: " Checking ", + }, "agent")).toEqual([{ + kind: "update-task", id: "27", group: "Inspect files", + patch: { status: "in_progress", content: "Test updates", activeForm: "Checking" }, + }]); + expect(tasks.get(liveThread)).toEqual([{ + id: "27", content: "Test updates", status: "in_progress", + activeForm: "Checking", group: "Inspect files", + }]); + expect(toolUse("delete", "TaskUpdate", { taskId: "27", status: "deleted" }, "agent")) + .toEqual([{ kind: "remove-task", id: "27", group: "Inspect files" }]); + expect(tasks.get(liveThread)).toEqual([]); + }); + + it("emits no writes for malformed and unsuccessful tool effects", () => { + expect(toolUse("bad-todos", "TodoWrite", { todos: [null, { content: " " }] })).toEqual([]); + expect(toolUse("bad-plan", "update_plan", { plan: [], tasks: [{ title: "ignored" }] })).toEqual([]); + expect(toolUse("bad-update", "TaskUpdate", { taskId: "", status: "completed" })).toEqual([]); + toolUse("create", "TaskCreate", { title: "Valid title" }); + expect(toolResult("create", "Created task #1", true)).toEqual([]); + expect(toolResult("create", "Created task without an id")).toEqual([]); + expect(toolResult("missing", "Created task #1")).toEqual([]); + expect(tasks.get(liveThread)).toBeNull(); + }); + + it("bounds malformed parent chains before selecting a sub-agent group", () => { + expect(reducer.reduce(execution, { + kind: "tool-use", toolName: "TodoWrite", + toolInput: { todos: [{ content: "Task" }] }, parentToolCallId: "first", + bufferedCalls: [ + { toolCallId: "first", toolName: "Other", parentToolCallId: "second", _rawToolInput: {} }, + { toolCallId: "second", toolName: "Other", parentToolCallId: "first", _rawToolInput: {} }, + ], + })).toEqual({ + kind: "intents", execution, + intents: [{ kind: "upsert-group", group: "Sub-agent", + tasks: [{ content: "Task", status: "pending", group: "Sub-agent" }] }], + }); + }); + + it("keeps colliding harness IDs scoped to their agent group", () => { + toolUse("agent-a", "Agent", { description: "Agent A" }); + toolUse("agent-b", "Agent", { description: "Agent B" }); + toolUse("create-a", "TaskCreate", { subject: "A task" }, "agent-a"); + toolUse("create-b", "TaskCreate", { subject: "B task" }, "agent-b"); + toolResult("create-a", "Created #1"); + toolResult("create-b", "Created #1"); + toolUse("ambiguous", "TaskUpdate", { taskId: "1", status: "completed" }, "missing-parent"); + expect(tasks.get(liveThread)).toEqual([ + { id: "1", content: "A task", status: "pending", group: "Agent A" }, + { id: "1", content: "B task", status: "pending", group: "Agent B" }, + ]); + toolUse("scoped", "TaskUpdate", { taskId: "1", status: "completed" }, "agent-a"); + expect(tasks.get(liveThread)).toEqual([ + { id: "1", content: "A task", status: "completed", group: "Agent A" }, + { id: "1", content: "B task", status: "pending", group: "Agent B" }, + ]); + toolUse("remove-b", "TaskUpdate", { taskId: "1", status: "deleted" }, "agent-b"); + toolUse("sole-match", "TaskUpdate", { taskId: "1", status: "cancelled" }, "missing-parent"); + expect(tasks.get(liveThread)).toEqual([ + { id: "1", content: "A task", status: "cancelled", group: "Agent A" }, + ]); + }); + + it("rejects stale execution identities before emitting writer intents", () => { + const command: TaskToolCommand = { + kind: "tool-use", toolName: "TodoWrite", + toolInput: { todos: [{ content: "must not appear" }] }, bufferedCalls: [], + }; + expect(reducer.reduce({ ...execution, executionId: "previous" }, command)) + .toEqual({ kind: "stale-execution" }); + expect(reducer.reduce({ ...execution, turnId: "previous" }, command)) + .toEqual({ kind: "stale-execution" }); + expect(reducer.reduce({ ...execution, threadId: intentThread }, command)) + .toEqual({ kind: "stale-execution" }); + const result: TaskToolCommand = { + kind: "tool-result", toolCallId: "create", output: "Created #9", isError: false, + bufferedCalls: [{ toolCallId: "create", toolName: "TaskCreate", _rawToolInput: { subject: "late task" } }], + }; + expect(reducer.reduce({ ...execution, executionId: "previous" }, result)) + .toEqual({ kind: "stale-execution" }); + expect(reducer.reduce(execution, result)).toEqual({ + kind: "intents", execution, + intents: [{ kind: "append-task", task: { id: "9", content: "late task", status: "pending", group: "Tasks" } }], + }); + expect(tasks.get(liveThread)).toBeNull(); + }); +}); diff --git a/apps/server/src/features/agents/tasks/task-persistence-service.ts b/apps/server/src/features/agents/tasks/task-persistence-service.ts index 8624a0dab..6aeba43ae 100644 --- a/apps/server/src/features/agents/tasks/task-persistence-service.ts +++ b/apps/server/src/features/agents/tasks/task-persistence-service.ts @@ -1,7 +1,8 @@ import { inject, injectable } from "tsyringe"; import { logger } from "@mcode/shared"; import { NarrativeStore } from "../conversation/narrative/narrative-store.js"; -import { TaskRepo, type StoredTask } from "../orchestration/persistence/task-repo.js"; +import { TaskRepo } from "../orchestration/persistence/task-repo.js"; +import { taskToolWriteIntents, type TaskToolWriteIntent } from "./task-tool-intent-reducer.js"; /** Persists provider task-tool state for reconnect hydration. */ @injectable() @@ -16,134 +17,52 @@ export class TaskPersistenceService { threadId: string, event: { toolName: string; toolInput: Record; parentToolCallId?: string }, ): void { - const group = event.parentToolCallId ? this.groupFor(threadId, event.parentToolCallId) : "Tasks"; - if (event.toolName === "TodoWrite") { - this.persistTodoWrite(threadId, event.toolInput, group); - return; - } - if (event.toolName === "TaskUpdate") { - this.applyTaskUpdate(threadId, event.toolInput, group); - return; - } - if (event.toolName === "update_plan") this.persistPlanTasks(threadId, event.toolInput, group); + const intents = taskToolWriteIntents({ + kind: "tool-use", + ...event, + bufferedCalls: this.narrative.getBufferedToolCalls(threadId), + }); + const label = event.toolName === "TodoWrite" ? "TodoWrite tasks" + : event.toolName === "update_plan" ? "update_plan tasks" : "TaskUpdate"; + this.applyIntents(threadId, intents, label); } /** Persist one TaskCreate after its result supplies the stable harness identity. */ onToolResult(threadId: string, toolCallId: string, output: string, isError: boolean): void { if (isError) return; - const buffered = this.narrative.getBufferedToolCalls(threadId) - .find((tool) => tool.toolCallId === toolCallId && tool.toolName === "TaskCreate"); - if (!buffered) return; - const id = this.parseHarnessId(output); - const input = buffered._rawToolInput ?? {}; - const content = this.taskContent(input); - if (!id || !content) return; - const group = buffered.parentToolCallId ? this.groupFor(threadId, buffered.parentToolCallId) : "Tasks"; - const activeForm = this.nonEmpty(input.activeForm); - this.tryPersist("TaskCreate task", threadId, () => this.tasks.appendTask(threadId, { - id, - content, - status: "pending", - ...(activeForm ? { activeForm } : {}), - group, - })); - } - - private persistTodoWrite(threadId: string, input: Record, group: string): void { - if (!Array.isArray(input.todos)) return; - const todos = input.todos.flatMap((value): StoredTask[] => { - if (!this.isRecord(value) || !this.nonEmpty(value.content)) return []; - return [{ content: String(value.content), status: this.status(value.status), group }]; - }); - if (todos.length > 0) this.tryPersist("TodoWrite tasks", threadId, () => this.tasks.upsertGroup(threadId, group, todos)); - } - - private persistPlanTasks(threadId: string, input: Record, group: string): void { - const values = Array.isArray(input.plan) - ? input.plan - : Array.isArray(input.tasks) - ? input.tasks - : Array.isArray(input.todos) - ? input.todos - : []; - const tasks = values.flatMap((value): StoredTask[] => { - const item: Record = this.isRecord(value) ? value : { step: value }; - const content = this.nonEmpty(item.step) ?? this.nonEmpty(item.content) - ?? this.nonEmpty(item.title) ?? this.nonEmpty(item.description); - return content ? [{ content, status: this.status(item.status), group }] : []; - }); - if (tasks.length > 0) this.tryPersist("update_plan tasks", threadId, () => this.tasks.upsertGroup(threadId, group, tasks)); - } - - private applyTaskUpdate(threadId: string, input: Record, group: string): void { - const id = input.taskId == null ? "" : String(input.taskId); - if (!id) return; - this.tryPersist("TaskUpdate", threadId, () => { - if (input.status === "deleted") { - this.tasks.removeTask(threadId, id, group); - return; - } - const patch: Partial> = {}; - if (input.status !== undefined) patch.status = this.status(input.status); - const content = this.nonEmpty(input.subject); - const activeForm = this.nonEmpty(input.activeForm); - if (content) patch.content = content; - if (activeForm) patch.activeForm = activeForm; - if (Object.keys(patch).length > 0) this.tasks.updateTask(threadId, id, patch, group); + const intents = taskToolWriteIntents({ + kind: "tool-result", + toolCallId, + output, + isError, + bufferedCalls: this.narrative.getBufferedToolCalls(threadId), }); + this.applyIntents(threadId, intents, "TaskCreate task"); } - private groupFor(threadId: string, parentToolCallId: string): string { - const calls = this.narrative.getBufferedToolCalls(threadId); - let current: string | undefined = parentToolCallId; - while (current) { - const call = calls.find((item) => item.toolCallId === current); - if (!call) break; - if (call.toolName === "Agent") { - const label = this.nonEmpty(call._rawToolInput?.description) ?? this.nonEmpty(call._rawToolInput?.prompt); - return label ? label.slice(0, 80) : "Sub-agent"; - } - current = call.parentToolCallId; + private applyIntents(threadId: string, intents: readonly TaskToolWriteIntent[], label: string): void { + for (const intent of intents) { + this.tryPersist(label, threadId, () => this.applyIntent(threadId, intent)); } - return "Sub-agent"; } - private status(value: unknown): StoredTask["status"] { - switch (value) { - case "inProgress": - case "in-progress": - return "in_progress"; - case "canceled": - return "cancelled"; - case "pending": - case "in_progress": - case "completed": - case "cancelled": - return value; - default: - return "pending"; + private applyIntent(threadId: string, intent: TaskToolWriteIntent): void { + switch (intent.kind) { + case "upsert-group": + this.tasks.upsertGroup(threadId, intent.group, intent.tasks); + return; + case "append-task": + this.tasks.appendTask(threadId, intent.task); + return; + case "update-task": + this.tasks.updateTask(threadId, intent.id, intent.patch, intent.group); + return; + case "remove-task": + this.tasks.removeTask(threadId, intent.id, intent.group); + return; } } - private taskContent(input: Record): string | null { - const subject = this.nonEmpty(input.subject) ?? this.nonEmpty(input.title) ?? this.nonEmpty(input.content); - const description = this.nonEmpty(input.description); - if (!subject) return description; - return description ? `${subject} - ${description}` : subject; - } - - private parseHarnessId(output: string): string | null { - return /#(\d+)/.exec(output)?.[1] ?? null; - } - - private nonEmpty(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; - } - - private isRecord(value: unknown): value is Record { - return value !== null && typeof value === "object"; - } - private tryPersist(label: string, threadId: string, persist: () => void): void { try { persist(); diff --git a/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts b/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts new file mode 100644 index 000000000..0387515d0 --- /dev/null +++ b/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts @@ -0,0 +1,165 @@ +import type { BufferedToolCall } from "../conversation/narrative/narrative-turn-state.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; +import type { StoredTask } from "../orchestration/persistence/task-repo.js"; + +/** Narrative fields needed to resolve TaskCreate and sub-agent groups. */ +export type TaskToolCall = Pick; +type TaskPatch = Partial>; + +/** Data-only task writes for the execution's sole durable writer. */ +export type TaskToolWriteIntent = + | { readonly kind: "upsert-group"; readonly group: string; readonly tasks: StoredTask[] } + | { readonly kind: "append-task"; readonly task: StoredTask } + | { readonly kind: "update-task"; readonly id: string; readonly group: string; readonly patch: TaskPatch } + | { readonly kind: "remove-task"; readonly id: string; readonly group: string }; + +/** Narrative attribution is resolved before a tool event reaches this reducer. */ +export type TaskToolCommand = + | { + readonly kind: "tool-use"; + readonly toolName: string; + readonly toolInput: Record; + readonly parentToolCallId?: string; + readonly bufferedCalls: readonly TaskToolCall[]; + } + | { + readonly kind: "tool-result"; + readonly toolCallId: string; + readonly output: string; + readonly isError: boolean; + readonly bufferedCalls: readonly TaskToolCall[]; + }; + +/** The writer receives intents only when the complete execution identity matches. */ +export type TaskToolReduction = + | { readonly kind: "intents"; readonly execution: ExecutionIdentity; readonly intents: TaskToolWriteIntent[] } + | { readonly kind: "stale-execution" }; + +/** Pure task-tool interpretation shared by the live adapter and execution owner. */ +export function taskToolWriteIntents(command: TaskToolCommand): TaskToolWriteIntent[] { + if (command.kind === "tool-result") return taskCreateIntent(command); + const { toolName, toolInput, parentToolCallId, bufferedCalls } = command; + if (toolName !== "TodoWrite" && toolName !== "TaskUpdate" && toolName !== "update_plan") return []; + const group = parentToolCallId ? groupFor(bufferedCalls, parentToolCallId) : "Tasks"; + if (toolName === "TodoWrite") return todoWriteIntents(toolInput, group); + if (toolName === "TaskUpdate") return taskUpdateIntents(toolInput, group); + return planIntents(toolInput, group); +} + +/** Fences commands to one turn attempt before returning cloneable writer data. */ +export class TaskToolIntentReducer { + readonly execution: ExecutionIdentity; + + constructor(execution: ExecutionIdentity) { + this.execution = { ...execution }; + } + + reduce(execution: ExecutionIdentity, command: TaskToolCommand): TaskToolReduction { + if (execution.threadId !== this.execution.threadId + || execution.turnId !== this.execution.turnId + || execution.executionId !== this.execution.executionId) { + return { kind: "stale-execution" }; + } + return { kind: "intents", execution: { ...this.execution }, intents: taskToolWriteIntents(command) }; + } +} + +function todoWriteIntents(input: Record, group: string): TaskToolWriteIntent[] { + if (!Array.isArray(input.todos)) return []; + const tasks = input.todos.flatMap((value): StoredTask[] => { + if (!isRecord(value) || !nonEmpty(value.content)) return []; + return [{ content: String(value.content), status: status(value.status), group }]; + }); + return tasks.length > 0 ? [{ kind: "upsert-group", group, tasks }] : []; +} + +function planIntents(input: Record, group: string): TaskToolWriteIntent[] { + const values = Array.isArray(input.plan) + ? input.plan + : Array.isArray(input.tasks) + ? input.tasks + : Array.isArray(input.todos) + ? input.todos + : []; + const tasks = values.flatMap((value): StoredTask[] => { + const item: Record = isRecord(value) ? value : { step: value }; + const content = nonEmpty(item.step) ?? nonEmpty(item.content) + ?? nonEmpty(item.title) ?? nonEmpty(item.description); + return content ? [{ content, status: status(item.status), group }] : []; + }); + return tasks.length > 0 ? [{ kind: "upsert-group", group, tasks }] : []; +} + +function taskUpdateIntents(input: Record, group: string): TaskToolWriteIntent[] { + const id = input.taskId == null ? "" : String(input.taskId); + if (!id) return []; + if (input.status === "deleted") return [{ kind: "remove-task", id, group }]; + const patch: TaskPatch = {}; + if (input.status !== undefined) patch.status = status(input.status); + const content = nonEmpty(input.subject); + const activeForm = nonEmpty(input.activeForm); + if (content) patch.content = content; + if (activeForm) patch.activeForm = activeForm; + return Object.keys(patch).length > 0 ? [{ kind: "update-task", id, group, patch }] : []; +} + +function taskCreateIntent(command: Extract): TaskToolWriteIntent[] { + if (command.isError) return []; + const buffered = command.bufferedCalls + .find((tool) => tool.toolCallId === command.toolCallId && tool.toolName === "TaskCreate"); + if (!buffered) return []; + const id = /#(\d+)/.exec(command.output)?.[1]; + const input = buffered._rawToolInput ?? {}; + const content = taskContent(input); + if (!id || !content) return []; + const group = buffered.parentToolCallId ? groupFor(command.bufferedCalls, buffered.parentToolCallId) : "Tasks"; + const activeForm = nonEmpty(input.activeForm); + return [{ + kind: "append-task", + task: { id, content, status: "pending", ...(activeForm ? { activeForm } : {}), group }, + }]; +} + +function groupFor(calls: readonly TaskToolCall[], parentToolCallId: string): string { + let current: string | undefined = parentToolCallId; + const seen = new Set(); + while (current && !seen.has(current)) { + seen.add(current); + const call = calls.find((item) => item.toolCallId === current); + if (!call) break; + if (call.toolName === "Agent") { + const label = nonEmpty(call._rawToolInput?.description) ?? nonEmpty(call._rawToolInput?.prompt); + return label ? label.slice(0, 80) : "Sub-agent"; + } + current = call.parentToolCallId; + } + return "Sub-agent"; +} + +function status(value: unknown): StoredTask["status"] { + switch (value) { + case "inProgress": + case "in-progress": return "in_progress"; + case "canceled": return "cancelled"; + case "pending": + case "in_progress": + case "completed": + case "cancelled": return value; + default: return "pending"; + } +} + +function taskContent(input: Record): string | null { + const subject = nonEmpty(input.subject) ?? nonEmpty(input.title) ?? nonEmpty(input.content); + const description = nonEmpty(input.description); + if (!subject) return description; + return description ? `${subject} - ${description}` : subject; +} + +function nonEmpty(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object"; +} diff --git a/apps/server/src/features/agents/tools/persistence/tool-call-record-repo.ts b/apps/server/src/features/agents/tools/persistence/tool-call-record-repo.ts index b326c6c37..e40cc97ee 100644 --- a/apps/server/src/features/agents/tools/persistence/tool-call-record-repo.ts +++ b/apps/server/src/features/agents/tools/persistence/tool-call-record-repo.ts @@ -155,15 +155,15 @@ export class ToolCallRecordRepo { } /** Create multiple tool call records in a single transaction. */ - bulkCreate(inputs: CreateToolCallRecordInput[]): void { + bulkCreate(inputs: CreateToolCallRecordInput[], replaceExisting = false): void { this.orm.transaction((tx) => { const now = new Date().toISOString(); for (const item of inputs) { const insert = prepareToolCallRecordInsert(item, now); - tx.insert(toolCallRecords) - .values(this.insertValues(item, insert)) - .onConflictDoNothing() - .run(); + const { id, ...rest } = this.insertValues(item, insert); + const write = tx.insert(toolCallRecords).values({ id, ...rest }); + if (replaceExisting) write.onConflictDoUpdate({ target: toolCallRecords.id, set: rest }).run(); + else write.onConflictDoNothing().run(); } }); } diff --git a/apps/server/src/features/agents/transport/agent-rpc.ts b/apps/server/src/features/agents/transport/agent-rpc.ts index f67920928..f3b7719b4 100644 --- a/apps/server/src/features/agents/transport/agent-rpc.ts +++ b/apps/server/src/features/agents/transport/agent-rpc.ts @@ -6,6 +6,7 @@ import type { } from "@mcode/contracts"; import { WS_METHODS } from "@mcode/contracts"; import { logger } from "@mcode/shared"; +import * as NodePerfHooks from "node:perf_hooks"; import type { z } from "zod"; import type { GitWatcherService } from "../../projects/git/git-watcher-service.js"; import type { ThreadControlService } from "../../thread-control/authority/thread-control-service.js"; @@ -21,6 +22,7 @@ import type { ThoughtSegmentRepo } from "../conversation/narrative/persistence/t import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; import type { HookExecutionRepo } from "../events/persistence/hook-execution-repo.js"; import type { AgentService } from "../orchestration/agent-service.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; import type { AgentTurnContinuationPort } from "../orchestration/agent-runtime-internal-ports.js"; import type { TaskRepo } from "../orchestration/persistence/task-repo.js"; import type { AgentPermissionService } from "../permissions/agent-permission-service.js"; @@ -107,11 +109,16 @@ const PREVIEW_ANNOTATION_FENCE_END = "mcode-preview-annotations:end -->"; const agentHandlers: AgentRpcHandlerMap = { "agent.send": async (deps, params) => { - await deps.agentService.sendMessage({ - ...params, - content: appendPreviewAnnotations(params.content, params.previewAnnotations), - displayContent: params.displayContent ?? params.content, - }); + const started = serverWorkTrace ? NodePerfHooks.performance.now() : 0; + try { + await deps.agentService.sendMessage({ + ...params, + content: appendPreviewAnnotations(params.content, params.previewAnnotations), + displayContent: params.displayContent ?? params.content, + }); + } finally { + if (serverWorkTrace) serverWorkTrace.record("agent-send", params.threadId, undefined, NodePerfHooks.performance.now() - started); + } }, "agent.recoveryIncident": (deps) => deps.turnRecoveryService.currentRecoveryIncident(), "agent.retry": async (deps, params) => { diff --git a/apps/server/src/features/agents/turns/__tests__/assistant-execution-state.test.ts b/apps/server/src/features/agents/turns/__tests__/assistant-execution-state.test.ts new file mode 100644 index 000000000..03bbb8057 --- /dev/null +++ b/apps/server/src/features/agents/turns/__tests__/assistant-execution-state.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { AssistantExecutionState } from "../assistant-execution-state.js"; + +describe("AssistantExecutionState", () => { + it("accumulates final text and clears it at a message boundary", () => { + const state = new AssistantExecutionState(); + state.appendStreamingText(" partial"); + state.appendStreamingText(" answer "); + + expect(state.getStreamingText()).toBe(" partial answer "); + expect(state.materializationInput("fallback-model")).toEqual({ + content: "partial answer", + model: "fallback-model", + attachments: [], + fromProvider: false, + }); + + state.resetStreamingText(); + expect(state.getStreamingText()).toBe(""); + expect(state.hasBufferedBody()).toBe(false); + }); + + it("uses the provider message body and merges generated attachments by id", () => { + const state = new AssistantExecutionState(); + const image = { id: "image", name: "image.png", mimeType: "image/png", sizeBytes: 12 }; + const updatedImage = { ...image, sizeBytes: 20 }; + const file = { id: "file", name: "notes.txt", mimeType: "text/plain", sizeBytes: 8 }; + state.appendStreamingText("draft text"); + state.bufferAttachments([image]); + state.bufferBody("final answer", "provider-model"); + state.bufferAttachments([updatedImage, file]); + + const input = state.materializationInput("fallback-model"); + expect(input).toEqual({ + content: "final answer", + model: "provider-model", + attachments: [updatedImage, file], + fromProvider: true, + }); + expect(structuredClone(input)).toEqual(input); + + state.commitMaterialization(); + expect(state.hasMaterialized()).toBe(true); + expect(state.getStreamingText()).toBe(""); + expect(state.getBufferedAttachments()).toEqual([]); + expect(state.hasBufferedBody()).toBe(false); + }); + + it("keeps a text-only empty turn empty after an undecided message boundary", () => { + const state = new AssistantExecutionState(); + state.appendStreamingText(" \n"); + expect(state.hasBufferedBody()).toBe(false); + + state.resetStreamingText(); + expect(state.materializationInput(null)).toEqual({ + content: "", + model: null, + attachments: [], + fromProvider: false, + }); + expect(state.hasMaterialized()).toBe(false); + }); +}); diff --git a/apps/server/src/features/agents/turns/__tests__/codex-parent-message-projection.test.ts b/apps/server/src/features/agents/turns/__tests__/codex-parent-message-projection.test.ts new file mode 100644 index 000000000..c9751b8bd --- /dev/null +++ b/apps/server/src/features/agents/turns/__tests__/codex-parent-message-projection.test.ts @@ -0,0 +1,149 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; +import { ThoughtSegmentRepo } from "../../conversation/narrative/persistence/thought-segment-repo.js"; +import { HookExecutionRepo } from "../../events/persistence/hook-execution-repo.js"; +import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; +import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; +import { SnapshotService } from "../../../projects/diffs/snapshots/snapshot-service.js"; +import { RealGitExecutor } from "../../../projects/git/execution/real-git-executor.js"; +import { TurnSnapshotRepo } from "../persistence/turn-snapshot-repo.js"; +import { TurnFinalizer } from "../turn-finalizer.js"; +import { CodexParentMessageProjection } from "../codex-parent-message-projection.js"; +import { deriveTurnAssistantMessageId } from "../turn-assistant-message-id.js"; + +const execution = { threadId: "thread-1", turnId: "turn-1", executionId: "execution-1" } as const; +const endedAt = "2026-09-24T10:00:00.000Z"; +const image = { id: "image-1", name: "image.png", mimeType: "image/png", sizeBytes: 12 }; + +describe("CodexParentMessageProjection", () => { + let directory: string; + let path: string; + let db: Database; + let messages: MessageRepo; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-codex-message-projection-")); + path = NodePath.join(directory, "mcode.db"); + db = openDatabase({ dbPath: path }); + messages = new MessageRepo(db); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-1", "Workspace", "C:/fixture", endedAt, endedAt); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(execution.threadId, "workspace-1", "Thread", "main", "codex", endedAt, endedAt); + messages.create(execution.threadId, "user", "Question", 1, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, undefined, "user-1"); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + function finalizer(): TurnFinalizer { + return new TurnFinalizer( + messages, + new ThreadRepo(db), + new NarrativeStore(messages, new ToolCallRecordRepo(db), new ThoughtSegmentRepo(db), new HookExecutionRepo(db)), + new SnapshotService(new RealGitExecutor()), + new TurnSnapshotRepo(db), + db, + ); + } + + function projection(): CodexParentMessageProjection { + return new CodexParentMessageProjection({ execution, turnKind: "ordinary" }); + } + + it("matches the live finalizer after an intervening notice and survives repeat and reload", () => { + const live = finalizer(); + const worker = projection(); + live.bufferAssistantAttachments(execution.threadId, [image]); + worker.bufferGeneratedAttachment(execution, image); + messages.create(execution.threadId, "system", "Notice", 2, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, undefined, "notice-1"); + + const publicMessage = worker.projectMessage({ + execution, content: "Answer", model: "codex-model", precedingMessageId: "notice-1", + postTurnGoalReceipt: false, + }); + const liveMessageId = live.bufferAssistantBody(execution.threadId, "Answer", "codex-model"); + expect(publicMessage).toEqual({ messageId: liveMessageId, model: "codex-model", attachments: [image] }); + expect(liveMessageId).toBe(deriveTurnAssistantMessageId(execution.threadId, "notice-1")); + + const terminal = worker.projectTerminal({ execution, outcome: "completed", endedAt, + fallbackModel: "later-model", narrative: [] }); + expect(structuredClone(terminal)).toEqual(terminal); + expect(terminal).toMatchObject({ fromProvider: true, + assistant: { messageId: publicMessage.messageId, content: "Answer", model: "codex-model", attachments: [image] } }); + expect(worker.projectTerminal({ execution, outcome: "completed", endedAt, + fallbackModel: "later-model", narrative: [] })).toEqual(terminal); + + messages.createAssistantIdempotent({ id: terminal.assistant.messageId, threadId: execution.threadId, + content: terminal.assistant.content, model: terminal.assistant.model, + attachments: [...terminal.assistant.attachments], sequence: 3 }); + db.close(true); + db = openDatabase({ dbPath: path }); + messages = new MessageRepo(db); + expect(messages.listByThread(execution.threadId, 10).messages.filter((message) => message.role === "assistant")) + .toEqual([expect.objectContaining({ id: publicMessage.messageId, model: "codex-model", attachments: [image] })]); + + const replayWorker = projection(); + replayWorker.bufferGeneratedAttachment(execution, image); + const replay = replayWorker.projectMessage({ execution, content: "Answer", model: "codex-model", + precedingMessageId: "notice-1", postTurnGoalReceipt: false }); + expect(replay).toEqual(publicMessage); + messages.createAssistantIdempotent({ id: replay.messageId, threadId: execution.threadId, + content: "Answer", model: "codex-model", sequence: 3 }); + expect(messages.listByThread(execution.threadId, 10).messages.filter((message) => message.role === "assistant")) + .toHaveLength(1); + }); + + it("keeps the published ID when a later notice changes the last visible row", () => { + const worker = projection(); + const publicMessage = worker.projectMessage({ execution, content: "Answer", model: null, + precedingMessageId: "user-1", postTurnGoalReceipt: false }); + messages.create(execution.threadId, "system", "Later notice", 2); + + const terminal = worker.projectTerminal({ execution, outcome: "completed", endedAt, + precedingMessageId: messages.listByThread(execution.threadId, 1).messages[0]?.id, + fallbackModel: "new-model", narrative: [] }); + expect(terminal.assistant.messageId).toBe(publicMessage.messageId); + expect(terminal.assistant.model).toBeNull(); + }); + + it("derives a text-only fallback ID at terminal time and fences a prior execution", () => { + const worker = projection(); + const stale = { ...execution, executionId: "old-execution" }; + expect(() => worker.appendFinalResponseText(stale, "wrong")).toThrow(/Stale Codex/); + expect(() => worker.bufferGeneratedAttachment(stale, image)).toThrow(/Stale Codex/); + expect(() => worker.projectMessage({ execution: stale, content: "wrong", model: null, + precedingMessageId: "user-1", postTurnGoalReceipt: false })).toThrow(/Stale Codex/); + worker.appendFinalResponseText(execution, " partial answer "); + const terminal = worker.projectTerminal({ execution, outcome: "cancelled", endedAt, + precedingMessageId: "user-1", fallbackModel: "codex-fallback", narrative: [] }); + expect(terminal).toMatchObject({ fromProvider: false, + assistant: { messageId: deriveTurnAssistantMessageId(execution.threadId, "user-1"), + content: "partial answer", model: "codex-fallback", attachments: [] } }); + expect(() => worker.projectTerminal({ execution: { ...execution, turnId: "later-turn" }, + outcome: "completed", endedAt, precedingMessageId: "user-1", fallbackModel: null, narrative: [] })) + .toThrow(/Stale Codex/); + }); + + it("rejects plan turns and post-turn goal receipts without mutating the ordinary message", () => { + expect(() => new CodexParentMessageProjection({ execution, turnKind: "plan" })) + .toThrow(/plan output needs early assistant materialization/); + const worker = projection(); + expect(() => worker.projectMessage({ execution, content: "Goal achieved in 3s.", model: null, + precedingMessageId: "user-1", postTurnGoalReceipt: true })).toThrow(/Post-turn goal receipts/); + expect(worker.projectTerminal({ execution, outcome: "completed", endedAt, + precedingMessageId: "user-1", fallbackModel: null, narrative: [] }).assistant.content).toBe(""); + }); +}); diff --git a/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts b/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts new file mode 100644 index 000000000..74664a98a --- /dev/null +++ b/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest"; +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; + +import { NarrativeRecoveryDelta, splitNarrativeRecoveryDelta } from "../narrative-recovery-delta.js"; + +const thought = { + kind: "narrationSegment", + record: { + id: "thought-1", message_id: "message-1", text: "First thought", + started_at: "2026-09-24T10:00:00.000Z", ended_at: null, sort_order: 1, + }, +} satisfies ParentNarrativeRecoveryItem; +const revisedThought = { ...thought, record: { ...thought.record, text: "Revised thought" } }; + +describe("NarrativeRecoveryDelta", () => { + it("returns changed items, then skips an unchanged acknowledged snapshot", () => { + const delta = new NarrativeRecoveryDelta(); + const first = delta.prepare([thought]); + expect(first).toMatchObject({ items: [thought], discardedItemIds: [] }); + first?.acknowledge(); + expect(delta.prepare([thought])).toBeNull(); + + const revised = delta.prepare([revisedThought]); + expect(revised).toMatchObject({ items: [revisedThought], discardedItemIds: [] }); + revised?.acknowledge(); + expect(delta.prepare([revisedThought])).toBeNull(); + }); + + it("reports an item removed from the complete snapshot", () => { + const delta = new NarrativeRecoveryDelta(); + delta.prepare([thought])?.acknowledge(); + const discard = delta.prepare([]); + expect(discard).toMatchObject({ items: [], discardedItemIds: ["narrationSegment:thought-1"] }); + discard?.acknowledge(); + expect(delta.prepare([])).toBeNull(); + }); + + it("retries the same delta when persistence failed before acknowledgement", () => { + const delta = new NarrativeRecoveryDelta(); + const failed = delta.prepare([thought]); + const retry = delta.prepare([thought]); + expect(retry).toMatchObject({ items: [thought], discardedItemIds: [] }); + retry?.acknowledge(); + expect(delta.prepare([thought])).toBeNull(); + expect(() => failed?.acknowledge()).toThrow("already superseded"); + }); + + it("keeps separate executions' fingerprints independent", () => { + const firstExecution = new NarrativeRecoveryDelta(); + const secondExecution = new NarrativeRecoveryDelta(); + firstExecution.prepare([thought])?.acknowledge(); + expect(firstExecution.prepare([thought])).toBeNull(); + expect(secondExecution.prepare([thought])).toMatchObject({ items: [thought] }); + expect(secondExecution.prepare([])).toBeNull(); + }); + + it("splits a large delta without acknowledging any chunk early", () => { + const delta = new NarrativeRecoveryDelta(); + const snapshot = Array.from({ length: 63 }, (_, index) => ({ + ...thought, + record: { ...thought.record, id: `thought-${index}` }, + })); + const prepared = delta.prepare(snapshot); + expect(prepared).not.toBeNull(); + if (!prepared) return; + + const chunks = splitNarrativeRecoveryDelta("execution-1", prepared); + expect(chunks.map((chunk) => chunk.items.length)).toEqual([62, 1]); + expect(chunks.flatMap((chunk) => chunk.items)).toEqual(snapshot); + expect(delta.prepare(snapshot)?.items).toHaveLength(63); + prepared.acknowledge(); + expect(delta.prepare(snapshot)).toBeNull(); + }); + + it("splits by serialized bytes and rejects an item larger than one command", () => { + const delta = new NarrativeRecoveryDelta(); + const snapshot = ["first", "second"].map((id) => ({ + ...thought, + record: { ...thought.record, id, text: "x".repeat(150_000) }, + })); + const prepared = delta.prepare(snapshot); + expect(prepared).not.toBeNull(); + if (!prepared) return; + expect(splitNarrativeRecoveryDelta("execution-1", prepared).map((chunk) => chunk.items.length)).toEqual([1, 1]); + const oversized = delta.prepare([{ ...thought, record: { ...thought.record, text: "x".repeat(300_000) } }]); + expect(oversized).not.toBeNull(); + if (!oversized) return; + expect(() => splitNarrativeRecoveryDelta("execution-1", oversized)).toThrow("exceeds one writer command"); + }); +}); diff --git a/apps/server/src/features/agents/turns/__tests__/narrative-tool-recovery.test.ts b/apps/server/src/features/agents/turns/__tests__/narrative-tool-recovery.test.ts new file mode 100644 index 000000000..3667d21db --- /dev/null +++ b/apps/server/src/features/agents/turns/__tests__/narrative-tool-recovery.test.ts @@ -0,0 +1,164 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; +import { NarrativeTurnState } from "../../conversation/narrative/narrative-turn-state.js"; +import { NarrativeRecoveryDelta, type PreparedNarrativeRecoveryDelta } from "../narrative-recovery-delta.js"; +import { ACTIVE_TURN_RECOVERY_RETAINED_LIMITS } from "../active-turn-recovery-retention-policy.js"; + +const execution = { threadId: "thread", turnId: "turn", executionId: "execution" }; +const thought = { kind: "narrationSegment", record: { + id: "thought", message_id: "", text: "", started_at: "2026-09-25T12:00:00.000Z", ended_at: null, sort_order: 0, +} } satisfies ParentNarrativeRecoveryItem; + +function setup(toolName = "Read") { + const state = new NarrativeTurnState(execution); + state.beginTurn(execution.threadId); + state.openOrExtendThought(execution.threadId, "Keep this thought"); + state.closeOpenThought(execution.threadId); + state.bufferToolCall(execution.threadId, { toolCallId: "tool", toolName, + parentToolCallId: "parent", toolInput: toolName === "Agent" ? { description: "Initial agent" } : { file_path: "first.txt" } }); + const partial = new NarrativeRecoveryDelta(); + const full = new NarrativeRecoveryDelta(); + const checkpoint = () => { + const snapshot = state.recoverySnapshot(execution.threadId); + const actual = partial.prepare(snapshot); + const expected = full.prepare(snapshot); + expect(changes(actual)).toEqual(changes(expected)); + actual?.acknowledge(); + expected?.acknowledge(); + return actual; + }; + checkpoint(); + const update = (toolCallId = "tool") => { + const actual = partial.prepareToolUpdate(state.toolRecoveryItem(execution.threadId, toolCallId)); + const expected = full.prepare(state.recoverySnapshot(execution.threadId)); + expect(changes(actual)).toEqual(changes(expected)); + actual?.acknowledge(); + expected?.acknowledge(); + return actual; + }; + return { state, partial, checkpoint, update }; +} + +function changes(delta: PreparedNarrativeRecoveryDelta | null) { + return delta && { items: delta.items, discardedItemIds: delta.discardedItemIds }; +} + +function toolRecord() { + const { state } = setup(); + const item = state.toolRecoveryItem(execution.threadId, "tool"); + if (!item) throw new Error("Expected buffered tool"); + return item; +} + +beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-09-25T12:00:00.000Z")); +}); +afterEach(() => { vi.useRealTimers(); }); + +describe("targeted tool recovery", () => { + it("matches full recovery for output, metadata removal, duplicates and later full checkpoints", () => { + const { state, update, checkpoint } = setup(); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "first output", false, + { file_path: "updated.txt" }, { exitCode: 0, outputTruncated: true, outputTotalBytes: 500 }); + const first = update(); + expect(first?.items).toMatchObject([{ kind: "toolCall", record: { + id: "tool", parent_tool_call_id: "parent", input_summary: "updated.txt", output_summary: "first output", + exit_code: 0, output_truncated: 1, output_total_bytes: 500, + } }]); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "second output", true); + expect(update()?.items).toMatchObject([{ record: { status: "failed", exit_code: null, output_total_bytes: null } }]); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "second output", true); + expect(update()).toBeNull(); + expect(checkpoint()).toBeNull(); + state.openOrExtendThought(execution.threadId, "New thought"); + expect(checkpoint()?.items).toHaveLength(1); + state.takeOpenThought(execution.threadId); + expect(checkpoint()?.discardedItemIds).toHaveLength(1); + }); + + it("preserves parent and Agent metadata through a targeted result after a duplicate tool use", () => { + const { state, update, checkpoint } = setup("Agent"); + state.bufferToolCall(execution.threadId, { toolCallId: "tool", toolName: "Agent", + parentToolCallId: "reconciled-parent", toolInput: { description: "Reviewer", agentId: "agent-7", prompt: "Review changes" } }); + checkpoint(); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "Reviewed", false, + { description: "Updated reviewer", agentId: "agent-7", prompt: "Review final changes", durationMs: 123 }); + expect(update()?.items).toMatchObject([{ record: { parent_tool_call_id: "reconciled-parent", + subagent_agent_id: "agent-7", subagent_prompt: "Review final changes", subagent_duration_ms: 123 } }]); + expect(checkpoint()).toBeNull(); + }); + + it("does nothing for an unknown tool and returns an owned projection", () => { + const { state, update, checkpoint } = setup(); + state.updateBufferedToolCallOutput(execution.threadId, "unknown", "ignored", false); + expect(update("unknown")).toBeNull(); + const projected = state.toolRecoveryItem(execution.threadId, "tool"); + if (!projected) throw new Error("Expected buffered tool"); + projected.record.output_summary = "Mutated consumer copy"; + expect(checkpoint()).toBeNull(); + }); + + it("keeps failed updates retryable and fences obsolete acknowledgements", () => { + const { state, partial } = setup(); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "done", false); + const item = state.toolRecoveryItem(execution.threadId, "tool"); + const first = partial.prepareToolUpdate(item); + const retry = partial.prepareToolUpdate(item); + expect(changes(retry)).toEqual(changes(first)); + retry?.acknowledge(); + expect(partial.prepareToolUpdate(item)).toBeNull(); + expect(() => first?.acknowledge()).toThrow("already superseded"); + expect(partial.prepare(state.recoverySnapshot(execution.threadId))).toBeNull(); + expect(() => new NarrativeRecoveryDelta().prepareToolUpdate(item)).toThrow("full checkpoint"); + }); + + it("checks total retained bytes and recovers the accounting after shrinkage and a full checkpoint", () => { + const tool = toolRecord(); + const padding = ACTIVE_TURN_RECOVERY_RETAINED_LIMITS.maxBytes + - Buffer.byteLength(JSON.stringify(tool)) - Buffer.byteLength(JSON.stringify(thought)) - 32; + const retainedThought = { ...thought, record: { ...thought.record, text: "x".repeat(padding) } }; + const delta = new NarrativeRecoveryDelta(); + delta.prepare([retainedThought, tool])?.acknowledge(); + const enlarged = { ...tool, record: { ...tool.record, output_summary: "x".repeat(100) } }; + expect(() => delta.prepareToolUpdate(enlarged)).toThrow("retained byte capacity"); + delta.prepare([thought, tool])?.acknowledge(); + delta.prepareToolUpdate(enlarged)?.acknowledge(); + delta.prepareToolUpdate(tool)?.acknowledge(); + expect(delta.prepare([thought, tool])).toBeNull(); + expect(() => delta.prepareToolUpdate({ ...tool, record: { ...tool.record, + output_summary: "x".repeat(ACTIVE_TURN_RECOVERY_RETAINED_LIMITS.maxBytes) } })).toThrow("active-turn byte limit"); + }); + + it("checks the retained record count even when only one tool changes", () => { + const tool = toolRecord(); + const delta = new NarrativeRecoveryDelta(); + const thoughts = Array.from({ length: ACTIVE_TURN_RECOVERY_RETAINED_LIMITS.maxRecords }, (_, index) => ({ + ...thought, record: { ...thought.record, id: `thought-${index}` }, + })); + delta.prepare([...thoughts, tool])?.acknowledge(); + expect(() => delta.prepareToolUpdate({ ...tool, record: { ...tool.record, output_summary: "done" } })) + .toThrow("retained record capacity"); + }); + + it("counts duplicate snapshot identities even when their full checkpoint has no changes", () => { + const tool = toolRecord(); + const delta = new NarrativeRecoveryDelta(); + const state = new NarrativeTurnState(execution); + state.openHook(execution.threadId, { hookName: "check", toolName: null, phase: "pre", payload: "", sortOrder: 0 }); + const hook = state.recoverySnapshot(execution.threadId).find((item) => item.kind === "hook"); + if (!hook) throw new Error("Expected hook recovery item"); + const repeated = { ...hook, record: { ...hook.record, payload: "x".repeat(130_900) } }; + delta.prepare([repeated, tool])?.acknowledge(); + expect(delta.prepare([repeated, repeated, tool])).toBeNull(); + expect(() => delta.prepareToolUpdate({ ...tool, record: { ...tool.record, output_summary: "done" } })) + .toThrow("retained byte capacity"); + expect(delta.prepare([repeated, tool])).toBeNull(); + expect(delta.prepareToolUpdate({ ...tool, record: { ...tool.record, output_summary: "done" } })).not.toBeNull(); + + const repeatedIds = new NarrativeRecoveryDelta(); + repeatedIds.prepare([thought, tool])?.acknowledge(); + expect(repeatedIds.prepare([...Array.from({ length: ACTIVE_TURN_RECOVERY_RETAINED_LIMITS.maxRecords }, () => thought), tool])).toBeNull(); + expect(() => repeatedIds.prepareToolUpdate(tool)).toThrow("retained record capacity"); + }); +}); diff --git a/apps/server/src/features/agents/turns/__tests__/parent-assistant-text-checkpoint-service.test.ts b/apps/server/src/features/agents/turns/__tests__/parent-assistant-text-checkpoint-service.test.ts index 7d86998bd..2fa03d885 100644 --- a/apps/server/src/features/agents/turns/__tests__/parent-assistant-text-checkpoint-service.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/parent-assistant-text-checkpoint-service.test.ts @@ -5,7 +5,7 @@ import * as NodePath from "node:path"; import type { Database } from "bun:sqlite"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { ParentAssistantTextCheckpointQueue, @@ -30,7 +30,7 @@ function seedParentAssistantTurn(db: Database): void { "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "workspace-1522", "Durability", "main", "claude", "active", NOW, NOW); const messages = new MessageRepo(db); - new CanonicalAgentEventSink(db, () => {}).startParentTurn({ + new CanonicalAgentBoundary(db, () => {}).startParentTurn({ thread: { id: THREAD_ID, workspaceId: "workspace-1522", providerId: "claude", createdAt: NOW }, turnId: TURN_ID, executionId: EXECUTION_ID, diff --git a/apps/server/src/features/agents/turns/__tests__/turn-diff-finalization.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-diff-finalization.test.ts index da3f0c489..9add876b7 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-diff-finalization.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-diff-finalization.test.ts @@ -10,7 +10,7 @@ import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-rep import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; import { SnapshotService } from "../../../projects/diffs/snapshots/snapshot-service.js"; import { RealGitExecutor } from "../../../projects/git/execution/real-git-executor.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { TurnDiffRepo } from "../persistence/turn-diff-repo.js"; import { TurnSnapshotRepo } from "../persistence/turn-snapshot-repo.js"; import { TurnDiffService } from "../turn-diff-service.js"; @@ -34,7 +34,7 @@ describe("native diff terminal persistence", () => { function harness(canonical: boolean) { const messages = new MessageRepo(db); const narrative = new NarrativeStore(messages, new ToolCallRecordRepo(db), new ThoughtSegmentRepo(db), new HookExecutionRepo(db)); - const sink = canonical ? new CanonicalAgentEventSink(db, () => {}) : undefined; + const sink = canonical ? new CanonicalAgentBoundary(db, () => {}) : undefined; if (sink) sink.startParentTurn({ thread: { id: identity.threadId, workspaceId: "ws-1", providerId: "codex", createdAt: new Date().toISOString() }, turnId: identity.turnId, executionId: identity.turnExecutionId, permissionMode: "supervised", providerIdentities: [], diff --git a/apps/server/src/features/agents/turns/__tests__/turn-diff-service.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-diff-service.test.ts index a2876b311..d4ccae5fd 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-diff-service.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-diff-service.test.ts @@ -4,7 +4,7 @@ import type { Database } from "bun:sqlite"; import type { ProviderTurnDiffUpdate, TurnFileEffectSummary } from "@mcode/contracts"; import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { TurnDiffRepo } from "../persistence/turn-diff-repo.js"; -import { TURN_DIFF_MAX_BYTES, TurnDiffService } from "../turn-diff-service.js"; +import { TURN_DIFF_MAX_BYTES, TurnDiffService, selectTurnDiffSettlement } from "../turn-diff-service.js"; const patch = "diff --git a/a.txt b/a.txt\nindex 1..2\n--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-old\n+new\n"; const identity = { threadId: "thread-1", turnId: "turn-1", turnExecutionId: "execution-1", deliveryAttempt: 1 }; @@ -72,6 +72,21 @@ describe("TurnDiffService production settlement", () => { expect(service.push(update({ revision: 99 }))).toBe("stale"); }); + it("transfers cloneable terminal evidence with exact execution fencing", () => { + expect(service.push(update())).toBe("accepted"); + expect(service.takeFinalizationEvidence(identity.threadId, "stale-execution")).toBeNull(); + const prepared = service.takeFinalizationEvidence(identity.threadId, identity.turnExecutionId); + expect(prepared).not.toBeNull(); + if (!prepared) throw new Error("Expected frozen terminal evidence"); + expect(service.liveComparison(identity.threadId)).toBeNull(); + expect(service.takeFinalizationEvidence(identity.threadId, identity.turnExecutionId)).toBeNull(); + expect(selectTurnDiffSettlement(structuredClone(prepared), "completed", effects)).toEqual({ + thread_id: identity.threadId, source: "native", patch, revision: 1, + }); + expect(selectTurnDiffSettlement(prepared, "interrupted", effects)).toBeNull(); + expect(repo.latest(identity.threadId)).toBeUndefined(); + }); + it.each(["invalidated", "interrupted", "cancelled", "errored"] as const)("preserves previous settlement on %s", (state) => { service.push(update()); service.prepareFinalization(identity.threadId, identity.turnExecutionId, "completed")("message-1", effects); diff --git a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts index b49e0718f..0ebc8ea2c 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts @@ -8,9 +8,11 @@ import { type TurnEventApplication, type TurnEventIngressFence, type TurnLifecycleControl, + type WorkerFileObservationPort, } from "../turn-event-pipeline.js"; import type { ProviderEventIngressEvent } from "../../../providers/composition/provider-event-ingress.js"; import { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "../parent-assistant-text-checkpoint-service.js"; +import type { TurnDiffService } from "../turn-diff-service.js"; const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; @@ -57,28 +59,77 @@ function createPipeline( previousFileFinalization: TurnEventApplication["previousFileFinalization"] = () => undefined, rejectForQueueCapacity = vi.fn(), ingressFence?: TurnEventIngressFence, + workerFileObserver?: WorkerFileObservationPort, + turnDiffs?: Pick, ): { pipeline: TurnEventPipeline; finalize: ReturnType; rejectForQueueCapacity: ReturnType; + publishCommitted: ReturnType; + observeFileMutation: ReturnType; } { const lifecycle: TurnLifecycleControl = { normalize: (event) => event, finalize, }; + const publishCommitted = vi.fn(); + const observeFileMutation = vi.fn(); const application: TurnEventApplication = { apply, - observeFileMutation: vi.fn(), + publishCommitted, + observeFileMutation, rejectForQueueCapacity, previousFileFinalization, beginResumedFileTracking: vi.fn(), observeToolUse: vi.fn(), observeToolResult: vi.fn(), }; - return { pipeline: new TurnEventPipeline(lifecycle, application, undefined, ingressFence), finalize, rejectForQueueCapacity }; + return { pipeline: new TurnEventPipeline(lifecycle, application, turnDiffs, ingressFence, workerFileObserver), + finalize, rejectForQueueCapacity, publishCommitted, observeFileMutation }; } describe("TurnEventPipeline", () => { + it("claims worker file callbacks before legacy attribution and keeps native diff routing", () => { + const workerFileObserver: WorkerFileObservationPort = { + ownsFileMutation: vi.fn((event) => event.threadId === "worker-thread"), + capture: vi.fn(() => false), + }; + const push = vi.fn((): "accepted" => "accepted"); + const { pipeline, observeFileMutation } = createPipeline( + () => true, undefined, undefined, undefined, undefined, workerFileObserver, { push }, + ); + const mutation = { + threadId: "worker-thread", turnExecutionId: EXECUTION_ID, deliveryAttempt: 1, + toolCallId: "tool-1", toolName: "Edit", toolInput: { file_path: "tracked.txt" }, + }; + + pipeline.handleProviderFileMutation(mutation); + pipeline.handleProviderFileMutation({ ...mutation, deliveryAttempt: 0 }); + pipeline.handleProviderFileMutation({ ...mutation, threadId: "legacy-thread" }); + pipeline.handleProviderTurnDiff({ + turnId: "turn-1", turnExecutionId: EXECUTION_ID, + deliveryAttempt: 1, revision: 1, state: "invalidated", + }); + + expect(workerFileObserver.capture).toHaveBeenCalledTimes(2); + expect(observeFileMutation).toHaveBeenCalledExactlyOnceWith({ ...mutation, threadId: "legacy-thread" }); + expect(push).toHaveBeenCalledExactlyOnceWith({ + turnId: "turn-1", turnExecutionId: EXECUTION_ID, + deliveryAttempt: 1, revision: 1, state: "invalidated", + }); + }); + + it("publishes projected worker output without legacy event application", () => { + const apply = vi.fn(() => true); + const { pipeline, publishCommitted } = createPipeline(apply); + const input = { ...textDelta("durable"), sourceKind: "worker-commit" as const }; + + pipeline.handleProjectedCommitted(input); + + expect(publishCommitted).toHaveBeenCalledWith(input.event); + expect(apply).not.toHaveBeenCalled(); + }); + it("keeps canonical receipt provenance and FIFO order while an earlier checkpoint delays publication", () => { let checkpointReady = false; const received: Array<{ input: ProviderEventIngressEvent; event: AgentEvent }> = []; @@ -129,6 +180,144 @@ describe("TurnEventPipeline", () => { expect(finalize).toHaveBeenCalledOnce(); }); + it("waits for an asynchronous application without holding another thread", async () => { + let acknowledge!: (accepted: boolean) => void; + const durable = new Promise((resolve) => { acknowledge = resolve; }); + const applied: string[] = []; + const { pipeline, finalize } = createPipeline((_input, event) => { + const delta = (event as Extract).delta; + applied.push(delta); + return delta === "first" ? durable : true; + }); + + pipeline.handleProviderEvent(textDelta("first")); + pipeline.handleProviderEvent(textDelta("second")); + pipeline.handleProviderEvent(textDelta("other", "thread-2")); + const finalization = pipeline.finalizeTurn({ + threadId: "thread-1", + executionId: EXECUTION_ID, + outcome: "completed", + source: "provider", + }); + + expect(applied).toEqual(["first", "other"]); + expect(finalize).not.toHaveBeenCalled(); + acknowledge(true); + await expect(finalization).resolves.toBe(true); + expect(applied).toEqual(["first", "other", "second"]); + }); + + it("waits for an in-flight acknowledgement before finalizing Stop", async () => { + let acknowledge!: (accepted: boolean) => void; + const durable = new Promise((resolve) => { acknowledge = resolve; }); + const apply = vi.fn(() => durable); + const { pipeline, finalize } = createPipeline(apply); + + pipeline.handleProviderEvent(textDelta("in flight")); + pipeline.handleProviderEvent(textDelta("discarded")); + const finalization = pipeline.finalizeTurn({ + threadId: "thread-1", + executionId: EXECUTION_ID, + outcome: "cancelled", + source: "user-stop", + }); + + expect(finalize).not.toHaveBeenCalled(); + acknowledge(true); + await expect(finalization).resolves.toBe(true); + expect(apply).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledOnce(); + }); + + it("keeps an asynchronous blocked event at the queue head until resume", async () => { + let acknowledge!: (accepted: boolean) => void; + const durable = new Promise((resolve) => { acknowledge = resolve; }); + const applied: string[] = []; + let ready = false; + const { pipeline } = createPipeline((_input, event) => { + const delta = (event as Extract).delta; + applied.push(delta); + return delta === "first" && !ready ? durable : true; + }); + + pipeline.handleProviderEvent(textDelta("first")); + pipeline.handleProviderEvent(textDelta("second")); + acknowledge(false); + await Promise.resolve(); + expect(applied).toEqual(["first"]); + + ready = true; + pipeline.resume("thread-1"); + expect(applied).toEqual(["first", "first", "second"]); + }); + + it("does not let a discarded acknowledgement consume the next execution's event", async () => { + const firstExecution = "00000000-0000-4000-8000-000000000010"; + const nextExecution = "00000000-0000-4000-8000-000000000011"; + let acknowledge!: (accepted: boolean) => void; + const durable = new Promise((resolve) => { acknowledge = resolve; }); + const applied: string[] = []; + const { pipeline } = createPipeline((_input, event) => { + applied.push(`${event.turnExecutionId}:${event.type}`); + return event.type === AgentEventType.TextDelta && event.turnExecutionId === firstExecution + ? durable + : true; + }); + + pipeline.handleProviderEvent(turnStarted(firstExecution)); + pipeline.handleProviderEvent({ + ...textDelta("old"), + event: { ...textDelta("old").event, turnExecutionId: firstExecution }, + }); + pipeline.discard("thread-1", firstExecution); + pipeline.handleProviderEvent(turnStarted(nextExecution)); + expect(applied).toEqual([`${firstExecution}:turnStarted`, `${firstExecution}:textDelta`]); + + acknowledge(true); + await vi.waitFor(() => { + expect(applied).toEqual([ + `${firstExecution}:turnStarted`, + `${firstExecution}:textDelta`, + `${nextExecution}:turnStarted`, + ]); + }); + }); + + it("does not finalize after an asynchronous application fails", async () => { + let fail!: (error: Error) => void; + const durable = new Promise((_resolve, reject) => { fail = reject; }); + const apply = vi.fn(() => durable); + const { pipeline, finalize } = createPipeline(apply); + + pipeline.handleProviderEvent(textDelta("write failure")); + const finalization = pipeline.finalizeTurn({ + threadId: "thread-1", + executionId: EXECUTION_ID, + outcome: "completed", + source: "provider", + }); + + fail(new Error("checkpoint failed")); + await expect(finalization).rejects.toThrow("checkpoint failed"); + pipeline.resume("thread-1"); + expect(apply).toHaveBeenCalledOnce(); + expect(finalize).not.toHaveBeenCalled(); + }); + + it("does not turn a failed checkpoint into a successful later Stop", async () => { + const { pipeline, finalize } = createPipeline(() => Promise.reject(new Error("checkpoint failed"))); + + pipeline.handleProviderEvent(textDelta("write failure")); + await Promise.resolve(); + await expect(pipeline.finalizeTurn({ + threadId: "thread-1", + executionId: EXECUTION_ID, + outcome: "cancelled", + source: "user-stop", + })).rejects.toThrow("checkpoint failed"); + expect(finalize).not.toHaveBeenCalled(); + }); + it("holds finalization until its thread-affine ingress worker is idle", async () => { let releaseIngress!: () => void; const ingressIdle = new Promise((resolve) => { releaseIngress = resolve; }); diff --git a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts index 7e608d470..ebaf6c62a 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts @@ -1,10 +1,14 @@ +import "reflect-metadata"; import { afterEach, describe, expect, it } from "vitest"; import * as NodeFSPromises from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeOS from "node:os"; import { AgentEventType, type TurnFileEffectSummary } from "@mcode/contracts"; import { TurnFileTracker } from "../turn-file-tracker.js"; +import { prepareExecutionFileEvidence } from "../turn-execution-file-evidence.js"; import { parseTurnDiff } from "../turn-diff-patch.js"; +import { SnapshotService } from "../../../projects/diffs/snapshots/snapshot-service.js"; +import { RealGitExecutor } from "../../../projects/git/execution/real-git-executor.js"; import { createCursorAcpTurnState, mapCursorAcpSessionNotification, @@ -90,16 +94,13 @@ describe("TurnFileTracker", () => { TEST_PLATFORM, ); tracker.beginTurn("t", root, "unavailable-ref"); - const pendingStarts = [tracker.observeToolUse( - "t", - "file-child", - "file_change", - { changes: [{ path: "tracked.txt", kind: "edit" }] }, - )]; + const event = { threadId: "t", toolCallId: "file-child", toolName: "file_change", toolInput: { changes: [{ path: "tracked.txt", kind: "edit" }] } }; + const captured = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected a captured baseline"); + expect(structuredClone(captured)).toEqual(captured); await NodeFSPromises.writeFile(trackedPath, "after\nextra\n"); - expect(pendingStarts).toHaveLength(1); + expect(await tracker.observeCapturedToolUse(event, structuredClone(captured))).toBe(true); await tracker.observeToolResult("t", "file-child"); - await Promise.all(pendingStarts); const summary = await tracker.finalizeTurn("t"); expect(summary).toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); @@ -111,6 +112,165 @@ describe("TurnFileTracker", () => { expect(updates.at(-1)).toEqual(summary); }); + it("transfers one execution's pre-edit capture to a separate tracker after a real edit", async () => { + const root = await tempDir("mcode-transferred-file-observation-"); + const path = NodePath.join(root, "tracked.txt"); + await NodeFSPromises.writeFile(path, "before\n"); + const baseline = async () => ({ kind: "unavailable" as const }); + const host = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const worker = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const expected = { threadId: "t", executionId: "execution-1", cwd: root }; + const handoff = host.beginExecutionTurn({ ...expected, baselineRef: null }); + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; + const captured = host.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected a pre-edit capture"); + + await NodeFSPromises.writeFile(path, "after\nextra\n"); + expect(worker.beginTurnFromHandoff(structuredClone(handoff), expected)).toBe(true); + expect(worker.beginTurnFromHandoff({ ...handoff, baselineRef: "other-ref" }, expected)).toBe(false); + await worker.setBaselineRef("t", handoff.generation, "later-ref"); + expect(worker.beginTurnFromHandoff(structuredClone(handoff), expected)).toBe(true); + expect(await worker.observeCapturedToolUse(event, { ...captured, executionId: "other-execution" })).toBe(false); + expect(await worker.observeCapturedToolUse(event, structuredClone(captured))).toBe(true); + await worker.observeToolResult("t", "edit"); + expect(await worker.finalizeTurn("t")).toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); + }); + + it("seals only the exact execution generation before a resumed turn reuses a tool ID", async () => { + const root = await tempDir("mcode-sealed-file-generation-"); + const path = NodePath.join(root, "tracked.txt"); + await NodeFSPromises.writeFile(path, "before\n"); + const tracker = trackerWithBaseline({}, []); + const first = tracker.beginExecutionTurn({ + threadId: "t", executionId: "execution-1", cwd: root, baselineRef: null, + }); + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; + const capture = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!capture) throw new Error("Expected first execution capture"); + await NodeFSPromises.writeFile(path, "after\n"); + expect(await tracker.observeCapturedToolUse(event, capture)).toBe(true); + await tracker.observeToolResult("t", "edit"); + const frozen = await tracker.finalEvidenceForExecution(first); + expect(frozen?.summary).toMatchObject({ fileCount: 1 }); + expect(await tracker.finalEvidenceForExecution(first)).toEqual(frozen); + expect(tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput)).toBeNull(); + expect(await tracker.observeCapturedToolUse(event, capture)).toBe(false); + + const second = tracker.beginExecutionTurn({ + threadId: "t", executionId: "execution-2", cwd: root, baselineRef: null, + }); + expect(await tracker.finalEvidenceForExecution({ ...first, generationToken: second.generationToken })).toBeNull(); + expect((await tracker.finalEvidenceForExecution(second))?.summary.fileCount).toBe(0); + tracker.clearTurn("t", first.generation); + expect(await tracker.finalEvidenceForExecution(first)).toBeNull(); + }); + + it("rejects native diff evidence from another delivery attempt before terminal preparation", async () => { + const root = await tempDir("mcode-file-diff-attempt-"); + const tracker = trackerWithBaseline({}, []); + const handoff = tracker.beginExecutionTurn({ + threadId: "t", executionId: "execution-1", cwd: root, baselineRef: null, + }); + const input = { + handoff, turnId: "turn-1", deliveryAttempt: 2, outcome: "completed" as const, + tracker, snapshots: new SnapshotService(new RealGitExecutor()), + nativeDiff: { + threadId: "t", turnId: "turn-1", turnExecutionId: "execution-1", deliveryAttempt: 1, + revision: 1, evidence: null, rejected: false, + }, + }; + await expect(prepareExecutionFileEvidence(input)).rejects.toThrow("Native diff does not belong"); + const prepared = await prepareExecutionFileEvidence({ ...input, nativeDiff: null }); + expect(prepared).toMatchObject({ + threadId: "t", executionId: "execution-1", filesChanged: [], snapshot: null, selectedTurnDiff: null, + }); + expect(() => structuredClone(prepared)).not.toThrow(); + }); + + it("rejects a handoff or capture for another execution or root", async () => { + const root = await tempDir("mcode-transferred-file-stale-"); + const otherRoot = await tempDir("mcode-transferred-file-other-root-"); + const baseline = async () => ({ kind: "unavailable" as const }); + const oldHost = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const currentHost = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const worker = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; + oldHost.beginExecutionTurn({ threadId: "t", executionId: "old-execution", cwd: root, baselineRef: null }); + const oldCapture = oldHost.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!oldCapture) throw new Error("Expected an old capture"); + const expected = { threadId: "t", executionId: "current-execution", cwd: root }; + const handoff = currentHost.beginExecutionTurn({ ...expected, baselineRef: null }); + + expect(worker.beginTurnFromHandoff(handoff, { ...expected, executionId: "old-execution" })).toBe(false); + expect(worker.beginTurnFromHandoff(handoff, { ...expected, cwd: otherRoot })).toBe(false); + expect(worker.beginTurnFromHandoff({ ...handoff, unrelatedThread: "secret" }, expected)).toBe(false); + expect(worker.beginTurnFromHandoff(handoff, expected)).toBe(true); + expect(await worker.observeCapturedToolUse(event, oldCapture)).toBe(false); + expect(await worker.observeCapturedToolUse(event, { ...oldCapture, executionId: expected.executionId })).toBe(false); + expect(await worker.finalizeTurn("t")).toMatchObject({ fileCount: 0 }); + }); + + it("rejects a captured baseline from another tool, root, or turn generation", async () => { + const root = await tempDir("mcode-stale-observation-"); + const path = NodePath.join(root, "tracked.txt"); + await NodeFSPromises.writeFile(path, "before\n"); + const tracker = new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, TEST_PLATFORM); + const event = { threadId: "t", toolCallId: "old-tool", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; + const firstGeneration = tracker.beginTurn("t", root, null); + const captured = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected a captured baseline"); + expect(await tracker.observeCapturedToolUse({ ...event, toolCallId: "other-tool" }, captured)).toBe(false); + expect(await tracker.observeCapturedToolUse({ ...event, toolName: "Write" }, captured)).toBe(false); + expect(await tracker.observeCapturedToolUse(event, { ...captured, canonicalRoot: "other-root" })).toBe(false); + expect(await tracker.observeCapturedToolUse({ ...event, toolInput: { file_path: "other.txt" } }, captured)).toBe(false); + const replacement = new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, TEST_PLATFORM); + expect(replacement.beginTurn("t", root, null)).toBe(firstGeneration); + expect(await replacement.observeCapturedToolUse(event, captured)).toBe(false); + + const nextGeneration = tracker.beginTurn("t", root, null); + await NodeFSPromises.writeFile(path, "after\n"); + expect(await tracker.observeCapturedToolUse(event, structuredClone(captured))).toBe(false); + await tracker.observeToolResult("t", event.toolCallId); + expect(await tracker.finalizeTurn("t", firstGeneration)).toMatchObject({ fileCount: 0 }); + expect(await tracker.finalizeTurn("t", nextGeneration)).toMatchObject({ fileCount: 0 }); + }); + + it("binds captured observations to the exact before and after evidence text", async () => { + const root = await tempDir("mcode-observation-evidence-"); + const tracker = trackerWithBaseline({}, []); + tracker.beginTurn("t", root, null); + const mutation = { path: "file.txt", kind: "edit", fullFileContent: true, beforeText: "before\n", afterText: "after\n" }; + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { _mcodeFileMutations: [mutation] } }; + const captured = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected captured text evidence"); + + expect(await tracker.observeCapturedToolUse({ + ...event, toolInput: { _mcodeFileMutations: [{ ...mutation, beforeText: "befoXe\n" }] }, + }, captured)).toBe(false); + expect(await tracker.observeCapturedToolUse({ + ...event, toolInput: { _mcodeFileMutations: [{ ...mutation, afterText: "aftEr\n" }] }, + }, captured)).toBe(false); + }); + + it("does not accept an observation if input extraction clears its turn", async () => { + const root = await tempDir("mcode-observation-cleared-turn-"); + const tracker = trackerWithBaseline({}, []); + tracker.beginTurn("t", root, null); + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "file.txt" } }; + const captured = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected captured tool observation"); + const clearingInput: Record = {}; + Object.defineProperty(clearingInput, "file_path", { + get: () => { + tracker.clearTurn("t"); + return "file.txt"; + }, + }); + + expect(await tracker.observeCapturedToolUse({ ...event, toolInput: clearingInput }, captured)).toBe(false); + expect(tracker.getCurrentTurnId("t")).toBeUndefined(); + }); + it("classifies added, edited, and removed files with net line totals", async () => { const root = await tempDir("mcode-file-effects-"); await NodeFSPromises.writeFile(NodePath.join(root, "edited.txt"), "one\ntwo"); @@ -386,6 +546,23 @@ describe("TurnFileTracker", () => { expect(synchronousDurationMs).toBeLessThan(250); }); + it("keeps captured pre-edit observations within the path and byte budgets", async () => { + const root = await tempDir("mcode-captured-observation-budget-"); + const tracker = trackerWithBaseline({}, []); + tracker.beginTurn("t", root, null); + const changes = Array.from({ length: 5 }, (_, index) => ({ path: `file-${index}.txt`, kind: "edit" })); + for (const change of changes) { + await NodeFSPromises.writeFile(NodePath.join(root, change.path), "x".repeat(350_000)); + } + const captured = tracker.captureToolUseObservation("t", "bulk", "file_change", { changes }); + if (!captured) throw new Error("Expected bounded observations"); + expect(captured.observations).toHaveLength(5); + expect(captured.observations.filter(Boolean)).toHaveLength(4); + const capturedBytes = captured.observations.reduce((total, observation) => + total + Buffer.byteLength(observation?.baseline?.text ?? "", "utf8"), 0); + expect(capturedBytes).toBeLessThanOrEqual(1_048_576); + }); + it("falls back to complete async observation when a rename exceeds the remaining path budget", async () => { const root = await tempDir("mcode-file-effects-mixed-budget-"); for (let index = 0; index < 3; index += 1) { diff --git a/apps/server/src/features/agents/turns/__tests__/turn-finalizer.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-finalizer.test.ts index 1e96bdbd9..fc0bf1f33 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-finalizer.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-finalizer.test.ts @@ -16,7 +16,7 @@ import { TurnSnapshotRepo } from "../persistence/turn-snapshot-repo.js"; import type { TurnOutcome } from "../turn-outcome.js"; import type { TurnFileTracker } from "../turn-file-tracker.js"; import { broadcast } from "../../../../application/transport/push.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { ParentAssistantTextCheckpointService } from "../parent-assistant-text-checkpoint-service.js"; vi.mock("../../../../application/transport/push.js", () => ({ broadcast: vi.fn() })); @@ -355,7 +355,7 @@ describe("TurnFinalizer canonical commit recovery", () => { thoughtRepo, hookRepo, ); - const sink = new CanonicalAgentEventSink(db, vi.fn()); + const sink = new CanonicalAgentBoundary(db, vi.fn()); sink.startParentTurn({ thread: { id: THREAD, diff --git a/apps/server/src/features/agents/turns/assistant-execution-state.ts b/apps/server/src/features/agents/turns/assistant-execution-state.ts new file mode 100644 index 000000000..716460bcd --- /dev/null +++ b/apps/server/src/features/agents/turns/assistant-execution-state.ts @@ -0,0 +1,95 @@ +import type { StoredAttachment } from "@mcode/contracts"; + +/** Data needed to decide which assistant body a completed execution persists. */ +export interface AssistantMaterializationInput { + content: string; + model: string | null; + attachments: StoredAttachment[]; + fromProvider: boolean; +} + +interface BufferedBody { + content: string; + model: string | null; + attachments: StoredAttachment[]; +} + +/** Volatile assistant state for one provider execution, with no persistence or transport dependencies. */ +export class AssistantExecutionState { + private streamingText = ""; + private bufferedBody: BufferedBody | undefined; + private bufferedAttachments: StoredAttachment[] = []; + private materialized = false; + + appendStreamingText(delta: string): void { + this.streamingText += delta; + } + + getStreamingText(): string { + return this.streamingText; + } + + resetStreamingText(): void { + this.streamingText = ""; + } + + bufferBody(content: string, model: string | null, attachments = this.bufferedAttachments): void { + this.bufferedBody = { content, model, attachments }; + } + + bufferAttachments(attachments: StoredAttachment[]): void { + if (attachments.length === 0) return; + const byId = new Map(this.bufferedAttachments.map((attachment) => [attachment.id, attachment])); + for (const attachment of attachments) byId.set(attachment.id, attachment); + this.bufferedAttachments = [...byId.values()]; + } + + getBufferedAttachments(): StoredAttachment[] { + return this.bufferedAttachments; + } + + hasBufferedBody(): boolean { + return Boolean(this.bufferedBody?.content.trim() || this.streamingText.trim()); + } + + hasProviderBody(): boolean { + return this.bufferedBody !== undefined; + } + + hasMaterialized(): boolean { + return this.materialized; + } + + materializationInput(fallbackModel: string | null): AssistantMaterializationInput { + const buffered = this.bufferedBody; + if (buffered) { + return { + content: buffered.content, + model: buffered.model, + attachments: mergeAttachments(buffered.attachments, this.bufferedAttachments), + fromProvider: true, + }; + } + return { + content: this.streamingText.trim(), + model: fallbackModel, + attachments: this.bufferedAttachments, + fromProvider: false, + }; + } + + commitMaterialization(): void { + this.streamingText = ""; + this.bufferedBody = undefined; + this.bufferedAttachments = []; + this.materialized = true; + } +} + +function mergeAttachments(first: StoredAttachment[], second: StoredAttachment[]): StoredAttachment[] { + if (first.length === 0) return second; + if (second.length === 0) return first; + const byId = new Map(first.map((attachment) => [attachment.id, attachment])); + for (const attachment of second) byId.set(attachment.id, attachment); + return [...byId.values()]; +} diff --git a/apps/server/src/features/agents/turns/codex-parent-message-projection.ts b/apps/server/src/features/agents/turns/codex-parent-message-projection.ts new file mode 100644 index 000000000..31272bb24 --- /dev/null +++ b/apps/server/src/features/agents/turns/codex-parent-message-projection.ts @@ -0,0 +1,117 @@ +import type { ParentNarrativeRecoveryItem, StoredAttachment, TurnOutcome } from "@mcode/contracts"; + +import type { DataOnlyParentTerminalProjectionInput } from "../canonical/canonical-parent-turn-write.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; +import { AssistantExecutionState } from "./assistant-execution-state.js"; +import { deriveTurnAssistantMessageId } from "./turn-assistant-message-id.js"; + +/** The mailbox identifies one turn; the current message anchor arrives at projection time. */ +export interface CodexParentMessageProjectionInput { + readonly execution: ExecutionIdentity; + readonly turnKind: "ordinary" | "plan"; +} + +/** Public fields attached to a provider message before it reaches the renderer. */ +export interface ProjectedCodexParentMessage { + readonly messageId: string; + readonly model: string | null; + readonly attachments?: readonly StoredAttachment[]; +} + +/** Cloneable terminal data, including the public ID the writer must persist. */ +export interface ProjectedCodexParentTerminal extends DataOnlyParentTerminalProjectionInput { + readonly assistant: DataOnlyParentTerminalProjectionInput["assistant"] & { readonly messageId: string }; + readonly fromProvider: boolean; +} + +/** Keeps the assistant projection for one Codex parent execution off the server loop. */ +export class CodexParentMessageProjection { + private readonly execution: ExecutionIdentity; + private messageId: string | undefined; + private readonly assistant = new AssistantExecutionState(); + + constructor(input: CodexParentMessageProjectionInput) { + if (input.turnKind === "plan") { + throw new Error("Codex plan output needs early assistant materialization and is not supported by this projection"); + } + this.execution = structuredClone(input.execution); + } + + /** Record final response text that can become an assistant body if no provider message arrives. */ + appendFinalResponseText(execution: ExecutionIdentity, delta: string): void { + this.assertExecution(execution); + this.assistant.appendStreamingText(delta); + } + + /** Buffer a generated attachment with the same ID replacement order as the live finalizer. */ + bufferGeneratedAttachment(execution: ExecutionIdentity, attachment: StoredAttachment): void { + this.assertExecution(execution); + this.assistant.bufferAttachments([structuredClone(attachment)]); + } + + /** Buffer a normal provider body and return exactly the public fields of the live message path. */ + projectMessage(input: { + readonly execution: ExecutionIdentity; + readonly content: string; + readonly model: string | null; + readonly precedingMessageId: string; + readonly postTurnGoalReceipt: boolean; + }): ProjectedCodexParentMessage { + this.assertExecution(input.execution); + if (input.postTurnGoalReceipt) { + throw new Error("Post-turn goal receipts need a separate persisted message and are not supported by this projection"); + } + const messageId = this.deriveMessageId(input.precedingMessageId); + const attachments = structuredClone(this.assistant.getBufferedAttachments()); + this.assistant.bufferBody(input.content, input.model, attachments); + this.assistant.resetStreamingText(); + this.messageId = messageId; + return { + messageId, + model: input.model, + ...(attachments.length > 0 ? { attachments } : {}), + }; + } + + /** Produce terminal data for a future single-writer mutation without reading or writing SQLite. */ + projectTerminal(input: { + readonly execution: ExecutionIdentity; + readonly outcome: TurnOutcome; + readonly endedAt: string; + readonly fallbackModel: string | null; + readonly precedingMessageId?: string; + readonly narrative: readonly ParentNarrativeRecoveryItem[]; + }): ProjectedCodexParentTerminal { + this.assertExecution(input.execution); + // A notice can change the last visible row after publication. Keep the + // published ID; only a text-only fallback derives its ID at terminal time. + const messageId = this.messageId ?? this.deriveMessageId(input.precedingMessageId); + const assistant = this.assistant.materializationInput(input.fallbackModel); + return { + threadId: this.execution.threadId, + executionId: this.execution.executionId, + outcome: input.outcome, + endedAt: input.endedAt, + fromProvider: assistant.fromProvider, + assistant: { + messageId, + content: assistant.content, + model: assistant.model, + attachments: structuredClone(assistant.attachments), + }, + narrative: structuredClone(input.narrative), + }; + } + + private assertExecution(execution: ExecutionIdentity): void { + if (execution.threadId !== this.execution.threadId || execution.turnId !== this.execution.turnId + || execution.executionId !== this.execution.executionId) { + throw new Error(`Stale Codex parent message projection command: ${execution.executionId}`); + } + } + + private deriveMessageId(precedingMessageId: string | undefined): string { + if (!precedingMessageId) throw new Error("Codex parent projection needs the current preceding message ID"); + return deriveTurnAssistantMessageId(this.execution.threadId, precedingMessageId); + } +} diff --git a/apps/server/src/features/agents/turns/narrative-recovery-delta.ts b/apps/server/src/features/agents/turns/narrative-recovery-delta.ts new file mode 100644 index 000000000..0dd12f3bf --- /dev/null +++ b/apps/server/src/features/agents/turns/narrative-recovery-delta.ts @@ -0,0 +1,119 @@ +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; +import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; +import type { ParentNarrativeRecoveryCommit } from "./parent-turn-durability.js"; +import { assertActiveTurnRecoveryRetention } from "./active-turn-recovery-retention-policy.js"; + +/** One full-snapshot difference awaiting confirmation of its durable write. */ +export interface PreparedNarrativeRecoveryDelta { + readonly items: readonly ParentNarrativeRecoveryItem[]; + readonly discardedItemIds: readonly string[]; + acknowledge(): void; +} + +/** Tracks one execution's committed narrative snapshot without owning persistence. */ +export class NarrativeRecoveryDelta { + private fingerprints = new Map(); + private retainedBytes = 0; + private retainedRecords = 0; + private revision = 0; + + /** Compare a complete snapshot to the last acknowledged one. */ + prepare(snapshot: readonly ParentNarrativeRecoveryItem[]): PreparedNarrativeRecoveryDelta | null { + const next = new Map(); + const items: ParentNarrativeRecoveryItem[] = []; + let retainedBytes = 0; + for (const item of snapshot) { + const id = `${item.kind}:${item.record.id}`; + const fingerprint = JSON.stringify(item); + retainedBytes += Buffer.byteLength(fingerprint, "utf8"); + next.set(id, fingerprint); + if (this.fingerprints.get(id) !== fingerprint) items.push(item); + } + const discardedItemIds = [...this.fingerprints.keys()] + .filter((id) => !next.has(id)).map(canonicalItemId); + if (items.length === 0 && discardedItemIds.length === 0) { + // Recovery limits count snapshot entries, including repeated hook/thought identities. + this.retainedBytes = retainedBytes; + this.retainedRecords = snapshot.length; + return null; + } + const preparedAt = this.revision; + return { + items, + discardedItemIds, + acknowledge: () => { + if (this.revision !== preparedAt) throw new Error("Narrative recovery delta was already superseded"); + this.fingerprints = next; + this.retainedBytes = retainedBytes; + this.retainedRecords = snapshot.length; + this.revision += 1; + }, + }; + } + + /** Update one already checkpointed tool without treating unrelated records as deleted. */ + prepareToolUpdate(item: Extract | null): PreparedNarrativeRecoveryDelta | null { + if (!item) return null; + const id = `${item.kind}:${item.record.id}`; + const previous = this.fingerprints.get(id); + if (previous === undefined) throw new Error("Tool recovery update requires an acknowledged full checkpoint"); + const fingerprint = JSON.stringify(item); + const bytes = Buffer.byteLength(fingerprint, "utf8"); + if (bytes > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes) throw new Error("Parent narrative recovery item exceeds the active-turn byte limit"); + const retainedBytes = this.retainedBytes + bytes - Buffer.byteLength(previous, "utf8"); + assertActiveTurnRecoveryRetention(this.retainedRecords, retainedBytes); + if (previous === fingerprint) return null; + const preparedAt = this.revision; + return { + items: [item], discardedItemIds: [], + acknowledge: () => { + if (this.revision !== preparedAt) throw new Error("Narrative recovery delta was already superseded"); + this.fingerprints.set(id, fingerprint); + this.retainedBytes = retainedBytes; + this.revision += 1; + }, + }; + } +} + +/** Split one prepared delta into writer-sized commands; acknowledge only after every command commits. */ +export function splitNarrativeRecoveryDelta( + executionId: string, + prepared: PreparedNarrativeRecoveryDelta, +): ParentNarrativeRecoveryCommit[] { + if (!executionId) throw new Error("Narrative recovery requires an execution ID"); + const chunks: ParentNarrativeRecoveryCommit[] = []; + let items: ParentNarrativeRecoveryItem[] = []; + let discardedItemIds: string[] = []; + const flush = (): void => { + if (items.length === 0 && discardedItemIds.length === 0) return; + chunks.push({ executionId, items, discardedItemIds }); + items = []; + discardedItemIds = []; + }; + const fits = (nextItems: ParentNarrativeRecoveryItem[], nextDiscarded: string[]): boolean => { + if (nextItems.length + nextDiscarded.length > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + return Buffer.byteLength(JSON.stringify({ executionId, items: nextItems, discardedItemIds: nextDiscarded }), "utf8") + <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; + }; + for (const item of prepared.items) { + if (!fits([...items, item], discardedItemIds)) flush(); + if (!fits([item], [])) throw new Error("Narrative recovery item exceeds one writer command"); + items.push(item); + } + for (const itemId of prepared.discardedItemIds) { + if (!fits(items, [...discardedItemIds, itemId])) flush(); + if (!fits([], [itemId])) throw new Error("Narrative recovery discard exceeds one writer command"); + discardedItemIds.push(itemId); + } + flush(); + return chunks; +} + +function canonicalItemId(key: string): string { + const separator = key.indexOf(":"); + if (separator <= 0 || separator === key.length - 1) { + throw new Error(`Invalid narrative recovery identity: ${key}`); + } + return key; +} diff --git a/apps/server/src/features/agents/turns/parent-assistant-text-checkpoint-service.ts b/apps/server/src/features/agents/turns/parent-assistant-text-checkpoint-service.ts index f42594f9e..5fb229922 100644 --- a/apps/server/src/features/agents/turns/parent-assistant-text-checkpoint-service.ts +++ b/apps/server/src/features/agents/turns/parent-assistant-text-checkpoint-service.ts @@ -13,6 +13,7 @@ import { import { inject, injectable } from "tsyringe"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "./active-turn-recovery-retention-policy.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; export { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "./active-turn-recovery-retention-policy.js"; @@ -1045,7 +1046,10 @@ export class ParentAssistantTextCheckpointQueue { return false; } try { - const result = this.checkpoints.appendChunk(chunk.entries.map((entry) => entry.input)); + const append = () => this.checkpoints.appendChunk(chunk.entries.map((entry) => entry.input)); + const result = serverWorkTrace + ? serverWorkTrace.measure("assistant-text-write", state.threadId, executionId, append) + : append(); if (result.outcome === "overflow") { this.rejectChunk(executionId, state, chunk, "Parent assistant text recovery capacity reached"); return false; diff --git a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts index a8df4d111..b297e1bd0 100644 --- a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts +++ b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts @@ -5,6 +5,8 @@ import { } from "@mcode/contracts"; import type { ParentTurnDurability } from "./parent-turn-durability.js"; import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; +import { NarrativeRecoveryDelta } from "./narrative-recovery-delta.js"; interface ParentNarrativeRecoveryCheckpoint { persist(): void; @@ -13,7 +15,7 @@ interface ParentNarrativeRecoveryCheckpoint { /** Commits changed structured narrative records before AgentService publishes their source event. */ export class ParentNarrativeRecoveryCoordinator { - private readonly fingerprintsByExecution = new Map>(); + private readonly deltasByExecution = new Map(); constructor( private readonly canonicalSink: ParentTurnDurability, @@ -22,10 +24,20 @@ export class ParentNarrativeRecoveryCoordinator { /** Commit only the semantic records changed by this accepted provider event. */ checkpoint(event: AgentEvent): void { - const checkpoint = this.prepareCheckpoint(event); + if (!serverWorkTrace) { + const checkpoint = this.prepareCheckpoint(event); + if (!checkpoint) return; + checkpoint.persist(); + checkpoint.confirm(); + return; + } + const checkpoint = serverWorkTrace.measure("narrative-prepare", event.threadId, + event.turnExecutionId, () => this.prepareCheckpoint(event)); if (!checkpoint) return; - checkpoint.persist(); - checkpoint.confirm(); + serverWorkTrace.measure("narrative-persist", event.threadId, + event.turnExecutionId, () => checkpoint.persist()); + serverWorkTrace.measure("narrative-confirm", event.threadId, + event.turnExecutionId, () => checkpoint.confirm()); } /** Prepare a recovery commit whose dedupe state advances only after its transaction commits. */ @@ -33,42 +45,32 @@ export class ParentNarrativeRecoveryCoordinator { event: AgentEvent, snapshot?: readonly ParentNarrativeRecoveryItem[], ): ParentNarrativeRecoveryCheckpoint | null { - if (!this.requiresStructuredRecovery(event) || !event.turnExecutionId) return null; - if (!this.canonicalSink.loadTurnByExecution(event.turnExecutionId)) return null; + const executionId = event.turnExecutionId; + if (!this.requiresStructuredRecovery(event) || !executionId) return null; + if (!this.canonicalSink.loadTurnByExecution(executionId)) return null; const snapshots = snapshot ?? this.narrativeStore.recoverySnapshot(event.threadId); - - const fingerprints = this.fingerprintsByExecution.get(event.turnExecutionId) ?? new Map(); - const nextFingerprints = new Map(); - const currentKeys = new Set(snapshots.map((item) => `${item.kind}:${item.record.id}`)); - const changed = snapshots.filter((item) => { - const key = `${item.kind}:${item.record.id}`; - const fingerprint = JSON.stringify(item); - nextFingerprints.set(key, fingerprint); - if (fingerprints.get(key) === fingerprint) return false; - return true; - }); - const discardedItemIds = [...fingerprints.keys()] - .filter((key) => !currentKeys.has(key)) - .map((key) => this.canonicalItemId(key)); - if (changed.length === 0 && discardedItemIds.length === 0) return null; + const delta = this.deltasByExecution.get(executionId) ?? new NarrativeRecoveryDelta(); + const prepared = delta.prepare(snapshots); + if (!prepared) return null; + this.deltasByExecution.set(executionId, delta); return { persist: () => { const committed = this.canonicalSink.recordParentNarrativeRecovery({ - executionId: event.turnExecutionId!, - items: changed, - discardedItemIds, + executionId, + items: prepared.items, + discardedItemIds: prepared.discardedItemIds, }); if (!committed) { - throw new Error(`Canonical parent turn was not found: ${event.turnExecutionId}`); + throw new Error(`Canonical parent turn was not found: ${executionId}`); } }, - confirm: () => this.fingerprintsByExecution.set(event.turnExecutionId!, nextFingerprints), + confirm: () => prepared.acknowledge(), }; } /** Forget volatile dedupe state after a terminal turn releases its buffers. */ clear(executionId: string | undefined): void { - if (executionId) this.fingerprintsByExecution.delete(executionId); + if (executionId) this.deltasByExecution.delete(executionId); } private requiresStructuredRecovery(event: AgentEvent): boolean { @@ -79,13 +81,4 @@ export class ParentNarrativeRecoveryCoordinator { || event.type === AgentEventType.HookStarted || event.type === AgentEventType.HookCompleted; } - - private canonicalItemId(key: string): string { - const separator = key.indexOf(":"); - const kind = separator < 0 ? "" : key.slice(0, separator); - const id = separator < 0 ? "" : key.slice(separator + 1); - if (!kind || !id) throw new Error(`Invalid narrative recovery identity: ${key}`); - return kind === "toolCall" ? `toolCall:${id}` : `${kind}:${id}`; - } - } diff --git a/apps/server/src/features/agents/turns/provider-turn-event-application.ts b/apps/server/src/features/agents/turns/provider-turn-event-application.ts index 31c2be3a8..ce74dbe0c 100644 --- a/apps/server/src/features/agents/turns/provider-turn-event-application.ts +++ b/apps/server/src/features/agents/turns/provider-turn-event-application.ts @@ -7,6 +7,7 @@ import { type ProviderId, } from "@mcode/contracts"; import { logger } from "@mcode/shared"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; import { broadcast } from "../../../application/transport/push.js"; import { BrowserNarrativeEventSanitizer } from "../../browser-automation/index.js"; @@ -111,6 +112,11 @@ export class ProviderTurnEventApplication implements TurnEventApplication { return this.applyPreparedEvent(input, event, publish); } + /** Publish an execution-writer event without repeating any turn persistence. */ + publishCommitted(event: AgentEvent): void { + this.publish(event); + } + /** Record a provider file mutation before its public event is available. */ observeFileMutation(event: import("@mcode/contracts").ProviderFileMutationStart): void { this.fileEffects.observeProviderMutation(event); @@ -588,7 +594,10 @@ export class ProviderTurnEventApplication implements TurnEventApplication { private checkpointNarrative(event: AgentEvent, publish: boolean): boolean { if (!publish || this.isUnsavedNarrationBoundary(event)) return true; try { - this.parentNarrativeRecovery.checkpoint(event); + if (serverWorkTrace) { + serverWorkTrace.measure("narrative-checkpoint", event.threadId, event.turnExecutionId, + () => this.parentNarrativeRecovery.checkpoint(event)); + } else this.parentNarrativeRecovery.checkpoint(event); return true; } catch { this.runtime.stopForEventApplicationFailure(event, "Parent narrative recovery checkpoint failed"); diff --git a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts index 68309a8f2..d695c8aec 100644 --- a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts +++ b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts @@ -36,6 +36,11 @@ import { } from "../../providers/availability/provider-availability-errors.js"; import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import { + projectParentUserMessage, + type DataOnlyParentTurnStartInput, + type ParentUserMessageWrite, +} from "../canonical/canonical-parent-turn-write.js"; import { GoalLifecycleService, type GoalCommandEffectReceipt, @@ -46,6 +51,7 @@ import { } from "./parent-turn-durability.js"; import { appendSelectedTextComments } from "./selected-text-comment-append.js"; import { ApprovalReviewPolicy, type ApprovalReviewDecision } from "./approval-review-policy.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; const FILE_INJECTION_SEPARATOR = "\n\n---\n"; @@ -120,6 +126,18 @@ export interface TurnRuntimeAdmissionAuthority { owns(lease: TurnRuntimeLease): boolean; } +/** Commits a new parent turn through its exact execution mailbox. */ +export interface TurnParentStartOwner { + start(input: { + readonly execution: ExecutionIdentity; + readonly ownerEpoch: number; + readonly providerId: string; + readonly parentTurn: DataOnlyParentTurnStartInput; + readonly parentLive?: import("../execution/provider-execution-event-state.js").ExecutionParentStartContext; + readonly publishParentStart?: boolean; + }): Promise; +} + /** The thread-control action selected during generic dispatch preparation. */ export type ThreadControlLeaseDirective = | { @@ -136,6 +154,7 @@ export type ThreadControlLeaseDirective = export interface PreparedTurnDispatch { readonly kind: "dispatch"; readonly lease: TurnRuntimeLease; + readonly parentStartInput: DataOnlyParentTurnStartInput; readonly provider: IAgentProvider; readonly providerId: ProviderId; readonly request: TurnRequest; @@ -144,6 +163,7 @@ export interface PreparedTurnDispatch { readonly threadControl: ThreadControlLeaseDirective | null; readonly contextSeed: number; readonly contextWindow: number | null; + readonly workerOwned?: true; } /** The stable outcome of handling a complete send command. */ @@ -188,6 +208,7 @@ export class TurnAdmissionDispatchCoordinator { private readonly environment: WorkspaceEnvironmentService | undefined, private readonly files: FileService | undefined, private readonly platform: NodeJS.Platform, + private readonly parentStartOwner?: TurnParentStartOwner, ) {} /** Admit one command and return an immutable package for the runtime owner. */ @@ -416,13 +437,15 @@ export class TurnAdmissionDispatchCoordinator { runtime.activate(lease); const attachmentData = await this.persistAttachments(prepared); const sourceTurnId = prepared.command.sourceTurnId ?? NodeCrypto.randomUUID(); - this.startParentTurn(prepared, lease, sourceTurnId, attachmentData, review); + const parentStartInput = this.prepareParentTurnStartInput(prepared, lease, sourceTurnId, attachmentData, review); + await this.commitParentStart(prepared, lease, sourceTurnId, parentStartInput); this.publishCommittedEffects(prepared, sourceTurnId); const wirePayload = this.buildWirePayload(prepared); const request = await this.buildTurnRequest(prepared, lease, sourceTurnId, attachmentData, cwd, wirePayload, review); return { kind: "dispatch", lease, + parentStartInput, provider: prepared.provider, providerId: prepared.providerId, request, @@ -431,6 +454,7 @@ export class TurnAdmissionDispatchCoordinator { threadControl: this.threadControlDirective(prepared, sourceTurnId), contextSeed: prepared.thread.last_context_tokens ?? 0, contextWindow: prepared.thread.context_window, + ...(this.parentStartOwner && prepared.providerId === "codex" ? { workerOwned: true as const } : {}), }; } @@ -682,19 +706,70 @@ export class TurnAdmissionDispatchCoordinator { return prepared.automaticAttachments ?? this.persistCommandAttachments(prepared.command); } - private startParentTurn( + private startParentTurn(input: DataOnlyParentTurnStartInput): void { + const { userMessage, reopenThread, answeredPlanQuestionMessageId, ...start } = input; + let reopenedThread: Exclude, null> | null = null; + this.parentTurns.startParentTurn({ + ...start, + projectUserMessage: () => { + if (reopenThread) reopenedThread = this.reopenThread(input.thread.id); + const message = projectParentUserMessage(this.messages, input.thread.id, userMessage); + if (answeredPlanQuestionMessageId) this.planAnswers.markAnswered(answeredPlanQuestionMessageId, input.thread.id); + return message; + }, + }); + if (reopenedThread) broadcast("thread.lifecycleChanged", { thread: reopenedThread }); + } + + private async commitParentStart( + prepared: PreparedCommand, + lease: TurnRuntimeLease, + sourceTurnId: string, + input: DataOnlyParentTurnStartInput, + ): Promise { + if (!this.parentStartOwner || prepared.providerId !== "codex") return this.startParentTurn(input); + const precedingMessageId = input.userMessage.messageId; + if (!precedingMessageId) throw new Error("Worker-owned turn needs its committed user message identity"); + const planFeature = prepared.command.planAction === "revise" ? "output" + : this.effectiveInteractionMode(prepared.command) === "plan" ? "questions" : "none"; + await this.parentStartOwner.start({ + execution: { threadId: lease.threadId, turnId: sourceTurnId, executionId: lease.turnExecutionId }, + ownerEpoch: lease.generation, + providerId: prepared.providerId, + parentTurn: input, + parentLive: { planFeature, precedingMessageId }, + publishParentStart: prepared.providerId === "codex", + }); + if (input.reopenThread) { + const reopened = this.threads.findById(lease.threadId); + if (reopened) broadcast("thread.lifecycleChanged", { thread: reopened }); + } + } + + private prepareParentTurnStartInput( prepared: PreparedCommand, lease: TurnRuntimeLease, sourceTurnId: string, attachments: PersistedAttachmentData, review: ApprovalReviewDecision, - ): void { + ): DataOnlyParentTurnStartInput { const command = prepared.command; - const wasUserCompleted = prepared.thread.user_completed_at !== null; - const nextSequence = command.persistedUserMessage?.sequence - ?? this.messages.getLatestSequenceIncludingInternal(command.threadId) + 1; - let reopenedThread: Exclude, null> | null = null; - this.parentTurns.startParentTurn({ + const userMessage: ParentUserMessageWrite = command.persistedUserMessage + ? { kind: "existing", messageId: command.persistedUserMessage.id } + : { + kind: "create", + messageId: command.messageId ?? NodeCrypto.randomUUID(), + content: command.displayContent ?? command.content, + sequence: this.messages.getLatestSequenceIncludingInternal(command.threadId) + 1, + attachments: attachments.stored.length > 0 ? [...attachments.stored] : undefined, + replyToMessageId: command.replyToMessageId, + quotedText: command.quotedText, + mentions: prepared.mentions.length > 0 ? [...prepared.mentions] : undefined, + previewAnnotations: command.previewAnnotations, + origin: this.messageOrigin(command), + selectedTextComments: command.selectedTextComments, + }; + return { thread: { id: prepared.thread.id, workspaceId: prepared.workspace.id, @@ -708,14 +783,10 @@ export class TurnAdmissionDispatchCoordinator { approvalReviewReason: review.reason, providerIdentities: this.resumeIdentities(prepared), retryOfExecutionId: command.retryOfExecutionId, - projectUserMessage: () => { - if (wasUserCompleted) reopenedThread = this.reopenThread(command.threadId); - const message = this.persistUserMessage(prepared, nextSequence, attachments); - this.markPlanAnswer(command, prepared.thread.id); - return message; - }, - }); - if (reopenedThread) broadcast("thread.lifecycleChanged", { thread: reopenedThread }); + userMessage, + reopenThread: prepared.thread.user_completed_at !== null, + answeredPlanQuestionMessageId: command.markPlanAnswerForMessageId, + }; } private effectivePermissionMode(requested: PermissionMode | "default" | undefined, persisted: PermissionMode): "full" | "supervised" { @@ -739,19 +810,6 @@ export class TurnAdmissionDispatchCoordinator { return reopened; } - private persistUserMessage(prepared: PreparedCommand, sequence: number, attachments: PersistedAttachmentData) { - const command = prepared.command; - if (command.persistedUserMessage) return this.persistedQueuedMessage(command); - const origin = this.messageOrigin(command); - return this.createUserMessage(prepared, sequence, attachments, origin); - } - - private persistedQueuedMessage(command: SendMessageCommand) { - const message = this.messages.findByIdInThread(command.threadId, command.persistedUserMessage!.id); - if (!message) throw new Error(`Queued Turn message was not found: ${command.persistedUserMessage!.id}`); - return message; - } - private messageOrigin(command: SendMessageCommand) { if (!command.sourceThreadId || !command.originSourceTurnId || !command.sourceProviderId) return undefined; return { @@ -762,39 +820,6 @@ export class TurnAdmissionDispatchCoordinator { }; } - private createUserMessage( - prepared: PreparedCommand, - sequence: number, - attachments: PersistedAttachmentData, - origin: ReturnType, - ) { - const command = prepared.command; - const args = [ - command.threadId, - "user" as const, - command.displayContent ?? command.content, - sequence, - attachments.stored.length > 0 ? [...attachments.stored] : undefined, - command.replyToMessageId, - command.quotedText, - undefined, - undefined, - prepared.mentions.length > 0 ? [...prepared.mentions] : undefined, - command.previewAnnotations, - ] as const; - if (command.selectedTextComments !== undefined) { - return this.messages.create(...args, origin, command.messageId, command.selectedTextComments); - } - if (command.messageId === undefined && origin === undefined) return this.messages.create(...args); - if (command.messageId === undefined) return this.messages.create(...args, origin); - if (origin === undefined) return this.messages.create(...args, undefined, command.messageId); - return this.messages.create(...args, origin, command.messageId); - } - - private markPlanAnswer(command: SendMessageCommand, threadId: string): void { - if (command.markPlanAnswerForMessageId) this.planAnswers.markAnswered(command.markPlanAnswerForMessageId, threadId); - } - private publishCommittedEffects(prepared: PreparedCommand, sourceTurnId: string): void { this.publishPlanAnswer(prepared.command); const command = prepared.command; @@ -812,9 +837,11 @@ export class TurnAdmissionDispatchCoordinator { private persistThreadSettings(prepared: PreparedCommand): void { const command = prepared.command; const model = command.model ?? "claude-sonnet-4-6"; - this.threads.updateModel(command.threadId, model); - if (command.provider !== undefined) this.threads.updateProvider(command.threadId, prepared.providerId); - this.threads.updateSettings(command.threadId, this.threadSettings(command, prepared.providerId)); + this.threads.updateSettings(command.threadId, { + ...this.threadSettings(command, prepared.providerId), + model, + ...(command.provider === undefined ? {} : { provider: prepared.providerId }), + }); broadcast("thread.modelUpdated", { threadId: command.threadId, model, diff --git a/apps/server/src/features/agents/turns/turn-diff-service.ts b/apps/server/src/features/agents/turns/turn-diff-service.ts index eb45cb945..ebc5e1113 100644 --- a/apps/server/src/features/agents/turns/turn-diff-service.ts +++ b/apps/server/src/features/agents/turns/turn-diff-service.ts @@ -17,6 +17,17 @@ interface ActiveDiff extends TurnDiffIdentity { rejected?: boolean; } +/** Cloneable native evidence frozen for one execution before terminal file settlement. */ +export interface PreparedTurnDiffEvidence extends TurnDiffIdentity { + readonly threadId: string; + readonly revision: number; + readonly evidence: ProviderTurnDiffUpdate | null; + readonly rejected: boolean; +} + +/** Data the terminal writer can store with its assigned assistant message. */ +export type SelectedTurnDiff = Pick; + /** Owns admission, volatile native evidence, and final source reconciliation. */ export class TurnDiffService { private readonly active = new Map(); @@ -57,19 +68,28 @@ export class TurnDiffService { /** Freeze native evidence at the terminal fence before asynchronous file settlement. */ prepareFinalization(threadId: string, executionId: string | undefined, outcome: TurnOutcome): SettleTurnDiff { - const current = [...this.active.values()].find((entry) => entry.threadId === threadId && entry.turnExecutionId === executionId); + const current = this.takeFinalizationEvidence(threadId, executionId); if (!current) return () => {}; - this.clear(current.turnId, executionId); return (messageId, effects, reconstructionPatch) => { - if (outcome !== "completed") return; - const selected = selectSettledEvidence(current, effects, reconstructionPatch); + const selected = selectTurnDiffSettlement(current, outcome, effects, reconstructionPatch); if (!selected) return; - this.repo.create({ id: NodeCrypto.randomUUID(), message_id: messageId, thread_id: threadId, - ...selected, revision: Math.max(0, current.revision) }); + this.repo.create({ id: NodeCrypto.randomUUID(), message_id: messageId, ...selected }); this.changed(threadId); }; } + /** Transfer terminal evidence only for its exact admitted execution. */ + takeFinalizationEvidence(threadId: string, executionId: string | undefined): PreparedTurnDiffEvidence | null { + const current = [...this.active.values()].find((entry) => entry.threadId === threadId && entry.turnExecutionId === executionId); + if (!current) return null; + this.clear(current.turnId, executionId); + return { + threadId: current.threadId, turnId: current.turnId, + turnExecutionId: current.turnExecutionId, deliveryAttempt: current.deliveryAttempt, + revision: current.revision, evidence: structuredClone(current.evidence), rejected: current.rejected ?? false, + }; + } + /** Read current Live metadata; native patch bytes stay on the file-diff endpoint. */ liveComparison(threadId: string): ReviewComparison | null { const current = [...this.active.values()].find((entry) => entry.threadId === threadId); @@ -119,14 +139,26 @@ function isNewRevision(update: ProviderTurnDiffUpdate, current: ActiveDiff): boo return matches(update, current) && Number.isSafeInteger(update.revision) && update.revision > current.revision; } -function selectSettledEvidence(current: ActiveDiff, effects: TurnFileEffectSummary | undefined, reconstructionPatch?: string): Pick | null { +/** Reconcile frozen native evidence with settled file effects without a repository connection. */ +export function selectTurnDiffSettlement( + current: PreparedTurnDiffEvidence, + outcome: TurnOutcome, + effects: TurnFileEffectSummary | undefined, + reconstructionPatch?: string, +): SelectedTurnDiff | null { + if (outcome !== "completed") return null; + const selected = selectSettledEvidence(current, effects, reconstructionPatch); + return selected ? { thread_id: current.threadId, ...selected, revision: Math.max(0, current.revision) } : null; +} + +function selectSettledEvidence(current: PreparedTurnDiffEvidence, effects: TurnFileEffectSummary | undefined, reconstructionPatch?: string): Pick | null { const native = nativeSettlement(current, effects); if (native !== undefined) return native; if (reconstructionPatch && parseTurnDiff(reconstructionPatch)) return { source: "tracked", patch: reconstructionPatch }; return effects?.fileCount ? { source: "git", patch: null } : null; } -function nativeSettlement(current: ActiveDiff, effects: TurnFileEffectSummary | undefined): Pick | null | undefined { +function nativeSettlement(current: PreparedTurnDiffEvidence, effects: TurnFileEffectSummary | undefined): Pick | null | undefined { const evidence = current.evidence; if (!evidence) return undefined; if (evidence.state === "snapshot") return { source: "native", patch: evidence.patch }; diff --git a/apps/server/src/features/agents/turns/turn-event-pipeline.ts b/apps/server/src/features/agents/turns/turn-event-pipeline.ts index 5df79c552..eeca23996 100644 --- a/apps/server/src/features/agents/turns/turn-event-pipeline.ts +++ b/apps/server/src/features/agents/turns/turn-event-pipeline.ts @@ -1,5 +1,7 @@ import type { AgentEvent, ProviderFileMutationStart, ProviderTurnDiffUpdate } from "@mcode/contracts"; import type { TurnDiffService } from "./turn-diff-service.js"; +import * as NodePerfHooks from "node:perf_hooks"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; import type { ProviderEventIngressConsumer, @@ -38,7 +40,9 @@ export interface TurnLifecycleControl { /** Applies one ordered event to the existing focused turn collaborators. */ export interface TurnEventApplication { /** Apply a validated event after the pipeline admits it in per-turn order. */ - apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean; + apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean | Promise; + /** Publish an event whose complete semantic work already committed on the execution writer. */ + publishCommitted(event: AgentEvent): void; /** Record a provider file mutation before its corresponding public event arrives. */ observeFileMutation(event: ProviderFileMutationStart): void; /** Abort one turn only when its completion cannot be compacted into the bounded queue. */ @@ -58,6 +62,12 @@ export interface TurnEventIngressFence { waitForThread(threadId: string): Promise; } +/** Claims every worker-owned file callback, including one rejected by its attempt fence. */ +export interface WorkerFileObservationPort { + ownsFileMutation(event: ProviderFileMutationStart): boolean; + capture(event: ProviderFileMutationStart): boolean; +} + interface QueuedTurnEvent { input: ProviderEventIngressEvent; event: AgentEvent; @@ -69,6 +79,8 @@ interface QueuedTurnEvent { export class TurnEventPipeline implements ProviderEventIngressConsumer { private readonly queues = new Map(); private readonly drainingThreads = new Set(); + private readonly inFlightApplications = new Map>(); + private readonly applicationErrors = new Map(); private readonly queuedBytesByThread = new Map(); private readonly finalizations = new Map>(); private readonly barriersByThread = new Map>(); @@ -77,13 +89,17 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { private readonly barrierGenerationByThread = new Map(); private readonly earlyFileEffects = new WeakSet(); private readonly fileBarrierDeferredEvents = new WeakSet(); - private readonly queueEmptyWaitersByThread = new Map void>>(); + private readonly queueEmptyWaitersByThread = new Map void; + reject: (error: unknown) => void; + }>>(); constructor( private readonly lifecycle: TurnLifecycleControl, private readonly application: TurnEventApplication, private readonly turnDiffs?: Pick, private readonly ingressFence?: TurnEventIngressFence, + private readonly workerFileObserver?: WorkerFileObservationPort, ) {} /** Accept an ingress envelope and preserve its source receipt through the turn queue. */ @@ -95,6 +111,12 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { this.drain(event.threadId); } + /** Publish committed worker output without passing it through legacy persistence. */ + handleProjectedCommitted(input: ProviderEventIngressEvent): void { + const event = this.lifecycle.normalize(input.event); + if (event) this.application.publishCommitted(event); + } + /** Stop only the affected turn when provider ingress cannot retain one of its events. */ handleProviderIngressOverflow(input: ProviderEventIngressEvent): void { this.application.rejectForQueueCapacity(input.event); @@ -103,6 +125,10 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { /** Observe one provider file mutation before public event attribution is available. */ handleProviderFileMutation(event: ProviderFileMutationStart): void { + if (this.workerFileObserver?.ownsFileMutation(event)) { + this.workerFileObserver.capture(event); + return; + } this.application.observeFileMutation(event); } @@ -113,18 +139,30 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { /** Materialize a terminal turn once, after every earlier event in its turn queue. */ finalizeTurn(command: FinalizeTurnCommand): Promise | null { + const inFlight = this.inFlightApplications.get(command.threadId); + const priorFailure = this.applicationErrors.get(command.threadId); if (cancelsDeferredWork(command.source)) this.discard(command.threadId, command.executionId); const existing = this.finalizations.get(command.threadId); if (existing) return existing; - const pending = this.ingressFence + // A cancelled queue may still have one durable write in progress. + const afterIngress = () => this.ingressFence ? this.ingressFence.waitForThread(command.threadId).then(() => this.finalizeAfterIngress(command)) : this.finalizeAfterIngress(command); + const pending = priorFailure !== undefined + ? Promise.reject(priorFailure) + : inFlight ? inFlight.then(afterIngress) : afterIngress(); this.finalizations.set(command.threadId, pending); - void pending.finally(() => this.finalizations.delete(command.threadId)); + void pending.then( + () => this.finalizations.delete(command.threadId), + () => this.finalizations.delete(command.threadId), + ); return pending; } private finalizeAfterIngress(command: FinalizeTurnCommand): Promise { + if (this.applicationErrors.has(command.threadId)) { + return Promise.reject(this.applicationErrors.get(command.threadId)); + } this.drain(command.threadId); return this.isReadyForFinalization(command.threadId) ? this.startFinalization(command) @@ -144,7 +182,8 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { this.deferredExecutionByThread.delete(threadId); this.queues.delete(threadId); this.queuedBytesByThread.delete(threadId); - this.completeQueue(threadId); + this.applicationErrors.delete(threadId); + if (!this.inFlightApplications.has(threadId)) this.completeQueue(threadId); } /** Return whether the pipeline already attributed this deferred event's file effect. */ @@ -186,8 +225,12 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { } private drain(threadId: string): void { - if (this.drainingThreads.has(threadId)) return; + if (this.drainingThreads.has(threadId) || this.applicationErrors.has(threadId)) return; this.drainingThreads.add(threadId); + this.drainOwnedQueue(threadId); + } + + private drainOwnedQueue(threadId: string): void { try { const queue = this.queues.get(threadId); while (queue && queue.length > 0) { @@ -197,33 +240,102 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { this.decrementQueuedBytes(threadId, next.byteLength); continue; } - if (!this.application.apply(next.input, next.event, next.publish)) return; + const applied = this.application.apply(next.input, next.event, next.publish); + if (typeof applied !== "boolean") { + const inFlight = Promise.resolve(applied); + this.inFlightApplications.set(threadId, inFlight); + void inFlight.then( + (accepted) => this.completeAsyncApplication(threadId, queue, next, inFlight, accepted), + (error: unknown) => this.failAsyncApplication(threadId, queue, inFlight, error), + ); + return; + } + if (!applied) return; queue.shift(); this.decrementQueuedBytes(threadId, next.byteLength); } - if (queue?.length === 0) this.completeQueue(threadId); + if (!queue || queue.length === 0) this.completeQueue(threadId); } finally { + if (!this.inFlightApplications.has(threadId)) this.drainingThreads.delete(threadId); + } + } + + private completeAsyncApplication( + threadId: string, + queue: QueuedTurnEvent[], + next: QueuedTurnEvent, + inFlight: Promise, + accepted: boolean, + ): void { + if (this.inFlightApplications.get(threadId) !== inFlight) return; + this.inFlightApplications.delete(threadId); + const currentQueue = this.queues.get(threadId); + if (accepted && currentQueue === queue && queue[0] === next) { + queue.shift(); + this.decrementQueuedBytes(threadId, next.byteLength); + } + if (!accepted && currentQueue === queue) { this.drainingThreads.delete(threadId); + return; + } + try { + this.drainOwnedQueue(threadId); + } catch (error) { + this.failApplication(threadId, error); + } + } + + private failAsyncApplication( + threadId: string, + queue: QueuedTurnEvent[], + inFlight: Promise, + error: unknown, + ): void { + if (this.inFlightApplications.get(threadId) !== inFlight) return; + this.inFlightApplications.delete(threadId); + if (this.queues.get(threadId) === queue) { + this.failApplication(threadId, error); + return; + } + try { + this.drainOwnedQueue(threadId); + } catch (nextError) { + this.failApplication(threadId, nextError); } } + private failApplication(threadId: string, error: unknown): void { + this.drainingThreads.delete(threadId); + this.applicationErrors.set(threadId, error); + const waiters = this.queueEmptyWaitersByThread.get(threadId) ?? []; + this.queueEmptyWaitersByThread.delete(threadId); + for (const waiter of waiters) waiter.reject(error); + } + private waitForQueue(threadId: string): Promise { this.drain(threadId); + if (this.applicationErrors.has(threadId)) return Promise.reject(this.applicationErrors.get(threadId)); if (!this.queues.has(threadId) && !this.drainingThreads.has(threadId)) return Promise.resolve(); - return new Promise((resolve) => { + return new Promise((resolve, reject) => { const waiters = this.queueEmptyWaitersByThread.get(threadId) ?? []; - waiters.push(resolve); + waiters.push({ resolve, reject }); this.queueEmptyWaitersByThread.set(threadId, waiters); }); } private isReadyForFinalization(threadId: string): boolean { const queuedEvents = this.queues.get(threadId)?.length ?? 0; - return queuedEvents === 0 || (queuedEvents === 1 && this.drainingThreads.has(threadId)); + return queuedEvents === 0 && !this.inFlightApplications.has(threadId); } private startFinalization(command: FinalizeTurnCommand): Promise { - return this.lifecycle.finalize(command) ?? Promise.resolve(false); + const trace = serverWorkTrace; + if (!trace) return this.lifecycle.finalize(command) ?? Promise.resolve(false); + const started = NodePerfHooks.performance.now(); + const result = this.lifecycle.finalize(command) ?? Promise.resolve(false); + return result.finally(() => trace.record( + "finalization", command.threadId, command.executionId, NodePerfHooks.performance.now() - started, + )); } private decrementQueuedBytes(threadId: string, byteLength: number): void { @@ -239,7 +351,7 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { this.queues.delete(threadId); const waiters = this.queueEmptyWaitersByThread.get(threadId) ?? []; this.queueEmptyWaitersByThread.delete(threadId); - for (const resolve of waiters) resolve(); + for (const waiter of waiters) waiter.resolve(); } private deferBehindFileFinalization(next: QueuedTurnEvent): boolean { diff --git a/apps/server/src/features/agents/turns/turn-execution-file-evidence.ts b/apps/server/src/features/agents/turns/turn-execution-file-evidence.ts new file mode 100644 index 000000000..5cadf5f0f --- /dev/null +++ b/apps/server/src/features/agents/turns/turn-execution-file-evidence.ts @@ -0,0 +1,94 @@ +import type { TurnFileEffectSummary, TurnOutcome } from "@mcode/contracts"; + +import type { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; +import type { CreateTurnSnapshotInput } from "./persistence/turn-snapshot-repo.js"; +import type { FileTurnHandoff, TurnFileTracker } from "./turn-file-tracker.js"; +import { + selectTurnDiffSettlement, + type PreparedTurnDiffEvidence, + type SelectedTurnDiff, +} from "./turn-diff-service.js"; + +/** Cloneable snapshot data whose message ID is assigned by the terminal writer. */ +export interface ExecutionTurnSnapshot extends Omit { + readonly executionId: string; +} + +/** File data retained through a failed writer attempt and retired only after its receipt. */ +export interface PreparedExecutionFileEvidence { + readonly threadId: string; + readonly turnId: string; + readonly executionId: string; + readonly deliveryAttempt: number; + readonly fileEffects: TurnFileEffectSummary; + readonly filesChanged: readonly string[]; + readonly snapshot: ExecutionTurnSnapshot | null; + readonly selectedTurnDiff: SelectedTurnDiff | null; +} + +/** Settle one sealed file generation before sending terminal data to the sole writer. */ +export async function prepareExecutionFileEvidence(input: { + readonly handoff: FileTurnHandoff; + readonly turnId: string; + readonly deliveryAttempt: number; + readonly outcome: TurnOutcome; + readonly nativeDiff: PreparedTurnDiffEvidence | null; + readonly tracker: TurnFileTracker; + readonly snapshots: SnapshotService; +}): Promise { + const { handoff, tracker, snapshots } = input; + const evidence = await tracker.finalEvidenceForExecution(handoff); + if (!evidence) throw new Error(`File evidence generation no longer belongs to execution ${handoff.executionId}`); + assertNativeDiffOwner(input); + + const fileEffects = evidence.summary; + const { snapshot, filesChanged } = await prepareSnapshot(handoff, evidence.baselineRef, fileEffects, snapshots); + return { + threadId: handoff.threadId, + turnId: input.turnId, + executionId: handoff.executionId, + deliveryAttempt: input.deliveryAttempt, + fileEffects, + filesChanged, + snapshot, + selectedTurnDiff: input.nativeDiff + ? selectTurnDiffSettlement(input.nativeDiff, input.outcome, fileEffects, evidence.reconstructionPatch) + : null, + }; +} + +function assertNativeDiffOwner(input: { + readonly nativeDiff: PreparedTurnDiffEvidence | null; + readonly handoff: FileTurnHandoff; + readonly turnId: string; + readonly deliveryAttempt: number; +}): void { + const diff = input.nativeDiff; + if (!diff) return; + if (diff.threadId !== input.handoff.threadId || diff.turnId !== input.turnId + || diff.turnExecutionId !== input.handoff.executionId + || diff.deliveryAttempt !== input.deliveryAttempt) { + throw new Error(`Native diff does not belong to execution ${input.handoff.executionId}`); + } +} + +async function prepareSnapshot( + handoff: FileTurnHandoff, + refBefore: string | null, + fileEffects: TurnFileEffectSummary, + snapshots: SnapshotService, +): Promise<{ snapshot: ExecutionTurnSnapshot | null; filesChanged: string[] }> { + const refAfter = refBefore ? await snapshots.captureRef(handoff.cwd) : null; + const filesChanged = fileEffects.effects.filter((effect) => effect.scope === "workspace") + .map((effect) => effect.path); + if (fileEffects.fileCount === 0 && !(refBefore && refAfter)) return { snapshot: null, filesChanged }; + return { filesChanged, snapshot: { + threadId: handoff.threadId, + executionId: handoff.executionId, + refBefore: refBefore ?? "", + refAfter: refAfter ?? "", + filesChanged, + fileEffects, + worktreePath: null, + } }; +} diff --git a/apps/server/src/features/agents/turns/turn-file-tracker.ts b/apps/server/src/features/agents/turns/turn-file-tracker.ts index 70e9c392d..5b70e6bb2 100644 --- a/apps/server/src/features/agents/turns/turn-file-tracker.ts +++ b/apps/server/src/features/agents/turns/turn-file-tracker.ts @@ -5,6 +5,7 @@ import * as NodePath from "node:path"; import { createTextPatch } from "@mcode/shared"; import type { FileEffect, TurnFileEffectSummary } from "@mcode/contracts"; import { MAX_TURN_FILE_EFFECTS } from "@mcode/contracts"; +import { z } from "zod"; import { normalizeFilesystemPath } from "../../../shared/filesystem/path-identity.js"; const MAX_FILE_BYTES = 1_048_576; @@ -32,10 +33,22 @@ interface MutationCandidate { } interface CandidateObservation { - resolvedPath: Pick | null; - resolvedOldPath: Pick | null; - baseline: FileState | null; - oldBaseline: FileState | null; + readonly resolvedPath: Readonly> | null; + readonly resolvedOldPath: Readonly> | null; + readonly baseline: Readonly | null; + readonly oldBaseline: Readonly | null; +} + +/** Plain, bounded pre-edit evidence that can cross a worker message boundary. */ +export interface CapturedToolUseObservation { + readonly threadId: string; + readonly executionId: string | null; + readonly toolCallId: string; + readonly generation: number; + readonly generationToken: string; + readonly canonicalRoot: string; + readonly candidateIdentity: string; + readonly observations: readonly (CandidateObservation | null)[]; } interface SyncObservationBudget { @@ -71,11 +84,15 @@ interface TrackedPath { } interface TurnState { + sealed: boolean; reconstructionRejected?: boolean; + executionId: string | null; generation: number; + generationToken: string; cwd: string; canonicalRoot: string; baselineRef: string | null; + initialBaselineRef: string | null; revision: number; tracked: Map; summary: TurnFileEffectSummary; @@ -99,6 +116,34 @@ export type TurnBaselineReader = ( /** Injection token for the composed live file-effect tracker. */ export const TURN_FILE_TRACKER = "TurnFileTracker"; +const fileTurnHandoffSchema = z.object({ + threadId: z.string().min(1), + executionId: z.string().min(1), + cwd: z.string().min(1).max(4096), + canonicalRoot: z.string().min(1).max(4096), + baselineRef: z.string().nullable(), + generation: z.number().int().positive().max(Number.MAX_SAFE_INTEGER - 1), + generationToken: z.string().uuid(), +}).strict(); + +/** One execution's cloneable file-tracker start state, with no other turn data. */ +export type FileTurnHandoff = Readonly>; +type FileTurnStart = Omit & { readonly executionId: string | null }; + +/** Frozen file evidence for one execution after its queued observations settle. */ +export interface ExecutionFileEvidence { + readonly baselineRef: string | null; + readonly summary: TurnFileEffectSummary; + readonly reconstructionPatch?: string; +} + +/** Identity supplied by the scheduler, independently of the handoff payload. */ +export interface ExpectedFileExecution { + readonly threadId: string; + readonly executionId: string; + readonly cwd: string; +} + /** * Tracks bounded, explicit agent file mutations and reduces them to net turn effects. * Git supplies the immutable in-project baseline; external paths use the state observed @@ -119,27 +164,60 @@ export class TurnFileTracker { /** Start a fresh tracker scope for an agent turn. */ beginTurn(threadId: string, cwd: string, baselineRef: string | null): number { + return this.openTurn(threadId, cwd, baselineRef, null).generation; + } + + /** Start one execution and return only the identity needed by its worker tracker. */ + beginExecutionTurn(input: ExpectedFileExecution & { readonly baselineRef: string | null }): FileTurnHandoff { + if (!input.threadId || !input.executionId) throw new Error("File turn execution identity is required"); + const turn = this.openTurn(input.threadId, input.cwd, input.baselineRef, input.executionId); + return Object.freeze({ + threadId: input.threadId, executionId: input.executionId, cwd: turn.cwd, + canonicalRoot: turn.canonicalRoot, baselineRef: turn.baselineRef, + generation: turn.generation, generationToken: turn.generationToken, + }); + } + + /** Open the same execution in a separate tracker after checking its scheduled identity and root. */ + beginTurnFromHandoff(value: unknown, expected: ExpectedFileExecution): boolean { + const parsed = fileTurnHandoffSchema.safeParse(value); + if (!parsed.success) return false; + const handoff = parsed.data; + if (handoff.threadId !== expected.threadId || handoff.executionId !== expected.executionId) return false; + const canonicalRoot = canonicalWorkingRoot(expected.cwd, this.platform); + if (!canonicalRoot || canonicalRoot !== handoff.canonicalRoot + || canonicalWorkingRoot(handoff.cwd, this.platform) !== canonicalRoot) return false; + const current = this.getTurn(handoff.threadId); + if (current) return sameFileTurnHandoff(current, handoff); + if (this.turns.get(handoff.threadId)?.has(handoff.generation)) return false; + this.installTurn(handoff.threadId, newTurnState(handoff)); + this.nextGeneration = Math.max(this.nextGeneration, handoff.generation + 1); + return true; + } + + private openTurn(threadId: string, cwd: string, baselineRef: string | null, executionId: string | null): TurnState { const canonicalRoot = normalizeFilesystemPath(NodeFS.realpathSync.native(cwd), this.platform); - const generation = this.nextGeneration++; + const start = { + threadId, executionId, cwd, canonicalRoot, baselineRef, + generation: this.nextGeneration, generationToken: NodeCrypto.randomUUID(), + }; + if (executionId !== null) fileTurnHandoffSchema.parse(start); + this.nextGeneration += 1; + const turn = newTurnState(start); + this.installTurn(threadId, turn); + return turn; + } + + private installTurn(threadId: string, turn: TurnState): void { const generations = this.turns.get(threadId) ?? new Map(); - generations.set(generation, { - generation, - cwd, - canonicalRoot, - baselineRef, - revision: 0, - tracked: new Map(), - summary: EMPTY_SUMMARY, - chain: Promise.resolve(), - }); + generations.set(turn.generation, turn); while (generations.size > 4) { const oldest = generations.keys().next().value as number | undefined; if (oldest === undefined) break; generations.delete(oldest); } this.turns.set(threadId, generations); - this.currentGeneration.set(threadId, generation); - return generation; + this.currentGeneration.set(threadId, turn.generation); } /** Return the authoritative tracker generation for the active turn. */ @@ -160,20 +238,54 @@ export class TurnFileTracker { toolName: string, toolInput: Record, ): Promise { - const candidates = extractMutationCandidates(toolName, toolInput); - if (candidates.length === 0) return Promise.resolve(); + const captured = this.captureToolUseObservation(threadId, toolCallId, toolName, toolInput); + return captured + ? this.observeCapturedToolUse({ threadId, toolCallId, toolName, toolInput }, captured).then(() => undefined) + : Promise.resolve(); + } + + /** Take the bounded filesystem baseline synchronously, before the provider may edit. */ + captureToolUseObservation( + threadId: string, + toolCallId: string, + toolName: string, + toolInput: Record, + ): CapturedToolUseObservation | null { const generation = this.currentGeneration.get(threadId); - if (generation === undefined) return Promise.resolve(); + if (generation === undefined) return null; const turn = this.getTurn(threadId, generation); - if (!turn) return Promise.resolve(); - const boundedCandidates = dedupeMutationCandidates(candidates).slice(0, MAX_TURN_FILE_EFFECTS); - const observations = this.synchronousObservations(turn, boundedCandidates); + if (!turn || turn.sealed) return null; + const candidates = boundedMutationCandidates(toolName, toolInput); + if (candidates.length === 0) return null; + const observations = this.synchronousObservations(turn, candidates).map(freezeCandidateObservation); + return Object.freeze({ + threadId, toolCallId, executionId: turn.executionId, + generation, generationToken: turn.generationToken, + canonicalRoot: turn.canonicalRoot, + candidateIdentity: mutationCandidateIdentity(toolName, candidates), + observations: Object.freeze(observations), + }); + } + + /** Apply a captured baseline only to its original thread, tool, root, and active generation. */ + observeCapturedToolUse( + event: { readonly threadId: string; readonly toolCallId: string; readonly toolName: string; readonly toolInput: Record }, + captured: CapturedToolUseObservation, + ): Promise { + const { threadId, toolCallId, toolName, toolInput } = event; + const turn = this.getTurn(threadId); + if (!turn || turn.sealed || !capturedMatchesTurn(captured, turn, threadId, toolCallId)) return Promise.resolve(false); + const generation = turn.generation; + const boundedCandidates = boundedMutationCandidates(toolName, toolInput); + if (boundedCandidates.length === 0 || captured.observations.length !== boundedCandidates.length + || captured.candidateIdentity !== mutationCandidateIdentity(toolName, boundedCandidates)) return Promise.resolve(false); + if (this.getTurn(threadId) !== turn) return Promise.resolve(false); this.generationByToolCall.set(toolGenerationKey(threadId, toolCallId), generation); return this.enqueue(threadId, async (queuedTurn) => { for (const [index, candidate] of boundedCandidates.entries()) { - await this.captureCandidate(queuedTurn, toolCallId, candidate, observations[index]); + await this.captureCandidate(queuedTurn, toolCallId, candidate, captured.observations[index] ?? undefined); } - }, generation); + }, generation).then(() => true); } /** Verify all paths attributed to a completed file tool and publish the net summary. */ @@ -212,11 +324,33 @@ export class TurnFileTracker { return turn.summary; } + /** Read an execution's settled evidence only while its original handoff still owns the generation. */ + async finalEvidenceForExecution(handoff: FileTurnHandoff): Promise { + const parsed = fileTurnHandoffSchema.safeParse(handoff); + if (!parsed.success) return null; + const turn = this.getTurn(parsed.data.threadId, parsed.data.generation); + if (!turn || !sameFileTurnHandoff(turn, parsed.data)) return null; + turn.sealed = true; + await turn.chain; + if (this.getTurn(parsed.data.threadId, parsed.data.generation) !== turn + || !sameFileTurnHandoff(turn, parsed.data)) return null; + return { + baselineRef: turn.baselineRef, + summary: structuredClone(turn.summary), + reconstructionPatch: this.reconstructionPatchForTurn(turn), + }; + } + /** Reconstruct a bounded patch only from complete explicit file-tool evidence. */ async reconstructionPatch(threadId: string, generation?: number): Promise { const turn = this.getTurn(threadId, generation); if (!turn || turn.reconstructionRejected) return undefined; await turn.chain; + return this.reconstructionPatchForTurn(turn); + } + + private reconstructionPatchForTurn(turn: TurnState): string | undefined { + if (turn.reconstructionRejected) return undefined; const tracked = [...turn.tracked.values()].filter((entry) => entry.scope === "workspace" && entry.effectCandidate); if (tracked.length !== turn.summary.fileCount || tracked.some((entry) => entry.evidenceRejected || entry.evidenceBefore === undefined || entry.evidenceAfter === undefined || entry.oldPath !== undefined)) return undefined; const patches = tracked.map((entry) => createTextPatch(entry.displayPath.replaceAll("\\", "/"), entry.evidenceBefore!, entry.evidenceAfter!, entry.effectCandidate!.effect.kind === "added" ? "added" : entry.effectCandidate!.effect.kind === "removed" ? "removed" : "edited")); @@ -242,7 +376,7 @@ export class TurnFileTracker { generation?: number, ): Promise { const turn = this.getTurn(threadId, generation); - if (!turn) return Promise.resolve(); + if (!turn || turn.sealed) return Promise.resolve(); const operation = turn.chain.then(() => work(turn)); turn.chain = operation.catch(() => undefined); return operation; @@ -674,6 +808,83 @@ function dedupeMutationCandidates(candidates: MutationCandidate[]): MutationCand return [...unique.values()]; } +function boundedMutationCandidates(toolName: string, toolInput: Record): MutationCandidate[] { + return dedupeMutationCandidates(extractMutationCandidates(toolName, toolInput)).slice(0, MAX_TURN_FILE_EFFECTS); +} + +function newTurnState(start: FileTurnStart): TurnState { + return { + sealed: false, + executionId: start.executionId, + generation: start.generation, + generationToken: start.generationToken, + cwd: start.cwd, + canonicalRoot: start.canonicalRoot, + baselineRef: start.baselineRef, + initialBaselineRef: start.baselineRef, + revision: 0, + tracked: new Map(), + summary: EMPTY_SUMMARY, + chain: Promise.resolve(), + }; +} + +function canonicalWorkingRoot(cwd: string, platform: NodeJS.Platform): string | null { + try { + return normalizeFilesystemPath(NodeFS.realpathSync.native(cwd), platform); + } catch { + return null; + } +} + +function sameFileTurnHandoff(turn: TurnState, handoff: FileTurnHandoff): boolean { + return turn.executionId === handoff.executionId && turn.generation === handoff.generation + && turn.generationToken === handoff.generationToken && turn.canonicalRoot === handoff.canonicalRoot + && turn.cwd === handoff.cwd && turn.initialBaselineRef === handoff.baselineRef; +} + +function capturedMatchesTurn( + captured: CapturedToolUseObservation, + turn: TurnState, + threadId: string, + toolCallId: string, +): boolean { + return captured.threadId === threadId && captured.toolCallId === toolCallId + && captured.executionId === turn.executionId + && captured.generation === turn.generation && captured.generationToken === turn.generationToken + && captured.canonicalRoot === turn.canonicalRoot; +} + +function mutationCandidateIdentity(toolName: string, candidates: readonly MutationCandidate[]): string { + const hash = NodeCrypto.createHash("sha256"); + hash.update(JSON.stringify([toolName])); + for (const candidate of candidates) { + hash.update(JSON.stringify([ + candidate.path, candidate.oldPath ?? null, candidate.operationHint, + candidate.providerConfirmed === true, candidate.beforeText !== undefined, + candidate.afterText !== undefined, + ])); + for (const text of [candidate.beforeText, candidate.afterText]) { + if (text === undefined) hash.update("u"); + else { + hash.update(`t${text.length}:`); + hash.update(text, "utf16le"); + } + } + } + return hash.digest("hex"); +} + +function freezeCandidateObservation(observation: CandidateObservation | undefined): CandidateObservation | null { + if (!observation) return null; + return Object.freeze({ + resolvedPath: observation.resolvedPath ? Object.freeze({ ...observation.resolvedPath }) : null, + resolvedOldPath: observation.resolvedOldPath ? Object.freeze({ ...observation.resolvedOldPath }) : null, + baseline: observation.baseline ? Object.freeze({ ...observation.baseline }) : null, + oldBaseline: observation.oldBaseline ? Object.freeze({ ...observation.oldBaseline }) : null, + }); +} + function observeCandidateSynchronously( canonicalRoot: string, candidate: MutationCandidate, diff --git a/apps/server/src/features/agents/turns/turn-finalizer.ts b/apps/server/src/features/agents/turns/turn-finalizer.ts index 3aecd61f3..2bba7bf1b 100644 --- a/apps/server/src/features/agents/turns/turn-finalizer.ts +++ b/apps/server/src/features/agents/turns/turn-finalizer.ts @@ -37,6 +37,7 @@ import type { ParentTurnDurability } from "./parent-turn-durability.js"; import type { ParentAssistantTextCheckpointService } from "./parent-assistant-text-checkpoint-service.js"; import { deriveTurnAssistantMessageId } from "./turn-assistant-message-id.js"; import type { TurnDiffService, SettleTurnDiff } from "./turn-diff-service.js"; +import { AssistantExecutionState, type AssistantMaterializationInput } from "./assistant-execution-state.js"; /** Pre-turn git ref captured at send time, used to diff the turn's file changes. */ interface TurnRef { @@ -45,13 +46,6 @@ interface TurnRef { fileTrackerGeneration?: number; } -/** Provider assistant body buffered during a turn, materialized at finalize. */ -interface BufferedBody { - content: string; - model: string | null; - attachments: StoredAttachment[]; -} - interface MaterializedAssistantRow { id: string; content: string; @@ -65,13 +59,6 @@ interface CanonicalProjection { narrative: ReturnType; } -interface AssistantMaterializationInput { - content: string; - model: string | null; - attachments: StoredAttachment[]; - fromProvider: boolean; -} - /** Durable snapshot operations required by terminal materialization. */ export type TurnSnapshotPersistence = Pick; @@ -88,14 +75,8 @@ export class TurnFinalizer { private readonly persistingThreads = new Set(); /** Last persisted assistant message id per thread, for late-hook attachment. */ private readonly lastPersistedMessageIdByThread = new Map(); - /** Streaming assistant text accumulated from textDelta events, per thread. */ - private readonly streamingAssistantTextByThread = new Map(); - /** Buffered provider assistant body awaiting materialization at finalize, per thread. */ - private readonly bufferedBodyByThread = new Map(); - /** Generated attachments awaiting materialization with the assistant row. */ - private readonly bufferedAttachmentsByThread = new Map(); - /** Threads whose assistant row was already materialized this turn (e.g. eagerly for a plan FK). */ - private readonly materializedThreads = new Set(); + /** One assistant decision state per active thread execution. */ + private readonly assistantStateByThread = new Map(); private readonly orm: BunSQLiteDatabase; /** Serializes finalize calls per thread so a slow git snapshot cannot drop a later turn. */ private readonly finalizeChainByThread = new Map>(); @@ -118,18 +99,17 @@ export class TurnFinalizer { /** Append a streaming assistant-text delta for the current turn. */ appendStreamingText(threadId: string, delta: string): void { - const prev = this.streamingAssistantTextByThread.get(threadId) ?? ""; - this.streamingAssistantTextByThread.set(threadId, prev + delta); + this.assistantState(threadId).appendStreamingText(delta); } /** Return the unmaterialized assistant text for the active turn. */ getStreamingText(threadId: string): string { - return this.streamingAssistantTextByThread.get(threadId) ?? ""; + return this.assistantStateByThread.get(threadId)?.getStreamingText() ?? ""; } /** Drop accumulated streaming text (a real assistant row now exists, or a new turn began). */ resetStreamingText(threadId: string): void { - this.streamingAssistantTextByThread.delete(threadId); + this.assistantStateByThread.get(threadId)?.resetStreamingText(); } /** @@ -147,24 +127,19 @@ export class TurnFinalizer { model: string | null, attachments = this.getBufferedAssistantAttachments(threadId), ): string { - this.bufferedBodyByThread.set(threadId, { content, model, attachments }); + this.assistantState(threadId).bufferBody(content, model, attachments); return deriveTurnAssistantMessageId(threadId, this.turnAnchorId(threadId)); } /** Buffer assistant-generated attachments until the turn's assistant row is materialized. */ bufferAssistantAttachments(threadId: string, attachments: StoredAttachment[]): void { if (attachments.length === 0) return; - const existing = this.bufferedAttachmentsByThread.get(threadId) ?? []; - const byId = new Map(existing.map((att) => [att.id, att])); - for (const att of attachments) { - byId.set(att.id, att); - } - this.bufferedAttachmentsByThread.set(threadId, [...byId.values()]); + this.assistantState(threadId).bufferAttachments(attachments); } /** Return generated attachments buffered for the current assistant turn. */ getBufferedAssistantAttachments(threadId: string): StoredAttachment[] { - return this.bufferedAttachmentsByThread.get(threadId) ?? []; + return this.assistantStateByThread.get(threadId)?.getBufferedAttachments() ?? []; } /** Record the pre-turn git ref so the turn's file changes can be diffed at finalize. */ @@ -206,20 +181,11 @@ export class TurnFinalizer { hasRecordableActivity(threadId: string): boolean { // An already-materialized row (e.g. eagerly written for a plan FK target) // is itself recordable activity even after its buffered body was consumed. - if (this.materializedThreads.has(threadId)) return true; - if (this.hasBufferedBody(threadId)) return true; - if (this.getBufferedAssistantAttachments(threadId).length > 0) return true; + const assistant = this.assistantStateByThread.get(threadId); + if (assistant?.hasMaterialized() || assistant?.hasBufferedBody() || assistant?.getBufferedAttachments().length) return true; return this.narrativeStore.hasBufferedNarrative(threadId); } - /** True when a non-empty assistant body is buffered (provider body or streaming text). */ - private hasBufferedBody(threadId: string): boolean { - const body = this.bufferedBodyByThread.get(threadId)?.content.trim(); - if (body) return true; - const streamed = this.streamingAssistantTextByThread.get(threadId)?.trim(); - return Boolean(streamed); - } - /** * Resolve the anchor id the turn's synthesized assistant row keys on — the id * of the message immediately preceding the assistant turn (normally the user @@ -731,16 +697,9 @@ export class TurnFinalizer { } private assistantMaterializationInput(threadId: string): AssistantMaterializationInput { - const buffered = this.bufferedBodyByThread.get(threadId); - const streamed = this.streamingAssistantTextByThread.get(threadId)?.trim(); - const fromProvider = buffered != null; - const content = fromProvider ? buffered.content : (streamed ?? ""); - const model = fromProvider ? buffered.model : (this.threadRepo.findById(threadId)?.model ?? null); - const bufferedAttachments = this.getBufferedAssistantAttachments(threadId); - const attachments = fromProvider - ? this.mergeAttachments(buffered.attachments, bufferedAttachments) - : bufferedAttachments; - return { content, model, attachments, fromProvider }; + const state = this.assistantState(threadId); + const fallbackModel = state.hasProviderBody() ? null : (this.threadRepo.findById(threadId)?.model ?? null); + return state.materializationInput(fallbackModel); } private broadcastMaterializedAssistant(threadId: string, materialized: MaterializedAssistantRow): void { @@ -756,10 +715,7 @@ export class TurnFinalizer { } private commitAssistantMaterialization(threadId: string): void { - this.streamingAssistantTextByThread.delete(threadId); - this.bufferedBodyByThread.delete(threadId); - this.bufferedAttachmentsByThread.delete(threadId); - this.materializedThreads.add(threadId); + this.assistantState(threadId).commitMaterialization(); } /** @@ -778,25 +734,15 @@ export class TurnFinalizer { if (turnRef !== undefined && this.turnRefBefore.get(threadId) === turnRef) { this.turnRefBefore.delete(threadId); } - this.streamingAssistantTextByThread.delete(threadId); - this.bufferedBodyByThread.delete(threadId); - this.bufferedAttachmentsByThread.delete(threadId); - this.materializedThreads.delete(threadId); + this.assistantStateByThread.delete(threadId); this.narrativeStore.clearTurn(threadId); this.turnFileTracker?.clearTurn(threadId, turnRef?.fileTrackerGeneration); this.persistingThreads.delete(threadId); } - private mergeAttachments( - first: StoredAttachment[], - second: StoredAttachment[], - ): StoredAttachment[] { - if (first.length === 0) return second; - if (second.length === 0) return first; - const byId = new Map(first.map((att) => [att.id, att])); - for (const att of second) { - byId.set(att.id, att); - } - return [...byId.values()]; + private assistantState(threadId: string): AssistantExecutionState { + const state = this.assistantStateByThread.get(threadId) ?? new AssistantExecutionState(); + this.assistantStateByThread.set(threadId, state); + return state; } } diff --git a/apps/server/src/features/projects/lifecycle/transport/workspace-thread-rpc.ts b/apps/server/src/features/projects/lifecycle/transport/workspace-thread-rpc.ts index 227714c07..7505b7557 100644 --- a/apps/server/src/features/projects/lifecycle/transport/workspace-thread-rpc.ts +++ b/apps/server/src/features/projects/lifecycle/transport/workspace-thread-rpc.ts @@ -1,7 +1,9 @@ import { WS_METHODS, type Thread, type Workspace, type WsMethodName } from "@mcode/contracts"; import { logger } from "@mcode/shared"; +import * as NodePerfHooks from "node:perf_hooks"; import type { z } from "zod"; import type { GoalLifecycleService } from "../../../agents/goals/goal-lifecycle-service.js"; +import { serverWorkTrace } from "../../../agents/diagnostics/server-work-trace.js"; import { broadcast } from "../../../../application/transport/push.js"; import type { GithubService } from "../../../pull-requests/github/github-service.js"; import type { CiWatcherService } from "../../../pull-requests/status/ci-watcher.js"; @@ -224,15 +226,22 @@ async function createThread( deps: WorkspaceThreadRouterDeps, params: { workspaceId: string; title: string; mode: string; branch: string }, ): Promise { - const thread = await deps.threadService.create( - params.workspaceId, - params.title, - params.mode, - params.branch, - { branchless: params.mode === "worktree" }, - ); - watchReturnedThreadWorktree(deps, thread); - return thread; + const started = serverWorkTrace ? NodePerfHooks.performance.now() : 0; + let threadId: string | undefined; + try { + const thread = await deps.threadService.create( + params.workspaceId, + params.title, + params.mode, + params.branch, + { branchless: params.mode === "worktree" }, + ); + threadId = thread.id; + watchReturnedThreadWorktree(deps, thread); + return thread; + } finally { + if (serverWorkTrace) serverWorkTrace.record("thread-create", threadId, undefined, NodePerfHooks.performance.now() - started); + } } function watchReturnedThreadWorktree( diff --git a/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts b/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts index 960e314f4..c545ec739 100644 --- a/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts +++ b/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts @@ -46,6 +46,8 @@ describe("ClaudeCanonicalEventPublisher", () => { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, + batchId: `claude:${routing.executionId}:attempt:1:event:1`, + deliveryAttempt: 1, events: [expect.objectContaining({ eventId: `claude:${routing.executionId}:attempt:1:event:1`, sourceProviderId: "claude", @@ -94,4 +96,22 @@ describe("ClaudeCanonicalEventPublisher", () => { `claude:${routing.executionId}:attempt:1:event:3`, ]); }); + + it("checks a completed turn without resetting sequence for late SDK events", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(undefined); + const publisher = new ClaudeCanonicalEventPublisher(createSink(submit)); + const event = providerRuntimeEvent({ + type: AgentEventType.System, + threadId: routing.threadId, + subtype: "late-hook", + turnExecutionId: routing.executionId, + }); + + publisher.publish(routing, event, []); + await publisher.flushForExecution(routing); + publisher.publish(routing, event, []); + await publisher.waitForExecution(routing); + + expect(submit.mock.calls.map(([batch]) => batch.events[0]?.sourceSequence)).toEqual([1, 2]); + }); }); diff --git a/apps/server/src/features/providers/adapters/claude/__tests__/claude-provider-result-error.test.ts b/apps/server/src/features/providers/adapters/claude/__tests__/claude-provider-result-error.test.ts index 1d98d92d0..e4967a1cb 100644 --- a/apps/server/src/features/providers/adapters/claude/__tests__/claude-provider-result-error.test.ts +++ b/apps/server/src/features/providers/adapters/claude/__tests__/claude-provider-result-error.test.ts @@ -291,4 +291,67 @@ describe("ClaudeProvider result is_error handling (#293)", () => { expect(submittedEvents.map((runtimeEvent) => runtimeEvent.event)) .not.toContainEqual(expect.objectContaining({ type: AgentEventType.TurnComplete })); }); + + it("reports canonical sink failure for the exact execution without direct event delivery", async () => { + const failure = vi.fn(async () => undefined); + const directEvents = vi.fn(); + provider = new ClaudeProvider( + stubEnvService(), + stubJobObject(), + undefined, + undefined, + undefined, + { + ...mockProviderHost(), + events: { submit: vi.fn(async () => { throw new Error("writer unavailable"); }) }, + }, + ); + provider.setCanonicalTurnDeliveryFailureHandler(failure); + provider.on("event", directEvents); + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { releaseStream = resolve; }); + const streamDone = vi.fn(); + provider.on("_streamDone:mcode-thread-failed", streamDone); + mockQuery.mockImplementation(({ prompt }: { prompt: AsyncIterable }) => Object.assign( + (async function* () { + await prompt[Symbol.asyncIterator]().next(); + yield { type: "system", subtype: "init", session_id: "sdk-abc" }; + yield { type: "result", is_error: true, errors: ["rate_limit_exceeded"] }; + await holdStream; + })(), + { ...queryMethodStubs(), close: vi.fn() }, + )); + + try { + await provider.sendTurn({ + turnId: "turn-failed", + turnExecutionId: "execution-failed", + deliveryAttempt: 2, + sessionId: "mcode-thread-failed", + workspaceId: "workspace-1", + threadId: "thread-failed", + message: "hi", + cwd: "/tmp", + model: "claude-sonnet-4-6", + permissionMode: "default", + interactionMode: "build", + providerOptions: {}, + }); + + await vi.waitFor(() => expect(failure).toHaveBeenCalledExactlyOnceWith( + { + threadId: "thread-failed", + turnId: "turn-failed", + executionId: "execution-failed", + deliveryAttempt: 2, + }, + expect.objectContaining({ message: "writer unavailable" }), + )); + expect(streamDone).not.toHaveBeenCalled(); + expect(directEvents).not.toHaveBeenCalled(); + } finally { + releaseStream(); + } + await vi.waitFor(() => expect(streamDone).toHaveBeenCalledOnce()); + }); }); diff --git a/apps/server/src/features/providers/adapters/claude/claude-provider.ts b/apps/server/src/features/providers/adapters/claude/claude-provider.ts index d787202c6..690c1eadd 100644 --- a/apps/server/src/features/providers/adapters/claude/claude-provider.ts +++ b/apps/server/src/features/providers/adapters/claude/claude-provider.ts @@ -509,6 +509,10 @@ export class ClaudeProvider /** Serializes canonical event submission when the adapter runs in the server composition. */ private readonly canonicalEventPublisher: ClaudeCanonicalEventPublisher | undefined; + private canonicalTurnDeliveryFailureHandler: + | ((routing: ClaudeCanonicalEventRouting, error: Error) => Promise) + | undefined; + private readonly reportedCanonicalDeliveryFailures = new WeakSet(); /** * Tail of the most recent stderr emitted by each session's Claude Code * subprocess, capped at {@link STDERR_CAPTURE_LIMIT}. The SDK's process-exit @@ -601,6 +605,13 @@ export class ClaudeProvider }); } + /** Reports canonical sink failure to the owner of the exact active turn. */ + setCanonicalTurnDeliveryFailureHandler( + handler: (routing: ClaudeCanonicalEventRouting, error: Error) => Promise, + ): void { + this.canonicalTurnDeliveryFailureHandler = handler; + } + /** * Merges {@link EnvService.getEnv} into `process.env` for the Claude SDK spawn window * only, then restores the previous environment. @@ -2054,7 +2065,12 @@ export class ClaudeProvider if (executionId && executionId !== state.currentTurnExecutionId) state.pendingPromptExecutionIds.push(executionId); }); - void this.consumeClaudeStream(sessionId, q, routing, state); + void this.consumeClaudeStream(sessionId, q, routing, state).catch((error: unknown) => { + logger.error("Claude stream finalization failed", { + executionId: state.currentTurnExecutionId, + error: error instanceof Error ? error.message : String(error), + }); + }); } private createClaudeStreamState( @@ -2151,6 +2167,7 @@ export class ClaudeProvider } const outcome = await state.mapper.map(message); if (outcome !== "none") { + await this.flushCanonicalExecution(state.currentTurnExecutionId); state.awaitingResume = outcome === "turn_complete"; state.resumedTurnStarted = false; } @@ -2410,15 +2427,43 @@ export class ClaudeProvider try { await this.canonicalEventPublisher.waitForExecution(routing); } catch (error: unknown) { - logger.error("Claude canonical event delivery failed", { - executionId, - error: error instanceof Error ? error.message : String(error), - }); + await this.reportCanonicalDeliveryFailure(routing, error); } finally { this.getCanonicalRoutings().delete(executionId); } } + private async flushCanonicalExecution(executionId: string): Promise { + const routing = this.getCanonicalRoutings().get(executionId); + if (!routing || !this.canonicalEventPublisher) return; + try { + await this.canonicalEventPublisher.flushForExecution(routing); + } catch (error: unknown) { + await this.reportCanonicalDeliveryFailure(routing, error); + } + } + + private async reportCanonicalDeliveryFailure( + routing: ClaudeCanonicalEventRouting, + error: unknown, + ): Promise { + if (this.reportedCanonicalDeliveryFailures.has(routing)) return; + this.reportedCanonicalDeliveryFailures.add(routing); + const deliveryError = error instanceof Error ? error : new Error(String(error)); + logger.error("Claude canonical event delivery failed", { + executionId: routing.executionId, + error: deliveryError.message, + }); + try { + await this.canonicalTurnDeliveryFailureHandler?.(routing, deliveryError); + } catch (handlerError: unknown) { + logger.error("Claude canonical failure handler failed", { + executionId: routing.executionId, + error: handlerError instanceof Error ? handlerError.message : String(handlerError), + }); + } + } + private getCanonicalRoutings(): Map { this.canonicalRoutings ??= new Map(); return this.canonicalRoutings; diff --git a/apps/server/src/features/providers/catalog/__tests__/provider-catalog-service.test.ts b/apps/server/src/features/providers/catalog/__tests__/provider-catalog-service.test.ts index b0d5505f0..6fc680344 100644 --- a/apps/server/src/features/providers/catalog/__tests__/provider-catalog-service.test.ts +++ b/apps/server/src/features/providers/catalog/__tests__/provider-catalog-service.test.ts @@ -1,12 +1,15 @@ import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import type { Database } from "bun:sqlite"; +import { Database } from "bun:sqlite"; import type { ProviderCatalogChange, ProviderCatalogRequest, ProviderCatalogSnapshot, } from "@mcode/contracts"; -import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { openDatabase, openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { ProviderCatalogSnapshotRepo } from "../persistence/provider-catalog-snapshot-repo.js"; import { ProviderCatalogService, @@ -48,6 +51,46 @@ describe("ProviderCatalogService", () => { return { repo, service: new ProviderCatalogService(repo) }; } + it("keeps a background refresh failure contained while another SQLite writer holds the database", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-catalog-busy-")); + const path = NodePath.join(directory, "app.sqlite"); + db = openDatabase({ dbPath: path }); + db.run("PRAGMA busy_timeout = 1"); + db.prepare("INSERT INTO workspaces (id, name, path) VALUES (?, ?, ?)") + .run("workspace-1", "Workspace 1", "C:/repo"); + const blocker = new Database(path, { strict: true }); + const repo = new ProviderCatalogSnapshotRepo(db); + const service = new ProviderCatalogService(repo); + const upsert = vi.spyOn(repo, "upsert"); + const changes: ProviderCatalogChange[] = []; + service.onChanged((change) => changes.push(change)); + const input = { + request: REQUEST, context: CACHED.context, cwd: "C:/repo", refresh: async () => CACHED, + }; + let locked = false; + + try { + blocker.run("BEGIN IMMEDIATE"); + locked = true; + service.request(input); + await vi.waitFor(() => expect(upsert).toHaveBeenCalledTimes(1)); + await new Promise((resolve) => setImmediate(resolve)); + expect(changes).toHaveLength(0); + + blocker.run("ROLLBACK"); + locked = false; + service.request(input); + await vi.waitFor(() => expect(changes).toHaveLength(1)); + expect(repo.get(providerCatalogContextKey(REQUEST, "C:/repo"))).toEqual(CACHED); + } finally { + if (locked) blocker.run("ROLLBACK"); + blocker.close(true); + db.close(true); + db = undefined; + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); + it("returns a stale persisted snapshot before background refresh completes", async () => { const { repo, service } = createService(); const key = providerCatalogContextKey(REQUEST, "C:/repo"); diff --git a/apps/server/src/features/providers/catalog/persistence/provider-catalog-snapshot-repo.ts b/apps/server/src/features/providers/catalog/persistence/provider-catalog-snapshot-repo.ts index 3666bc85e..5fdca191c 100644 --- a/apps/server/src/features/providers/catalog/persistence/provider-catalog-snapshot-repo.ts +++ b/apps/server/src/features/providers/catalog/persistence/provider-catalog-snapshot-repo.ts @@ -51,6 +51,7 @@ export class ProviderCatalogSnapshotRepo { snapshot: ProviderCatalogSnapshot, ): boolean { const validated = ProviderCatalogSnapshotSchema().parse(snapshot); + // Reading the workspace before acquiring a write lock can fail a later WAL snapshot upgrade. return this.orm.transaction((tx) => { // The legacy INSERT...SELECT only wrote when the referenced workspace // still existed, so a dangling workspaceId must abort with no changes. @@ -101,6 +102,6 @@ export class ProviderCatalogSnapshotRepo { ) .run(); return true; - }); + }, { behavior: "immediate" }); } } diff --git a/apps/server/src/features/providers/catalog/provider-catalog-service.ts b/apps/server/src/features/providers/catalog/provider-catalog-service.ts index 8a03adcaa..333b827bf 100644 --- a/apps/server/src/features/providers/catalog/provider-catalog-service.ts +++ b/apps/server/src/features/providers/catalog/provider-catalog-service.ts @@ -327,11 +327,22 @@ export class ProviderCatalogService { pendingSubscribers: new Map(), }; this.inflight.set(persistenceKey, job); - void new Promise((resolve) => { - setImmediate(() => { - void this.refresh(persistenceKey, job).finally(resolve); + setImmediate(() => { + void this.runRefresh(persistenceKey, job).catch((error: unknown) => { + logger.warn("Provider catalog background refresh could not persist", { + providerId: input.request.providerId, + workspaceId: input.request.workspaceId, + error: error instanceof Error ? error.message : String(error), + }); }); - }).finally(() => { + }); + return true; + } + + private async runRefresh(persistenceKey: string, job: CatalogRefreshJob): Promise { + try { + await this.refresh(persistenceKey, job); + } finally { this.inflight.delete(persistenceKey); for (const [pendingRequestKey, pending] of job.pendingSubscribers) { const persisted = this.snapshotRepo.get(persistenceKey); @@ -348,8 +359,7 @@ export class ProviderCatalogService { false, ); } - }); - return true; + } } private emitChange(change: ProviderCatalogChange): void { diff --git a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts index 05f96508c..4efa269f2 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts @@ -13,6 +13,7 @@ import type { ProviderHostPorts } from "@mcode/providers"; import { setupContainer } from "../../../../application/composition/container.js"; import { CanonicalAgentBoundary } from "../../../agents/canonical/canonical-agent-boundary.js"; +import { WorkerOwnedTurnRuntime } from "../../../agents/execution/worker-owned-turn-runtime.js"; import { MessageRepo } from "../../../agents/conversation/persistence/message-repo.js"; import { ProviderRegistry } from "../provider-registry.js"; import { SettingsService } from "../../../settings/settings-service.js"; @@ -90,6 +91,7 @@ describe("provider composition container", () => { afterEach(async () => { vi.restoreAllMocks(); await container.resolve(ProviderRegistry).shutdown(); + await container.resolve(WorkerOwnedTurnRuntime).close(); container.resolve(ProviderEventIngress).shutdown(); container.resolve(SettingsService).dispose(); database?.close(true); diff --git a/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts index e47cb267d..fe2a49ff7 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import { container, Lifecycle } from "tsyringe"; import { AgentEventType, + ProviderRuntimeEventSchema, type CanonicalAgentEventEnvelope, type IAgentProvider, type ProviderId, @@ -17,9 +18,10 @@ import { type ProviderEventIngressEvent, } from "../provider-event-ingress.js"; import type { ProviderEventAdapter } from "../provider-event-adapter.js"; -import type { - ProviderEventWorkerCallbacks, - ProviderEventWorkerPool, +import { + ThreadEventWorkerPool, + type ProviderEventWorkerCallbacks, + type ProviderEventWorkerPool, } from "../provider-event-worker-pool.js"; import { processProviderEventWorkerTask, @@ -27,11 +29,13 @@ import { } from "../provider-event-worker-protocol.js"; import { CodexCollaborationEventAdapter } from "../../../agents/collaboration/adapters/codex-collaboration-event-adapter.js"; import type { CodexCollaborationDurability } from "../../../agents/collaboration/codex-collaboration-durability.js"; +import type { ProjectedCommittedProviderEvent } from "../../../agents/execution/execution-worker-handler.js"; const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; -function runtimeEvent(delta: string, threadId = "thread-1"): ProviderRuntimeEvent { +function runtimeEvent(delta: string, threadId = "thread-1", deliveryAttempt?: number): ProviderRuntimeEvent { return { + ...(deliveryAttempt === undefined ? {} : { deliveryAttempt }), event: { type: AgentEventType.TextDelta, threadId, @@ -68,7 +72,7 @@ function oversizedToolResult(threadId: string): ProviderRuntimeEvent { }; } -function committedEnvelope(eventId: string, delta: string): CanonicalAgentEventEnvelope { +function committedEnvelope(eventId: string, delta: string, deliveryAttempt?: number, sourceProviderId: ProviderId = "claude"): CanonicalAgentEventEnvelope { return { eventId, routing: { @@ -77,7 +81,7 @@ function committedEnvelope(eventId: string, delta: string): CanonicalAgentEventE executionId: EXECUTION_ID, itemId: `item-${eventId}`, }, - sourceProviderId: "claude", + sourceProviderId, sourceIdentities: [], acceptedSequence: 1, durableRevision: 1, @@ -90,7 +94,7 @@ function committedEnvelope(eventId: string, delta: string): CanonicalAgentEventE turnId: "turn-1", kind: "system", providerIdentities: [], - payload: { projection: "providerRuntimeEvent", runtimeEvent: runtimeEvent(delta) }, + payload: { projection: "providerRuntimeEvent", runtimeEvent: runtimeEvent(delta, "thread-1", deliveryAttempt) }, createdAt: "2026-08-27T12:00:00.000Z", updatedAt: "2026-08-27T12:00:00.000Z", }, @@ -98,6 +102,20 @@ function committedEnvelope(eventId: string, delta: string): CanonicalAgentEventE }; } +function projectedEvent(eventId: string, delta: string): ProjectedCommittedProviderEvent { + return { + providerId: "claude", + sourceKind: "canonical-commit", + event: runtimeEvent(delta).event, + canonicalReceipt: { + eventId, + acceptedSequence: 1, + durableRevision: 1, + serverTimestamps: { acceptedAt: "2026-08-27T12:00:00.000Z" }, + }, + }; +} + function createProvider(id: ProviderId): IAgentProvider { return Object.assign(new NodeEvents.EventEmitter(), { id }) as unknown as IAgentProvider; } @@ -170,6 +188,7 @@ function createIngress( ) { const diagnostics: ProviderEventIngressDiagnostic[] = []; const received: ProviderEventIngressEvent[] = []; + const projected: ProviderEventIngressEvent[] = []; const overflowed: ProviderEventIngressEvent[] = []; const registry = { resolveAll: () => providers } as never; const ingress = new ProviderEventIngress( @@ -179,10 +198,11 @@ function createIngress( ); ingress.start(registry, { handleProviderEvent: (event) => received.push(event), + handleProjectedCommitted: (event) => projected.push(event), handleProviderFileMutation: vi.fn(), handleProviderIngressOverflow: (event) => overflowed.push(event), }); - return { diagnostics, ingress, overflowed, provider: providers[0], received }; + return { diagnostics, ingress, overflowed, projected, provider: providers[0], received }; } async function flushIngress(): Promise { @@ -197,6 +217,41 @@ async function drainIngress(ingress: ProviderEventIngress): Promise { } describe("ProviderEventIngress", () => { + it("starts legacy workers only when the first legacy event needs preprocessing", async () => { + const createWorker = vi.fn(() => new Worker(new URL("../provider-event.worker.ts", import.meta.url), { type: "module" })); + const workerPool = new ThreadEventWorkerPool({ workerCount: 2, createWorker }); + const { ingress, projected, received } = createIngress(undefined, undefined, workerPool); + try { + expect(createWorker).not.toHaveBeenCalled(); + ingress.acceptProjectedCommitted([{ ...projectedEvent("owned-codex", "projected"), providerId: "codex" }]); + await ingress.waitForThread("thread-1"); + expect(projected).toEqual([expect.objectContaining({ providerId: "codex" })]); + expect(createWorker).not.toHaveBeenCalled(); + + ingress.acceptProviderRuntime("claude", runtimeEvent("legacy")); + await ingress.waitForThread("thread-1"); + expect(createWorker).toHaveBeenCalledTimes(2); + expect(received).toEqual([expect.objectContaining({ + providerId: "claude", event: expect.objectContaining({ delta: "legacy" }), + })]); + expect(ingress.queueMetrics().pendingEvents).toBe(0); + } finally { + ingress.shutdown(); + } + }); + + it("shuts down before the first submission without creating workers", async () => { + const createWorker = vi.fn(() => new Worker(new URL("../provider-event.worker.ts", import.meta.url), { type: "module" })); + const workerPool = new ThreadEventWorkerPool({ workerCount: 2, createWorker }); + const { ingress, diagnostics, received } = createIngress(undefined, undefined, workerPool); + ingress.shutdown(); + ingress.acceptProviderRuntime("claude", runtimeEvent("after shutdown")); + await ingress.waitForThread("thread-1"); + expect(createWorker).not.toHaveBeenCalled(); + expect(received).toEqual([]); + expect(diagnostics).toEqual([expect.objectContaining({ reason: "worker-shutdown" })]); + }); + it("resolves the diagnostic sink through its explicit injection token", () => { const child = container.createChildContainer(); const provider = createProvider("claude"); @@ -229,6 +284,20 @@ describe("ProviderEventIngress", () => { expect(received[0]?.canonicalReceipt).toBeUndefined(); }); + it("keeps private delivery attempt metadata through runtime and committed ingress", async () => { + const provider = createProvider("codex"); + const { ingress, received } = createIngress([provider]); + const runtime = ProviderRuntimeEventSchema().parse(runtimeEvent("live", "thread-1", 2)); + + (provider as unknown as NodeEvents.EventEmitter).emit("event", structuredClone(runtime)); + ingress.acceptCommitted([committedEnvelope("committed-attempt", "committed", 2, "codex")]); + await flushIngress(); + + expect(received.map((event) => event.deliveryAttempt)).toEqual([2, 2]); + expect(received.map((event) => event.event.type)).toEqual([AgentEventType.TextDelta, AgentEventType.TextDelta]); + expect(ProviderRuntimeEventSchema().safeParse({ ...runtime, deliveryAttempt: 0 }).success).toBe(false); + }); + it("keeps direct canonical commits and runtime events in arrival order", async () => { const cursor = createProvider("cursor"); const { ingress, received } = createIngress([createProvider("claude"), cursor]); @@ -261,6 +330,22 @@ describe("ProviderEventIngress", () => { expect(received).toHaveLength(1); }); + it("queues projected writer events in order without calling the legacy consumer", async () => { + const { ingress, projected, received } = createIngress(); + const first = projectedEvent("worker-first", "first"); + const second = projectedEvent("worker-second", "second"); + + ingress.acceptProjectedCommitted([first, second]); + expect(projected).toEqual([]); + await ingress.waitForThread("thread-1"); + + expect(projected.map((item) => item.canonicalReceipt?.eventId)).toEqual(["worker-first", "worker-second"]); + expect(received).toEqual([]); + ingress.acceptProjectedCommitted([first]); + await ingress.waitForThread("thread-1"); + expect(projected).toHaveLength(2); + }); + it("does not invoke the consumer on the provider callback stack", async () => { const { provider, received } = createIngress(); diff --git a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts index 4e94cef32..2bb9d276b 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it, vi } from "vitest"; -import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; +import { ProviderIdSchema, type CanonicalAgentEventEnvelope } from "@mcode/contracts"; +import type { ProviderEventBatch } from "@mcode/providers"; -import { createProviderHostPorts } from "../provider-host-ports.js"; +import { + createProviderHostPorts, + type ProviderEventOwnershipRoute, + type WorkerOwnedProviderEventResult, +} from "../provider-host-ports.js"; const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; @@ -35,6 +40,65 @@ function committedRuntimeEnvelope(): CanonicalAgentEventEnvelope { }; } +function ownedBatch(providerId: "claude" | "cursor", deliveryAttempt: number, sequence: number): ProviderEventBatch { + const eventId = `${providerId}:${EXECUTION_ID}:attempt:${deliveryAttempt}:event:${sequence}`; + const itemId = `${providerId}:${EXECUTION_ID}:attempt:${deliveryAttempt}:item:${sequence}`; + const envelope = committedRuntimeEnvelope(); + if (envelope.payload.type !== "item.recorded") throw new Error("Expected recorded item fixture"); + return { + threadId: "thread-1", + turnId: "turn-1", + executionId: EXECUTION_ID, + batchId: eventId, + deliveryAttempt, + phase: "running", + events: [{ + eventId, + routing: { ...envelope.routing, itemId }, + sourceProviderId: providerId, + sourceIdentities: [], + sourceSequence: sequence, + payload: { + type: "item.recorded", + item: { ...envelope.payload.item, id: itemId }, + }, + }], + }; +} + +function workerCommit(batch: ProviderEventBatch, sequence: number): WorkerOwnedProviderEventResult { + const [draft] = batch.events; + if (!draft || !batch.batchId || !batch.deliveryAttempt) throw new Error("Expected owned batch fixture"); + return { + batchId: batch.batchId, + deliveryAttempt: batch.deliveryAttempt, + commit: { + outcome: "committed", + conversationRevision: sequence, + rosterRevision: 0, + acceptedThrough: sequence, + durableThrough: sequence, + eventCount: 1, + }, + providerEvents: [{ + providerId: ProviderIdSchema.parse(draft.sourceProviderId), + sourceKind: "canonical-commit", + event: { + type: "textDelta", + threadId: batch.threadId, + turnExecutionId: batch.executionId, + delta: `output ${sequence}`, + }, + canonicalReceipt: { + eventId: draft.eventId, + acceptedSequence: sequence, + durableRevision: sequence, + serverTimestamps: { acceptedAt: "2026-08-27T12:00:00.000Z" }, + }, + }], + }; +} + describe("createProviderHostPorts", () => { it("adapts the server browser grant to the Provider contract", () => { const ports = createProviderHostPorts({ @@ -143,4 +207,124 @@ describe("createProviderHostPorts", () => { await expect(ports.events.submit(batch)).rejects.toThrow("commit failed"); expect(acceptCommitted).not.toHaveBeenCalled(); }); + + it.each(["claude", "cursor"] as const)("delivers %s worker batches only after ordered commit replies", async (providerId) => { + const commit = vi.fn(); + const accepted: string[] = []; + let acknowledge: ((value: WorkerOwnedProviderEventResult) => void) | undefined; + const firstReply = new Promise((resolve) => { acknowledge = resolve; }); + const submit = vi.fn() + .mockImplementationOnce(() => firstReply) + .mockImplementation((batch: ProviderEventBatch) => Promise.resolve(workerCommit(batch, 2))); + const route: ProviderEventOwnershipRoute = { + kind: "worker", + threadId: "thread-1", + turnId: "turn-1", + executionId: EXECUTION_ID, + deliveryAttempt: 2, + submit, + }; + const ports = createProviderHostPorts({ + events: { commit }, + ingress: { acceptProjectedCommitted: (events: WorkerOwnedProviderEventResult["providerEvents"]) => + accepted.push(...events.map((event) => event.canonicalReceipt.eventId)) }, + eventOwnership: { resolve: () => route }, + } as never); + const first = ownedBatch(providerId, 2, 1); + const second = ownedBatch(providerId, 2, 2); + + const pending = ports.events.submit(first); + expect(commit).not.toHaveBeenCalled(); + expect(accepted).toEqual([]); + acknowledge?.(workerCommit(first, 1)); + await expect(pending).resolves.toMatchObject({ + commit: { outcome: "committed", acceptedThrough: 1 }, + delivery: { ingress: "queued" }, + }); + await ports.events.submit(second); + expect(accepted).toEqual([first.events[0]?.eventId, second.events[0]?.eventId]); + expect(submit).toHaveBeenCalledTimes(2); + expect(commit).not.toHaveBeenCalled(); + }); + + it("rejects a stale attempt or mismatched receipt without falling back to the legacy writer", async () => { + const batch = ownedBatch("claude", 1, 1); + const commit = vi.fn(); + const acceptProjectedCommitted = vi.fn(); + const submit = vi.fn(() => Promise.resolve({ ...workerCommit(batch, 1), deliveryAttempt: 2 })); + const route: ProviderEventOwnershipRoute = { + kind: "worker", + threadId: batch.threadId, + turnId: batch.turnId, + executionId: batch.executionId, + deliveryAttempt: 2, + submit, + }; + const ports = createProviderHostPorts({ + events: { commit }, + ingress: { acceptProjectedCommitted }, + eventOwnership: { resolve: () => route }, + } as never); + + await expect(ports.events.submit(batch)).rejects.toThrow("does not match execution ownership"); + expect(submit).not.toHaveBeenCalled(); + route.deliveryAttempt = 1; + await expect(ports.events.submit(batch)).rejects.toThrow("receipt does not match submitted batch"); + batch.batchId = undefined; + await expect(ports.events.submit(batch)).rejects.toThrow("does not match execution ownership"); + expect(commit).not.toHaveBeenCalled(); + expect(acceptProjectedCommitted).not.toHaveBeenCalled(); + }); + + it("propagates a worker commit conflict without publishing or using the legacy writer", async () => { + const batch = ownedBatch("cursor", 1, 1); + const commit = vi.fn(); + const acceptProjectedCommitted = vi.fn(); + const result = workerCommit(batch, 1); + const ports = createProviderHostPorts({ + events: { commit }, + ingress: { acceptProjectedCommitted }, + eventOwnership: { resolve: () => ({ + kind: "worker", + threadId: batch.threadId, + turnId: batch.turnId, + executionId: batch.executionId, + deliveryAttempt: 1, + submit: () => Promise.resolve({ + ...result, + commit: { ...result.commit, outcome: "conflict" }, + providerEvents: [], + }), + }) }, + } as never); + + await expect(ports.events.submit(batch)).rejects.toThrow("Canonical worker batch failed: conflict"); + expect(commit).not.toHaveBeenCalled(); + expect(acceptProjectedCommitted).not.toHaveBeenCalled(); + }); + + it("propagates worker failure and rejected ownership without a legacy commit", async () => { + const batch = ownedBatch("cursor", 1, 1); + const commit = vi.fn(); + const submit = vi.fn(() => Promise.reject(new Error("worker unavailable"))); + let route: ProviderEventOwnershipRoute = { + kind: "worker", + threadId: batch.threadId, + turnId: batch.turnId, + executionId: batch.executionId, + deliveryAttempt: 1, + submit, + }; + const ports = createProviderHostPorts({ + events: { commit }, + ingress: { acceptProjectedCommitted: vi.fn() }, + eventOwnership: { resolve: () => route }, + } as never); + + await expect(ports.events.submit(batch)).rejects.toThrow("worker unavailable"); + route = { kind: "rejected" }; + await expect(ports.events.submit(batch)).rejects.toThrow("no longer admitted"); + expect(submit).toHaveBeenCalledTimes(1); + expect(commit).not.toHaveBeenCalled(); + }); }); diff --git a/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts b/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts index 91651f3a7..2a5b3c3a8 100644 --- a/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts +++ b/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts @@ -55,6 +55,8 @@ export class CanonicalLiveEventPublisher { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, + batchId: draft.eventId, + deliveryAttempt: routing.deliveryAttempt, phase: "running", events: [draft], }); @@ -67,16 +69,24 @@ export class CanonicalLiveEventPublisher { }); } - /** Waits for all queued drafts, then reports a sink failure to the caller. */ - async waitForExecution(routing: CanonicalLiveEventRouting): Promise { + /** Checks delivery so a pooled provider can report turn failure before stream teardown. */ + async flushForExecution(routing: CanonicalLiveEventRouting): Promise { const key = this.queueKey(routing); const queue = this.queues.get(key); if (!queue) return; await queue.tail; - this.queues.delete(key); if (queue.failure) throw queue.failure; } + /** Waits for all queued drafts and retires the execution queue. */ + async waitForExecution(routing: CanonicalLiveEventRouting): Promise { + try { + await this.flushForExecution(routing); + } finally { + this.queues.delete(this.queueKey(routing)); + } + } + private queueFor(routing: CanonicalLiveEventRouting): ProviderExecutionQueue { const key = this.queueKey(routing); const existing = this.queues.get(key); diff --git a/apps/server/src/features/providers/composition/provider-event-ingress.ts b/apps/server/src/features/providers/composition/provider-event-ingress.ts index 4261ef24e..3f7c8c5e9 100644 --- a/apps/server/src/features/providers/composition/provider-event-ingress.ts +++ b/apps/server/src/features/providers/composition/provider-event-ingress.ts @@ -1,4 +1,5 @@ import { logger } from "@mcode/shared"; +import * as NodePerfHooks from "node:perf_hooks"; import { AgentEventType, isTurnDiffSource, @@ -30,6 +31,8 @@ import { type ProviderEventWorkerTask, } from "./provider-event-worker-protocol.js"; import { normalizeProviderError } from "./provider-error-normalize.js"; +import { eventApplyType, serverWorkTrace } from "../../agents/diagnostics/server-work-trace.js"; +import type { ProjectedCommittedProviderEvent } from "../../agents/execution/execution-worker-handler.js"; const MAX_PENDING_NON_TERMINAL_EVENTS = 8_192; const MAX_PENDING_NON_TERMINAL_EVENTS_PER_THREAD = 2_048; @@ -65,7 +68,7 @@ const MAX_OVERFLOWED_TURNS = 16_384; export const PROVIDER_EVENT_INGRESS_DIAGNOSTIC_SINK = Symbol("ProviderEventIngressDiagnosticSink"); /** Identifies the transport path that delivered one provider runtime event. */ -export type ProviderEventSourceKind = "provider-runtime" | "canonical-commit"; +export type ProviderEventSourceKind = "provider-runtime" | "canonical-commit" | "worker-commit"; /** Durable canonical metadata that must remain attached to a projected live event. */ export interface CanonicalProviderEventReceipt { @@ -81,6 +84,7 @@ export interface ProviderEventIngressEvent { providerId: ProviderId; sourceKind: ProviderEventSourceKind; event: AgentEvent; + deliveryAttempt?: number; runtimeExtension?: ProviderRuntimeEvent["extension"]; canonicalReceipt?: CanonicalProviderEventReceipt; } @@ -88,6 +92,8 @@ export interface ProviderEventIngressEvent { /** Downstream contract for applying accepted provider events in the turn pipeline. */ export interface ProviderEventIngressConsumer { handleProviderEvent(event: ProviderEventIngressEvent): void; + /** Publish a writer-owned event without running the legacy turn write pipeline. */ + handleProjectedCommitted?(event: ProviderEventIngressEvent): void; handleProviderFileMutation(event: ProviderFileMutationStart): void; handleProviderTurnDiff(event: ProviderTurnDiffUpdate): void; /** Stop one turn when ingress cannot retain one of its provider events. */ @@ -128,6 +134,7 @@ export interface ProviderEventIngressQueueMetrics { interface QueuedProviderEvent { event: ProviderEventIngressEvent; queuedAt: number; + traceQueuedAt: number; byteLength: number; terminal: boolean; } @@ -189,7 +196,6 @@ export class ProviderEventIngress { if (this.started) return; this.started = true; this.consumer = consumer; - this.workerPool.start(); for (const provider of providerRegistry.resolveAll()) { provider.on("file_mutation_start", (event) => consumer.handleProviderFileMutation(event)); if (isTurnDiffSource(provider)) provider.onTurnDiff((event) => consumer.handleProviderTurnDiff(event)); @@ -224,6 +230,15 @@ export class ProviderEventIngress { /** Accept one provider-originated runtime event that has no canonical receipt. */ acceptProviderRuntime(providerId: ProviderId, runtimeEvent: unknown): void { + if (serverWorkTrace) { + const threadId = providerEventWorkerThreadId({ kind: "provider-runtime", providerId, runtimeEvent }); + serverWorkTrace.measure("provider-callback", threadId, undefined, () => this.acceptProviderRuntimeCore(providerId, runtimeEvent)); + return; + } + this.acceptProviderRuntimeCore(providerId, runtimeEvent); + } + + private acceptProviderRuntimeCore(providerId: ProviderId, runtimeEvent: unknown): void { const parsedProviderId = ProviderIdSchema.safeParse(providerId); if (!parsedProviderId.success) { this.report({ reason: "provider-identity-mismatch", sourceKind: "provider-runtime", providerId: String(providerId) }); @@ -251,6 +266,24 @@ export class ProviderEventIngress { for (const envelope of envelopes) this.acceptCanonicalEnvelope(envelope); } + /** Deliver writer-projected events after their durable reply without reapplying old turn writes. */ + acceptProjectedCommitted(events: readonly ProjectedCommittedProviderEvent[]): void { + if (this.stopped || !this.consumer?.handleProjectedCommitted) { + throw new Error("Worker-owned provider publication is unavailable"); + } + for (const event of events) { + const eventId = event.canonicalReceipt.eventId; + if (this.seenCanonicalEventIds.has(eventId) || this.pendingCanonicalEventIds.has(eventId)) { + this.report({ reason: "duplicate-event", sourceKind: "worker-commit", eventId }); + continue; + } + if (!this.enqueue({ ...event, sourceKind: "worker-commit" })) { + throw new Error("Worker-owned provider publication queue is full"); + } + this.rememberCanonicalEvent(eventId); + } + } + private acceptCanonicalEnvelope(envelope: unknown): void { const task: ProviderEventWorkerTask = { kind: "canonical-commit", envelope }; const eventId = canonicalEventIdentity(task); @@ -299,7 +332,19 @@ export class ProviderEventIngress { this.report({ reason: "worker-shutdown", sourceKind: task.kind, eventId: canonicalEventIdentity(task) }); return false; } - const accepted = this.workerPool.submit(providerEventWorkerThreadId(task), task, { onOutcome }); + const threadId = providerEventWorkerThreadId(task); + let accepted: boolean; + const trace = serverWorkTrace; + if (trace) { + const started = NodePerfHooks.performance.now(); + accepted = this.workerPool.submit(threadId, task, { onOutcome: (outcome) => { + trace.record("worker-wait", threadId, + outcome.status === "accepted" ? outcome.event.event.turnExecutionId : undefined, + NodePerfHooks.performance.now() - started); + onOutcome(outcome); + } }); + trace.record("worker-admission", threadId, undefined, NodePerfHooks.performance.now() - started); + } else accepted = this.workerPool.submit(threadId, task, { onOutcome }); if (!accepted) this.rejectForWorkerCapacity(task); return accepted; } @@ -372,6 +417,10 @@ export class ProviderEventIngress { queueCapacity: "global" | "thread" | "global-bytes" | "thread-bytes" | "terminal", ): false { this.rejectedQueueEvents += 1; + if (event.sourceKind === "worker-commit") { + this.reportQueueDiagnostic("queue-capacity", event, queue?.length ?? 0, queueCapacity); + return false; + } if (!this.rememberOverflowedTurn(event)) return false; this.reportQueueDiagnostic("queue-capacity", event, queue?.length ?? 0, queueCapacity); this.discardPendingExecution(event); @@ -392,7 +441,7 @@ export class ProviderEventIngress { this.pendingByThread.set(threadId, queued); this.readyThreadIds.push(threadId); } - queued.push({ event, queuedAt: Date.now(), byteLength, terminal }); + queued.push({ event, queuedAt: Date.now(), traceQueuedAt: serverWorkTrace ? NodePerfHooks.performance.now() : 0, byteLength, terminal }); this.pendingEventCount += 1; this.pendingByteCount += byteLength; this.pendingBytesByThread.set(threadId, (this.pendingBytesByThread.get(threadId) ?? 0) + byteLength); @@ -431,11 +480,22 @@ export class ProviderEventIngress { for (let delivered = 0; delivered < MAX_PROVIDER_EVENTS_PER_DRAIN; delivered += 1) { const queued = this.takeNextPendingEvent(); if (!queued) break; - this.consumer.handleProviderEvent(queued.event); + if (serverWorkTrace) { + serverWorkTrace.record("mailbox-wait", queued.event.event.threadId, + queued.event.event.turnExecutionId, NodePerfHooks.performance.now() - queued.traceQueuedAt); + serverWorkTrace.measure("event-apply", queued.event.event.threadId, + queued.event.event.turnExecutionId, () => this.deliverQueuedEvent(queued.event), + eventApplyType(queued.event.event.type)); + } else this.deliverQueuedEvent(queued.event); } if (this.pendingEventCount > 0) this.scheduleYieldedDrain(); } + private deliverQueuedEvent(event: ProviderEventIngressEvent): void { + if (event.sourceKind === "worker-commit") this.consumer?.handleProjectedCommitted?.(event); + else this.consumer?.handleProviderEvent(event); + } + private takeNextPendingEvent(): QueuedProviderEvent | undefined { while (this.readyThreadIds.length > 0) { const threadId = this.readyThreadIds.shift(); diff --git a/apps/server/src/features/providers/composition/provider-event-worker-protocol.ts b/apps/server/src/features/providers/composition/provider-event-worker-protocol.ts index e21da498f..04f75f811 100644 --- a/apps/server/src/features/providers/composition/provider-event-worker-protocol.ts +++ b/apps/server/src/features/providers/composition/provider-event-worker-protocol.ts @@ -203,6 +203,7 @@ function runtimeIngressEvent( providerId, sourceKind, event: normalizeEvent(runtimeEvent.event), + ...(runtimeEvent.deliveryAttempt !== undefined ? { deliveryAttempt: runtimeEvent.deliveryAttempt } : {}), ...(runtimeEvent.extension ? { runtimeExtension: runtimeEvent.extension } : {}), }; } diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index 890f578d9..400e9ebf3 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -1,4 +1,10 @@ -import type { ProviderHostPorts } from "@mcode/providers"; +import type { + ProviderEventBatch, + ProviderEventCommitReceipt, + ProviderHostPorts, + ProviderEventSubmissionReceipt, +} from "@mcode/providers"; +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; import type { HostRuntime } from "@mcode/shared/node/host-runtime"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; import type { EnvService } from "../../../runtime/environment/env-service.js"; @@ -8,6 +14,39 @@ import type { BrowserAutomationSessionLease } from "../../browser-automation/ind import type { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; import { killProcessTree } from "../../../runtime/process/containment/process-kill.js"; import type { ProviderEventIngress } from "./provider-event-ingress.js"; +import type { ProjectedCommittedProviderEvent } from "../../agents/execution/execution-worker-handler.js"; + +/** A provider batch with the stable identity required by a worker owner. */ +export type WorkerOwnedProviderEventBatch = ProviderEventBatch & { + batchId: string; + deliveryAttempt: number; +}; + +/** One acknowledged worker commit and its projected live events, bound to the batch. */ +export interface WorkerOwnedProviderEventResult { + batchId: string; + deliveryAttempt: number; + commit: ProviderEventCommitReceipt; + providerEvents: readonly ProjectedCommittedProviderEvent[]; +} + +/** One durable route decision for an execution. Retired executions must resolve to rejected. */ +export type ProviderEventOwnershipRoute = + | { kind: "legacy" } + | { kind: "rejected" } + | { + kind: "worker"; + threadId: string; + turnId: string; + executionId: string; + deliveryAttempt: number; + submit(batch: WorkerOwnedProviderEventBatch): Promise; + }; + +/** Selects one commit owner for every batch of an exact execution. */ +export interface ProviderEventOwnership { + resolve(executionId: string, sourceProviderId?: string): ProviderEventOwnershipRoute; +} /** Server services used to compose the narrow Provider host-port boundary. */ export interface ProviderHostPortDependencies { @@ -19,6 +58,7 @@ export interface ProviderHostPortDependencies { grants: ScopedPreGrantService; events: CanonicalAgentBoundary; ingress: ProviderEventIngress; + eventOwnership?: ProviderEventOwnership; } /** Adapts server-owned services to the only host operations exposed to Providers. */ @@ -82,38 +122,112 @@ export function createProviderHostPorts( consume: (request) => dependencies.grants.tryConsume(request), }, events: { - submit: async (batch) => { - const result = dependencies.events.commit({ - threadId: batch.threadId, - turnId: batch.turnId, - executionId: batch.executionId, - phase: batch.phase, - nativeCursor: batch.nativeCursor, - events: batch.events, - }); - if (result.outcome === "committed" && result.events.length > 0) { - dependencies.ingress.acceptCommitted(result.events); - } - return { - commit: { - outcome: providerCommitOutcome(result.outcome), - conversationRevision: result.conversationRevision, - rosterRevision: result.rosterRevision, - acceptedThrough: result.acceptedThrough, - durableThrough: result.durableThrough, - eventCount: result.events.length, - }, - delivery: { - ingress: result.outcome === "committed" && result.events.length > 0 - ? "queued" - : "not-required", - }, - }; - }, + submit: (batch) => submitProviderEvents(dependencies, batch), }, }; } +async function submitProviderEvents( + dependencies: ProviderHostPortDependencies, + batch: ProviderEventBatch, +): Promise { + const route: ProviderEventOwnershipRoute = dependencies.eventOwnership?.resolve( + batch.executionId, batch.events[0]?.sourceProviderId, + ) + ?? { kind: "legacy" }; + if (route.kind === "rejected") throw new Error("Canonical event execution is no longer admitted"); + if (route.kind === "worker") return await submitWorkerEvents(dependencies, route, batch); + return submitLegacyEvents(dependencies, batch); +} + +async function submitWorkerEvents( + dependencies: ProviderHostPortDependencies, + route: Extract, + batch: ProviderEventBatch, +): Promise { + const ownedBatch = checkedWorkerBatch(batch, route); + const result = await route.submit(ownedBatch); + if (!matchesWorkerReceipt(ownedBatch, result)) { + throw new Error("Canonical worker receipt does not match submitted batch"); + } + if (result.commit.outcome === "conflict" || result.commit.outcome === "ingest-overflow") { + throw new Error(`Canonical worker batch failed: ${result.commit.outcome}`); + } + if (result.commit.outcome === "committed" && result.providerEvents.length > 0) { + dependencies.ingress.acceptProjectedCommitted(result.providerEvents); + } + return { + commit: result.commit, + delivery: { ingress: result.commit.outcome === "committed" && result.providerEvents.length > 0 + ? "queued" : "not-required" }, + }; +} + +function submitLegacyEvents( + dependencies: ProviderHostPortDependencies, + batch: ProviderEventBatch, +): ProviderEventSubmissionReceipt { + const result = dependencies.events.commit({ + threadId: batch.threadId, + turnId: batch.turnId, + executionId: batch.executionId, + phase: batch.phase, + nativeCursor: batch.nativeCursor, + events: batch.events, + }); + return deliverCommitted(dependencies.ingress, { + outcome: providerCommitOutcome(result.outcome), + conversationRevision: result.conversationRevision, + rosterRevision: result.rosterRevision, + acceptedThrough: result.acceptedThrough, + durableThrough: result.durableThrough, + eventCount: result.events.length, + }, result.events); +} + +function deliverCommitted( + ingress: ProviderEventIngress, + commit: ProviderEventCommitReceipt, + events: readonly CanonicalAgentEventEnvelope[], +): ProviderEventSubmissionReceipt { + const handoff = commit.outcome === "committed" && events.length > 0; + if (handoff) ingress.acceptCommitted(events); + return { commit, delivery: { ingress: handoff ? "queued" : "not-required" } }; +} + +function matchesWorkerReceipt( + batch: WorkerOwnedProviderEventBatch, + result: WorkerOwnedProviderEventResult, +): boolean { + if (result.batchId !== batch.batchId || result.deliveryAttempt !== batch.deliveryAttempt) return false; + if (result.commit.outcome !== "committed") return result.providerEvents.length === 0; + const submittedIds = new Set(batch.events.map((event) => event.eventId)); + return result.providerEvents.length <= result.commit.eventCount + && result.providerEvents.every((event) => submittedIds.has(event.canonicalReceipt.eventId)); +} + +function checkedWorkerBatch( + batch: ProviderEventBatch, + route: Extract, +): WorkerOwnedProviderEventBatch { + if (!batch.batchId || !Number.isSafeInteger(batch.deliveryAttempt) || (batch.deliveryAttempt ?? 0) < 1 + || batch.deliveryAttempt !== route.deliveryAttempt + || !matchesExecution(batch, route) + || batch.events.length === 0 + || !batch.events.every((event) => matchesExecution(event.routing, route))) { + throw new Error("Canonical worker batch does not match execution ownership"); + } + return { ...batch, batchId: batch.batchId, deliveryAttempt: batch.deliveryAttempt }; +} + +function matchesExecution( + routing: { threadId: string; turnId?: string; executionId: string }, + route: Pick, +): boolean { + return routing.threadId === route.threadId && routing.turnId === route.turnId + && routing.executionId === route.executionId; +} + function providerCommitOutcome( outcome: "committed" | "duplicate" | "conflict" | "terminal-outcome-confirmed" | "ingest-overflow", ): "committed" | "duplicate" | "conflict" | "ingest-overflow" { diff --git a/apps/server/src/features/providers/composition/register-providers.ts b/apps/server/src/features/providers/composition/register-providers.ts index 6f7011140..abf34f656 100644 --- a/apps/server/src/features/providers/composition/register-providers.ts +++ b/apps/server/src/features/providers/composition/register-providers.ts @@ -9,6 +9,7 @@ import { createProviderHostPorts } from "./provider-host-ports.js"; import { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; import { CanonicalAgentBoundary } from "../../agents/index.js"; +import { WorkerOwnedTurnRuntime } from "../../agents/execution/worker-owned-turn-runtime.js"; import { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; import { EnvService } from "../../../runtime/environment/env-service.js"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; @@ -90,6 +91,7 @@ export function registerProviderAdapters(container: DependencyContainer): void { grants: c.resolve(ScopedPreGrantService), events: c.resolve(CanonicalAgentBoundary), ingress: c.resolve(ProviderEventIngress), + eventOwnership: c.resolve(WorkerOwnedTurnRuntime).providerEvents, }), }); } diff --git a/apps/server/src/features/terminal/cleanup/__tests__/terminal-cleanup-ledger.test.ts b/apps/server/src/features/terminal/cleanup/__tests__/terminal-cleanup-ledger.test.ts index 22465569e..f4818d297 100644 --- a/apps/server/src/features/terminal/cleanup/__tests__/terminal-cleanup-ledger.test.ts +++ b/apps/server/src/features/terminal/cleanup/__tests__/terminal-cleanup-ledger.test.ts @@ -23,16 +23,19 @@ async function openLedgerTestDatabase(): Promise { `); return { prepare: (sql: string) => database.prepare(sql), - transaction: (action: () => Result) => () => { - database.exec("BEGIN"); - try { - const result = action(); - database.exec("COMMIT"); - return result; - } catch (error) { - database.exec("ROLLBACK"); - throw error; - } + transaction: (action: () => Result) => { + const run = (begin: "BEGIN" | "BEGIN IMMEDIATE") => { + database.exec(begin); + try { + const result = action(); + database.exec("COMMIT"); + return result; + } catch (error) { + database.exec("ROLLBACK"); + throw error; + } + }; + return Object.assign(() => run("BEGIN"), { immediate: () => run("BEGIN IMMEDIATE") }); }, close: () => database.close(), } as unknown as Database; diff --git a/apps/server/src/features/terminal/cleanup/terminal-cleanup-ledger.ts b/apps/server/src/features/terminal/cleanup/terminal-cleanup-ledger.ts index 84cb37f80..a96382daf 100644 --- a/apps/server/src/features/terminal/cleanup/terminal-cleanup-ledger.ts +++ b/apps/server/src/features/terminal/cleanup/terminal-cleanup-ledger.ts @@ -50,6 +50,8 @@ export class PtyHostCleanupLedger implements PtyHostCleanupLedgerStore { /** Adds one process identity or refreshes the matching generation record. */ record(record: PtyHostCleanupRecord): void { const parsed = validateRecord(record); + // Reserve the write lock before reading; a deferred read can lose its WAL snapshot + // when the execution writer commits between this ledger's SELECT and INSERT. this.db.transaction(() => { const existing = this.db .prepare( @@ -108,7 +110,7 @@ export class PtyHostCleanupLedger implements PtyHostCleanupLedgerStore { now, now, ); - })(); + }).immediate(); } /** Removes one record only when its session and host generation match. */ diff --git a/apps/server/src/features/terminal/transport/terminal-rpc.ts b/apps/server/src/features/terminal/transport/terminal-rpc.ts index 55767acd3..baffcef29 100644 --- a/apps/server/src/features/terminal/transport/terminal-rpc.ts +++ b/apps/server/src/features/terminal/transport/terminal-rpc.ts @@ -8,6 +8,8 @@ import { TerminalBackendError, type TerminalBackend } from "../backends/terminal import type { TerminalProfileService } from "../profiles/terminal-profile-service.js"; import type { WorkspaceTerminalPreferencesService } from "../preferences/workspace-terminal-preferences-service.js"; import type { SettingsService } from "../../settings/settings-service.js"; +import * as NodePerfHooks from "node:perf_hooks"; +import { serverWorkTrace } from "../../agents/diagnostics/server-work-trace.js"; type TerminalManagementMethod = | "terminal.profile.list" @@ -111,7 +113,15 @@ const terminalManagementHandlers: TerminalManagementHandlers = { const terminalClassicHandlers: TerminalClassicHandlers = { "terminal.capabilities": (deps) => deps.terminalService.capabilities(), - "terminal.create": (deps, params) => deps.terminalService.create(params.threadId), + "terminal.create": async (deps, params) => { + if (!serverWorkTrace) return deps.terminalService.create(params.threadId); + const started = NodePerfHooks.performance.now(); + try { + return await deps.terminalService.create(params.threadId); + } finally { + serverWorkTrace.record("terminal-create", params.threadId, undefined, NodePerfHooks.performance.now() - started); + } + }, "terminal.write": async (deps, params) => { await deps.terminalService.write(params.ptyId, params.data); }, diff --git a/apps/server/src/features/thread-control/cleanup/__tests__/cleanup-worker.test.ts b/apps/server/src/features/thread-control/cleanup/__tests__/cleanup-worker.test.ts index c6d15f9db..57ec42d4e 100644 --- a/apps/server/src/features/thread-control/cleanup/__tests__/cleanup-worker.test.ts +++ b/apps/server/src/features/thread-control/cleanup/__tests__/cleanup-worker.test.ts @@ -295,6 +295,88 @@ describe("CleanupWorker sandbox worktrees", { timeout: 20_000 }, () => { expect(gitWorktrees.removeWorktree).toHaveBeenCalledOnce(); }); + it("restricts startup recovery and cleanup admission to the fixture workspace", async () => { + const fixture = workspaces.create("Fixture", "/fixture"); + const user = workspaces.create("Copied user project", "/fixture-other"); + const deleting = workspaces.create("Deleting user project", "/user-deleting"); + const empty = workspaces.create("Empty deleting user project", "/user-empty"); + workspaces.softDelete(deleting.id); + workspaces.softDelete(empty.id); + addThread(deleting.id, "user-missing-job", "/user-worktree", "user", null); + + for (const workspace of [fixture, user]) { + const expiredId = `${workspace.id}-expired`; + const explicitId = `${workspace.id}-explicit`; + addThread(workspace.id, expiredId, `${workspace.path}/expired`, "expired", new Date(0).toISOString()); + addThread(workspace.id, explicitId, `${workspace.path}/explicit`, "explicit", null); + threads.softDelete(explicitId); + const job = cleanupJobs.insert({ + thread_id: explicitId, + workspace_path: workspace.path, + worktree_path: `${workspace.path}/explicit`, + branch: "explicit", + }); + database.prepare("UPDATE cleanup_jobs SET attempts = 5, last_error = 'original error' WHERE id = ?") + .run(job.id); + } + for (const kind of ["explicit", "retention"] as const) { + cleanupJobs.insert({ + thread_id: `user-orphan-${kind}`, + workspace_path: user.path, + worktree_path: null, + branch: null, + kind, + }); + } + const userRows = () => ({ + workspaces: database.prepare("SELECT * FROM workspaces WHERE id != ? ORDER BY id").all(fixture.id), + threads: database.prepare("SELECT * FROM threads WHERE workspace_id != ? ORDER BY id").all(fixture.id), + jobs: database.prepare("SELECT * FROM cleanup_jobs WHERE workspace_path != ? ORDER BY id").all(fixture.path), + }); + const before = userRows(); + + worker.start(fixture.path); + try { + await worker.reconcileOnStartup(); + await worker.poll(); + + expect(userRows()).toEqual(before); + expect(threads.findById(`${fixture.id}-expired`)).toBeNull(); + expect(threads.findById(`${fixture.id}-explicit`)).toBeNull(); + expect(cleanupJobs.countByWorkspacePath(fixture.path)).toBe(0); + expect(gitWorktrees.removeWorktree).toHaveBeenCalledTimes(2); + expect(cleanupJobs.getDueCounts(Date.now(), fixture.path)).toEqual({ explicit: 0, retention: 0 }); + expect(await worker.processOneJob()).toBe(false); + expect(userRows()).toEqual(before); + } finally { + await worker.shutdown(); + } + }); + + it("leaves all copied rows untouched when the fixture path has no matching workspace", async () => { + const user = workspaces.create("Copied user project", "/user"); + addThread(user.id, "user-expired", "/user/expired", "expired", new Date(0).toISOString()); + const job = cleanupJobs.insert({ + thread_id: "user-orphan", workspace_path: user.path, worktree_path: null, branch: null, + }); + database.prepare("UPDATE cleanup_jobs SET attempts = 5 WHERE id = ?").run(job.id); + const threadBefore = threads.findById("user-expired"); + const jobBefore = cleanupJobs.findById(job.id); + + worker.start("/missing-fixture"); + try { + await worker.reconcileOnStartup(); + await worker.poll(); + expect(await worker.processOneJob()).toBe(false); + expect(threads.findById("user-expired")).toEqual(threadBefore); + expect(cleanupJobs.findById(job.id)).toEqual(jobBefore); + expect(cleanupJobs.count()).toBe(1); + expect(gitWorktrees.removeWorktree).not.toHaveBeenCalled(); + } finally { + await worker.shutdown(); + } + }); + it("deletes exhausted orphan job rows on the next poll after startup requeue", async () => { const job = cleanupJobs.insert({ thread_id: "thread-already-gone", diff --git a/apps/server/src/features/thread-control/cleanup/cleanup-worker.ts b/apps/server/src/features/thread-control/cleanup/cleanup-worker.ts index 1f918e76a..a3b18d9bc 100644 --- a/apps/server/src/features/thread-control/cleanup/cleanup-worker.ts +++ b/apps/server/src/features/thread-control/cleanup/cleanup-worker.ts @@ -65,6 +65,7 @@ export class CleanupWorker { private running = false; private stopped = false; private pollPromise: Promise | null = null; + private workspacePath: string | undefined; private readonly mutationReservations: ThreadControlMutationReservationService; constructor( @@ -89,10 +90,12 @@ export class CleanupWorker { } /** - * Start the worker and begin polling for due jobs. + * Start polling, optionally restricted to the agent runtime's fixture workspace. */ - start(): void { + start(workspacePath?: string): void { if (this.pollTimer !== null) return; + // Agent setup copies user projects into its database; cleanup must leave them untouched. + this.workspacePath = workspacePath; const removedArtifacts = pruneStaleToolOutputArtifacts(); if (removedArtifacts > 0) { logger.info("Pruned stale tool-output artifacts", { removed: removedArtifacts }); @@ -156,11 +159,13 @@ export class CleanupWorker { // that arrives during the async job execution sees running=true. this.running = true; try { - const retentionJobsEnqueued = this.cleanupJobRepo.enqueueExpiredCompleted(new Date().toISOString()); + const retentionJobsEnqueued = this.cleanupJobRepo.enqueueExpiredCompleted( + new Date().toISOString(), undefined, this.workspacePath, + ); const nowMs = Date.now(); - const jobs = this.cleanupJobRepo.findDue(nowMs); + const jobs = this.cleanupJobRepo.findDue(nowMs, undefined, this.workspacePath); if (jobs.length > 0 || retentionJobsEnqueued > 0) { - const dueCounts = this.cleanupJobRepo.getDueCounts(nowMs); + const dueCounts = this.cleanupJobRepo.getDueCounts(nowMs, this.workspacePath); const selectedExplicitJobs = jobs.filter((job) => job.kind === "explicit").length; const selectedRetentionJobs = jobs.length - selectedExplicitJobs; logger.info("CleanupWorker batch selected", { @@ -429,12 +434,13 @@ export class CleanupWorker { * hard-deletes it immediately. */ async reconcileOnStartup(): Promise { - const requeued = this.cleanupJobRepo.requeueExhaustedJobs(); + const requeued = this.cleanupJobRepo.requeueExhaustedJobs(this.workspacePath); if (requeued > 0) { logger.info("Requeued exhausted cleanup jobs", { requeued }); } - const deletingWorkspaces = this.workspaceRepo.findDeleting(); + const deletingWorkspaces = this.workspaceRepo.findDeleting() + .filter((workspace) => this.workspacePath === undefined || workspace.path === this.workspacePath); for (const ws of deletingWorkspaces) { const threads = this.threadRepo.listAllByWorkspace(ws.id); @@ -534,7 +540,7 @@ export class CleanupWorker { /** Process a single due cleanup job. Returns true if a job was processed. Exported for testing. */ async processOneJob(): Promise { - const jobs = this.cleanupJobRepo.findDue(Date.now()); + const jobs = this.cleanupJobRepo.findDue(Date.now(), undefined, this.workspacePath); if (jobs.length === 0) return false; await this.executeJob(jobs[0]); return true; diff --git a/apps/server/src/features/thread-control/cleanup/persistence/cleanup-job-repo.ts b/apps/server/src/features/thread-control/cleanup/persistence/cleanup-job-repo.ts index 07b8d38b8..90b4ac2d3 100644 --- a/apps/server/src/features/thread-control/cleanup/persistence/cleanup-job-repo.ts +++ b/apps/server/src/features/thread-control/cleanup/persistence/cleanup-job-repo.ts @@ -167,16 +167,16 @@ export class CleanupJobRepo { * Return jobs that are due to run: next_retry_at <= now and attempts < max. * Ordered by created_at ascending so oldest jobs are processed first. */ - findDue(nowMs: number, limit = CLEANUP_BATCH_LIMIT): CleanupJob[] { + findDue(nowMs: number, limit = CLEANUP_BATCH_LIMIT, workspacePath?: string): CleanupJob[] { const boundedLimit = Math.max(1, Math.min(100, Math.trunc(limit))); - const counts = this.getDueCounts(nowMs); + const counts = this.getDueCounts(nowMs, workspacePath); if (counts.explicit === 0 || counts.retention === 0) { - return this.findDueFromAvailableKind(nowMs, boundedLimit, counts); + return this.findDueFromAvailableKind(nowMs, boundedLimit, counts, workspacePath); } const limits = calculateDueLimits(counts, boundedLimit); - const explicitJobs = this.findDueByKind(nowMs, "explicit", limits.explicit); - const retentionJobs = this.findDueByKind(nowMs, "retention", limits.retention); + const explicitJobs = this.findDueByKind(nowMs, "explicit", limits.explicit, workspacePath); + const retentionJobs = this.findDueByKind(nowMs, "retention", limits.retention, workspacePath); return interleaveDueJobs(retentionJobs, explicitJobs, boundedLimit); } @@ -184,12 +184,13 @@ export class CleanupJobRepo { nowMs: number, limit: number, counts: CleanupJobDueCounts, + workspacePath?: string, ): CleanupJob[] { const kind = counts.retention > 0 ? "retention" : "explicit"; - return this.findDueByKind(nowMs, kind, limit); + return this.findDueByKind(nowMs, kind, limit, workspacePath); } - private findDueByKind(nowMs: number, kind: CleanupJob["kind"], limit: number): CleanupJob[] { + private findDueByKind(nowMs: number, kind: CleanupJob["kind"], limit: number, workspacePath?: string): CleanupJob[] { const rows = this.orm .select() .from(cleanupJobs) @@ -197,6 +198,7 @@ export class CleanupJobRepo { lte(cleanupJobs.nextRetryAt, nowMs), lt(cleanupJobs.attempts, MAX_CLEANUP_ATTEMPTS), eq(cleanupJobs.kind, kind), + workspacePath === undefined ? undefined : eq(cleanupJobs.workspacePath, workspacePath), )) .orderBy(asc(cleanupJobs.createdAt)) .limit(limit) @@ -205,7 +207,7 @@ export class CleanupJobRepo { } /** Return due cleanup job counts grouped by processing kind. */ - getDueCounts(nowMs: number): CleanupJobDueCounts { + getDueCounts(nowMs: number, workspacePath?: string): CleanupJobDueCounts { const counts: CleanupJobDueCounts = { explicit: 0, retention: 0 }; const rows = this.orm .select({ kind: cleanupJobs.kind, count: sql`COUNT(*)` }) @@ -214,6 +216,7 @@ export class CleanupJobRepo { lte(cleanupJobs.nextRetryAt, nowMs), lt(cleanupJobs.attempts, MAX_CLEANUP_ATTEMPTS), inArray(cleanupJobs.kind, ["explicit", "retention"]), + workspacePath === undefined ? undefined : eq(cleanupJobs.workspacePath, workspacePath), )) .groupBy(cleanupJobs.kind) .all(); @@ -222,7 +225,7 @@ export class CleanupJobRepo { } /** Queue a bounded set of expired completed threads in one database transaction. */ - enqueueExpiredCompleted(nowIso: string, limit = CLEANUP_BATCH_LIMIT): number { + enqueueExpiredCompleted(nowIso: string, limit = CLEANUP_BATCH_LIMIT, workspacePath?: string): number { const boundedLimit = Math.max(1, Math.min(100, Math.trunc(limit))); const due = this.orm .select({ @@ -240,6 +243,7 @@ export class CleanupJobRepo { isNotNull(threads.scheduledDeletionAt), lte(threads.scheduledDeletionAt, nowIso), isNull(threads.cleanupState), + workspacePath === undefined ? undefined : eq(workspaces.path, workspacePath), )) .orderBy(asc(threads.scheduledDeletionAt), asc(threads.id)) .limit(boundedLimit) @@ -316,12 +320,15 @@ export class CleanupJobRepo { * deleting workspaces. Orphaned rows self-clean on the next poll because * the worker deletes explicit jobs whose thread is gone. */ - requeueExhaustedJobs(): number { + requeueExhaustedJobs(workspacePath?: string): number { const result = runChanges( this.orm .update(cleanupJobs) .set({ attempts: 0, nextRetryAt: 0 }) - .where(gte(cleanupJobs.attempts, MAX_CLEANUP_ATTEMPTS)), + .where(and( + gte(cleanupJobs.attempts, MAX_CLEANUP_ATTEMPTS), + workspacePath === undefined ? undefined : eq(cleanupJobs.workspacePath, workspacePath), + )), ); return result.changes; } diff --git a/apps/server/src/features/thread-control/persistence/__tests__/thread-repo-settings.test.ts b/apps/server/src/features/thread-control/persistence/__tests__/thread-repo-settings.test.ts index 785ce0751..ebb3b3ea5 100644 --- a/apps/server/src/features/thread-control/persistence/__tests__/thread-repo-settings.test.ts +++ b/apps/server/src/features/thread-control/persistence/__tests__/thread-repo-settings.test.ts @@ -42,6 +42,53 @@ describe("ThreadRepo.updateSettings", () => { expect(thread?.permission_mode).toBeNull(); }); + it("persists model, provider, and supplied settings together while retaining omitted values", () => { + repo.updateSettings(threadId, { reasoning_level: "high", thinking: true, codex_fast_mode: true }); + + expect(repo.updateSettings(threadId, { + model: "gpt-5.6-luna", + provider: "codex", + permission_mode: "full", + thinking: false, + codex_fast_mode: null, + })).toBe(true); + + expect(repo.findById(threadId)).toMatchObject({ + model: "gpt-5.6-luna", + provider: "codex", + permission_mode: "full", + reasoning_level: "high", + thinking: false, + codex_fast_mode: null, + }); + repo.updateSettings(threadId, { model: "gpt-5.6-sol" }); + expect(repo.findById(threadId)).toMatchObject({ model: "gpt-5.6-sol", provider: "codex", permission_mode: "full" }); + }); + + it("leaves all thread settings unchanged when the combined update fails", () => { + repo.updateSettings(threadId, { model: "original-model", provider: "claude", permission_mode: "supervised" }); + const before = repo.findById(threadId); + db.exec(` + CREATE TRIGGER reject_full_permission BEFORE UPDATE OF permission_mode ON threads + WHEN NEW.permission_mode = 'full' + BEGIN + SELECT RAISE(ABORT, 'permission update rejected'); + END; + `); + + expect(() => repo.updateSettings(threadId, { + model: "gpt-5.6-luna", provider: "codex", permission_mode: "full", + })).toThrow("permission update rejected"); + + expect(repo.findById(threadId)).toEqual(before); + }); + + it("retains the timestamp when no settings are supplied", () => { + const before = repo.findById(threadId); + expect(repo.updateSettings(threadId, {})).toBe(false); + expect(repo.findById(threadId)).toEqual(before); + }); + it("returns false for nonexistent thread", () => { const ok = repo.updateSettings("nonexistent", { reasoning_level: "low" }); expect(ok).toBe(false); diff --git a/apps/server/src/features/thread-control/persistence/thread-repo.ts b/apps/server/src/features/thread-control/persistence/thread-repo.ts index 5ecf53657..ec760ce3e 100644 --- a/apps/server/src/features/thread-control/persistence/thread-repo.ts +++ b/apps/server/src/features/thread-control/persistence/thread-repo.ts @@ -307,6 +307,8 @@ function serializeBooleanOverride(value: boolean | null): number | null { } type ThreadSettings = { + model?: string; + provider?: string; reasoning_level?: string; interaction_mode?: string; orchestration_mode?: string; @@ -320,6 +322,8 @@ type ThreadSettings = { }; const TEXT_SETTING_COLUMNS = [ + ["model", "model"], + ["provider", "provider"], ["reasoning_level", "reasoningLevel"], ["interaction_mode", "interactionMode"], ["orchestration_mode", "orchestrationMode"], @@ -1007,7 +1011,7 @@ export class ThreadRepo { return result.changes > 0; } - /** Persist per-thread composer settings (reasoning, mode, permission, copilot agent). */ + /** Persist model, provider, and supplied composer settings in one atomic update. */ updateSettings(id: string, settings: ThreadSettings): boolean { const set = buildSettingsSet(settings); if (Object.keys(set).length === 0) return false; diff --git a/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts b/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts index a566df2db..c047fc4d2 100644 --- a/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts +++ b/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts @@ -1,9 +1,15 @@ import "reflect-metadata"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { gc } from "bun"; import { getDefaultSettings } from "@mcode/contracts"; import { MemoryPressureService } from "../memory-pressure-service.js"; import type { RuntimeMemoryMeasurement } from "../runtime-memory-sampler.js"; +vi.mock("bun", async (importOriginal) => ({ + ...await importOriginal(), + gc: vi.fn(), +})); + const db = { run: vi.fn() } as unknown as import("bun:sqlite").Database; function settingsWithHeapBudget(getHeapMb: () => number) { @@ -64,6 +70,18 @@ describe("MemoryPressureService", () => { expect(service.currentPressure.level).toBe("normal"); }); + it("requests collection once per pressure transition after notifying memory owners", () => { + service.markActive("thread-1"); + service.sampleActiveMemoryForTest(v8Measurement(85, 100)); + vi.mocked(gc).mockClear(); + const collectionsAtNotification: number[] = []; + service.onPressureChange(() => collectionsAtNotification.push(vi.mocked(gc).mock.calls.length)); + service.sampleActiveMemoryForTest(v8Measurement(95, 100)); + service.sampleActiveMemoryForTest(v8Measurement(96, 100)); + expect(collectionsAtNotification).toEqual([0]); + expect(gc).toHaveBeenCalledExactlyOnceWith(false); + }); + it("notifies listeners when a turn becomes idle under sustained pressure", () => { const levels: string[] = []; service.onPressureChange((snapshot) => { @@ -80,6 +98,31 @@ describe("MemoryPressureService", () => { expect(levels).toEqual(["warning"]); }); + it("reclaims elevated pressure after completion callbacks without forcing collection during peer work", async () => { + service.markActive("thread-1"); + service.markActive("thread-2"); + service.sampleActiveMemoryForTest(v8Measurement(85, 100)); + expect(gc).toHaveBeenCalledWith(false); + vi.mocked(gc).mockClear(); + + service.markIdle("thread-1"); + expect(gc).not.toHaveBeenCalled(); + service.markIdle("thread-2"); + expect(gc).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(0); + expect(gc).toHaveBeenCalledExactlyOnceWith(true); + }); + + it("does not force an idle collection when another turn starts before callbacks finish", async () => { + service.markActive("thread-1"); + service.sampleActiveMemoryForTest(v8Measurement(85, 100)); + service.markIdle("thread-1"); + service.markActive("thread-2"); + vi.mocked(gc).mockClear(); + await vi.advanceTimersByTimeAsync(0); + expect(gc).not.toHaveBeenCalled(); + }); + it("uses Bun RSS for thresholds, recovery, and updated settings", async () => { let heapMb = 256; const initialBudgetBytes = heapMb * 1024 * 1024; diff --git a/apps/server/src/runtime/memory/memory-pressure-service.ts b/apps/server/src/runtime/memory/memory-pressure-service.ts index f663a178a..408248ae5 100644 --- a/apps/server/src/runtime/memory/memory-pressure-service.ts +++ b/apps/server/src/runtime/memory/memory-pressure-service.ts @@ -7,6 +7,7 @@ */ import { injectable, inject } from "tsyringe"; +import { gc } from "bun"; import type { Database } from "bun:sqlite"; import { logger } from "@mcode/shared"; import { SettingsService } from "../../features/settings/settings-service.js"; @@ -141,6 +142,10 @@ export class MemoryPressureService { } if (!threadId && this.activeTurns.size > 0) return; this.stopActiveMemoryPolling(); + if (this.pressure.level !== "normal") { + // Completion callbacks still hold turn data until their stack unwinds. + setImmediate(() => { if (this.activeTurns.size === 0) gc(true); }); + } this.setPressure({ level: "normal", source: this.pressure.source, usedBytes: 0, budgetBytes: 0, ratio: 0 }); this.clearIdleTimers(); if (this.isWindowBackground) { @@ -249,6 +254,7 @@ export class MemoryPressureService { budgetBytes: snapshot.budgetBytes, }); this.notifyPressureListeners(snapshot); + if (snapshot.level !== "normal") gc(false); } private notifyPressureListeners(snapshot: MemoryPressureSnapshot): void { @@ -300,9 +306,7 @@ export class MemoryPressureService { error: err instanceof Error ? err.message : String(err), }); } - if (typeof global.gc === "function") { - global.gc(); - } + gc(false); // These synchronous ops block the event loop; the duration ties stalls to // the health-probe failures they can cause. logger.info("Warm idle maintenance completed", { @@ -326,9 +330,9 @@ export class MemoryPressureService { }); } const now = Date.now(); - if (typeof global.gc === "function" && now - this.lastFullGcAt > MIN_FULL_GC_INTERVAL_MS) { + if (now - this.lastFullGcAt > MIN_FULL_GC_INTERVAL_MS) { this.lastFullGcAt = now; - global.gc(true); + gc(true); } this.state = "background-idle"; logger.info("Background idle maintenance completed", { diff --git a/apps/server/src/runtime/persistence/sqlite/bounded-write-batches.ts b/apps/server/src/runtime/persistence/sqlite/bounded-write-batches.ts index a6df31afd..92dc26e97 100644 --- a/apps/server/src/runtime/persistence/sqlite/bounded-write-batches.ts +++ b/apps/server/src/runtime/persistence/sqlite/bounded-write-batches.ts @@ -37,6 +37,8 @@ export interface RunBoundedWriteBatchesInput { onBatchStarted?: () => void; onBatchFinishing?: () => void; onBatchCommitted?: (result: WriteBatchResult) => void; + /** Acquire the write lock before a batch reads, avoiding failed read-to-write upgrades across connections. */ + beginImmediate?: boolean; } function assertPositiveLimit(value: number, name: keyof WriteBatchLimits): void { @@ -72,7 +74,8 @@ export async function runBoundedWriteBatches( batchRows = result.batchRows; input.onBatchFinishing?.(); }); - transaction(); + if (input.beginImmediate) transaction.immediate(); + else transaction(); batches += 1; totalBytes += batchBytes; totalRows += batchRows; @@ -110,7 +113,8 @@ export function runBoundedWriteBatchesSync( batchRows = result.batchRows; input.onBatchFinishing?.(); }); - transaction(); + if (input.beginImmediate) transaction.immediate(); + else transaction(); batches += 1; totalBytes += batchBytes; totalRows += batchRows; diff --git a/apps/server/src/runtime/persistence/sqlite/schema.ts b/apps/server/src/runtime/persistence/sqlite/schema.ts index b026777a7..72b6cd142 100644 --- a/apps/server/src/runtime/persistence/sqlite/schema.ts +++ b/apps/server/src/runtime/persistence/sqlite/schema.ts @@ -5,7 +5,7 @@ import { sql } from "drizzle-orm"; import { asc, desc } from "drizzle-orm"; -import { type AnySQLiteColumn, index, integer, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core"; +import { type AnySQLiteColumn, index, integer, primaryKey, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core"; const timestampDefault = sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`; @@ -741,6 +741,31 @@ export const canonicalAgentEvents = sqliteTable( ], ); +/** Writer receipts survive a lost worker reply and follow their canonical turn on deletion. */ +export const canonicalWriterOperationReceipts = sqliteTable( + "canonical_writer_operation_receipts", + { + executionId: text("execution_id").notNull() + .references(() => canonicalAgentTurns.executionId, { onDelete: "cascade" }), + operationId: text("operation_id").notNull(), + kind: text("kind").notNull(), + inputHash: text("input_hash").notNull(), + receiptJson: text("receipt_json").notNull(), + createdAt: text("created_at").notNull().default(timestampDefault), + }, + (table) => [primaryKey({ columns: [table.executionId, table.operationId] })], +); + +/** Monotonic live publication identity survives execution changes and receipt pruning. */ +export const canonicalWriterLivePublicationHeads = sqliteTable( + "canonical_writer_live_publication_heads", + { + threadId: text("thread_id").primaryKey().notNull() + .references(() => threads.id, { onDelete: "cascade" }), + lastSequence: integer("last_sequence").notNull(), + }, +); + /** Durable accepted and committed progress for one canonical execution. */ export const canonicalAgentIngestCheckpoints = sqliteTable( "canonical_agent_ingest_checkpoints", diff --git a/apps/server/tsconfig.json b/apps/server/tsconfig.json index 21045ad40..e099d1c96 100644 --- a/apps/server/tsconfig.json +++ b/apps/server/tsconfig.json @@ -13,7 +13,7 @@ "declaration": true, "declarationMap": true, "sourceMap": true, - "types": ["bun-types/sqlite"], + "types": ["bun-types/sqlite", "bun-types/bun"], "noUnusedLocals": true, "noUnusedParameters": true, "noFallthroughCasesInSwitch": true, diff --git a/apps/web/src/__tests__/agent-event-branches.test.ts b/apps/web/src/__tests__/agent-event-branches.test.ts index dfb45def1..792e1091f 100644 --- a/apps/web/src/__tests__/agent-event-branches.test.ts +++ b/apps/web/src/__tests__/agent-event-branches.test.ts @@ -515,6 +515,76 @@ describe("handleAgentEvent branches", () => { .toBe("00000000-0000-4000-8000-000000000002"); }); + it("does not reapply a stable publication after thread rehydration and a server restart", () => { + const threadId = "thread-stable-publication"; + const executionId = "00000000-0000-4000-8000-000000000001"; + const publicationId = "2"; + const firstEpoch = "00000000-0000-4000-8000-000000000002"; + const nextEpoch = "00000000-0000-4000-8000-000000000003"; + const key = `mcode-agent-publication-v2:${threadId}`; + sessionStorage.removeItem(key); + try { + resetThreadStoreForTests({ currentThreadId: threadId, + runningThreadIds: new Set([threadId]), + records: new Map([[threadId, { ...createEmptyThreadRecord(), runtimePhase: "running", + turnExecutionId: executionId }]]), + }); + const original: Extract = { type: "toolUse", threadId, turnExecutionId: executionId, + publicationId, epoch: firstEpoch, sequence: 8, toolCallId: "stable-call", + toolName: "Read", toolInput: { path: "/original" } }; + useThreadStore.getState().handleAgentEvent(original); + const persistedCalls = readThreadField(threadId, (record) => record.toolCalls); + expect(persistedCalls).toHaveLength(1); + + resetThreadStoreForTests({ currentThreadId: threadId, + runningThreadIds: new Set([threadId]), + records: new Map([[threadId, { ...createEmptyThreadRecord(), runtimePhase: "running", + turnExecutionId: executionId, toolCalls: persistedCalls }]]), + }); + useThreadStore.getState().handleAgentEvent({ ...original, epoch: nextEpoch, + sequence: 1, toolInput: { path: "/replayed" } }); + expect(readThreadField(threadId, (record) => record.toolCalls)).toEqual(persistedCalls); + expect(readThreadField(threadId, (record) => record.lastAgentEventEpoch)).toBe(nextEpoch); + useThreadStore.getState().handleAgentEvent({ ...original, publicationId: undefined, + epoch: nextEpoch, sequence: 2, toolCallId: "legacy-call" }); + expect(readThreadField(threadId, (record) => record.toolCalls)).toHaveLength(2); + } finally { + sessionStorage.removeItem(key); + } + }); + + it("does not repeat terminal projection after a stable completion is replayed", () => { + const threadId = "thread-stable-terminal"; + const executionId = "00000000-0000-4000-8000-000000000011"; + const key = `mcode-agent-publication-v2:${threadId}`; + sessionStorage.removeItem(key); + try { + resetThreadStoreForTests({ currentThreadId: threadId, + runningThreadIds: new Set([threadId]), + records: new Map([[threadId, { ...createEmptyThreadRecord(), runtimePhase: "running", + turnExecutionId: executionId, streaming: "completed answer" }]]), + }); + const completion = { type: "turnComplete", threadId, turnExecutionId: executionId, + publicationId: "3", sequence: 3, + epoch: "00000000-0000-4000-8000-000000000012", reason: "end_turn", + costUsd: null, tokensIn: 0, tokensOut: 0 } as AgentEvent; + useThreadStore.getState().handleAgentEvent(completion); + const completed = readThreadField(threadId, (record) => record); + const messageCount = completed.messages.length; + expect(messageCount).toBeGreaterThan(0); + + resetThreadStoreForTests({ currentThreadId: threadId, + runningThreadIds: new Set(), records: new Map([[threadId, completed]]), + }); + useThreadStore.getState().handleAgentEvent({ ...completion, + epoch: "00000000-0000-4000-8000-000000000013", sequence: 1 }); + expect(readThreadField(threadId, (record) => record.messages)).toHaveLength(messageCount); + expect(readThreadField(threadId, (record) => record.runtimePhase)).toBe(completed.runtimePhase); + } finally { + sessionStorage.removeItem(key); + } + }); + it("retains hook progress for an inactive thread", () => { const backgroundThreadId = "thread-background-hook"; resetThreadStoreForTests({ diff --git a/apps/web/src/__tests__/threadStore-followup-runtime.test.ts b/apps/web/src/__tests__/threadStore-followup-runtime.test.ts new file mode 100644 index 000000000..2e5640d02 --- /dev/null +++ b/apps/web/src/__tests__/threadStore-followup-runtime.test.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createEmptyThreadRecord } from "@/stores/thread-record"; +import { resetThreadStoreForTests } from "@/stores/thread-store-test-utils"; +import { useThreadStore } from "@/stores/threadStore"; +import { useWorkspaceStore } from "@/features/projects/state/workspaceStore"; +import { createMockThread, mockTransport } from "./mocks/transport"; + +vi.mock("@/transport", async () => ({ + ...(await vi.importActual("@/transport")), + getTransport: () => mockTransport, +})); + +const THREAD_ID = "followup-runtime"; +const OTHER_THREAD_ID = "other-runtime"; + +function record() { + return useThreadStore.getState().records.get(THREAD_ID); +} + +function completeFirstTurn(): void { + const handle = useThreadStore.getState().handleAgentEvent; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "first" }); + handle({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: "first", delta: "First reply" }); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "first", outcome: "completed" }); + expect(record()?.runtimePhase).toBe("completed"); +} + +describe("follow-up runtime ownership", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.mocked(mockTransport.sendMessage).mockResolvedValue(undefined); + useWorkspaceStore.setState({ + activeThreadId: THREAD_ID, + threads: [createMockThread({ id: THREAD_ID }), createMockThread({ id: OTHER_THREAD_ID })], + }); + resetThreadStoreForTests({ + currentThreadId: THREAD_ID, + runningThreadIds: new Set(), + records: new Map([ + [THREAD_ID, createEmptyThreadRecord()], + [OTHER_THREAD_ID, createEmptyThreadRecord()], + ]), + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it("completes a follow-up when the publication cursor write fails", async () => { + completeFirstTurn(); + await useThreadStore.getState().sendMessage(THREAD_ID, "Second prompt"); + const original = Storage.prototype.setItem; + vi.spyOn(Storage.prototype, "setItem").mockImplementation(function (this: Storage, key, value) { + if (key.startsWith("mcode-agent-publication-v2:")) throw new DOMException("quota", "QuotaExceededError"); + original.call(this, key, value); + }); + const handle = useThreadStore.getState().handleAgentEvent; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "second", sequence: 1, publicationId: "1" }); + handle({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: "second", sequence: 2, publicationId: "2", delta: "Visible reply" }); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "second", sequence: 3, publicationId: "3", outcome: "completed" }); + expect(record()?.runtimePhase).toBe("completed"); + expect(record()?.messages.map((message) => message.content)).toContain("Visible reply"); + expect(useThreadStore.getState().runningThreadIds.has(THREAD_ID)).toBe(false); + }); + + it.each([false, true])("completes a sent follow-up without file-effect metadata, background=%s", async (background) => { + completeFirstTurn(); + expect(await useThreadStore.getState().sendMessage(THREAD_ID, "Second prompt")).toBe(true); + expect(record()?.turnExecutionId).toBeNull(); + if (background) { + useThreadStore.setState({ currentThreadId: OTHER_THREAD_ID }); + useWorkspaceStore.setState({ activeThreadId: OTHER_THREAD_ID }); + } + + const handle = useThreadStore.getState().handleAgentEvent; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "second" }); + expect(record()?.turnExecutionId).toBe("second"); + handle({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: "second", delta: "Second reply" }); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "second", outcome: "completed" }); + + expect(record()?.runtimePhase).toBe("completed"); + expect(useThreadStore.getState().runningThreadIds.has(THREAD_ID)).toBe(false); + expect(record()?.messages.map((message) => message.content)).toContain("Second reply"); + expect(record()?.messages.map((message) => message.content)).toContain("First reply"); + expect(useThreadStore.getState().records.get(OTHER_THREAD_ID)?.runtimePhase).toBe("idle"); + }); + + it("retains the follow-up response on a duplicate start and ignores the previous turn's completion", async () => { + completeFirstTurn(); + await useThreadStore.getState().sendMessage(THREAD_ID, "Second prompt"); + const handle = useThreadStore.getState().handleAgentEvent; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "second" }); + handle({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: "second", delta: "Kept reply" }); + handle({ type: "contextEstimate", threadId: THREAD_ID, turnExecutionId: "second", tokensIn: 20 }); + const responseKey = record()?.currentTurnResponseKey; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "second" }); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "first", outcome: "completed" }); + + expect(record()?.turnExecutionId).toBe("second"); + expect(record()?.runtimePhase).toBe("running"); + expect(record()?.currentTurnResponseKey).toBe(responseKey); + expect(record()?.streaming).toBe("Kept reply"); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "second", outcome: "completed" }); + expect(record()?.runtimePhase).toBe("completed"); + }); +}); diff --git a/apps/web/src/__tests__/threadStore-turn-persist.test.ts b/apps/web/src/__tests__/threadStore-turn-persist.test.ts index 27894a2e3..48feaf163 100644 --- a/apps/web/src/__tests__/threadStore-turn-persist.test.ts +++ b/apps/web/src/__tests__/threadStore-turn-persist.test.ts @@ -450,6 +450,38 @@ describe("handleTurnPersisted", () => { expect(readThreadField(THREAD_ID, (record) => record.streaming)).toBe(""); }); + it("settles a stopped turn before any assistant message was written", () => { + const user = createMockMessage({ + id: "user-stop", + thread_id: THREAD_ID, + role: "user", + content: "Stop this turn", + }); + useThreadStore.setState({ + records: seedThreadRecord(THREAD_ID, { + messages: [user], + runtimePhase: "finalizing", + turnExecutionId: "execution-early-stop", + awaitingUserStopPersist: true, + }), + runningThreadIds: new Set([THREAD_ID]), + }); + + useThreadStore.getState().handleTurnPersisted({ + threadId: THREAD_ID, + messageId: null, + toolCallCount: 0, + filesChanged: [], + outcome: "cancelled", + executionId: "execution-early-stop", + }); + + expect(readThreadField(THREAD_ID, (record) => record.messages)).toEqual([user]); + expect(readThreadField(THREAD_ID, (record) => record.awaitingUserStopPersist)).toBeUndefined(); + expect(readThreadField(THREAD_ID, (record) => record.runtimePhase)).toBe("cancelled"); + expect(useThreadStore.getState().runningThreadIds.has(THREAD_ID)).toBe(false); + }); + it("materializes buffered streaming text when a terminal runtime snapshot arrives without a terminal event", () => { useThreadStore.setState({ records: seedThreadRecord(THREAD_ID, { diff --git a/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts b/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts new file mode 100644 index 000000000..01efb4692 --- /dev/null +++ b/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts @@ -0,0 +1,98 @@ +import type { AgentEvent } from "@mcode/contracts"; +import { describe, expect, it } from "vitest"; +import { StableAgentEventPublications } from "../stable-agent-event-publications"; + +const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; +const NEXT_EXECUTION_ID = "00000000-0000-4000-8000-000000000002"; + +function event(sequence: number, executionId = EXECUTION_ID): AgentEvent { + return { type: "toolUse", threadId: "thread-publication", turnExecutionId: executionId, + publicationId: String(sequence), toolCallId: `call-${sequence}`, + toolName: "Read", toolInput: {} }; +} + +function storage(): Pick { + const items = new Map(); + return { + getItem: (key) => items.get(key) ?? null, + setItem: (key, value) => { items.set(key, value); }, + }; +} + +describe("stable AgentEvent publications", () => { + it("rejects a publish-then-crash replay after recreating the client and changing server epoch", () => { + const shared = storage(); + const firstClient = new StableAgentEventPublications(() => shared); + const published = event(7); + expect(firstClient.accept({ ...published, epoch: "00000000-0000-4000-8000-000000000004" })) + .toBe(true); + const reloadedClient = new StableAgentEventPublications(() => shared); + expect(reloadedClient.accept({ ...published, epoch: "00000000-0000-4000-8000-000000000005" })) + .toBe(false); + expect(reloadedClient.accept(event(8))).toBe(true); + expect(reloadedClient.accept(event(7))).toBe(false); + }); + + it("rejects an arbitrarily late prior-execution replay after an all-day stream", () => { + const shared = storage(); + const cursor = new StableAgentEventPublications(() => shared); + for (let sequence = 1; sequence <= 8_193; sequence++) { + expect(cursor.accept(event(sequence))).toBe(true); + } + expect(cursor.accept(event(8_194, NEXT_EXECUTION_ID))).toBe(true); + expect(cursor.accept(event(1))).toBe(false); + expect(cursor.accept(event(8_193))).toBe(false); + }); + + it("lets two independent tabs each apply the same publication", () => { + const firstStore = storage(); + const secondStore = storage(); + const firstTab = new StableAgentEventPublications(() => firstStore); + const secondTab = new StableAgentEventPublications(() => secondStore); + expect(firstTab.accept(event(1))).toBe(true); + expect(secondTab.accept(event(1))).toBe(true); + expect(firstTab.accept(event(1))).toBe(false); + expect(secondTab.accept(event(1))).toBe(false); + }); + + it("applies each publication once when its cursor write fails", () => { + const cursor = new StableAgentEventPublications(() => ({ + getItem: () => null, + setItem: () => { throw new Error("quota"); }, + })); + expect(cursor.accept(event(1))).toBe(true); + expect(cursor.accept(event(1))).toBe(false); + expect(cursor.accept(event(2))).toBe(true); + expect(cursor.accept(event(1))).toBe(false); + }); + + it("retains a failed write over the older stored cursor and persists again after recovery", () => { + const shared = storage(); + let failWrites = false; + const cursor = new StableAgentEventPublications(() => ({ + getItem: shared.getItem, + setItem: (key, value) => { + if (failWrites) throw new Error("quota"); + shared.setItem(key, value); + }, + })); + expect(cursor.accept(event(6))).toBe(true); + failWrites = true; + expect(cursor.accept(event(7))).toBe(true); + expect(cursor.accept(event(7))).toBe(false); + failWrites = false; + expect(cursor.accept(event(8))).toBe(true); + const reloaded = new StableAgentEventPublications(() => shared); + expect(reloaded.accept(event(7))).toBe(false); + expect(reloaded.accept(event(8))).toBe(false); + expect(reloaded.accept(event(9))).toBe(true); + }); + + it("fails closed on an oversized stored cursor", () => { + const cursor = new StableAgentEventPublications(() => ({ + getItem: () => "1".repeat(17), + setItem: () => { throw new Error("unexpected write"); }, + })); + expect(cursor.accept(event(1))).toBe(false); + }); +}); diff --git a/apps/web/src/stores/thread-store/agent-event-preflight.ts b/apps/web/src/stores/thread-store/agent-event-preflight.ts index 5bd7a6ce3..f5df6ca2c 100644 --- a/apps/web/src/stores/thread-store/agent-event-preflight.ts +++ b/apps/web/src/stores/thread-store/agent-event-preflight.ts @@ -23,6 +23,7 @@ export type AgentEventHandlerTable = { /** Narrow store operations needed to validate and sequence one agent event. */ export interface AgentEventPreflightContext { clearApiRetry: (threadId: string) => void; + acceptPublication?: (event: AgentEvent) => boolean; flushPendingTextDeltas: () => void; getCurrentThreadId: () => string | null; getRecord: (threadId: string) => ThreadRecord; @@ -187,6 +188,7 @@ export function prepareAgentEvent( const incomingExecutionId = eventExecutionId(event); if (!acceptsExecution(event, runtimeRecord, incomingExecutionId)) return null; if (!acceptsSequence(context, event)) return null; + if (context.acceptPublication && !context.acceptPublication(event)) return null; const currentThreadId = context.getCurrentThreadId(); const activeThread = isActiveThread( diff --git a/apps/web/src/stores/thread-store/stable-agent-event-publications.ts b/apps/web/src/stores/thread-store/stable-agent-event-publications.ts new file mode 100644 index 000000000..16b518063 --- /dev/null +++ b/apps/web/src/stores/thread-store/stable-agent-event-publications.ts @@ -0,0 +1,64 @@ +import type { AgentEvent } from "@mcode/contracts"; + +const STORAGE_PREFIX = "mcode-agent-publication-v2:"; +const MAX_SEQUENCE_BYTES = 16; + +type PublicationStorage = Pick; + +function publicationSequence(value: string): number | null { + if (value.length > MAX_SEQUENCE_BYTES || !/^[1-9]\d*$/.test(value)) return null; + const sequence = Number(value); + return Number.isSafeInteger(sequence) ? sequence : null; +} + +/** Retains a per-thread cursor, surviving reloads when storage writes succeed. */ +export class StableAgentEventPublications { + private readonly pendingWrites = new Map(); + private warnedAboutWriteFailure = false; + + constructor(private readonly storage: () => PublicationStorage | undefined) {} + + /** Reserve a publication before applying its UI effects, retaining failed writes in memory. */ + accept(event: AgentEvent): boolean { + if (!event.publicationId) return true; + if (!event.turnExecutionId || publicationSequence(event.publicationId) === null) return false; + try { + return this.reserve(event.threadId, event.publicationId); + } catch { + return false; + } + } + + private reserve(threadId: string, publicationId: string): boolean { + const previous = this.pendingWrites.get(threadId) ?? this.readCursor(threadId); + if (previous === null || Number(publicationId) <= previous) return false; + this.pendingWrites.set(threadId, Number(publicationId)); + this.persistCursor(threadId, publicationId); + return true; + } + + private readCursor(threadId: string): number | null { + const store = this.storage(); + if (!store) return null; + const stored = store.getItem(`${STORAGE_PREFIX}${threadId}`); + return stored === null ? 0 : publicationSequence(stored); + } + + private persistCursor(threadId: string, publicationId: string): void { + try { + const store = this.storage(); + if (!store) throw new Error("Publication cursor storage is unavailable"); + store.setItem(`${STORAGE_PREFIX}${threadId}`, publicationId); + this.pendingWrites.delete(threadId); + } catch { + if (this.warnedAboutWriteFailure) return; + this.warnedAboutWriteFailure = true; + console.warn("Live event cursor could not be saved. Duplicate protection remains active in this renderer; reload recovery may replay older events."); + } + } +} + +/** Browser-tab cursor with constant retained bytes per thread. */ +export const stableAgentEventPublications = new StableAgentEventPublications( + () => typeof sessionStorage === "undefined" ? undefined : sessionStorage, +); diff --git a/apps/web/src/stores/threadStore.ts b/apps/web/src/stores/threadStore.ts index f7ada856d..e4616371e 100644 --- a/apps/web/src/stores/threadStore.ts +++ b/apps/web/src/stores/threadStore.ts @@ -78,6 +78,7 @@ import { prepareAgentEvent, type AgentEventHandlerTable, } from "./thread-store/agent-event-preflight"; +import { stableAgentEventPublications } from "./thread-store/stable-agent-event-publications"; import { hydrateRunningThreads as hydrateRunningThreadRecords, transferThreadRuntime as transferOptimisticThreadRuntime, @@ -242,7 +243,7 @@ interface ThreadState { /** Handle server-side tool call persistence confirmation. */ handleTurnPersisted: (payload: { threadId: string; - messageId: string; + messageId: string | null; turnId?: string | null; executionId?: string | null; outcome?: TurnOutcome | null; @@ -1384,7 +1385,9 @@ export const useThreadStore = create((zustandSet, get) => { ): void => { const fileEffectTurnId = typeof event.fileEffectTurnId === "string" ? event.fileEffectTurnId : ""; const record = getRec(event.threadId); - if (get().runningThreadIds.has(event.threadId) && record.fileEffectTurnId === fileEffectTurnId) return; + const sameExecution = runtime.incomingExecutionId === undefined + || record.turnExecutionId === runtime.incomingExecutionId; + if (get().runningThreadIds.has(event.threadId) && sameExecution && record.fileEffectTurnId === fileEffectTurnId) return; clearStreamingTextUsage(event.threadId); useTaskStore.getState().prepareTaskBubbleForNewTurn(event.threadId); patchRec(event.threadId, (current) => ({ @@ -2553,6 +2556,16 @@ export const useThreadStore = create((zustandSet, get) => { currentThreadId: string | null, terminalPhase: ThreadRecord["runtimePhase"] | undefined, ): Partial => { + if (payload.messageId === null) { + return { + ...turnPersistStopState(record, payload), + ...(terminalPhase ? terminalRuntimePatch(record, terminalPhase) : {}), + ...(payload.fileEffects && payload.turnId === record.fileEffectTurnId + && payload.fileEffects.revision >= record.fileEffectSummary.revision + ? { fileEffectSummary: payload.fileEffects } + : {}), + }; + } const localMessageId = resolveTurnPersistLocalMessageId(record, payload.messageId); const messages = persistedTurnMessages(record, payload, localMessageId); const stopState = turnPersistStopState(record, payload); @@ -3737,6 +3750,7 @@ export const useThreadStore = create((zustandSet, get) => { handleAgentEvent: (event) => { if (!hasAgentEventHandler(agentEventHandlers, event)) return; const runtime = prepareAgentEvent({ + acceptPublication: (incoming) => stableAgentEventPublications.accept(incoming), clearApiRetry: (id) => patchRec(id, { apiRetry: undefined }), flushPendingTextDeltas, getCurrentThreadId: () => get().currentThreadId, diff --git a/apps/web/src/transport/ws-events.ts b/apps/web/src/transport/ws-events.ts index 1e9cba13e..2a8c4f556 100644 --- a/apps/web/src/transport/ws-events.ts +++ b/apps/web/src/transport/ws-events.ts @@ -468,16 +468,9 @@ export function startPushListeners(): void { // turn.persisted: server has persisted tool calls for a completed turn unsubs.push( pushEmitter.on("turn.persisted", (data) => { - const payload = data as { - threadId: string; - turnId?: string | null; - executionId?: string | null; - messageId: string; - outcome?: "completed" | "cancelled" | "interrupted" | "errored" | null; - toolCallCount: number; - filesChanged: string[]; - fileEffects?: TurnFileEffectSummary; - }; + const parsed = WS_CHANNELS["turn.persisted"].safeParse(data); + if (!parsed.success) return; + const payload = parsed.data; useThreadStore.getState().handleTurnPersisted(payload); refreshTurnSnapshotsAfterPersist(payload.threadId, payload.filesChanged); diff --git a/docs/internals/narrative-pipeline.md b/docs/internals/narrative-pipeline.md index 93299334f..88635c1a6 100644 --- a/docs/internals/narrative-pipeline.md +++ b/docs/internals/narrative-pipeline.md @@ -38,9 +38,10 @@ the specific traps we hit so the next person doesn't trip on them. If you are about to touch any of these files, **read this first**: - `apps/server/src/features/providers/` (runtime event source and provider projection) -- `apps/server/src/features/agents/conversation/narrative/narrative-store.ts` (write seam: enrichment + - classification + persistence; owns the per-turn buffers and the - `agentCallStack`. Also owns the read seam, `load`.) +- `apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts` (per-turn + buffers, `agentCallStack`, enrichment, classification, and recovery snapshot) +- `apps/server/src/features/agents/conversation/narrative/narrative-store.ts` (SQLite + persistence, data-only effects from turn state, and the read seam, `load`) - `apps/server/src/features/agents/orchestration/agent-service.ts` (turn orchestration; delegates the write seam to NarrativeStore and retains turn-level concerns: turn snapshots, `turn.persisted` broadcast, and late-hook flushing) @@ -127,21 +128,12 @@ Provider-native event Provider runtime event keeps native evidence separate from AgentEvent data │ ▼ -Provider ingress selects a thread-affine worker for cloneable validation - │ - ▼ -Provider ingress queues accepted results fairly and selects a provider adapter - │ - ▼ -Adapter forwards a provider-neutral AgentEvent or consumes private provider work - │ - ▼ -Turn event pipeline ToolUse handler → narrative-store.ts bufferToolCall - (writes to the per-turn tool-call buffer for later persist) - │ - ▼ -Turn event pipeline enriches missing parentToolCallId via - narrativeStore.getCurrentParentToolCallId (agentCallStack fallback) +Provider event ownership selects the admitted execution's route + ├─ Worker-owned: ordered task mailbox → parent event and file reduction + │ → single execution writer commits event, turn effects, and publication receipt + │ → server releases the acknowledged AgentEvent + └─ Legacy: canonical commit → provider ingress and adapter + → turn event pipeline and parent turn application │ ▼ broadcast("agent.event", enrichedEvent) @@ -262,7 +254,7 @@ SDK doesn't surface this field (older paths, edge cases). - `index.ts` checks `if (event.parentToolCallId)` first — if set, leave it. Only falls back to `narrativeStore.getCurrentParentToolCallId` when the SDK omitted it. -- `narrative-store.ts bufferToolCall` does the same dance for the persistence +- `narrative-turn-state.ts bufferToolCall` does the same dance for the persistence buffer: SDK value wins, stack is a fallback. (AgentService's `bufferToolCall` is a thin wrapper that delegates here, then persists TodoWrite task state.) @@ -300,7 +292,7 @@ child `toolUse` events to lose their fallback parent ID. 3. When the session ends. **Never on `textDelta`. Never on streaming events.** The stack now lives on -`narrative-store.ts`; its `openOrExtendThought` (the textDelta path) never +`narrative-turn-state.ts`; its `openOrExtendThought` (the textDelta path) never touches `agentCallStack`. See the explanatory comment in the AgentService `TextDelta` handler. diff --git a/docs/internals/provider-architecture.md b/docs/internals/provider-architecture.md index a0d45c2df..eb9431815 100644 --- a/docs/internals/provider-architecture.md +++ b/docs/internals/provider-architecture.md @@ -72,32 +72,33 @@ A provider emits a `ProviderRuntimeEvent`. Its `event` is provider-neutral data that may reach the renderer. Its optional extension contains provider-native evidence that must not reach the renderer. -The server accepts runtime events in this order: - -```text -Provider runtime event → provider ingress → event worker → provider adapter → turn event pipeline → AgentEvent -``` - -Ingress validates the provider identity and sends cloneable event validation to -a fixed worker pool. A thread stays on one worker until its queued work drains; -the pool limits concurrent work and returns results in that thread's order. -Ingress then queues accepted results fairly across threads and preserves the -receipt when it came from a canonical commit. It waits for both the worker and -the ingress queue before finalizing a turn. The worker does not own the -database: canonical commits and turn state still run in the server process. -An adapter may forward a generic event, consume private provider work, or -reject malformed native evidence with a diagnostic. Only a forwarded -`AgentEvent` enters narration, lifecycle, and -renderer publication. +An admitted worker-owned execution sends its canonical provider drafts to the +execution's ordered mailbox. One of a fixed number of workers owns that task +for the execution lifetime. It reduces parent events and file observations, +then awaits the single execution SQLite writer for the event, turn state, and +publication receipt. The server publishes the committed result without +applying that event again through `ProviderTurnEventApplication`. Stop, +terminalization, and late events use the same execution fence. The mailbox has +bounded capacity and schedules tasks fairly when they share a worker. +Worker ownership currently admits user-dispatched Codex turns. Claude, Cursor, +and provider-originated resumes still use the legacy route. + +Executions without a worker owner still use provider ingress, cloneable event +preprocessing, the provider adapter, and `TurnEventPipeline`. That route +preserves the existing canonical receipt and finalization fence. A provider +adapter may forward a generic event, consume private native evidence, or +reject malformed evidence with a diagnostic. Only a forwarded `AgentEvent` +reaches the renderer. Codex collaboration evidence uses a Codex adapter. Claude, Cursor, and Copilot send generic runtime events and do not invoke that adapter. A new provider adds an adapter only when it has private native evidence that requires server-side projection. -Canonical commits hand their committed runtime envelopes directly to ingress. -The resulting receipt means durable acceptance or queueing. It never means that -the event was published to the renderer. +Canonical commits on the legacy route hand their committed runtime envelopes +to ingress. A worker-owned receipt instead covers the parent event and its +turn effects together. Neither receipt alone proves the renderer displayed the +event. `AgentService` owns provider selection and narrow parent-turn durability. It does not import a concrete provider, a provider adapter, or a canonical diff --git a/docs/performance/message-list-baseline.md b/docs/performance/message-list-baseline.md deleted file mode 100644 index 4335da3b6..000000000 --- a/docs/performance/message-list-baseline.md +++ /dev/null @@ -1,180 +0,0 @@ -# Message List Performance Baseline - -## Scope - -This report records the final issue #1546 baseline from these raw artifacts only: - -- `.dev/verification/performance/issue-1546-profiling-final.json` -- `.dev/verification/performance/issue-1546-production-final.json` - -It records no candidate win and no dependency change. The pinned versions remain `@tanstack/react-virtual` 3.13.23 and `@tanstack/virtual-core` 3.13.23. - -## Run contract - -| Field | Value | -| --- | --- | -| Source revision | `250a153964347c7792b9a48ff39802c760540a48` | -| Source dirty | `true` in both artifacts and both runtimes | -| Viewport | 1440 x 1000 | -| Warmups | 1 | -| Raw samples | 7 per workload and runtime | -| Workload order | `message100`, `message1000`, `threadSwitch`, `streaming`, `messageListBehavior`, `denseNarrative`, `markdownShiki`, `panelTransitions` | - -Overall, profiling accepted 107 samples and rejected 5. Production accepted 110 samples and rejected 2. - -```powershell -$env:PLAYWRIGHT_BROWSERS_PATH = Join-Path (Resolve-Path '.dev') 'playwright-browsers'; -bun run perf:frontend -- --mode profiling --sample-count 7 --output .dev/verification/performance/issue-1546-profiling-final.json - -$env:PLAYWRIGHT_BROWSERS_PATH = Join-Path (Resolve-Path '.dev') 'playwright-browsers'; -bun run perf:frontend -- --mode production --sample-count 7 --output .dev/verification/performance/issue-1546-production-final.json -``` - -## Environment and runtime signals - -The artifacts record Windows `win32` release `10.0.26200`, an x64 12th Gen Intel(R) Core(TM) i7-12700 with 20 CPUs, and 68,373,008,384 bytes of host memory. They record Node 22.16.0, Electron 35.7.5, and Playwright 1.62.1. - -| Runtime | Browser identity | Reported device memory | Fixed acceleration field | -| --- | --- | ---: | --- | -| standalone-web | HeadlessChrome 151.0.7922.34 | 32 GiB | `null` | -| electron | Chrome 134.0.6998.205 in Electron 35.7.5 | 8 GiB | `false` | - -Profiling records no Chromium trace object for either runtime, so ResizeObserver, GC, layout, and paint trace values are unavailable there. The production Electron process signal is available and records `accelerationMode: "disabled"`; the standalone-web process signal is `null`. OS GPU counters are unavailable in both runtimes with the signal: `OS GPU counters are not available in the paired un-packaged runner`. - -## Behavior baseline - -All required MessageList checks passed except click-driven sticky jump to an initially unmounted row, which missed the 3.6s observation window in 5/14 profiling samples and 2/14 production samples; focused runs could pass, so this is load-sensitive current-baseline behavior, not a claimed migration result. - -The cache contract exercised by this baseline is: - -- A cache hit restores the saved reading anchor. -- Simulated cache eviction forgets scroll memory, holds the outgoing transcript during the cold load, then positions the restored transcript at the tail. - -Each cell gives accepted/rejected raw samples. All seven samples ran for every workload and runtime. - -| Workload | Profiling standalone-web | Profiling Electron | Production standalone-web | Production Electron | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 7/0 | 7/0 | 7/0 | -| message1000 | 7/0 | 7/0 | 7/0 | 7/0 | -| threadSwitch | 7/0 | 7/0 | 7/0 | 7/0 | -| streaming | 7/0 | 7/0 | 7/0 | 7/0 | -| messageListBehavior | 3/4 | 6/1 | 6/1 | 6/1 | -| denseNarrative | 7/0 | 7/0 | 7/0 | 7/0 | -| markdownShiki | 7/0 | 7/0 | 7/0 | 7/0 | -| panelTransitions | 7/0 | 7/0 | 7/0 | 7/0 | - -## Workload durations - -All values are milliseconds. The artifact summaries include accepted samples only, so the `messageListBehavior` summaries have three or six samples as shown by their accepted counts. - -### Profiling, standalone-web - -| Workload | Accepted/rejected | Min | Median | Max | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 120.3 | 127.6 | 181.2 | -| message1000 | 7/0 | 173.7 | 195.8 | 212.0 | -| threadSwitch | 7/0 | 96.3 | 112.7 | 137.6 | -| streaming | 7/0 | 16721.2 | 16737.8 | 16781.8 | -| messageListBehavior | 3/4 | 2664.8 | 2734.0 | 2739.7 | -| denseNarrative | 7/0 | 24.8 | 29.4 | 31.7 | -| markdownShiki | 7/0 | 3997.9 | 4063.1 | 4651.0 | -| panelTransitions | 7/0 | 56.0 | 58.3 | 65.8 | - -### Profiling, Electron - -| Workload | Accepted/rejected | Min | Median | Max | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 196.3 | 211.0 | 242.7 | -| message1000 | 7/0 | 208.9 | 213.5 | 223.1 | -| threadSwitch | 7/0 | 152.9 | 155.5 | 204.7 | -| streaming | 7/0 | 13435.2 | 13487.7 | 13948.9 | -| messageListBehavior | 6/1 | 3108.9 | 3365.7 | 3519.8 | -| denseNarrative | 7/0 | 34.8 | 40.7 | 44.6 | -| markdownShiki | 7/0 | 5687.2 | 5825.8 | 6204.3 | -| panelTransitions | 7/0 | 102.9 | 111.0 | 120.6 | - -### Production, standalone-web - -| Workload | Accepted/rejected | Min | Median | Max | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 75.6 | 109.5 | 124.9 | -| message1000 | 7/0 | 148.5 | 152.5 | 166.7 | -| threadSwitch | 7/0 | 106.8 | 111.7 | 148.5 | -| streaming | 7/0 | 16749.4 | 16774.7 | 16816.2 | -| messageListBehavior | 6/1 | 3181.4 | 3279.8 | 3319.7 | -| denseNarrative | 7/0 | 63.3 | 64.2 | 81.9 | -| markdownShiki | 7/0 | 5666.4 | 6134.6 | 7008.2 | -| panelTransitions | 7/0 | 84.5 | 94.1 | 95.9 | - -### Production, Electron - -| Workload | Accepted/rejected | Min | Median | Max | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 185.8 | 214.3 | 274.1 | -| message1000 | 7/0 | 159.0 | 185.3 | 217.4 | -| threadSwitch | 7/0 | 139.6 | 154.1 | 176.1 | -| streaming | 7/0 | 13438.8 | 13488.0 | 13901.5 | -| messageListBehavior | 6/1 | 2558.2 | 2574.0 | 2639.4 | -| denseNarrative | 7/0 | 42.3 | 45.5 | 51.4 | -| markdownShiki | 7/0 | 3940.5 | 4419.6 | 4665.5 | -| panelTransitions | 7/0 | 55.2 | 79.8 | 86.4 | - -## MessageList attribution - -This table records the narrow MessageList stages for `messageListBehavior` in profiling mode. Stage counts include every measured stage observation, not only the accepted workload samples. - -| Runtime | Narrative stage count | Narrative median | Narrative p95 | TanStack stage count | TanStack median | TanStack p95 | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| standalone-web | 39 | 0.2 | 0.6 | 384 | 0.0 | 0.1 | -| electron | 78 | 0.5 | 2.5 | 782 | 0.0 | 0.1 | - -The React profiler is present for all 14 profiling `messageListBehavior` samples. It records 14 commits per runtime, and its actual and base duration minimum, median, and maximum are all 0.0 ms. React attribution is `null` for the corresponding production samples. - -Production Chromium trace attribution is present for `messageListBehavior`. The table gives min, median, and max across its seven raw samples. ResizeObserver callback trace duration is `null` in both runtimes. - -| Runtime | GC min/median/max | Layout min/median/max | Paint min/median/max | -| --- | ---: | ---: | ---: | -| standalone-web | 423.730 / 533.545 / 574.810 | 74.686 / 96.521 / 98.244 | 99.005 / 116.156 / 121.028 | -| electron | 879.900 / 923.831 / 997.533 | 97.836 / 112.967 / 118.839 | 106.737 / 114.805 / 118.057 | - -## Shiki attribution - -`codeToHtml` is the largest Shiki stage in every runtime and mode. Values below are medians in milliseconds. A dash means the artifact reports `null` for that stage. - -### Profiling Shiki medians - -| Runtime | Phase | Worker startup | Highlighter creation | Grammar load | Code to HTML | Worker delivery | React commit | HTML insertion | Total completion | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| standalone-web | cold, n=7 | 0.5 | 1.1 | 3.0 | 494.1 | 7.0 | 1.3 | 0.0 | 574.9 | -| standalone-web | warm, n=63 | - | 0.0 | 0.0 | 372.3 | 8.0 | 0.8 | 0.0 | 2507.7 | -| electron | cold, n=7 | 3.9 | 1.6 | 4.9 | 559.4 | 3.0 | 1.5 | 0.0 | 726.9 | -| electron | warm, n=63 | - | 0.0 | 0.0 | 553.2 | 4.0 | 1.6 | 0.0 | 3534.7 | - -Profiling Shiki style and layout fields are `null`. - -### Production Shiki medians - -| Runtime | Phase | Worker startup | Highlighter creation | Grammar load | Code to HTML | Worker delivery | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| standalone-web | cold, n=7 | 5.0 | 1.7 | 4.8 | 616.1 | 7.0 | -| standalone-web | warm, n=63 | - | 0.0 | 0.0 | 562.2 | 10.0 | -| electron | cold, n=7 | 4.7 | 1.0 | 3.0 | 503.9 | 4.0 | -| electron | warm, n=63 | - | 0.0 | 0.0 | 386.6 | 4.0 | - -Production `reactCommit`, `htmlInsertion`, and `totalCompletion` fields are `null`. Production workload style and layout fields are available: - -| Runtime | Style min/median/max | Layout min/median/max | -| --- | ---: | ---: | -| standalone-web | 97.951 / 109.194 / 127.527 | 46.340 / 50.839 / 57.788 | -| electron | 101.389 / 113.301 / 127.111 | 45.503 / 48.114 / 54.967 | - -The response-byte fields are present. Cold samples have n=7 and warm samples have n=63 in every runtime and mode. Their median bytes are 32,485 and 32,485 for profiling standalone-web, 32,345 and 32,293 for profiling Electron, 32,310 and 32,345 for production standalone-web, and 32,485 and 32,485 for production Electron. - -## Limits - -- This is a baseline from a dirty source tree, not a before-and-after comparison. -- Rejected behavior samples remain in the raw artifacts but not in duration summaries. -- The standalone-web acceleration field is `null`; it does not establish an acceleration setting. -- The profiling artifacts contain no Chromium trace attribution. -- GPU counters are unavailable in the paired un-packaged runner. -- The artifacts do not provide Electron process measurements except the production Electron signal. diff --git a/docs/performance/tanstack-virtual-current-trial.md b/docs/performance/tanstack-virtual-current-trial.md deleted file mode 100644 index 0288d5c5f..000000000 --- a/docs/performance/tanstack-virtual-current-trial.md +++ /dev/null @@ -1,181 +0,0 @@ -# TanStack Virtual Current-Stable Trial - -## Decision - -Keep `@tanstack/react-virtual` 3.13.23 and `@tanstack/virtual-core` 3.13.23. -The 3.14.10 trial did not pass the MessageList behavior gate. The duration -samples therefore do not support an upgrade. - -This report records a reversible dependency trial for issue #1547. The trial -changed only `bun.lock` while it ran. The original lock file and installed -packages were restored before this report was added. - -## Candidate - -The release state was frozen at approximately `2026-08-25T14:59Z`. - -| Package | Candidate version | Published | Tarball integrity | Release commit | -| --- | --- | --- | --- | --- | -| `@tanstack/react-virtual` | 3.14.10 | 2026-08-18 15:06:28.045 UTC | `sha512-SRyoUbdFMRHuYXMijV5H4ZarQWpXkj3iANq8OFre+pybeVap8ZJjZ3Nz9bVjx4d8PfobVUQUdKyyyHYk3E+djw==` | `e9874f033c74afd3251eeb9f3e60b2530cc7ae88` | -| `@tanstack/virtual-core` | 3.17.8 | 2026-08-18 15:06:21.826 UTC | `sha512-BfEvehNpOT75r5Ksc5xW6NZuXujTfb7nlSEyVu4XHG3gdxNg1KqXruWbDewXOUaUYIo4oRbSfkjIajz4MAT8tA==` | `e9874f033c74afd3251eeb9f3e60b2530cc7ae88` | - -The candidate lock entries used the official tarballs: - -- `https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.14.10.tgz` -- `https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.8.tgz` - -Official sources: [React change log](https://github.com/TanStack/virtual/blob/main/packages/react-virtual/CHANGELOG.md), -[core change log](https://github.com/TanStack/virtual/blob/main/packages/virtual-core/CHANGELOG.md), -and [release commit](https://github.com/TanStack/virtual/commit/e9874f033c74afd3251eeb9f3e60b2530cc7ae88). - -The existing application range, `^3.13.23`, accepted 3.14.10. The candidate -lock resolved React Virtual 3.14.10 and its nested Virtual Core dependency -3.17.8. A frozen install and both candidate runner builds succeeded with no -product-source change or public API incompatibility. The candidate tree differed -from the clean baseline only in `bun.lock`. - -## Run contract - -All four runs used source revision `b04b632061f335e010c24de0b2edaf17e1360688`. -The runner used a 1440 by 1000 viewport, one warmup, and seven samples for each -workload and runtime. The workload order was `message100`, `message1000`, -`threadSwitch`, `streaming`, `messageListBehavior`, `denseNarrative`, -`markdownShiki`, and `panelTransitions`. - -All four artifacts record Playwright 1.62.1. Standalone web reported no -acceleration field. Electron reported disabled acceleration in every artifact. -The baseline artifacts were clean at `b04b6320`. The candidate artifacts had -`sourceDirty: true` because the temporary dependency resolution changed only -`bun.lock`. - -```powershell -bun run perf:frontend -- --mode profiling --sample-count 7 --output .dev/verification/performance/issue-1547-baseline-profiling.json -bun run perf:frontend -- --mode production --sample-count 7 --output .dev/verification/performance/issue-1547-baseline-production.json -bun run perf:frontend -- --mode profiling --sample-count 7 --output .dev/verification/performance/issue-1547-candidate-profiling.json -bun run perf:frontend -- --mode production --sample-count 7 --output .dev/verification/performance/issue-1547-candidate-production.json -``` - -Each command wrote its raw artifact but exited 1 when the runner rejected one -or more behavior samples. The exit status preserves the runner contract. It -does not discard accepted samples from the artifact. - -## Behavior results - -The fresh issue #1547 baseline records the known load-sensitive sticky jump. It -missed the 3.6-second observation window in six of 14 profiling samples and -two of 14 production samples. The accepted issue #1546 baseline recorded five -of 14 profiling misses and two of 14 production misses. This fresh rerun shows -load-sensitive variation. It does not replace or restate the accepted #1546 -artifacts. The candidate run did not prove parity: - -- Candidate profiling rejected all 56 Electron samples after a WebSocket - closure and connection refusal. Treat its Electron React data as invalid. -- Candidate production observed two standalone-web sticky jumps and five - Electron sticky jumps. The baseline production observed zero standalone-web - and two Electron sticky jumps. The candidate run therefore observed five - more sticky failures in total, including three more in Electron. -- Candidate production also observed two Electron streaming failures. Each - failed sample did not commit all 200 visible updates, did not keep the tail in - view, and moved a user who had left the tail. Baseline production accepted all - 14 streaming samples. - -The message-list behavior samples that reached their assertions still recorded -dynamic Markdown heights, older and newer anchors, cache-hit restoration, -cache-miss tail restoration, live-to-persisted identity, focus, and interactive -controls, resident thread switch, and text selection. The sticky jump targets -were initially unmounted. The failed sticky samples did not reach the required -transcript row within the observation window. - -| Mode | Runtime | Baseline accepted/rejected | Candidate accepted/rejected | Result | -| --- | --- | ---: | ---: | --- | -| Profiling | standalone web | 54/2 | 55/1 | Candidate sticky behavior remained load-sensitive. | -| Profiling | Electron | 52/4 | 0/56 | Candidate Electron run was invalid after socket failures. | -| Production | standalone web | 56/0 | 54/2 | Candidate run observed two sticky-jump rejections. | -| Production | Electron | 54/2 | 49/7 | Candidate run observed three more sticky and two streaming rejections. | -| Total | both runtimes | 216/8 | 158/66 | Candidate run did not prove parity. | - -The reported candidate profiling Electron socket errors are a runner -observation, not evidence that TanStack Virtual caused a product failure. They -still prevent that run from proving parity or a performance result. The -production behavior observations alone fail the candidate under the predeclared -behavior-parity gate. - -## Accepted-sample duration comparison - -These values are medians in milliseconds from accepted standalone-web samples. -They are descriptive only. Candidate behavior failed, and sample pools differ, -so no row supports a performance claim. - -### Profiling - -| Workload | Baseline | Candidate | -| --- | ---: | ---: | -| `message100` | 81.5 | 128.1 | -| `message1000` | 81.5 | 149.3 | -| `threadSwitch` | 59.3 | 123.2 | -| `streaming` | 16738.1 | 16745.3 | -| `messageListBehavior` | 3159.3 | 3162.3 | -| `denseNarrative` | 35.2 | 32.0 | -| `markdownShiki` | 8237.7 | 6095.4 | -| `panelTransitions` | 76.0 | 72.0 | - -### Production - -| Workload | Baseline | Candidate | -| --- | ---: | ---: | -| `message100` | 123.9 | 124.4 | -| `message1000` | 137.1 | 150.7 | -| `threadSwitch` | 114.3 | 127.1 | -| `streaming` | 16729.4 | 16720.6 | -| `messageListBehavior` | 2927.6 | 2986.6 | -| `denseNarrative` | 37.6 | 42.3 | -| `markdownShiki` | 5744.1 | 4425.8 | -| `panelTransitions` | 55.7 | 67.8 | - -## Attribution boundaries - -React data came from profiling builds. In accepted standalone-web -`messageListBehavior` samples, React actual duration had a median of 0.0 ms in -both baseline and candidate data. The narrow MessageList stages also had the -same median: narrative-item projection was 0.4 ms and `getVirtualItems()` was -0.0 ms. Their p95 values were 0.7 ms and 0.1 ms. The sample counts differed, -so these values do not establish an improvement. - -Chromium trace data came from production builds. Electron process CPU and -memory data came from production Electron samples. OS GPU counters were -unavailable in all four artifacts. The candidate did not pass behavior, and its -profiling Electron samples were invalid, so the Chromium and process signals -are recorded in the raw artifacts but are not compared as upgrade evidence. - -## Upstream review - -The candidate includes upstream work for scroll compensation above or across -the fold, dynamic `scrollToIndex`, DOM-node identity cleanup, end anchoring, -cached measurements, skipping redundant unchanged-offset scroll events, -allocation reduction, ResizeObserver and count-shrink safety, pending -observer-reset cancellation, and observer teardown state reset. - -React Virtual 3.14.0 also adds opt-in `directDomUpdates`. Mcode does not set -that option. The candidate build succeeded with the existing Mcode options and -showed no public API break. This trial did not edit product code or opt in to -new APIs. - -## Restoration and limits - -The temporary lockfile was restored byte-for-byte. Its saved and restored -SHA-256 value was `54D9671BAB3EC886AF5BBCA8FFB205C9978C44CEC6F47993AECF4CF4CB708825`. -A frozen install then resolved React Virtual and Virtual Core back to 3.13.23. -No dependency manifest or lockfile change remains. - -The candidate fails the behavior gate, so this trial cannot decide whether a -future version can improve Mcode. The profiling Electron socket failure also -limits attribution. A future trial needs a candidate that first passes the -full behavior matrix in both runtimes. Only then can matched accepted samples -support an upgrade decision. - -## Raw artifacts - -- `.dev/verification/performance/issue-1547-baseline-profiling.json` -- `.dev/verification/performance/issue-1547-baseline-production.json` -- `.dev/verification/performance/issue-1547-candidate-profiling.json` -- `.dev/verification/performance/issue-1547-candidate-production.json` diff --git a/docs/performance/vlist-react-adapter-prototype.md b/docs/performance/vlist-react-adapter-prototype.md deleted file mode 100644 index fa5238f53..000000000 --- a/docs/performance/vlist-react-adapter-prototype.md +++ /dev/null @@ -1,73 +0,0 @@ -# vlist React adapter prototype - -## Scope - -This development-only probe tests an architectural bridge between React and vlist. One React root portals logical rows into hosts that vlist owns. - -The bridge uses a vlist plugin with priority `100`. It performs these operations: - -1. `onCalculate` removes outgoing React rows before vlist changes the DOM. -2. `onCalculate` parks retained portal hosts when their logical rows move to new indices. -3. vlist updates and recycles its item shells. -4. `onCommit` moves parked hosts into their new shells and renders the visible React rows. -5. The adapter unmounts React before it destroys vlist. - -The probe uses `overscan: 0` and fixed 84-pixel rows. It does not change the production TanStack message list or compare performance. - -## Frozen baseline - -| Package | Version | Published | Source revision | -| --- | --- | --- | --- | -| `vlist` | 2.6.1 | 2026-07-15 | [`7c39284b964a4d4e7b67821d17a622cf599eaed8`](https://github.com/floor/vlist/tree/7c39284b964a4d4e7b67821d17a622cf599eaed8) | -| `vlist-react` | 2.6.0 | 2026-07-08 | [`4f8a94334f32386b4921dfd8384368506c24f8d3`](https://github.com/floor/vlist-react/tree/4f8a94334f32386b4921dfd8384368506c24f8d3) | - -The vlist-react wrapper creates a vanilla vlist instance in a React effect and updates its items. It does not define React-owned virtual rows. [Source](https://github.com/floor/vlist-react/blob/4f8a94334f32386b4921dfd8384368506c24f8d3/src/index.ts#L42-L104) - -## Browser observation - -The Electron renderer ran three profiling samples. The run wrote disposable raw output to `.dev/verification/performance/issue-1548-vlist-lifecycle-bridge-prepend-electron.json`. Repository policy keeps live verification artifacts under the ignored `.dev/verification/` directory. The raw file is not part of this document. See [ADR 0017](../adr/0017-separate-maintained-tests-from-disposable-verification.md). - -Run this command to create a new local artifact: - -```powershell -bun run perf:frontend -- --mode profiling --sample-count 3 --workload vlistLifecycle --runtime electron --output .dev/verification/performance/issue-1548-vlist-lifecycle-bridge-prepend-electron.json -``` - -All three samples returned this decision: - -```json -{"status":"accepted","candidateEligible":true,"reason":null} -``` - -The runner derives the decision from raw browser facts. It does not trust a summary from the probe. - -The lifecycle checks cover heterogeneous rows, local state, effects, refs, body portals, focus, controls, and host reuse. The checks also cover native scroll recycling and final disposal. - -The prepend check inserts two older rows above a visible anchor. In each sample, the anchor stayed at pixel `774`. Its edited draft stayed `edited-before-prepend`. Its effect and ref counts stayed at one. Its React portal host token stayed `portal-host-0`. - -The standalone Chromium process failed to start twice within its 180-second limit. Later Electron profiles also failed during app startup. Those failures occurred before probe code ran. - -The runner ignores timing fields for this workload. This probe makes no performance claim. - -## Gate matrix - -| Gate | Status | Evidence | -| --- | --- | --- | -| React portal lifecycle under vlist host reuse | Passed in Electron | Three samples show React cleanup before vlist detaches the old host. Native scroll also recycles pool shells. | -| Fixed-height prepend anchor and logical identity | Passed in Electron | Three samples keep the anchor position, draft, mount count, ref count, and portal host. | -| Dynamic-height measurement | Not run | The fixed-height prepend bridge does not prove the autosize path. [Source](https://github.com/floor/vlist/blob/7c39284b964a4d4e7b67821d17a622cf599eaed8/src/plugins/autosize/plugin.ts) | -| Tail follow and user-away posture | Not run | No browser artifact covers append behavior after the bridge change. | -| Sticky user-message jump | Not run | vlist sticky support covers group headers, not the MessageList contract. [Source](https://github.com/floor/vlist/blob/7c39284b964a4d4e7b67821d17a622cf599eaed8/src/plugins/groups/plugin.ts) | -| Selection and scroll-to-message | Not run | The probe covers focus and controls only. | -| Thread-switch anchor restore | Not run | No browser artifact covers per-thread anchor storage after the bridge change. | -| Arbitrary retained-row range | Does not fit the core model | Core rendering calculates one contiguous visible range plus overscan. It has no API for an unrelated retained range. [Source](https://github.com/floor/vlist/blob/7c39284b964a4d4e7b67821d17a622cf599eaed8/src/core/pipeline.ts#L225-L265) | -| Equivalent retained-state mechanism | Not run | The bridge can keep moved visible hosts, but it does not prove an off-screen retained-state design. | -| Production integration and comparison | Not run | The production TanStack route is unchanged. | - -## Architectural result - -The original direct-portal topology failed because vlist detached the portal target before React cleanup. The bridge changes that ownership handoff. React now removes outgoing rows before vlist updates its shells. - -Prepend needs more work than cleanup order. vlist keys rendered shells by index. The bridge parks retained portal hosts, shifts the pending fixed-height range, and restores each host under its new index. This keeps the React component mounted. - -The tested bridge removes the original lifecycle rejection. It also proves fixed-height prepend behavior in Electron. The candidate remains incomplete until the other behavior gates pass. diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 56d481a6a..3d6b36c8f 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "private": true, "type": "module", + "sideEffects": false, "exports": { ".": "./src/index.ts", "./snapshot": "./src/snapshot.ts" diff --git a/packages/contracts/src/events/__tests__/agent-event.test.ts b/packages/contracts/src/events/__tests__/agent-event.test.ts index 7a9eaf5f8..7df55d6d4 100644 --- a/packages/contracts/src/events/__tests__/agent-event.test.ts +++ b/packages/contracts/src/events/__tests__/agent-event.test.ts @@ -2,6 +2,16 @@ import { describe, it, expect } from "vitest"; import { AgentEventSchema, AgentEventType } from "../agent-event.js"; describe("AgentEvent provider_unavailable", () => { + it("preserves a bounded durable publication identity", () => { + const event = { type: "turnStarted", threadId: "t-1", + turnExecutionId: "00000000-0000-4000-8000-000000000001", + publicationId: "1" }; + expect(AgentEventSchema().parse(event)).toEqual(event); + expect(AgentEventSchema().safeParse({ ...event, publicationId: "x" }).success).toBe(false); + expect(AgentEventSchema().safeParse({ ...event, publicationId: "1".repeat(17) }).success) + .toBe(false); + }); + it("parses a disabled-provider event", () => { const parsed = AgentEventSchema().parse({ type: "providerUnavailable", diff --git a/packages/contracts/src/events/agent-event.ts b/packages/contracts/src/events/agent-event.ts index 89578645e..bd2a1829c 100644 --- a/packages/contracts/src/events/agent-event.ts +++ b/packages/contracts/src/events/agent-event.ts @@ -384,6 +384,8 @@ const AgentEventSequenceSchema = z.object({ epoch: AgentEventEpochSchema.optional(), /** Mcode-owned execution identity for the turn that produced this event. */ turnExecutionId: z.string().uuid().optional(), + /** Durable semantic receipt identity; survives a server event-epoch change. */ + publicationId: z.string().regex(/^[1-9]\d*$/).max(16).optional(), }); /** Validated agent event payload, including an optional server ordering sequence. */ diff --git a/packages/contracts/src/events/provider-runtime-event.ts b/packages/contracts/src/events/provider-runtime-event.ts index cab378144..7a526c900 100644 --- a/packages/contracts/src/events/provider-runtime-event.ts +++ b/packages/contracts/src/events/provider-runtime-event.ts @@ -63,6 +63,7 @@ export const ProviderRuntimeExtensionSchema = lazySchema(() => z export const ProviderRuntimeEventSchema = lazySchema(() => z.object({ event: AgentEventSchema(), + deliveryAttempt: z.number().int().positive().optional(), extension: ProviderRuntimeExtensionSchema().optional(), }).strict(), ); diff --git a/packages/contracts/src/providers/interfaces.ts b/packages/contracts/src/providers/interfaces.ts index e213a5b30..6e3060bfa 100644 --- a/packages/contracts/src/providers/interfaces.ts +++ b/packages/contracts/src/providers/interfaces.ts @@ -32,6 +32,10 @@ export interface CompletionOptions { /** Explicit provider file-tool start used to capture a mutation baseline without publishing narrative UI. */ export interface ProviderFileMutationStart { threadId: string; + /** Exact dispatched attempt, when the provider can bind this private notification. */ + turnExecutionId?: string; + /** Provider-bound retry ordinal for this private notification. */ + deliveryAttempt?: number; toolCallId: string; toolName: string; toolInput: Record; diff --git a/packages/contracts/src/ws/channels.ts b/packages/contracts/src/ws/channels.ts index d6999c550..1a7ad67cc 100644 --- a/packages/contracts/src/ws/channels.ts +++ b/packages/contracts/src/ws/channels.ts @@ -163,7 +163,7 @@ export const WS_CHANNELS = { "turn.persisted": z.object({ threadId: z.string(), turnId: z.string().nullable().optional(), - messageId: z.string(), + messageId: z.string().nullable(), toolCallCount: z.number(), filesChanged: z.array(z.string()), fileEffects: TurnFileEffectSummarySchema().optional(), diff --git a/packages/providers/src/__tests__/codex/codex-canonical-event-publisher.test.ts b/packages/providers/src/__tests__/codex/codex-canonical-event-publisher.test.ts new file mode 100644 index 000000000..a869ff653 --- /dev/null +++ b/packages/providers/src/__tests__/codex/codex-canonical-event-publisher.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it, vi } from "vitest"; +import { AgentEventRoutingSchema } from "@mcode/agent-model"; +import { AgentEventType, providerRuntimeEvent } from "@mcode/contracts"; +import type { ProviderEventBatch, ProviderEventSinkPort, ProviderEventSubmissionReceipt } from "../../host-ports.js"; +import { + CodexCanonicalEventPublisher, + type CodexCanonicalEventRouting, +} from "../../private/codex/codex-canonical-event-publisher.js"; + +const routing: CodexCanonicalEventRouting = { + threadId: "thread-1", + turnId: "turn-1", + executionId: "00000000-0000-4000-8000-000000000001", + deliveryAttempt: 1, +}; + +const acceptedReceipt: ProviderEventSubmissionReceipt = { + commit: { + outcome: "committed", + conversationRevision: 1, + rosterRevision: 1, + acceptedThrough: 1, + durableThrough: 1, + eventCount: 1, + }, + delivery: { ingress: "queued" }, +}; + +function sink(submit: ProviderEventSinkPort["submit"]): ProviderEventSinkPort { + return { submit }; +} + +describe("CodexCanonicalEventPublisher", () => { + it("submits exact ordered drafts with stable batch, item and event IDs", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedReceipt); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + + publisher.publish(routing, { + ...providerRuntimeEvent({ + type: AgentEventType.TextDelta, + threadId: routing.threadId, + turnExecutionId: routing.executionId, + delta: "Hello", + }), + deliveryAttempt: 1, + }); + publisher.publish(routing, { + ...providerRuntimeEvent({ + type: AgentEventType.Ended, + threadId: routing.threadId, + turnExecutionId: routing.executionId, + }), + deliveryAttempt: 1, + }); + + await publisher.waitForExecution(routing); + expect(submit).toHaveBeenCalledTimes(2); + const [textBatch, endedBatch] = submit.mock.calls.map(([batch]) => batch); + expect(textBatch).toMatchObject({ + threadId: routing.threadId, + turnId: routing.turnId, + executionId: routing.executionId, + deliveryAttempt: 1, + phase: "running", + batchId: `codex:${routing.executionId}:attempt:1:event:1`, + events: [{ + eventId: `codex:${routing.executionId}:attempt:1:event:1`, + sourceProviderId: "codex", + sourceSequence: 1, + ingestClass: "volatile", + }], + }); + expect(textBatch.events[0]?.routing.itemId).toBe(`codex:${routing.executionId}:attempt:1:item:1`); + AgentEventRoutingSchema.parse(textBatch.events[0]?.routing); + expect(endedBatch.events[0]?.sourceSequence).toBe(2); + expect(endedBatch.events[0]?.payload).toMatchObject({ + type: "item.recorded", + item: { payload: { projection: "providerRuntimeEvent", runtimeEvent: { + deliveryAttempt: 1, + event: { type: AgentEventType.Ended, turnExecutionId: routing.executionId }, + } } }, + }); + }); + + it("uses a distinct sequence for a retry of the same execution", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedReceipt); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + await publisher.waitForExecution(routing); + const retry = { ...routing, deliveryAttempt: 2 }; + publisher.publish(retry, { ...event, deliveryAttempt: 2 }); + await publisher.waitForExecution(retry); + + expect(submit.mock.calls.map(([batch]) => batch.batchId)).toEqual([ + `codex:${routing.executionId}:attempt:1:event:1`, + `codex:${routing.executionId}:attempt:2:event:1`, + ]); + }); + + it("continues the sequence for an event after the first delivery drain", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedReceipt); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + await publisher.waitForExecution(routing); + publisher.publish(routing, event); + await publisher.retireExecution(routing); + + expect(submit.mock.calls.map(([batch]) => batch.batchId)).toEqual([ + `codex:${routing.executionId}:attempt:1:event:1`, + `codex:${routing.executionId}:attempt:1:event:2`, + ]); + }); + + it("discards queued events at Stop while awaiting the current sink call", async () => { + let completeFirst!: (receipt: ProviderEventSubmissionReceipt) => void; + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockImplementation(() => new Promise((resolve) => { completeFirst = resolve; })); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const onFailure = vi.fn(); + publisher.setFailureHandler(onFailure); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + await vi.waitFor(() => expect(submit).toHaveBeenCalledOnce()); + for (let index = 0; index < 40; index++) publisher.publish(routing, event); + const drained = publisher.discardQueuedForExecution(routing); + publisher.publish(routing, event); + completeFirst(acceptedReceipt); + await drained; + + expect(submit).toHaveBeenCalledOnce(); + expect(onFailure).not.toHaveBeenCalled(); + await publisher.retireExecution(routing); + }); + + it("stops submitting later events after the sink fails", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockRejectedValueOnce(new Error("canonical sink unavailable")); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + publisher.publish(routing, event); + + await expect(publisher.waitForExecution(routing)).rejects.toThrow("canonical sink unavailable"); + expect(submit).toHaveBeenCalledTimes(1); + }); + + it("reports the first sink failure for the exact attempt before a delivery drain", async () => { + let rejectSubmit!: (error: Error) => void; + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockImplementation(() => new Promise((_, reject) => { rejectSubmit = reject; })); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const onFailure = vi.fn(async () => undefined); + publisher.setFailureHandler(onFailure); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + await vi.waitFor(() => expect(submit).toHaveBeenCalledOnce()); + rejectSubmit(new Error("writer unavailable")); + await vi.waitFor(() => expect(onFailure).toHaveBeenCalledExactlyOnceWith( + routing, + expect.objectContaining({ message: "writer unavailable" }), + )); + + publisher.publish(routing, event); + await expect(publisher.waitForExecution(routing)).rejects.toThrow("writer unavailable"); + expect(submit).toHaveBeenCalledOnce(); + expect(onFailure).toHaveBeenCalledOnce(); + }); + + it.each(["conflict", "ingest-overflow"] as const)("rejects a resolved %s receipt", async (outcome) => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockResolvedValue({ ...acceptedReceipt, commit: { ...acceptedReceipt.commit, outcome } }); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + publisher.publish(routing, event); + + await expect(publisher.waitForExecution(routing)).rejects.toThrow(outcome); + expect(submit).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/providers/src/__tests__/codex/codex-event-mapper.test.ts b/packages/providers/src/__tests__/codex/codex-event-mapper.test.ts index c69670f65..99f5ce345 100644 --- a/packages/providers/src/__tests__/codex/codex-event-mapper.test.ts +++ b/packages/providers/src/__tests__/codex/codex-event-mapper.test.ts @@ -3065,6 +3065,21 @@ describe("CodexEventMapper", () => { ]); }); + it("reports native turn identity synchronously with an early child file mutation start", () => { + const observed: Array<{ nativeThreadId?: string; nativeTurnId?: string; toolCallId: string }> = []; + mapper = new CodexEventMapper("test-thread", "parent-thread", (event) => observed.push(event)); + mapper.mapNotification({ + jsonrpc: "2.0", method: "item/started", + params: { threadId: "child-thread", turnId: "native-turn", item: { + type: "fileChange", id: "file-child", changes: [{ path: "src/child.ts", kind: "edit" }], + } }, + }); + expect(observed).toEqual([{ nativeThreadId: "child-thread", nativeTurnId: "native-turn", + toolCallId: "file-child", threadId: "test-thread", toolName: "file_change", + toolInput: expect.objectContaining({ changes: [{ path: "src/child.ts", kind: "edit" }] }), + }]); + }); + it("drops an unrelated unknown-thread notification instead of replaying it", () => { mapper = new CodexEventMapper("test-thread", "parent-thread"); const unknown = mapper.mapNotification({ diff --git a/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts b/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts index 639909dde..a78c691a3 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts @@ -81,9 +81,17 @@ vi.mock("../../private/codex/codex-app-server.js", async () => { import { BrowserAutomationSessionLease, CodexProvider, stubEnvService } from "./codex-provider-test-fixture.js"; import { AgentEventSchema, AgentEventType } from "@mcode/contracts"; -import type { ProviderRuntimeEvent, ProviderTurnDiffUpdate } from "@mcode/contracts"; +import type { ProviderFileMutationStart, ProviderRuntimeEvent, ProviderTurnDiffUpdate } from "@mcode/contracts"; +import type { ProviderEventBatch, ProviderEventSinkPort, ProviderEventSubmissionReceipt } from "../../host-ports.js"; const schemaValidExecutionId = "00000000-0000-4000-8000-000000000001"; +const acceptedEventReceipt: ProviderEventSubmissionReceipt = { + commit: { + outcome: "committed", conversationRevision: 1, rosterRevision: 1, + acceptedThrough: 1, durableThrough: 1, eventCount: 1, + }, + delivery: { ingress: "queued" }, +}; function makeProvider( catalogService: { @@ -108,6 +116,7 @@ function makeProvider( createCodexConfiguration?: () => Promise; close?: (sessionId: string) => Promise; } = undefined as never, + eventSink?: ProviderEventSinkPort, ): CodexProvider { return new CodexProvider( { get: async () => ({ provider: { cli: { codex: "codex" } } }) } as never, @@ -116,6 +125,7 @@ function makeProvider( catalogService as never, browserAutomationLease, threadControlMcp as never, + eventSink, ); } @@ -217,6 +227,63 @@ describe("CodexProvider first turn on new session", () => { provider.shutdown(); }); + it("keeps repeated public item IDs on their native attempts and rejects old server callbacks", async () => { + const provider = makeProvider(); + const events: ProviderRuntimeEvent[] = []; + const mutations: ProviderFileMutationStart[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => events.push(event)); + provider.on("file_mutation_start", (event: ProviderFileMutationStart) => mutations.push(event)); + sendTurnMock.mockResolvedValueOnce("native-attempt-1").mockResolvedValueOnce("native-attempt-2"); + const request = { + turnId: "mcode-turn", turnExecutionId: schemaValidExecutionId, deliveryAttempt: 1, + sessionId, workspaceId: "workspace-test", threadId, message: "edit", cwd: process.cwd(), + model: "gpt-5.4", interactionMode: "build" as const, providerOptions: {}, + permissionMode: "supervised" as const, approvalReviewMode: "automatic" as const, + }; + const notifyTool = (server: (typeof appServers)[number], turnId: string) => server.emit("notification", { + method: "item/started", + params: { + threadId: "sdk-thread-1", turnId, + item: { type: "fileChange", id: "same-native-item", changes: [{ path: "tracked.txt", kind: "edit" }] }, + }, + }); + const notifyUnknownChild = (server: (typeof appServers)[number]) => server.emit("notification", { + method: "item/started", + params: { + threadId: "unknown-child", + item: { type: "fileChange", id: "unbound-child-edit", changes: [{ path: "child.txt", kind: "edit" }] }, + }, + }); + + await provider.sendTurn(request); + await new Promise((resolve) => setImmediate(resolve)); + const firstServer = appServers[0]!; + notifyTool(firstServer, "native-attempt-1"); + notifyUnknownChild(firstServer); + await provider.discardSession(sessionId); + + await provider.sendTurn({ ...request, deliveryAttempt: 2, resumeFrom: undefined }); + await new Promise((resolve) => setImmediate(resolve)); + const retryServer = appServers[1]!; + notifyTool(firstServer, "native-attempt-1"); + notifyUnknownChild(firstServer); + notifyTool(retryServer, "native-attempt-2"); + notifyUnknownChild(retryServer); + retryServer.emit("notification", { + method: "turn/completed", + params: { threadId: "sdk-thread-1", turn: { id: "native-attempt-2", status: "completed", usage: {} } }, + }); + await new Promise((resolve) => setImmediate(resolve)); + + expect(events.filter((event) => event.event.type === AgentEventType.ToolUse + && event.event.toolCallId === "same-native-item").map((event) => event.deliveryAttempt)).toEqual([1, 2]); + expect(mutations.filter((event) => event.toolCallId === "unbound-child-edit") + .map((event) => event.deliveryAttempt)).toEqual([undefined, undefined]); + expect(events.find((event) => event.event.type === AgentEventType.Ended + && event.event.turnExecutionId === schemaValidExecutionId && event.deliveryAttempt === 2)).toBeDefined(); + await provider.shutdown(); + }); + it("passes loopback browser MCP config and a child-only bearer token", async () => { const lease = new BrowserAutomationSessionLease(); const inheritedBrowserToken = process.env.MCODE_BROWSER_MCP_TOKEN; @@ -585,6 +652,177 @@ describe("CodexProvider first turn on new session", () => { await ended; }); + it("submits an attempt-bound parent event through the canonical sink only when enabled", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + + await provider.sendTurn({ + turnId: "test-turn", + turnExecutionId: schemaValidExecutionId, + deliveryAttempt: 2, + sessionId, + workspaceId: "workspace-test", + threadId, + message: "hey", + cwd: process.cwd(), + model: "gpt-5.4", + interactionMode: "build", + providerOptions: {}, + permissionMode: "auto", + }); + await new Promise((resolve) => setImmediate(resolve)); + const server = appServers[0]!; + server.emit("notification", { + method: "item/agentMessage/delta", + params: { threadId: "sdk-thread-1", turnId: "turn-test-id", delta: "Hello" }, + }); + await vi.waitFor(() => expect(submit).toHaveBeenCalled()); + + const batches = submit.mock.calls.map(([batch]) => batch); + expect(batches.some((batch) => batch.events.some((event) => event.payload.type === "item.recorded" + && event.payload.item.payload.projection === "providerRuntimeEvent" + && event.payload.item.payload.runtimeEvent.event.type === AgentEventType.TextDelta))).toBe(true); + expect(batches.every((batch) => batch.deliveryAttempt === 2 && batch.turnId === "test-turn")).toBe(true); + expect(legacy.some((event) => event.event.type === AgentEventType.TextDelta)).toBe(false); + }); + + it("keeps Codex on the existing event channel until canonical delivery is enabled", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + + await provider.sendTurn({ + turnId: "test-turn", + turnExecutionId: schemaValidExecutionId, + deliveryAttempt: 2, + sessionId, + workspaceId: "workspace-test", + threadId, + message: "hey", + cwd: process.cwd(), + model: "gpt-5.4", + interactionMode: "build", + providerOptions: {}, + permissionMode: "auto", + }); + await new Promise((resolve) => setImmediate(resolve)); + appServers[0]!.emit("notification", { + method: "item/agentMessage/delta", + params: { threadId: "sdk-thread-1", turnId: "turn-test-id", delta: "Hello" }, + }); + + expect(legacy.some((event) => event.event.type === AgentEventType.TextDelta)).toBe(true); + expect(submit).not.toHaveBeenCalled(); + }); + + it("fences duplicate parent Ended callbacks for an owned attempt", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + await provider.sendTurn({ + turnId: "test-turn", turnExecutionId: schemaValidExecutionId, deliveryAttempt: 2, + sessionId, workspaceId: "workspace-test", threadId, message: "hey", cwd: process.cwd(), + model: "gpt-5.4", interactionMode: "build", providerOptions: {}, permissionMode: "auto", + }); + const emit = (provider as unknown as { emitRuntimeEvent(event: ProviderRuntimeEvent): void }).emitRuntimeEvent.bind(provider); + const ended: ProviderRuntimeEvent = { event: { type: AgentEventType.Ended, + threadId, turnExecutionId: schemaValidExecutionId, outcome: "completed" } }; + emit(ended); + emit(ended); + await provider.waitForCanonicalTurnEvents({ + threadId, turnId: "test-turn", executionId: schemaValidExecutionId, deliveryAttempt: 2, + }); + + expect(submit.mock.calls.flatMap(([batch]) => batch.events).filter((event) => + event.payload.type === "item.recorded" + && event.payload.item.payload.projection === "providerRuntimeEvent" + && event.payload.item.payload.runtimeEvent.event.type === AgentEventType.Ended)).toHaveLength(1); + expect(legacy.some((event) => event.event.type === AgentEventType.Ended)).toBe(false); + }); + + it("stamps the admitted attempt on an early parent event before native turn binding", async () => { + let resolveNativeTurn!: (turnId: string) => void; + sendTurnMock.mockImplementationOnce(() => new Promise((resolve) => { resolveNativeTurn = resolve; })); + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + + await provider.sendTurn({ + turnId: "test-turn", + turnExecutionId: schemaValidExecutionId, + deliveryAttempt: 3, + sessionId, + workspaceId: "workspace-test", + threadId, + message: "hey", + cwd: process.cwd(), + model: "gpt-5.4", + interactionMode: "build", + providerOptions: {}, + permissionMode: "auto", + }); + await vi.waitFor(() => expect(sendTurnMock).toHaveBeenCalled()); + const server = appServers[0]!; + server.emit("notification", { + method: "item/agentMessage/delta", + params: { threadId: "sdk-thread-1", turnId: "early-native-turn", delta: "Early" }, + }); + await vi.waitFor(() => expect(submit).toHaveBeenCalled()); + expect(submit.mock.calls[0]?.[0]).toMatchObject({ + deliveryAttempt: 3, + events: [{ payload: { item: { payload: { runtimeEvent: { + deliveryAttempt: 3, + event: { type: AgentEventType.TextDelta, turnExecutionId: schemaValidExecutionId }, + } } } } }], + }); + expect(legacy.some((event) => event.event.type === AgentEventType.TextDelta)).toBe(false); + + resolveNativeTurn("early-native-turn"); + server.emit("notification", { + method: "turn/completed", + params: { threadId: "sdk-thread-1", turn: { id: "early-native-turn", status: "completed" } }, + }); + await provider.waitForCanonicalTurnEvents({ + threadId, turnId: "test-turn", executionId: schemaValidExecutionId, deliveryAttempt: 3, + }); + }); + + it("fences new parent events synchronously while draining already queued batches", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + await provider.sendTurn({ + turnId: "test-turn", turnExecutionId: schemaValidExecutionId, deliveryAttempt: 2, + sessionId, workspaceId: "workspace-test", threadId, message: "hey", cwd: process.cwd(), + model: "gpt-5.4", interactionMode: "build", providerOptions: {}, permissionMode: "auto", + }); + await new Promise((resolve) => setImmediate(resolve)); + const emitText = (delta: string) => appServers[0]!.emit("notification", { + method: "item/agentMessage/delta", + params: { threadId: "sdk-thread-1", turnId: "turn-test-id", delta }, + }); + emitText("Before stop"); + const drained = provider.fenceCanonicalTurnEvents({ + threadId, turnId: "test-turn", executionId: schemaValidExecutionId, deliveryAttempt: 2, + }); + emitText("After stop"); + await drained; + + expect(submit).toHaveBeenCalledTimes(1); + expect(legacy.some((event) => event.event.type === AgentEventType.TextDelta)).toBe(false); + }); + it("cancels the main turn and reuses its app-server for the next turn", async () => { const provider = makeProvider(); const events: ProviderRuntimeEvent[] = []; @@ -917,6 +1155,7 @@ describe("CodexProvider first turn on new session", () => { expect(interruptRejected).toBe(true); expect(server.isAlive).toBe(false); expect(events).toContainEqual({ + deliveryAttempt: 1, event: { type: AgentEventType.Ended, threadId, diff --git a/packages/providers/src/__tests__/codex/codex-provider-lifecycle.test.ts b/packages/providers/src/__tests__/codex/codex-provider-lifecycle.test.ts index fc2cf8053..b29c34ba5 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-lifecycle.test.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-lifecycle.test.ts @@ -88,7 +88,7 @@ function createProvider() { const complete = async (number: number, executionId: string) => { await vi.waitFor(() => expect(starts()).toHaveLength(number)); child.complete(`native-turn-${number}`); - await vi.waitFor(() => expect(events).toContainEqual({ event: { + await vi.waitFor(() => expect(events).toContainEqual({ deliveryAttempt: 1, event: { type: AgentEventType.Ended, threadId: request.threadId, turnExecutionId: executionId, outcome: "completed", } })); }; diff --git a/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts b/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts index cb66dbdcc..2aff53038 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts @@ -51,7 +51,7 @@ vi.mock("../../private/codex/codex-app-server.js", async () => { import { CodexProvider, stubEnvService } from "./codex-provider-test-fixture.js"; import { AgentEventType } from "@mcode/contracts"; -import type { ProviderRuntimeEvent } from "@mcode/contracts"; +import type { ProviderFileMutationStart, ProviderRuntimeEvent } from "@mcode/contracts"; interface PoolEntry { pendingTurnId: string | null; @@ -127,6 +127,40 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { loggerWarnMock.mockClear(); }); + it("binds a private early child file mutation only to a known native turn and retry attempt", async () => { + const provider = makeProvider(); + const mutations: ProviderFileMutationStart[] = []; + provider.on("file_mutation_start", (event: ProviderFileMutationStart) => mutations.push(event)); + const entry = await startSession(provider, "mcode-file-attempt", "file-attempt"); + const state = entry as PoolEntry & { + turnExecutionIdsByNativeTurn: Map; + turnDeliveryAttemptsByNativeTurn: Map; + nativeExecutionConflictKeys: Set; + turnDiffRouting?: { turnId: string; turnExecutionId: string; deliveryAttempt: number }; + }; + state.turnExecutionIdsByNativeTurn.set("bound-native", "same-execution"); + state.turnDeliveryAttemptsByNativeTurn.set("bound-native", 1); + state.turnDiffRouting = { turnId: "test-turn", turnExecutionId: "same-execution", deliveryAttempt: 2 }; + const item = { type: "fileChange", id: "edit", changes: [{ path: "tracked.txt", kind: "edit" }] }; + + entry.server.emit("notification", { method: "item/started", + params: { threadId: "early-child", turnId: "old-native", item } }); + entry.server.emit("notification", { method: "item/started", + params: { threadId: "early-child", turnId: "bound-native", item: { ...item, id: "edit-current" } } }); + + expect(mutations).toHaveLength(2); + expect(mutations[0]).toMatchObject({ toolCallId: "edit" }); + expect(mutations[0]?.turnExecutionId).toBeUndefined(); + expect(mutations[0]?.deliveryAttempt).toBeUndefined(); + expect(mutations[1]).toMatchObject({ + toolCallId: "edit-current", turnExecutionId: "same-execution", deliveryAttempt: 1, + }); + state.nativeExecutionConflictKeys.add("bound-native"); + entry.server.emit("notification", { method: "item/started", + params: { threadId: "early-child", turnId: "bound-native", item: { ...item, id: "edit-conflicted" } } }); + expect(mutations[2]?.deliveryAttempt).toBeUndefined(); + }); + it("assigns the active execution to main tool and text events before turn/start binds", async () => { const provider = makeProvider(); const events: ProviderRuntimeEvent[] = []; @@ -466,6 +500,8 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { turnBindingPhase: "idle" | "awaiting" | "bound"; currentNativeTurnId?: string; turnExecutionIdsByNativeTurn: Map; + turnDeliveryAttemptsByNativeTurn: Map; + turnDiffRouting?: { turnId: string; turnExecutionId: string; deliveryAttempt: number }; childExecutionGenerations: Map; nativeThreadExecutionIds: Map; pendingChildEvents: unknown[]; @@ -476,6 +512,8 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { state.turnBindingPhase = "bound"; state.currentNativeTurnId = "main-native-turn"; state.turnExecutionIdsByNativeTurn.set("main-native-turn", state.currentTurnExecutionId); + state.turnDeliveryAttemptsByNativeTurn.set("main-native-turn", 2); + state.turnDiffRouting = { turnId: "test-turn", turnExecutionId: state.currentTurnExecutionId, deliveryAttempt: 2 }; entry.server.emit("notification", { method: "item/started", @@ -551,6 +589,7 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { "child-two-layer-item", ]); expect(childEvents).toHaveLength(3); + expect(childEvents.every((runtimeEvent) => runtimeEvent.deliveryAttempt === undefined)).toBe(true); expect(childEvents.every((runtimeEvent) => ( runtimeEvent.event.turnExecutionId === "exec-mcode-child-two-layer-replay" diff --git a/packages/providers/src/__tests__/codex/codex-provider-test-fixture.ts b/packages/providers/src/__tests__/codex/codex-provider-test-fixture.ts index 6fbd27220..0d3564489 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-test-fixture.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-test-fixture.ts @@ -5,6 +5,7 @@ import type { ProviderBrowserLeaseGrant, ProviderBrowserLeaseHandle, ProviderBrowserLeaseRequest, + ProviderEventSinkPort, ProviderHostPorts, } from "../../host-ports.js"; import type { TurnRequest } from "@mcode/contracts"; @@ -167,6 +168,7 @@ export class CodexProvider extends PackageCodexProvider { createCodexConfiguration?: (sessionId: string) => Promise; close?: (sessionId: string) => Promise; }, + eventSink?: ProviderEventSinkPort, ) { const delegatedSettingsService = { get: () => settings.get() }; const host: ProviderHostPorts = { @@ -186,7 +188,7 @@ export class CodexProvider extends PackageCodexProvider { close: (sessionId) => threadControl?.close?.(sessionId) ?? Promise.resolve(), }, grants: { consume: () => false }, - events: { submit: async () => undefined }, + events: eventSink ?? { submit: async () => undefined }, }; const codexPorts: CodexProviderPorts = { settings: { diff --git a/packages/providers/src/factory-types.ts b/packages/providers/src/factory-types.ts index 3f17e105c..53dedd5f8 100644 --- a/packages/providers/src/factory-types.ts +++ b/packages/providers/src/factory-types.ts @@ -6,6 +6,7 @@ import type { StoredAttachment, } from "@mcode/contracts"; import type { ProviderHostPorts } from "./host-ports.js"; +import type { CodexCanonicalEventRouting } from "./private/codex/codex-canonical-event-publisher.js"; /** Configuration validated by every inert Provider factory. */ export interface ProviderFactoryConfiguration { @@ -63,7 +64,13 @@ export interface ProviderBoundary { } /** Usable Codex Provider returned by its public factory. */ -export type CodexProviderBoundary = IAgentProvider & ProviderBoundary; +export type CodexProviderBoundary = IAgentProvider & ProviderBoundary & { + setCanonicalTurnEventDeliveryEnabled(enabled: boolean): void; + setCanonicalTurnDeliveryFailureHandler(handler: (routing: CodexCanonicalEventRouting, error: Error) => void | Promise): void; + fenceCanonicalTurnEvents(routing: CodexCanonicalEventRouting, options?: { discardQueued?: boolean }): Promise; + waitForCanonicalTurnEvents(routing: CodexCanonicalEventRouting): Promise; + retireCanonicalTurnEvents(routing: CodexCanonicalEventRouting): Promise; +}; /** Usable Cursor Provider returned by its public factory. */ export type CursorProviderBoundary = IAgentProvider & ProviderBoundary & { diff --git a/packages/providers/src/host-ports.ts b/packages/providers/src/host-ports.ts index 5fa605369..03e7320f5 100644 --- a/packages/providers/src/host-ports.ts +++ b/packages/providers/src/host-ports.ts @@ -124,6 +124,10 @@ export interface ProviderEventBatch { threadId: string; turnId: string; executionId: string; + /** Stable across retries of this submission when an execution has a worker owner. */ + batchId?: string; + /** Provider delivery attempt that produced this batch. */ + deliveryAttempt?: number; phase: string; nativeCursor?: unknown; events: readonly ProviderEventDraft[]; diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index 8f47a6442..75c035dbb 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -16,6 +16,7 @@ export type { ProviderFactoryConfiguration, ProviderFactoryInput, } from "./factory-types.js"; +export type { CodexCanonicalEventRouting } from "./private/codex/codex-canonical-event-publisher.js"; export type { ProviderBrowserLeaseHandle, ProviderBrowserLeaseGrant, diff --git a/packages/providers/src/private/codex/codex-canonical-event-publisher.ts b/packages/providers/src/private/codex/codex-canonical-event-publisher.ts new file mode 100644 index 000000000..273636666 --- /dev/null +++ b/packages/providers/src/private/codex/codex-canonical-event-publisher.ts @@ -0,0 +1,166 @@ +import { AgentEventType } from "@mcode/contracts"; +import type { ProviderRuntimeEvent } from "@mcode/contracts"; +import { logger } from "@mcode/shared"; +import type { ProviderEventDraft, ProviderEventSinkPort } from "../../host-ports.js"; + +const MAX_PENDING_EVENTS_PER_EXECUTION = 1_024; + +/** Exact Mcode turn and delivery attempt that produced a Codex event. */ +export interface CodexCanonicalEventRouting { + threadId: string; + turnId: string; + executionId: string; + deliveryAttempt: number; +} + +interface ExecutionQueue { + nextSourceSequence: number; + pendingEventCount: number; + tail: Promise; + failure: Error | undefined; + discardQueued: boolean; +} + +/** Serializes Codex parent events through the server-owned canonical sink. */ +export class CodexCanonicalEventPublisher { + private readonly queues = new Map(); + private failureHandler: + | ((routing: CodexCanonicalEventRouting, error: Error) => void | Promise) + | undefined; + + constructor(private readonly sink: ProviderEventSinkPort) {} + + /** Reports the first failed delivery for an exact execution without waiting for its next turn. */ + setFailureHandler( + handler: (routing: CodexCanonicalEventRouting, error: Error) => void | Promise, + ): void { + this.failureHandler = handler; + } + + /** Queues one event; the caller must have verified its exact execution and attempt. */ + publish(routing: CodexCanonicalEventRouting, runtimeEvent: ProviderRuntimeEvent): void { + const queue = this.queueFor(routing); + if (queue.failure || queue.discardQueued) return; + if (queue.pendingEventCount >= MAX_PENDING_EVENTS_PER_EXECUTION) { + this.reportFailure(routing, queue, new Error(`Codex canonical event queue overflowed for execution ${routing.executionId}`)); + return; + } + + const sourceSequence = queue.nextSourceSequence++; + queue.pendingEventCount++; + const draft = this.createDraft(routing, runtimeEvent, sourceSequence); + queue.tail = queue.tail + .then(async () => { + if (queue.failure || queue.discardQueued) return; + const receipt = await this.sink.submit({ + threadId: routing.threadId, + turnId: routing.turnId, + executionId: routing.executionId, + batchId: draft.eventId, + deliveryAttempt: routing.deliveryAttempt, + phase: "running", + events: [draft], + }); + if (receipt.commit.outcome === "conflict" || receipt.commit.outcome === "ingest-overflow") { + throw new Error(`Codex canonical event ${draft.eventId} was ${receipt.commit.outcome}`); + } + }) + .catch((error: unknown) => { this.reportFailure(routing, queue, toError(error)); }) + .finally(() => { queue.pendingEventCount--; }); + } + + /** Waits for acknowledged delivery and exposes a failed or overflowing queue. */ + async waitForExecution(routing: CodexCanonicalEventRouting): Promise { + const key = this.queueKey(routing); + const queue = this.queues.get(key); + if (!queue) return; + await queue.tail; + if (queue.failure) throw queue.failure; + } + + /** Stop queued deliveries at a cancellation cut, then await only the current sink call. */ + async discardQueuedForExecution(routing: CodexCanonicalEventRouting): Promise { + const queue = this.queues.get(this.queueKey(routing)); + if (queue) queue.discardQueued = true; + await this.waitForExecution(routing); + } + + /** Releases a completed route after its final acknowledged event. */ + async retireExecution(routing: CodexCanonicalEventRouting): Promise { + await this.waitForExecution(routing); + this.queues.delete(this.queueKey(routing)); + } + + private queueFor(routing: CodexCanonicalEventRouting): ExecutionQueue { + const key = this.queueKey(routing); + const existing = this.queues.get(key); + if (existing) return existing; + const queue: ExecutionQueue = { + nextSourceSequence: 1, + pendingEventCount: 0, + tail: Promise.resolve(), + failure: undefined, + discardQueued: false, + }; + this.queues.set(key, queue); + return queue; + } + + private queueKey(routing: CodexCanonicalEventRouting): string { + return `${routing.executionId}:attempt:${routing.deliveryAttempt}`; + } + + private reportFailure(routing: CodexCanonicalEventRouting, queue: ExecutionQueue, error: Error): void { + if (queue.failure) return; + queue.failure = error; + try { + void Promise.resolve(this.failureHandler?.(routing, error)).catch((handlerError: unknown) => { + logger.error("Codex canonical failure handler failed", { + executionId: routing.executionId, + error: toError(handlerError).message, + }); + }); + } catch (handlerError: unknown) { + logger.error("Codex canonical failure handler failed", { + executionId: routing.executionId, + error: toError(handlerError).message, + }); + } + } + + private createDraft( + routing: CodexCanonicalEventRouting, + runtimeEvent: ProviderRuntimeEvent, + sourceSequence: number, + ): ProviderEventDraft { + const eventId = `codex:${routing.executionId}:attempt:${routing.deliveryAttempt}:event:${sourceSequence}`; + const itemId = `codex:${routing.executionId}:attempt:${routing.deliveryAttempt}:item:${sourceSequence}`; + const timestamp = new Date().toISOString(); + return { + eventId, + routing: { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, itemId }, + sourceProviderId: "codex", + sourceIdentities: [], + sourceSequence, + providerTimestamp: timestamp, + ...(runtimeEvent.event.type === AgentEventType.TextDelta ? { ingestClass: "volatile" as const } : {}), + payload: { + type: "item.recorded", + item: { + id: itemId, + threadId: routing.threadId, + turnId: routing.turnId, + kind: "system", + providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent }, + createdAt: timestamp, + updatedAt: timestamp, + }, + }, + }; + } +} + +function toError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/packages/providers/src/private/codex/codex-event-mapper.ts b/packages/providers/src/private/codex/codex-event-mapper.ts index 6ef45de6f..34a2d7ce8 100644 --- a/packages/providers/src/private/codex/codex-event-mapper.ts +++ b/packages/providers/src/private/codex/codex-event-mapper.ts @@ -37,6 +37,12 @@ type ChildNotificationContext = { nativeTurnId: string | undefined; }; +/** Native identity retained until the provider binds this private notification to an execution. */ +export interface CodexPendingFileMutationStart extends ProviderFileMutationStart { + nativeThreadId?: string; + nativeTurnId?: string; +} + const NOTICE_KIND_BY_SUBTYPE: Record = { "provider.notice.unknown-event": "diagnostic", }; @@ -225,7 +231,7 @@ export class CodexEventMapper { constructor( threadId: string, mainCodexThreadId?: string, - private readonly onPendingMutationStart?: (event: ProviderFileMutationStart) => void, + private readonly onPendingMutationStart?: (event: CodexPendingFileMutationStart) => void, ) { this.threadId = threadId; this.mainCodexThreadId = mainCodexThreadId; @@ -542,6 +548,8 @@ export class CodexEventMapper { if (!toolUse || toolUse.type !== AgentEventType.ToolUse) return; this.onPendingMutationStart({ threadId: toolUse.threadId, + nativeThreadId: this.notificationThreadId(notification), + nativeTurnId: this.nativeTurnId(notification), toolCallId: toolUse.toolCallId, toolName: toolUse.toolName, toolInput: toolUse.toolInput, diff --git a/packages/providers/src/private/codex/codex-provider.ts b/packages/providers/src/private/codex/codex-provider.ts index c9d649b53..dd1121903 100644 --- a/packages/providers/src/private/codex/codex-provider.ts +++ b/packages/providers/src/private/codex/codex-provider.ts @@ -42,6 +42,7 @@ import type { ProviderModelInfo, ProviderUsageInfo, ProviderRuntimeEvent, + ProviderFileMutationStart, ProviderTurnDiffUpdate, ProviderCapabilityName, } from "@mcode/contracts"; @@ -54,6 +55,10 @@ import { checkCodexVersion, meetsMinVersion } from "./codex-version.js"; import { CodexAppServer, warmCodexAppServer } from "./codex-app-server.js"; import type { CodexApprovalRequest } from "./codex-app-server.js"; import { CodexEventMapper } from "./codex-event-mapper.js"; +import { + CodexCanonicalEventPublisher, + type CodexCanonicalEventRouting, +} from "./codex-canonical-event-publisher.js"; import { buildCodexInput, hasCodexInternalThreadControlMcp, @@ -328,6 +333,8 @@ interface CodexSessionState { pendingTurnStartNotification?: { nativeTurnId: string; executionId: string }; /** Immutable Mcode execution identity keyed by native Codex turn id. */ turnExecutionIdsByNativeTurn: Map; + /** Immutable dispatch attempt keyed by the same authoritative native turn id. */ + turnDeliveryAttemptsByNativeTurn: Map; /** Current generation execution identity keyed by authoritative child thread id. */ nativeThreadExecutionIds: Map; /** Execution currently owning the active main turn, retained through drains. */ @@ -337,7 +344,7 @@ interface CodexSessionState { /** Bounded conflict fingerprints already logged for this session. */ nativeExecutionConflictKeys: Set; /** Current generation for each authoritative child thread linkage. */ - childExecutionGenerations: Map; + childExecutionGenerations: Map; /** Monotonic child generation counter, bounded by the child map lifetime. */ nextChildGeneration: number; /** Child threads already queried for authoritative model metadata this session. */ @@ -628,9 +635,68 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv } private emitRuntimeEvent(runtimeEvent: ProviderRuntimeEvent): void { + const state = this.canonicalTurnRoutingsByThread.get(runtimeEvent.event.threadId); + const routing = state?.routing; + if (routing && runtimeEvent.extension?.child === undefined + && runtimeEvent.event.turnExecutionId === routing.executionId) { + if (state.kind === "fenced") return; + if (runtimeEvent.deliveryAttempt !== undefined && runtimeEvent.deliveryAttempt !== routing.deliveryAttempt) return; + this.canonicalEventPublisher.publish(routing, { + ...runtimeEvent, + deliveryAttempt: routing.deliveryAttempt, + }); + // Ended closes this exact parent attempt; late SDK callbacks must stay fenced. + if (runtimeEvent.event.type === AgentEventType.Ended) state.kind = "fenced"; + return; + } super.emit("event", runtimeEvent); } + private readonly canonicalEventPublisher: CodexCanonicalEventPublisher; + private readonly canonicalTurnRoutingsByThread = new Map(); + private canonicalTurnEventDeliveryEnabled = false; + + /** Enables the server-owned event route for newly dispatched Codex parent turns. */ + setCanonicalTurnEventDeliveryEnabled(enabled: boolean): void { + this.canonicalTurnEventDeliveryEnabled = enabled; + } + + /** Report a failed canonical sink promptly for the exact turn attempt. */ + setCanonicalTurnDeliveryFailureHandler( + handler: (routing: CodexCanonicalEventRouting, error: Error) => void | Promise, + ): void { + this.canonicalEventPublisher.setFailureHandler(handler); + } + + /** Reports acknowledged canonical delivery or its failure for one exact attempt. */ + waitForCanonicalTurnEvents(routing: CodexCanonicalEventRouting): Promise { + return this.canonicalEventPublisher.waitForExecution(routing); + } + + /** Stops admission for one exact parent attempt before draining its queued batches. */ + fenceCanonicalTurnEvents(routing: CodexCanonicalEventRouting, options?: { discardQueued?: boolean }): Promise { + const state = this.canonicalTurnRoutingsByThread.get(routing.threadId); + if (!state || state.routing.turnId !== routing.turnId + || state.routing.executionId !== routing.executionId + || state.routing.deliveryAttempt !== routing.deliveryAttempt) return Promise.resolve(); + this.canonicalTurnRoutingsByThread.set(routing.threadId, { kind: "fenced", routing: state.routing }); + return options?.discardQueued + ? this.canonicalEventPublisher.discardQueuedForExecution(routing) + : this.canonicalEventPublisher.waitForExecution(routing); + } + + /** Drop the drained publisher queue while retaining a fence against late SDK events. */ + retireCanonicalTurnEvents(routing: CodexCanonicalEventRouting): Promise { + const state = this.canonicalTurnRoutingsByThread.get(routing.threadId); + if (state?.kind !== "fenced" || state.routing.turnId !== routing.turnId + || state.routing.executionId !== routing.executionId + || state.routing.deliveryAttempt !== routing.deliveryAttempt) return Promise.resolve(); + return this.canonicalEventPublisher.retireExecution(routing); + } + /** Owns the session pool, idle eviction (with busy guard), and JobObject/kill. */ private readonly runtime: SessionRuntime; private sdkSessionIds = new Map(); @@ -668,6 +734,8 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv threadControlEligible: boolean; } >(); + /** Guards callbacks before pool registration and after a retry replaces the app-server. */ + private activeCodexServers = new Map(); /** Browser scope staged only until a fresh main app-server process starts. */ private pendingBrowserAccess = new Map(this, { jobObject: { isWindowsJob: false, @@ -1043,6 +1112,16 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv async sendTurn(req: TurnRequest<"codex">): Promise { const preparingSession = this.runtime.get(req.sessionId); if (preparingSession) preparingSession.nextTurnExecutionId = req.turnExecutionId; + if (this.canonicalTurnEventDeliveryEnabled) { + const threadId = this.threadIdForSession(req.sessionId); + await this.retirePreviousCanonicalTurn(threadId); + this.canonicalTurnRoutingsByThread.set(threadId, { kind: "active", routing: { + threadId, + turnId: req.turnId, + executionId: req.turnExecutionId, + deliveryAttempt: req.deliveryAttempt ?? 1, + } }); + } const turn = await this.prepareCodexTurn(req); if (!turn) return; if (this.consumePendingCodexStop(turn)) return; @@ -1170,6 +1249,16 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv this.pendingBrowserAccess.set(sessionId, { stage, workspaceId: request.workspaceId, permissionCapability: turn.browserPermissionCapability }); } + private async retirePreviousCanonicalTurn(threadId: string): Promise { + const state = this.canonicalTurnRoutingsByThread.get(threadId); + if (!state) return; + this.canonicalTurnRoutingsByThread.set(threadId, { kind: "fenced", routing: state.routing }); + await this.canonicalEventPublisher.retireExecution(state.routing); + if (this.canonicalTurnRoutingsByThread.get(threadId)?.routing === state.routing) { + this.canonicalTurnRoutingsByThread.delete(threadId); + } + } + private async acquireCodexTurn(turn: PreparedCodexTurn): Promise { const { request, threadId } = turn; try { @@ -1283,20 +1372,25 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv const mcodeInstructions = this.codexSpawnInstructions(context); const server = this.createCodexAppServer(context, mcodeInstructions, browserTokenEnvName); - const mapper = this.createCodexEventMapper(threadId); - - this.attachCodexServerEvents(context, server, mapper); - const internalMcpStartup = internalMcp ? observeCodexInternalMcpStartup(server) : undefined; - const internalMcpStartupOutcome = await this.startCodexAppServer(context, server, internalMcpStartup, browserTokenEnvName); - this.reportCodexInternalMcpSetupFailure(context, server, internalMcpStartupOutcome); - await this.verifyCodexInternalMcpSetup(context, server, internalMcpStartup, internalMcpStartupOutcome); - this.refreshUsageFromServer(server, threadId); + const mapper = this.createCodexEventMapper(threadId, context.sessionId); + this.activeCodexServers.set(context.sessionId, server); + try { + this.attachCodexServerEvents(context, server, mapper); + const internalMcpStartup = internalMcp ? observeCodexInternalMcpStartup(server) : undefined; + const internalMcpStartupOutcome = await this.startCodexAppServer(context, server, internalMcpStartup, browserTokenEnvName); + this.reportCodexInternalMcpSetupFailure(context, server, internalMcpStartupOutcome); + await this.verifyCodexInternalMcpSetup(context, server, internalMcpStartup, internalMcpStartupOutcome); + this.refreshUsageFromServer(server, threadId); - this.attachCodexThreadIdentity(context, server, mapper); - const state = this.createCodexSessionState(context, server, mapper); - this.scheduleStagedCodexTurn(context, server); + this.attachCodexThreadIdentity(context, server, mapper); + const state = this.createCodexSessionState(context, server, mapper); + this.scheduleStagedCodexTurn(context, server); - return { state, pids: [] }; + return { state, pids: [] }; + } catch (error) { + if (this.activeCodexServers.get(context.sessionId) === server) this.activeCodexServers.delete(context.sessionId); + throw error; + } } private async prepareCodexSpawn(args: SpawnArgs): Promise { @@ -1405,8 +1499,18 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv return [...(context.internalMcp?.configOverrides ?? []), ...browserOverrides]; } - private createCodexEventMapper(threadId: string): CodexEventMapper { - const mapper = new CodexEventMapper(threadId, undefined, (event) => this.emit("file_mutation_start", event)); + private createCodexEventMapper(threadId: string, sessionId: string): CodexEventMapper { + const mapper = new CodexEventMapper(threadId, undefined, (event) => { + const state = this.runtime.get(sessionId); + const nativeTurnId = event.nativeTurnId; + const bound = state?.mapper === mapper ? this.nativeTurnAttempt(state, nativeTurnId) : undefined; + this.emit("file_mutation_start", { + threadId: event.threadId, + ...(bound ? { turnExecutionId: bound.executionId, deliveryAttempt: bound.deliveryAttempt } : {}), + toolCallId: event.toolCallId, + toolName: event.toolName, toolInput: event.toolInput, + } satisfies ProviderFileMutationStart); + }); mapper.setOutputTruncationMode(this.outputTruncationMode); this.emitRuntimeEvent(mapper.sessionStartedEvent()); return mapper; @@ -1414,6 +1518,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv private attachCodexServerEvents(context: CodexSpawnContext, server: CodexAppServer, mapper: CodexEventMapper): void { server.on("invalidNotification", () => { + if (this.activeCodexServers.get(context.sessionId) !== server) return; for (const event of mapper.mapNotification(undefined)) this.emitRuntimeEvent(event); }); server.on("notification", (notification: CodexNotification) => this.handleCodexServerNotification(context, server, mapper, notification)); @@ -1436,12 +1541,12 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv ): void { const rawNotification = notification; const entry = this.runtime.get(context.sessionId); + if (!this.isActiveCodexServer(context.sessionId, server, mapper, entry)) return; const mainNotification = isMainThreadNotification(server, rawNotification.params); const nativeThreadId = nativeThreadIdFromParams(rawNotification.params); const nativeTurnId = nativeTurnIdFromParams(rawNotification.params); - const knownExecutionId = nativeTurnId ? entry?.turnExecutionIdsByNativeTurn.get(nativeTurnId) : undefined; // Reject old main-turn events before they can reset the shared mapper for a new dispatch. - if (mainNotification && knownExecutionId && knownExecutionId !== entry?.currentTurnExecutionId) return; + if (mainNotification && !this.isCurrentCodexMainTurn(entry, nativeTurnId)) return; this.handleCodexUsageNotification(rawNotification, server, context.threadId); const replayThreadId = this.bindCodexTurnStarted(entry, rawNotification.method, mainNotification, nativeThreadId, nativeTurnId); const events = this.mapCodexNotificationEvents(context.threadId, notification, mapper, rawNotification); @@ -1452,7 +1557,24 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv this.deliverCodexNotificationEvents({ entry, sessionId: context.sessionId, mapper, mappedEvents: events, mainNotification, nativeThreadId, nativeTurnId, eventExecutionId: executionId, startupEventExecutionId: startupExecutionId }); if (entry && replayThreadId) this.replayPendingChildEvents(entry, context.sessionId, replayThreadId); this.fetchCompletedChildMetadata(context.sessionId, context.threadId, server, mapper, rawNotification.method, nativeThreadId, executionId); - this.handleDiscoveredCodexChildren({ entry, sessionId: context.sessionId, threadId: context.threadId, server, mapper, notification: rawNotification, nativeThreadId, eventExecutionId: executionId }); + this.handleDiscoveredCodexChildren({ entry, sessionId: context.sessionId, threadId: context.threadId, server, mapper, notification: rawNotification, nativeThreadId, nativeTurnId, eventExecutionId: executionId }); + } + + private isActiveCodexServer(sessionId: string, server: CodexAppServer, mapper: CodexEventMapper, entry: CodexSessionState | undefined): boolean { + return this.activeCodexServers.get(sessionId) === server + && (!entry || (entry.server === server && entry.mapper === mapper)); + } + + private isCurrentCodexMainTurn(entry: CodexSessionState | undefined, nativeTurnId: string | undefined): boolean { + if (!entry || !nativeTurnId) return true; + const knownExecutionId = entry.turnExecutionIdsByNativeTurn.get(nativeTurnId); + return (!knownExecutionId || knownExecutionId === entry.currentTurnExecutionId) + && this.matchesCurrentCodexAttempt(entry, nativeTurnId); + } + + private matchesCurrentCodexAttempt(entry: CodexSessionState, nativeTurnId: string | undefined): boolean { + const nativeAttempt = this.nativeTurnAttempt(entry, nativeTurnId); + return !nativeAttempt || nativeAttempt.deliveryAttempt === entry.turnDiffRouting?.deliveryAttempt; } /** Subscribe to complete native turn diffs without routing their bytes through renderer events. */ @@ -1485,10 +1607,13 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv mapper: CodexEventMapper, error: string, ): void { + if (this.activeCodexServers.get(context.sessionId) !== server) return; logger.error("CodexAppServer fatal", { sessionId: context.sessionId, error, breadcrumb: server.lastTransportBreadcrumb }); - const executionId = this.runtime.get(context.sessionId)?.activeParentTurnExecutionId ?? context.stagedExecutionId; + const entry = this.runtime.get(context.sessionId); + const executionId = entry?.activeParentTurnExecutionId ?? context.stagedExecutionId; for (const event of mapper.drainPendingAssistantBoundary(false)) { - this.emitRuntimeEvent(providerRuntimeEvent(executionId ? { ...event, turnExecutionId: executionId } : event)); + const runtimeEvent = providerRuntimeEvent(executionId ? { ...event, turnExecutionId: executionId } : event); + this.emitRuntimeEvent(entry ? this.withCodexTurnAttempt(entry, runtimeEvent, executionId) : runtimeEvent); } this.emitTurnFailure(context.threadId, error, undefined, true, executionId); if (this.runtime.get(context.sessionId)?.server === server) { @@ -1560,7 +1685,9 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv } private attachCodexThreadIdentity(context: CodexSpawnContext, server: CodexAppServer, mapper: CodexEventMapper): void { - server.on("threadIdChanged", (newThreadId: string) => this.recordCodexThreadIdentity(context, mapper, newThreadId)); + server.on("threadIdChanged", (newThreadId: string) => { + if (this.activeCodexServers.get(context.sessionId) === server) this.recordCodexThreadIdentity(context, mapper, newThreadId); + }); if (server.resumeFailed) { logger.warn("Codex session context lost; resume failed, started fresh thread", { sessionId: context.sessionId }); this.emitRuntimeEvent(providerRuntimeEvent({ type: AgentEventType.System, threadId: context.threadId, subtype: "context_lost" } satisfies AgentEvent)); @@ -1579,7 +1706,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv sessionId: context.sessionId, threadId: context.threadId, cwd: context.cwd, server, mapper, nextTurnExecutionId: this.pendingSpawnTurns.get(context.sessionId)?.turnExecutionId, lastUsedAt: Date.now(), sandboxMode: context.sandbox, runTurnSeq: 0, pendingTurnId: null, - turnBindingPhase: "idle", turnStartResponsePending: false, turnExecutionIdsByNativeTurn: new Map(), + turnBindingPhase: "idle", turnStartResponsePending: false, turnExecutionIdsByNativeTurn: new Map(), turnDeliveryAttemptsByNativeTurn: new Map(), nativeThreadExecutionIds: new Map(), nativeExecutionConflictKeys: new Set(), childExecutionGenerations: new Map(), turnDiffRevision: 0, nextChildGeneration: 0, pendingChildEvents: [], deliveredChildEventKeys: new Set(), workspaceId: context.browserAccess?.workspaceId ?? "unknown-workspace", @@ -1677,18 +1804,21 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv nativeTurnId: string | undefined, ): string | undefined { if (!state.activeParentTurnExecutionId) return undefined; - const executionId = nativeThreadId ? state.childExecutionGenerations.get(nativeThreadId)?.executionId : undefined; + const child = nativeThreadId ? state.childExecutionGenerations.get(nativeThreadId) : undefined; + const executionId = child?.executionId; const replayThreadId = nativeTurnId && executionId && nativeThreadId - ? this.bindChildNativeTurnStart(state, nativeThreadId, nativeTurnId, executionId) + ? this.bindChildNativeTurnStart(state, nativeThreadId, nativeTurnId, executionId, child.deliveryAttempt) : undefined; pruneExecutionMap(state.nativeThreadExecutionIds); pruneExecutionMap(state.turnExecutionIdsByNativeTurn); + this.pruneNativeTurnAttempts(state); return replayThreadId; } - private bindChildNativeTurnStart(state: CodexSessionState, nativeThreadId: string, nativeTurnId: string, executionId: string): string | undefined { + private bindChildNativeTurnStart(state: CodexSessionState, nativeThreadId: string, nativeTurnId: string, executionId: string, deliveryAttempt?: number): string | undefined { const assignment = assignNativeExecution(state.turnExecutionIdsByNativeTurn, nativeTurnId, executionId, state.nativeExecutionConflictKeys); - return assignment === "conflict" ? undefined : nativeThreadId; + return assignment === "conflict" || !this.rememberNativeTurnAttempt(state, nativeTurnId, executionId, deliveryAttempt) + ? undefined : nativeThreadId; } private mapCodexNotificationEvents( @@ -1714,7 +1844,32 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv ): void { if (!state || !nativeThreadId || mainNotification || !mapper.hasReceiverThread(nativeThreadId) || !state.activeParentTurnExecutionId) return; state.nativeThreadExecutionIds.set(nativeThreadId, state.activeParentTurnExecutionId); - if (nativeTurnId && method === "turn/started") state.turnExecutionIdsByNativeTurn.set(nativeTurnId, state.activeParentTurnExecutionId); + if (nativeTurnId && method === "turn/started") { + const assignment = assignNativeExecution(state.turnExecutionIdsByNativeTurn, nativeTurnId, state.activeParentTurnExecutionId, state.nativeExecutionConflictKeys); + if (assignment !== "conflict") this.rememberNativeTurnAttempt(state, nativeTurnId, state.activeParentTurnExecutionId, state.childExecutionGenerations.get(nativeThreadId)?.deliveryAttempt); + } + } + + private nativeTurnAttempt(state: CodexSessionState, nativeTurnId: string | undefined): { executionId: string; deliveryAttempt: number } | undefined { + if (!nativeTurnId || state.nativeExecutionConflictKeys.has(nativeTurnId)) return undefined; + const executionId = state.turnExecutionIdsByNativeTurn.get(nativeTurnId); + const deliveryAttempt = state.turnDeliveryAttemptsByNativeTurn.get(nativeTurnId); + return executionId && deliveryAttempt !== undefined ? { executionId, deliveryAttempt } : undefined; + } + + private rememberNativeTurnAttempt(state: CodexSessionState, nativeTurnId: string, executionId: string, deliveryAttempt: number | undefined): boolean { + if (deliveryAttempt === undefined) return true; + if (state.turnExecutionIdsByNativeTurn.get(nativeTurnId) !== executionId) return false; + const existing = state.turnDeliveryAttemptsByNativeTurn.get(nativeTurnId); + if (existing !== undefined && existing !== deliveryAttempt) return false; + state.turnDeliveryAttemptsByNativeTurn.set(nativeTurnId, deliveryAttempt); + return true; + } + + private pruneNativeTurnAttempts(state: CodexSessionState): void { + for (const nativeTurnId of state.turnDeliveryAttemptsByNativeTurn.keys()) { + if (!state.turnExecutionIdsByNativeTurn.has(nativeTurnId)) state.turnDeliveryAttemptsByNativeTurn.delete(nativeTurnId); + } } private codexNotificationExecutionId( @@ -1767,13 +1922,28 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv args: Omit[0], "mappedEvents">, event: ProviderRuntimeEvent, ): void { - const eventKey = childEventIdentity(event); + const boundEvent = this.withCodexDeliveryAttempt(args, event); + const eventKey = childEventIdentity(boundEvent); if (this.isDuplicateCodexChildEvent(args.entry, eventKey)) return; - if (this.shouldBufferCodexChildEvent(args, event)) { - if (args.entry && args.nativeThreadId) bufferPendingChildEvent(args.entry, event, args.nativeThreadId, args.nativeTurnId, eventKey); + if (this.shouldBufferCodexChildEvent(args, boundEvent)) { + if (args.entry && args.nativeThreadId) bufferPendingChildEvent(args.entry, boundEvent, args.nativeThreadId, args.nativeTurnId, eventKey); return; } - this.emitCodexNotificationEvent(args, event, eventKey); + this.emitCodexNotificationEvent(args, boundEvent, eventKey); + } + + private withCodexDeliveryAttempt( + args: Omit[0], "mappedEvents">, + event: ProviderRuntimeEvent, + ): ProviderRuntimeEvent { + if (!args.entry) return event; + // Replayed child events share the parent notification batch. Their own native turn is the only safe attempt source. + const nativeTurnId = event.extension?.child + ? event.extension.child.nativeTurnId + : args.nativeTurnId; + const bound = this.nativeTurnAttempt(args.entry, nativeTurnId); + if (!bound || (args.eventExecutionId && bound.executionId !== args.eventExecutionId)) return event; + return { ...event, deliveryAttempt: bound.deliveryAttempt }; } private isDuplicateCodexChildEvent(state: CodexSessionState | undefined, eventKey: string | undefined): boolean { @@ -1837,6 +2007,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv mapper: CodexEventMapper; notification: { method?: string; params?: Record }; nativeThreadId: string | undefined; + nativeTurnId: string | undefined; eventExecutionId: string | undefined; }): void { const childThreadId = nativeSubAgentThreadId(args.notification); @@ -1850,22 +2021,27 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv args: Parameters[0], childThreadId: string, ): void { - if (this.isCurrentCodexParentNotification(args)) this.registerCodexChildGeneration(args.entry as CodexSessionState, args.sessionId, childThreadId); + if (this.isCurrentCodexParentNotification(args)) this.registerCodexChildGeneration(args.entry as CodexSessionState, args.sessionId, childThreadId, args.nativeTurnId); this.fetchChildThreadMetadata(args.sessionId, args.threadId, args.server, args.mapper, childThreadId, args.eventExecutionId, true); } private isCurrentCodexParentNotification(args: Parameters[0]): boolean { const executionId = args.entry?.currentTurnExecutionId; if (!args.entry || !executionId || args.eventExecutionId !== executionId) return false; + if (!this.matchesCurrentCodexAttempt(args.entry, args.nativeTurnId)) return false; if (args.server.threadId && args.nativeThreadId === args.server.threadId) return true; return Boolean(args.nativeThreadId && args.entry.nativeThreadExecutionIds.get(args.nativeThreadId) === executionId); } - private registerCodexChildGeneration(state: CodexSessionState, sessionId: string, childThreadId: string): void { + private registerCodexChildGeneration(state: CodexSessionState, sessionId: string, childThreadId: string, parentNativeTurnId: string | undefined): void { const executionId = state.currentTurnExecutionId; if (!executionId) return; state.nextChildGeneration += 1; - state.childExecutionGenerations.set(childThreadId, { executionId, generation: state.nextChildGeneration }); + const parentAttempt = this.nativeTurnAttempt(state, parentNativeTurnId); + state.childExecutionGenerations.set(childThreadId, { + executionId, generation: state.nextChildGeneration, + ...(parentAttempt?.executionId === executionId ? { deliveryAttempt: parentAttempt.deliveryAttempt } : {}), + }); pruneChildGenerationMap(state.childExecutionGenerations); state.nativeThreadExecutionIds.set(childThreadId, executionId); pruneExecutionMap(state.nativeThreadExecutionIds); @@ -1903,9 +2079,11 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv private matchesChildEventGeneration( item: PendingChildEvent, - childMapping: { executionId: string; generation: number }, + childMapping: { executionId: string; generation: number; deliveryAttempt?: number }, executionId: string, ): boolean { + if (item.event.deliveryAttempt !== undefined && childMapping.deliveryAttempt !== undefined + && item.event.deliveryAttempt !== childMapping.deliveryAttempt) return false; if (item.childGeneration !== undefined) return item.childGeneration === childMapping.generation; return !item.executionIdAtBuffer || item.executionIdAtBuffer === executionId; } @@ -2002,9 +2180,8 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv async interrupt(state: CodexSessionState): Promise { const turnExecutionId = executionForDrain(state); for (const event of state.mapper.drainPendingAssistantBoundary(false)) { - this.emitRuntimeEvent(providerRuntimeEvent( - turnExecutionId ? { ...event, turnExecutionId } : event, - )); + const runtimeEvent = providerRuntimeEvent(turnExecutionId ? { ...event, turnExecutionId } : event); + this.emitRuntimeEvent(this.withCodexTurnAttempt(state, runtimeEvent, turnExecutionId)); } const nativeTurnId = state.currentNativeTurnId; if (nativeTurnId) { @@ -2012,24 +2189,24 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv await state.server.interruptTurnAndDrain(nativeTurnId); } catch (error) { if (turnExecutionId) { - this.emitRuntimeEvent(providerRuntimeEvent({ + this.emitCodexTurnEvent(state, { type: AgentEventType.Ended, threadId: state.threadId, turnExecutionId, reason: "provider_lost", - } satisfies AgentEvent)); + } satisfies AgentEvent, turnExecutionId); } throw error; } } else { await state.server.interruptTurn(); if (turnExecutionId) { - this.emitRuntimeEvent(providerRuntimeEvent({ + this.emitCodexTurnEvent(state, { type: AgentEventType.Ended, threadId: state.threadId, turnExecutionId, reason: "provider_lost", - } satisfies AgentEvent)); + } satisfies AgentEvent, turnExecutionId); } } state.pendingTurnStartNotification = undefined; @@ -2048,12 +2225,12 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv * Drives shutdown, eviction, and stale-discard, but not turn cancellation. */ async close(state: CodexSessionState): Promise { + if (this.activeCodexServers.get(state.sessionId) === state.server) this.activeCodexServers.delete(state.sessionId); await this.host.threadControl.close(state.sessionId); const turnExecutionId = executionForDrain(state); for (const event of state.mapper.drainPendingAssistantBoundary(false)) { - this.emitRuntimeEvent(providerRuntimeEvent( - turnExecutionId ? { ...event, turnExecutionId } : event, - )); + const runtimeEvent = providerRuntimeEvent(turnExecutionId ? { ...event, turnExecutionId } : event); + this.emitRuntimeEvent(this.withCodexTurnAttempt(state, runtimeEvent, turnExecutionId)); } this.liveSessionIds.delete(state.sessionId); state.pendingTurnStartNotification = undefined; @@ -2307,11 +2484,17 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv } private emitCodexPendingAssistantBoundary(entry: CodexSessionState, turnExecutionId: string): void { - for (const event of entry.mapper.drainPendingAssistantBoundary(false)) this.emitCodexTurnEvent(event, turnExecutionId); + for (const event of entry.mapper.drainPendingAssistantBoundary(false)) this.emitCodexTurnEvent(entry, event, turnExecutionId); } - private emitCodexTurnEvent(event: AgentEvent, turnExecutionId: string): void { - this.emitRuntimeEvent(providerRuntimeEvent({ ...event, turnExecutionId })); + private emitCodexTurnEvent(entry: CodexSessionState, event: AgentEvent, turnExecutionId: string): void { + this.emitRuntimeEvent(this.withCodexTurnAttempt(entry, providerRuntimeEvent({ ...event, turnExecutionId }), turnExecutionId)); + } + + private withCodexTurnAttempt(entry: CodexSessionState, event: ProviderRuntimeEvent, turnExecutionId: string | undefined): ProviderRuntimeEvent { + const nativeAttempt = this.nativeTurnAttempt(entry, entry.currentNativeTurnId); + return nativeAttempt && nativeAttempt.executionId === turnExecutionId + ? { ...event, deliveryAttempt: nativeAttempt.deliveryAttempt } : event; } private async waitForCodexTurn( @@ -2478,10 +2661,16 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv } const assignment = assignNativeExecution(entry.turnExecutionIdsByNativeTurn, turnId, turnExecutionId, entry.nativeExecutionConflictKeys); if (assignment === "conflict") throw new Error("Codex turn/start response reused native turn id"); + const deliveryAttempt = entry.turnDiffRouting?.turnExecutionId === turnExecutionId + ? entry.turnDiffRouting.deliveryAttempt : undefined; + if (!this.rememberNativeTurnAttempt(entry, turnId, turnExecutionId, deliveryAttempt)) { + throw new Error("Codex turn/start response reused native turn id across dispatch attempts"); + } entry.currentNativeTurnId = turnId; entry.pendingTurnId = turnId; entry.turnBindingPhase = "bound"; pruneExecutionMap(entry.turnExecutionIdsByNativeTurn); + this.pruneNativeTurnAttempts(entry); } private handleCodexTurnWaitError( @@ -2531,7 +2720,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv ): void { if (run.seq === run.entry.runTurnSeq) this.clearCodexTurnRunState(run.entry); if (!completion.serverDied && run.seq === run.entry.runTurnSeq) { - this.emitCodexTurnEvent(completion.deferredEnded ?? { + this.emitCodexTurnEvent(run.entry, completion.deferredEnded ?? { type: AgentEventType.Ended, threadId, turnExecutionId, @@ -2775,6 +2964,8 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv this.pendingSpawnTurns.clear(); this.pendingBrowserAccess.clear(); this.liveSessionIds.clear(); + this.activeCodexServers.clear(); + this.canonicalTurnRoutingsByThread.clear(); logger.info("CodexProvider shutdown complete"); } } diff --git a/packages/providers/src/private/cursor/__tests__/cursor-canonical-event-publisher.test.ts b/packages/providers/src/private/cursor/__tests__/cursor-canonical-event-publisher.test.ts index 726ee0a68..968d1f56e 100644 --- a/packages/providers/src/private/cursor/__tests__/cursor-canonical-event-publisher.test.ts +++ b/packages/providers/src/private/cursor/__tests__/cursor-canonical-event-publisher.test.ts @@ -48,6 +48,8 @@ describe("CursorCanonicalEventPublisher", () => { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, + batchId: `cursor:${routing.executionId}:attempt:1:event:1`, + deliveryAttempt: 1, phase: "running", events: [{ eventId: `cursor:${routing.executionId}:attempt:1:event:1`, @@ -65,6 +67,8 @@ describe("CursorCanonicalEventPublisher", () => { }); AgentEventRoutingSchema.parse(textBatch.events[0]?.routing); expect(terminalBatch).toMatchObject({ + batchId: `cursor:${routing.executionId}:attempt:1:event:2`, + deliveryAttempt: 1, events: [{ eventId: `cursor:${routing.executionId}:attempt:1:event:2`, sourceSequence: 2, diff --git a/packages/providers/src/private/cursor/__tests__/cursor-session-continuity.test.ts b/packages/providers/src/private/cursor/__tests__/cursor-session-continuity.test.ts index ffafeadb1..ed6afc771 100644 --- a/packages/providers/src/private/cursor/__tests__/cursor-session-continuity.test.ts +++ b/packages/providers/src/private/cursor/__tests__/cursor-session-continuity.test.ts @@ -1,7 +1,7 @@ import * as NodeEvents from "node:events"; import type * as NodeChildProcess from "node:child_process"; import type { ClientSideConnection, SessionNotification } from "@agentclientprotocol/sdk"; -import { getDefaultSettings, type TurnRequest } from "@mcode/contracts"; +import { AgentEventType, getDefaultSettings, type TurnRequest } from "@mcode/contracts"; import { describe, expect, it, vi } from "vitest"; import type { ProviderHostPorts } from "../../../host-ports.js"; import { AcpSessionRuntime, SessionRecoveryFailedError } from "../../protocols/acp/acp-session-runtime.js"; @@ -125,6 +125,54 @@ function submittedRuntimeEvents(host: ProviderHostPorts): unknown[] { } describe("CursorProvider session continuity", () => { + it("delivers the turn through the canonical host without EventEmitter duplicates", async () => { + const host = createHost(); + const fake = createFakeRuntime("cursor-session-1", 101); + const start = vi.spyOn(AcpSessionRuntime, "start").mockResolvedValue(fake.runtime); + const provider = new CursorProvider(host, { + settings: { get: () => getDefaultSettings() }, + skills: { list: () => [] }, + }, 60_000); + const directEvents = vi.fn(); + provider.on("event", directEvents); + + try { + await provider.sendTurn(turn("prompt", "execution-1")); + + expect(submittedRuntimeEvents(host)).toEqual([ + expect.objectContaining({ type: AgentEventType.System, turnExecutionId: "execution-1" }), + expect.objectContaining({ type: AgentEventType.TurnComplete, turnExecutionId: "execution-1" }), + expect.objectContaining({ type: AgentEventType.Ended, turnExecutionId: "execution-1" }), + ]); + expect(directEvents).not.toHaveBeenCalled(); + } finally { + start.mockRestore(); + await (provider as unknown as { runtime: { shutdown(): Promise } }).runtime.shutdown(); + } + }); + + it("rejects the turn when canonical host delivery fails", async () => { + const host = createHost(); + host.events.submit = vi.fn(async () => { throw new Error("writer unavailable"); }); + const fake = createFakeRuntime("cursor-session-1", 101); + const start = vi.spyOn(AcpSessionRuntime, "start").mockResolvedValue(fake.runtime); + const provider = new CursorProvider(host, { + settings: { get: () => getDefaultSettings() }, + skills: { list: () => [] }, + }, 60_000); + const directEvents = vi.fn(); + provider.on("event", directEvents); + + try { + await expect(provider.sendTurn(turn("prompt", "execution-1"))) + .rejects.toThrow("writer unavailable"); + expect(directEvents).not.toHaveBeenCalled(); + } finally { + start.mockRestore(); + await (provider as unknown as { runtime: { shutdown(): Promise } }).runtime.shutdown(); + } + }); + it("reuses a healthy ACP child for normal turns and replaces it only for an explicit fresh session", async () => { const host = createHost(); const first = createFakeRuntime("cursor-session-1", 101); diff --git a/packages/providers/src/private/cursor/cursor-canonical-event-publisher.ts b/packages/providers/src/private/cursor/cursor-canonical-event-publisher.ts index ad8fe4aa4..9ccbe4175 100644 --- a/packages/providers/src/private/cursor/cursor-canonical-event-publisher.ts +++ b/packages/providers/src/private/cursor/cursor-canonical-event-publisher.ts @@ -50,6 +50,8 @@ export class CursorCanonicalEventPublisher { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, + batchId: draft.eventId, + deliveryAttempt: routing.deliveryAttempt, phase: "running", events: [draft], }); diff --git a/scripts/build-server-dev-bundle.mjs b/scripts/build-server-dev-bundle.mjs index 70038022a..d2e0aa6e0 100644 --- a/scripts/build-server-dev-bundle.mjs +++ b/scripts/build-server-dev-bundle.mjs @@ -574,12 +574,14 @@ export function findCopilotSdkPath(serverCjsOut, platform, arch) { return NodeFS.existsSync(copilotIndex) && NodeFS.existsSync(platformEntry) ? copilotIndex : undefined; } -/** Bundle the server and its isolated PTY host from one compiled server tree. */ +/** Bundle the server and its isolated workers from one compiled server tree. */ export async function buildServerRuntimeBundles({ serverRoot, serverOutFile, ptyHostOutFile, providerEventWorkerOutFile = NodePath.resolve(NodePath.dirname(serverOutFile), "provider-event.worker.cjs"), + canonicalWriterWorkerOutFile = NodePath.resolve(NodePath.dirname(serverOutFile), "canonical-agent-writer.worker.cjs"), + executionWorkerOutFile = NodePath.resolve(NodePath.dirname(serverOutFile), "execution.worker.cjs"), production = false, }) { const shared = { @@ -588,7 +590,7 @@ export async function buildServerRuntimeBundles({ target: "node20", sourcemap: true, format: "cjs", - external: ["bun:sqlite", "node-pty", "electron", "koffi"], + external: ["bun", "bun:sqlite", "node-pty", "electron", "koffi"], banner: { js: 'var __importMetaUrl = require("url").pathToFileURL(__filename).href;', }, @@ -620,6 +622,16 @@ export async function buildServerRuntimeBundles({ entryPoints: [providerEventWorkerEntry], outfile: providerEventWorkerOutFile, })); + bundles.push(build({ + ...shared, + entryPoints: [NodePath.resolve(serverRoot, "dist-tsc/features/agents/canonical/canonical-agent-writer.worker.js")], + outfile: canonicalWriterWorkerOutFile, + })); + bundles.push(build({ + ...shared, + entryPoints: [NodePath.resolve(serverRoot, "dist-tsc/features/agents/execution/execution.worker.js")], + outfile: executionWorkerOutFile, + })); await Promise.all(bundles); } @@ -657,5 +669,5 @@ export async function rebuildServerDevBundle(options = {}) { console.log(`[server-dev-bundle] Copied Drizzle migrations -> ${drizzleDst}`); } - console.log(`[server-dev-bundle] Complete: ${serverOutFile}, ${ptyHostOutFile}`); + console.log(`[server-dev-bundle] Complete: ${serverOutFile}, ${ptyHostOutFile}, execution.worker.cjs`); } diff --git a/scripts/perf/seven-thread-live-harness.mjs b/scripts/perf/seven-thread-live-harness.mjs index 8a9137c16..04e813240 100644 --- a/scripts/perf/seven-thread-live-harness.mjs +++ b/scripts/perf/seven-thread-live-harness.mjs @@ -26,10 +26,13 @@ import { renderFixtureWrapper } from "../../.agents/skills/verify-mcode/scripts/ import { openRuntimeVerificationSocket } from "../../.agents/skills/verify-mcode/scripts/runtime.mjs"; export const THREAD_COUNT = 7; +const STOP_ONE_ORDINAL = 3; const TERMINAL_CONTROL_COUNT = 1; export const WORKLOAD_MODEL = "gpt-5.6-luna"; const PROVIDER_ID = "codex"; const HEALTH_SAMPLE_INTERVAL_MS = 250; +const MAX_WORKER_QUEUE_SAMPLES = 512; +const MAX_THROUGHPUT_INTERVALS = 600; const EVENT_LOOP_INTERVAL_MS = 50; const TURN_TIMEOUT_MS = 120_000; const CONTROL_RPC_TIMEOUT_MS = 90_000; @@ -37,7 +40,7 @@ const CLEANUP_RPC_TIMEOUT_MS = 30_000; const PERFORMANCE_DIRECTORY = [".dev", "verification", "performance", "seven-thread-live"]; const FIXTURE_SOURCE = [".agents", "skills", "verify-mcode", "scripts", "transcript-provider-fixture.mjs"]; const FIXTURE_PROMPT = "Seven concurrent performance verifier long narrative"; -const FLAG_OPTIONS = new Set(["--run", "--confirm-run", "--confirm-cleanup"]); +const FLAG_OPTIONS = new Set(["--run", "--confirm-run", "--confirm-cleanup", "--stop-one"]); const VALUE_OPTIONS = new Set(["--label", "--cleanup", "--cleanup-receipt"]); const TERMINAL_TRANSPORTS = new Map([ ["legacy", { @@ -58,6 +61,7 @@ const HELP = `Seven concurrent thread live performance harness Usage: bun scripts/perf/seven-thread-live-harness.mjs --run --confirm-run --label + bun scripts/perf/seven-thread-live-harness.mjs --run --confirm-run --label after --stop-one bun scripts/perf/seven-thread-live-harness.mjs --cleanup --confirm-cleanup bun scripts/perf/seven-thread-live-harness.mjs --cleanup-receipt --confirm-run @@ -65,6 +69,8 @@ The run command creates exactly seven direct threads in this worktree's .dev/fixture-repo. It temporarily routes Codex through the checked-in transcript fixture and restores the previous setting during cleanup. It never falls back to a real Codex model or prints runtime credentials. +--stop-one creates seven fixture threads, stops the third while all seven are +active, and verifies the other six complete and reload durably. `; /** Parses the small explicit command surface before any runtime mutation. */ @@ -135,11 +141,13 @@ function parseRunCommand(flags, values) { } const label = values.get("--label"); if (label !== "before" && label !== "after") throw new Error("--run requires --label before or --label after"); - return { command: "run", label }; + return flags.has("--stop-one") + ? { command: "run", label, stopOne: true } + : { command: "run", label }; } function parseCleanupReceiptCommand(flags, values, receiptPath) { - if (!flags.has("--confirm-run") || flags.has("--confirm-cleanup") || values.has("--label")) { + if (!flags.has("--confirm-run") || flags.has("--confirm-cleanup") || flags.has("--stop-one") || values.has("--label")) { throw new Error("--cleanup-receipt requires --confirm-run and cannot include run options"); } return { command: "cleanup", receiptPath }; @@ -147,7 +155,7 @@ function parseCleanupReceiptCommand(flags, values, receiptPath) { function parseCleanupCommand(flags, values, receiptPath) { if (!flags.has("--confirm-cleanup")) throw new Error("--cleanup requires --confirm-cleanup"); - if (flags.has("--confirm-run") || values.has("--label")) throw new Error("--cleanup cannot include run options"); + if (flags.has("--confirm-run") || flags.has("--stop-one") || values.has("--label")) throw new Error("--cleanup cannot include run options"); return { command: "cleanup", receiptPath }; } @@ -252,37 +260,54 @@ export function normalizeTerminalSessions(transport, sessions) { /** Extracts only Mcode server stall entries from JSONL server diagnostics. */ export function parseServerStallEntries(contents, startedAtMs, endedAtMs) { if (typeof contents !== "string") return []; - return contents.split(/\r?\n/).flatMap((line) => parseServerStallLine(line, startedAtMs, endedAtMs)); + const byTimestamp = new Map(); + for (const line of contents.split(/\r?\n/)) { + const sample = parseServerStallLine(line, startedAtMs, endedAtMs); + if (!sample) continue; + const previous = byTimestamp.get(sample.timestamp); + if (!previous || sample.source === "server-work-stall") byTimestamp.set(sample.timestamp, sample); + } + return [...byTimestamp.values()].map(({ timestamp, stalledMs }) => ({ timestamp, stalledMs })); } function parseServerStallLine(line, startedAtMs, endedAtMs) { - if (!line) return []; + if (!line) return null; try { const entry = JSON.parse(line); - if (!isValidServerStall(entry, startedAtMs, endedAtMs)) return []; - return [{ timestamp: entry.timestamp, stalledMs: entry.stalledMs }]; + const sample = serverStallSample(entry); + if (!sample || !isValidServerStallTime(entry.timestamp, startedAtMs, endedAtMs)) return null; + return { timestamp: entry.timestamp, ...sample }; } catch { - return []; + return null; + } +} + +function serverStallSample(entry) { + if (entry?.kind === "server-work-stall" && entry.message === "Server work trace" && Number.isFinite(entry.delayMs)) { + return { source: "server-work-stall", stalledMs: entry.delayMs }; + } + if (entry?.message === "Event loop stalled" && Number.isFinite(entry.stalledMs)) { + return { source: "event-loop-stalled", stalledMs: entry.stalledMs }; } + return null; } -function isValidServerStall(entry, startedAtMs, endedAtMs) { - const timestampMs = Date.parse(entry?.timestamp); - return entry?.message === "Event loop stalled" - && Number.isFinite(timestampMs) - && timestampMs >= startedAtMs - && timestampMs <= endedAtMs - && Number.isFinite(entry?.stalledMs); +function isValidServerStallTime(timestamp, startedAtMs, endedAtMs) { + const timestampMs = Date.parse(timestamp); + return Number.isFinite(timestampMs) && timestampMs >= startedAtMs && timestampMs <= endedAtMs; } /** Creates the stable names used to prove that only this harness owns a thread. */ -export function expectedThreadTitle(runId, ordinal) { - return `Seven-thread live performance ${runId} ${ordinal}/${THREAD_COUNT}`; +export function expectedThreadTitle(runId, ordinal, threadCount = THREAD_COUNT, scenario = "baseline") { + const name = scenario === "stop-one" + ? `${threadCount === 6 ? "Six" : "Seven"}-thread Stop verification` + : "Seven-thread live performance"; + return `${name} ${runId} ${ordinal}/${threadCount}`; } /** Runs the fixed seven-thread workload and writes its recovery-capable receipt. */ -export async function runLiveHarness({ repoRoot = resolveRepoRoot(), label }) { - const context = createLiveHarnessContext(repoRoot, label); +export async function runLiveHarness({ repoRoot = resolveRepoRoot(), label, stopOne = false }) { + const context = createLiveHarnessContext(repoRoot, label, stopOne); startLiveHarness(context); try { await executeLiveHarness(context); @@ -295,16 +320,16 @@ export async function runLiveHarness({ repoRoot = resolveRepoRoot(), label }) { return { receiptPath: context.run.receiptPath, receipt: context.receipt }; } -function createLiveHarnessContext(repoRoot, label) { +function createLiveHarnessContext(repoRoot, label, stopOne) { const paths = getRuntimePaths(repoRoot); const ports = requireLocalRuntime(repoRoot); - const run = createRun(repoRoot, label); - const receipt = createReceipt(run, label); + const run = createRun(repoRoot, stopOne ? `${label}-stop-one` : label); + const receipt = createReceipt(run, label, stopOne); const eventLoop = new EventLoopStallSampler(EVENT_LOOP_INTERVAL_MS); const serverStalls = new ServerLogStallReader(paths.logsDir); - const healthSampler = new HealthSampler(ports.healthUrl, receipt.metrics.health); + const healthSampler = new HealthSampler(ports.healthUrl, receipt.metrics.health, receipt.metrics.workerQueue); const eventState = new Map(); - const terminalEvents = new TerminalEventWaiter(eventState); + const terminalEvents = new TerminalEventWaiter(eventState, stopOne, () => receipt.state.stop); const turnStarts = new TurnStartWaiter(eventState); return { repoRoot, @@ -321,6 +346,7 @@ function createLiveHarnessContext(repoRoot, label) { socket: null, primaryError: null, workspace: null, + stopOne, }; } @@ -338,7 +364,11 @@ async function executeLiveHarness(context) { await createAndSubscribeThreads(context); const terminal = await preflightTerminalTransport(context); const { completed, sends } = await dispatchFixtureTurns(context); + if (context.stopOne) await waitForAllSevenActive(context); + const stop = context.stopOne ? stopOneActiveTurn(context) : null; + void stop?.catch(() => undefined); await sampleActiveControls(context, terminal); + if (stop) await stop; throwFirstRejected(await sends, "Dispatching the controlled fixture turns failed"); context.receipt.state.phase = "waiting-for-events"; await completed; @@ -350,6 +380,7 @@ async function prepareLiveHarness(context) { const { ports, receipt } = context; const initialHealth = await sampleHealth(ports.healthUrl); receipt.metrics.health.samples.push(initialHealth.durationMs); + recordWorkerQueueSample(receipt.metrics.workerQueue, initialHealth.workerQueue); assertHealthyRuntime(initialHealth); context.socket = await openRuntimeVerificationSocket(context.repoRoot, (push) => capturePush(push, context.eventState, context.terminalEvents, context.turnStarts)); await assertRuntimeIdle(context); @@ -414,14 +445,14 @@ async function startWorkloadMeasurement(context) { async function createAndSubscribeThreads(context) { const branch = currentFixtureBranch(context.paths.fixtureRepoDir); context.receipt.state.phase = "creating-threads"; - const creations = await Promise.allSettled(Array.from({ length: THREAD_COUNT }, (_, index) => createOwnedThread(context, branch, index + 1))); + const creations = await Promise.allSettled(Array.from({ length: context.receipt.workload.threadCount }, (_, index) => createOwnedThread(context, branch, index + 1))); throwFirstRejected(creations, "Creating verifier-owned direct threads failed"); assertCreatedThreadCount(context.receipt); await subscribeToThreadEvents(context); } async function createOwnedThread(context, branch, ordinal) { - const title = expectedThreadTitle(context.run.id, ordinal); + const title = expectedThreadTitle(context.run.id, ordinal, context.receipt.workload.threadCount, context.receipt.workload.scenario); const thread = await measuredRpc(context.socket, context.receipt.metrics.rpc, "thread.create", { workspaceId: context.workspace.id, title, @@ -437,7 +468,7 @@ async function createOwnedThread(context, branch, ordinal) { } function assertCreatedThreadCount(receipt) { - if (receipt.state.threads.length !== THREAD_COUNT) throw new Error("The harness did not create exactly seven threads"); + if (receipt.state.threads.length !== receipt.workload.threadCount) throw new Error("The harness did not create the requested fixture thread count"); } async function subscribeToThreadEvents(context) { @@ -467,6 +498,52 @@ async function dispatchFixtureTurns(context) { return { completed, sends }; } +async function waitForAllSevenActive(context) { + const deadline = Date.now() + TURN_TIMEOUT_MS; + const threads = context.receipt.state.threads; + while (Date.now() < deadline) { + if (threads.some((thread) => thread.completedAtMs !== null || thread.persistedAtMs !== null)) { + throw new Error("A fixture turn ended before all seven became active; Stop proof is inconclusive"); + } + const target = threads[STOP_ONE_ORDINAL - 1]; + if (threads.every((thread) => thread.startedAtMs !== null) + && target.events.filter((event) => event.type === "assistantMessageBoundary").length >= 10) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error("All seven fixture turns and ten narrative boundaries in the stopped turn were not observed before the deadline"); +} + +async function stopOneActiveTurn(context) { + const target = context.receipt.state.threads[STOP_ONE_ORDINAL - 1]; + const launchedAtMs = NodePerfHooks.performance.now(); + const allSevenActiveAtLaunch = context.receipt.state.threads.every(isActiveFixtureThread); + if (!allSevenActiveAtLaunch) throw new Error("Stop did not launch while all seven fixture turns were active"); + context.receipt.state.stop = { ordinal: STOP_ONE_ORDINAL, threadId: target.id, launchedAtMs, allSevenActiveAtLaunch }; + writeReceipt(context.run.receiptPath, context.receipt, context.paths.devDir); + const result = await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.stop", { threadId: target.id }, undefined, CONTROL_RPC_TIMEOUT_MS); + context.receipt.state.stop = { + ...context.receipt.state.stop, + completedAtMs: NodePerfHooks.performance.now(), + status: result?.status, + dispatchState: result?.dispatchState, + turnExecutionId: result?.turnExecutionId, + snapshotPhase: result?.snapshot?.phase, + }; + if (!isCancelledStopResult(result, target.id)) { + throw new Error("Stop did not return the targeted execution's cancelled outcome"); + } + context.terminalEvents.notify(); + writeReceipt(context.run.receiptPath, context.receipt, context.paths.devDir); +} + +function isActiveFixtureThread(thread) { + return thread.startedAtMs !== null && thread.completedAtMs === null && thread.persistedAtMs === null; +} + +function isCancelledStopResult(result, threadId) { + return result?.status === "cancelled" && result?.snapshot?.phase === "cancelled" && result?.threadId === threadId; +} + async function sendFixtureTurn(context, thread) { thread.sentAtMs = NodePerfHooks.performance.now(); await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.send", { @@ -538,17 +615,35 @@ async function readDurableConversations(context) { async function readDurableConversation(context, thread) { const tail = await measuredRpc(context.socket, context.receipt.metrics.rpc, "conversation.tail", { threadId: thread.id, limit: 2 }); - thread.durable = auditConversationTail(tail); + const expectedOutcome = context.stopOne && thread.ordinal === STOP_ONE_ORDINAL ? "cancelled" : "completed"; + const assistant = tail?.messages?.find((message) => message?.role === "assistant"); + const narrative = expectedOutcome === "cancelled" && assistant + ? await measuredRpc(context.socket, context.receipt.metrics.rpc, "narrative.list", { messageId: assistant.id }) + : null; + thread.durable = context.stopOne + ? auditConversationOutcome(tail, narrative, expectedOutcome, context.receipt.state.stop?.turnExecutionId) + : auditConversationTail(tail); if (!thread.durable.ok) throw new Error(`Durable conversation proof failed for thread ${thread.ordinal}`); } async function verifyCompletedWorkload(context) { const activeAgents = await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.activeCount", {}); if (activeAgents !== 0) throw new Error("The controlled turns completed but the runtime still reports active agents"); + if (context.stopOne) await verifyStoppedRuntimeSnapshot(context); assertConclusiveControlSamples(context.receipt); if (!auditRun(context.receipt)) throw new Error("The controlled workload completed with event-loss, order, completion, or durability failures"); } +async function verifyStoppedRuntimeSnapshot(context) { + const snapshots = await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.listRunning", {}); + const stop = context.receipt.state.stop; + stop.reconnectSnapshot = Array.isArray(snapshots) && snapshots.some((snapshot) => + snapshot?.threadId === stop.threadId + && snapshot?.turnExecutionId === stop.turnExecutionId + && snapshot?.phase === "cancelled"); + if (!stop.reconnectSnapshot) throw new Error("The stopped execution did not retain its cancelled reconnect snapshot"); +} + function assertConclusiveControlSamples(receipt) { if (receipt.modelList?.inconclusive || receipt.terminalCapabilities?.inconclusive || receipt.state.terminalCreateInconclusive) { throw new Error("The active-turn control samples were inconclusive because all fixture turns finished before a required RPC could begin"); @@ -586,7 +681,8 @@ async function recordFinalReceiptMetrics(context) { receipt.metrics.turnCompletion = summarizeLatency(receipt.state.threads.map((thread) => elapsedSinceSend(thread, "completedAtMs"))); receipt.metrics.turnDurability = summarizeLatency(receipt.state.threads.map((thread) => elapsedSinceSend(thread, "persistedAtMs"))); receipt.state.activeControlLaunch = attributeControlLaunchToTurnCompletion(receipt.state.activeControlLaunch, receipt.state.threads); - receipt.metrics.events = summarizeEventAudits(receipt.state.threads); + receipt.metrics.events = summarizeEventAudits(receipt); + receipt.metrics.eventThroughput = summarizeEventThroughput(receipt.state.threads); receipt.metrics.harnessEventLoopStalls = { scope: "harness-process", intervalMs: EVENT_LOOP_INTERVAL_MS, @@ -647,7 +743,7 @@ function createRun(repoRoot, label) { }; } -function createReceipt(run, label) { +function createReceipt(run, label, stopOne) { return { schemaVersion: 1, runId: run.id, @@ -657,6 +753,7 @@ function createReceipt(run, label) { ok: false, workload: { threadCount: THREAD_COUNT, + scenario: stopOne ? "stop-one" : "baseline", terminalControlCount: TERMINAL_CONTROL_COUNT, provider: PROVIDER_ID, model: WORKLOAD_MODEL, @@ -678,14 +775,23 @@ function createReceipt(run, label) { activeTurnAssertions: [], activeControlLaunch: null, terminalCreateInconclusive: false, + stop: null, threads: [], }, metrics: { health: { samples: [], summary: null }, + workerQueue: { + intervalMs: HEALTH_SAMPLE_INTERVAL_MS, + samples: [], + omittedSamples: 0, + unavailableSamples: 0, + summary: { sampleCount: 0, maxPending: 0, maxPendingBytes: 0, maxActiveExecutions: 0, maxQueuedPerWorker: 0, maxInFlightWorkers: 0 }, + }, rpc: {}, turnCompletion: null, turnDurability: null, events: null, + eventThroughput: null, harnessEventLoopStalls: null, serverEventLoopStalls: null, memory: [], @@ -827,6 +933,9 @@ function recordThreadPush(push, thread, turnStarts) { recordAgentEvent(push.data, thread, now, turnStarts); } else if (push.channel === "turn.persisted") { thread.persistedAtMs ??= now; + thread.persistedOutcome ??= push.data.outcome ?? null; + } else if (push.channel === "thread.status") { + thread.status = push.data.status; } } @@ -877,8 +986,10 @@ export function attributeControlLaunchToTurnCompletion(controlLaunch, threads) { } class TerminalEventWaiter { - constructor(state) { + constructor(state, stopOne = false, getStop = () => null) { this.state = state; + this.stopOne = stopOne; + this.getStop = getStop; this.resolve = null; } @@ -889,7 +1000,7 @@ class TerminalEventWaiter { reject(new Error("Not every controlled thread produced both turn completion and durable persistence events before the deadline")); }, timeoutMs); this.resolve = () => { - if (![...this.state.values()].every((thread) => thread.completedAtMs !== null && thread.persistedAtMs !== null)) return; + if (![...this.state.values()].every((thread) => this.isComplete(thread))) return; this.resolve = null; clearTimeout(timer); resolve(); @@ -901,6 +1012,19 @@ class TerminalEventWaiter { notify() { this.resolve?.(); } + + isComplete(thread) { + if (thread.persistedAtMs === null) return false; + if (!this.stopOne || thread.ordinal !== STOP_ONE_ORDINAL) return thread.completedAtMs !== null; + if (thread.status === "paused" || thread.status === "cancelled") return true; + return isInterruptedAfterStop(thread, this.getStop()); + } +} + +function isInterruptedAfterStop(thread, stop) { + return thread.status === "interrupted" && stop?.threadId === thread.id + && stop.status === "cancelled" && stop.snapshotPhase === "cancelled" + && typeof stop.turnExecutionId === "string"; } class TurnStartWaiter { @@ -942,14 +1066,69 @@ function auditConversationTail(tail) { }; } +function auditConversationOutcome(tail, narrative, expectedOutcome, expectedExecutionId) { + const messages = Array.isArray(tail?.messages) ? tail.messages : []; + const assistant = messages.find((message) => message?.role === "assistant"); + const hasUserMessage = messages.some((message) => message?.role === "user"); + const assistantSummary = summarizeAssistantOutcome(assistant); + const narrativeSummary = summarizeNarrativeOutcome(narrative); + const summary = { messageCount: messages.length, hasUserMessage, ...assistantSummary, ...narrativeSummary }; + return { + ...summary, + ok: hasExpectedDurableOutcome(summary, expectedOutcome, expectedExecutionId), + }; +} + +function summarizeAssistantOutcome(assistant) { + return { + hasFixtureAssistantMessage: assistant?.content?.includes("Fixture answer:") === true, + assistantContentLength: assistant?.content?.length ?? 0, + outcome: assistant?.outcome ?? null, + outcomeExecutionId: assistant?.outcomeExecutionId ?? null, + }; +} + +function summarizeNarrativeOutcome(narrative) { + const thoughts = narrative?.thoughts ?? []; + const tools = narrative?.tools ?? []; + return { + hasRetainedNarrative: thoughts.some((segment) => segment.text?.includes("Fixture step 1:")), + thoughtSegmentCount: thoughts.length, + toolCount: tools.length, + }; +} + +function hasExpectedDurableOutcome(summary, expectedOutcome, expectedExecutionId) { + if (!summary.hasUserMessage || summary.outcome !== expectedOutcome) return false; + if (expectedOutcome !== "cancelled") return summary.hasFixtureAssistantMessage; + return summary.outcomeExecutionId === expectedExecutionId + && summary.hasRetainedNarrative + && !summary.hasFixtureAssistantMessage; +} + function auditRun(receipt) { + if (receipt.state.threads.length !== THREAD_COUNT) return false; return receipt.state.threads.every((thread) => { thread.eventAudit = auditAgentEvents(thread.events); - return thread.eventAudit.sequenceValid && thread.eventAudit.completed && thread.durable?.ok === true; + return thread.eventAudit.sequenceValid && isExpectedTerminal(receipt, thread) && thread.durable?.ok === true; }); } -function summarizeEventAudits(threads) { +function isExpectedTerminal(receipt, thread) { + if (receipt.workload.scenario !== "stop-one" || thread.ordinal !== STOP_ONE_ORDINAL) return thread.eventAudit.completed; + return hasConfirmedStoppedTerminal(receipt, thread); +} + +function hasConfirmedStoppedTerminal(receipt, thread) { + const stop = receipt.state.stop; + return stop?.threadId === thread.id && stop.status === "cancelled" + && stop.snapshotPhase === "cancelled" && stop.reconnectSnapshot === true + && typeof stop.turnExecutionId === "string" && thread.persistedAtMs !== null + && (thread.status === "paused" || thread.status === "cancelled" || thread.status === "interrupted"); +} + +function summarizeEventAudits(receipt) { + const threads = receipt.state.threads; const audits = threads.map((thread) => thread.eventAudit ?? auditAgentEvents(thread.events ?? [])); for (let index = 0; index < threads.length; index += 1) threads[index].eventAudit = audits[index]; const flatten = (field) => audits.flatMap((audit) => audit[field] ?? []); @@ -960,7 +1139,28 @@ function summarizeEventAudits(threads) { missingSequences: flatten("missingSequences"), duplicateSequences: flatten("duplicateSequences"), arrivalOrderViolations: flatten("arrivalOrderViolations"), - ok: audits.length === THREAD_COUNT && audits.every((audit) => audit.sequenceValid && audit.completed), + ok: audits.length === THREAD_COUNT && audits.every((audit) => audit.sequenceValid) + && threads.every((thread) => thread.durable?.ok === true) + && threads.every((thread) => isExpectedTerminal(receipt, thread)), + }; +} + +/** Counts public event arrivals in bounded time buckets across the workload. */ +export function summarizeEventThroughput(threads) { + const arrivals = threads.flatMap((thread) => (thread.events ?? []).map((event) => event.atMs)).filter(Number.isFinite); + if (arrivals.length === 0) return { scope: "public-agent-event-push", intervalMs: 1_000, totalEvents: 0, eventsPerInterval: [], peakEventsPerSecond: 0 }; + const sentAt = threads.map((thread) => thread.sentAtMs).filter(Number.isFinite); + const first = Math.min(...arrivals, ...sentAt); + const last = Math.max(...arrivals); + const intervalMs = Math.max(1_000, Math.ceil((last - first + 1) / MAX_THROUGHPUT_INTERVALS / 1_000) * 1_000); + const counts = Array.from({ length: Math.floor((last - first) / intervalMs) + 1 }, () => 0); + for (const atMs of arrivals) counts[Math.floor((atMs - first) / intervalMs)] += 1; + return { + scope: "public-agent-event-push", + intervalMs, + totalEvents: arrivals.length, + eventsPerInterval: counts, + peakEventsPerSecond: roundMs(Math.max(...counts) * 1_000 / intervalMs), }; } @@ -1001,7 +1201,7 @@ function resolveCleanupTransport(receipt, threads, cleanup) { async function cleanupWorkloadResources(socket, receipt, threads, workspace, terminalTransport, cleanup) { if (cleanup.failures.length > 0) return; if (terminalTransport) await cleanupPtys(socket, threads, terminalTransport, cleanup); - await cleanupThreads(socket, threads, workspace.id, receipt.runId, cleanup); + await cleanupThreads(socket, threads, workspace.id, receipt.runId, receipt.workload.threadCount, receipt.workload.scenario, cleanup); } async function cleanupPtys(socket, threads, transport, cleanup) { @@ -1091,11 +1291,11 @@ function cleanupRpc(socket, method, params) { return socket.rpc(method, params, cleanupDeadline()); } -async function cleanupThreads(socket, threads, workspaceId, runId, cleanup) { +async function cleanupThreads(socket, threads, workspaceId, runId, threadCount, scenario, cleanup) { const current = await currentWorkspaceThreads(socket, workspaceId, cleanup); if (current === null) return; for (const thread of threads) { - await cleanupThread(socket, current, thread, runId, cleanup); + await cleanupThread(socket, current, thread, runId, threadCount, scenario, cleanup); } await verifyThreadsRemoved(socket, threads, workspaceId, cleanup); } @@ -1109,14 +1309,14 @@ async function currentWorkspaceThreads(socket, workspaceId, cleanup) { } } -async function cleanupThread(socket, current, thread, runId, cleanup) { +async function cleanupThread(socket, current, thread, runId, threadCount, scenario, cleanup) { if (!isReceiptThread(thread)) return; const found = current.find((candidate) => candidate?.id === thread.id); if (!found) { cleanup.threads.push({ threadId: thread.id, outcome: "already-deleted" }); return; } - if (!matchesReceiptThread(found, thread, runId)) { + if (!matchesReceiptThread(found, thread, runId, threadCount, scenario)) { cleanup.failures.push(`Refusing to delete thread ${thread.id}: title does not match this receipt`); return; } @@ -1133,8 +1333,8 @@ function isReceiptThread(thread) { return typeof thread?.id === "string" && Number.isInteger(thread?.ordinal); } -function matchesReceiptThread(found, thread, runId) { - return found.title === expectedThreadTitle(runId, thread.ordinal) && found.title === thread.title; +function matchesReceiptThread(found, thread, runId, threadCount, scenario) { + return found.title === expectedThreadTitle(runId, thread.ordinal, threadCount, scenario) && found.title === thread.title; } async function verifyThreadsRemoved(socket, threads, workspaceId, cleanup) { @@ -1205,13 +1405,51 @@ async function sampleHealth(healthUrl) { const started = NodePerfHooks.performance.now(); const response = await fetch(healthUrl, { signal: AbortSignal.timeout(15_000) }); const payload = await response.json(); - return { durationMs: NodePerfHooks.performance.now() - started, status: response.ok ? payload?.status : `http-${response.status}` }; + return { + durationMs: NodePerfHooks.performance.now() - started, + status: response.ok ? payload?.status : `http-${response.status}`, + workerQueue: response.ok ? payload?.workerQueue : null, + }; +} + +/** Copies only queue counts from health, with a fixed receipt sample limit. */ +export function recordWorkerQueueSample(target, raw) { + if (!validWorkerQueueDepth(raw)) { + target.unavailableSamples += 1; + return; + } + const workers = raw.workers.map((worker) => ({ index: worker.index, queued: worker.queued, inFlight: worker.inFlight })); + const sample = { + atMs: roundMs(NodePerfHooks.performance.now()), + pending: raw.pending, + pendingBytes: raw.pendingBytes, + activeExecutions: raw.activeExecutions, + workers, + }; + target.summary.sampleCount += 1; + target.summary.maxPending = Math.max(target.summary.maxPending, sample.pending); + target.summary.maxPendingBytes = Math.max(target.summary.maxPendingBytes, sample.pendingBytes); + target.summary.maxActiveExecutions = Math.max(target.summary.maxActiveExecutions, sample.activeExecutions); + target.summary.maxQueuedPerWorker = Math.max(target.summary.maxQueuedPerWorker, ...workers.map((worker) => worker.queued)); + target.summary.maxInFlightWorkers = Math.max(target.summary.maxInFlightWorkers, workers.filter((worker) => worker.inFlight).length); + if (target.samples.length < MAX_WORKER_QUEUE_SAMPLES) target.samples.push(sample); + else target.omittedSamples += 1; +} + +function validWorkerQueueDepth(value) { + const count = (number) => Number.isSafeInteger(number) && number >= 0; + return value !== null && typeof value === "object" + && count(value.pending) && count(value.pendingBytes) && count(value.activeExecutions) + && Array.isArray(value.workers) && value.workers.length <= 16 + && value.workers.every((worker) => worker !== null && typeof worker === "object" + && count(worker.index) && count(worker.queued) && typeof worker.inFlight === "boolean"); } class HealthSampler { - constructor(healthUrl, target) { + constructor(healthUrl, target, queueTarget) { this.healthUrl = healthUrl; this.target = target; + this.queueTarget = queueTarget; this.timer = null; this.inFlight = null; } @@ -1220,7 +1458,10 @@ class HealthSampler { const sample = async () => { if (this.inFlight) return; this.inFlight = sampleHealth(this.healthUrl) - .then((result) => this.target.samples.push(result.durationMs)) + .then((result) => { + this.target.samples.push(result.durationMs); + recordWorkerQueueSample(this.queueTarget, result.workerQueue); + }) .catch(() => this.target.failures = (this.target.failures ?? 0) + 1) .finally(() => { this.inFlight = null; }); }; @@ -1457,7 +1698,7 @@ async function main() { return; } const result = args.command === "run" - ? await runLiveHarness({ label: args.label }) + ? await runLiveHarness({ label: args.label, stopOne: args.stopOne }) : await cleanupReceipt({ receiptPath: args.receiptPath }); process.stdout.write(`${JSON.stringify({ ok: true, receiptPath: relativePath(resolveRepoRoot(), result.receiptPath) })}\n`); } diff --git a/scripts/perf/seven-thread-live-harness.test.mjs b/scripts/perf/seven-thread-live-harness.test.mjs index 5d697b0ff..c67495110 100644 --- a/scripts/perf/seven-thread-live-harness.test.mjs +++ b/scripts/perf/seven-thread-live-harness.test.mjs @@ -11,7 +11,9 @@ import { normalizeTerminalSessions, parseArguments, parseServerStallEntries, + recordWorkerQueueSample, selectTerminalTransport, + summarizeEventThroughput, summarizeLatency, } from "./seven-thread-live-harness.mjs"; @@ -20,6 +22,10 @@ NodeTest.test("requires an explicit before or after run confirmation", () => { parseArguments(["--run", "--confirm-run", "--label", "before"]), { command: "run", label: "before" }, ); + NodeAssertStrict.deepEqual( + parseArguments(["--run", "--confirm-run", "--label", "after", "--stop-one"]), + { command: "run", label: "after", stopOne: true }, + ); NodeAssertStrict.throws( () => parseArguments(["--run", "--label", "before"]), /requires --confirm-run/, @@ -36,6 +42,10 @@ NodeTest.test("requires an explicit before or after run confirmation", () => { () => parseArguments(["--cleanup-receipt", "receipt.json", "--confirm-cleanup"]), /requires --confirm-run/, ); + NodeAssertStrict.throws( + () => parseArguments(["--cleanup-receipt", "receipt.json", "--confirm-run", "--stop-one"]), + /cannot include run options/, + ); }); NodeTest.test("uses a stable nearest-rank latency summary", () => { @@ -82,6 +92,7 @@ NodeTest.test("names each of the exact verifier-owned threads", () => { const names = Array.from({ length: THREAD_COUNT }, (_, index) => expectedThreadTitle("run-id", index + 1)); NodeAssertStrict.equal(new Set(names).size, THREAD_COUNT); NodeAssertStrict.deepEqual(names.at(-1), "Seven-thread live performance run-id 7/7"); + NodeAssertStrict.equal(expectedThreadTitle("run-id", 3, THREAD_COUNT, "stop-one"), "Seven-thread Stop verification run-id 3/7"); }); NodeTest.test("uses the same legacy and modern Terminal lifecycle families as the web transport", () => { @@ -167,3 +178,54 @@ NodeTest.test("keeps only server diagnostic stall entries inside the workload wi ].join("\n"), Date.parse("2026-09-24T10:00:00.500Z"), Date.parse("2026-09-24T10:00:02.500Z")); NodeAssertStrict.deepEqual(entries, [{ timestamp: "2026-09-24T10:00:02.000Z", stalledMs: 750 }]); }); + +NodeTest.test("reads structured server work stalls and prefers them over duplicate legacy warnings", () => { + const entries = parseServerStallEntries([ + JSON.stringify({ timestamp: "2026-09-24T19:07:27.116Z", message: "Event loop stalled", stalledMs: 240 }), + JSON.stringify({ timestamp: "2026-09-24T19:07:27.116Z", message: "Server work trace", kind: "server-work-stall", delayMs: 260.15 }), + JSON.stringify({ timestamp: "2026-09-24T19:07:37.927Z", message: "Server work trace", kind: "server-work-stall", delayMs: 3639.19 }), + JSON.stringify({ timestamp: "2026-09-24T19:07:39.550Z", message: "Server work trace", kind: "other", delayMs: 1603 }), + JSON.stringify({ timestamp: "2026-09-24T19:07:50.000Z", message: "Server work trace", kind: "server-work-stall", delayMs: 9000 }), + ].join("\n"), Date.parse("2026-09-24T19:07:26.326Z"), Date.parse("2026-09-24T19:07:46.625Z")); + NodeAssertStrict.deepEqual(entries, [ + { timestamp: "2026-09-24T19:07:27.116Z", stalledMs: 260.15 }, + { timestamp: "2026-09-24T19:07:37.927Z", stalledMs: 3639.19 }, + ]); +}); + +NodeTest.test("records only bounded worker queue counts and retains the peak after the sample limit", () => { + const target = { + samples: [], omittedSamples: 0, unavailableSamples: 0, + summary: { sampleCount: 0, maxPending: 0, maxPendingBytes: 0, maxActiveExecutions: 0, maxQueuedPerWorker: 0, maxInFlightWorkers: 0 }, + }; + for (let index = 0; index < 513; index += 1) { + recordWorkerQueueSample(target, { + pending: index, pendingBytes: index * 2, activeExecutions: 7, + workers: [{ index: 0, queued: index, inFlight: true, command: "private prompt" }], + token: "private token", + }); + } + recordWorkerQueueSample(target, { pending: "wrong shape", workers: [] }); + NodeAssertStrict.equal(target.samples.length, 512); + NodeAssertStrict.equal(target.omittedSamples, 1); + NodeAssertStrict.equal(target.unavailableSamples, 1); + NodeAssertStrict.deepEqual(target.summary, { + sampleCount: 513, maxPending: 512, maxPendingBytes: 1024, + maxActiveExecutions: 7, maxQueuedPerWorker: 512, maxInFlightWorkers: 1, + }); + NodeAssertStrict.doesNotMatch(JSON.stringify(target), /private|prompt|token/); +}); + +NodeTest.test("summarizes public event arrival throughput in bounded one-second buckets", () => { + const result = summarizeEventThroughput([ + { sentAtMs: 0, events: [{ atMs: 100 }, { atMs: 900 }, { atMs: 1_100 }] }, + { sentAtMs: 0, events: [{ atMs: 2_500 }] }, + ]); + NodeAssertStrict.deepEqual(result, { + scope: "public-agent-event-push", intervalMs: 1_000, + totalEvents: 4, eventsPerInterval: [2, 1, 1], peakEventsPerSecond: 2, + }); + const longRun = summarizeEventThroughput([{ sentAtMs: 0, events: [{ atMs: 0 }, { atMs: 3_600_000 }] }]); + NodeAssertStrict.ok(longRun.eventsPerInterval.length <= 600); + NodeAssertStrict.equal(longRun.totalEvents, 2); +}); diff --git a/scripts/perf/subagent-detail-fixture.ts b/scripts/perf/subagent-detail-fixture.ts index 75db4a4d0..790591272 100644 --- a/scripts/perf/subagent-detail-fixture.ts +++ b/scripts/perf/subagent-detail-fixture.ts @@ -1,7 +1,7 @@ import "../../apps/server/node_modules/reflect-metadata/Reflect.js"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; -import { CanonicalAgentEventSink } from "../../apps/server/src/features/agents/canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../apps/server/src/features/agents/canonical/canonical-agent-boundary.js"; import { MessageRepo } from "../../apps/server/src/features/agents/conversation/persistence/message-repo.js"; import { ToolCallRecordRepo } from "../../apps/server/src/features/agents/tools/persistence/tool-call-record-repo.js"; import { WorkspaceRepo } from "../../apps/server/src/features/projects/persistence/workspace-repo.js"; @@ -58,7 +58,7 @@ function databasePath(repoRoot: string): string { return requested; } -function recordCompletedBashCalls(sink: CanonicalAgentEventSink, childThreadId: string, nativeTurnId: string): void { +function recordCompletedBashCalls(sink: CanonicalAgentBoundary, childThreadId: string, nativeTurnId: string): void { for (let index = 0; index < COMPLETED_TOOL_COUNT; index += 1) { const nativeItemId = `fixture-completed-${index}`; const isError = index === 124; @@ -68,13 +68,13 @@ function recordCompletedBashCalls(sink: CanonicalAgentEventSink, childThreadId: } } -function recordActiveBashCalls(sink: CanonicalAgentEventSink, childThreadId: string, nativeTurnId: string): void { +function recordActiveBashCalls(sink: CanonicalAgentBoundary, childThreadId: string, nativeTurnId: string): void { for (let index = 0; index < ACTIVE_TOOL_COUNT; index += 1) { sink.recordCodexChildItem({ childThreadId, nativeTurnId, nativeItemId: `fixture-active-${index}`, eventKey: "started", kind: "tool-call", payload: { projection: "codexChildToolCall", toolName: "Bash", toolInput: { command: `printf fixture-active-${index}` } } }); } } -function verifyFixtureRecovery(sink: CanonicalAgentEventSink, parentThreadId: string, childThreadId: string): void { +function verifyFixtureRecovery(sink: CanonicalAgentBoundary, parentThreadId: string, childThreadId: string): void { const recovery = sink.recoverThread(childThreadId, { conversationRevision: 0, rosterRevision: 0 }); const canonicalItems = recovery.mode === "snapshot" ? Object.values(recovery.snapshot.state.items) : []; const narrativeItemCount = canonicalItems.filter((item) => item.kind !== "message").length; @@ -126,7 +126,7 @@ function seed(repoRoot: string, path: string, dbPath: string): Descriptor { const fixturePath = NodePath.join(repoRoot, ".dev", "fixture-repo"); const workspace = workspaces.findByPath(fixturePath) ?? workspaces.create("fixture-repo", fixturePath); const parent = threads.create(workspace.id, MARKER, "local", "main", false, "codex"); - const sink = new CanonicalAgentEventSink(db, () => {}); + const sink = new CanonicalAgentBoundary(db, () => {}); const parentTurnId = crypto.randomUUID(); const parentExecutionId = crypto.randomUUID(); const parentItemId = "toolCall:fixture-subagent";