diff --git a/docs/data-model.md b/docs/data-model.md index 64e2d125cb..387f678428 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -78,30 +78,39 @@ The `agents/{sanitized-cwd}/` directory name is derived from the agent's `cwd` b Each agent is stored as a separate JSON file, grouped by project directory. -| Field | Type | Description | -| -------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `id` | `string` | UUID, primary key | -| `provider` | `string` | Agent provider (`"claude"`, `"codex"`, `"opencode"`, etc.) | -| `cwd` | `string` | Working directory the agent operates in | -| `workspaceId` | `string?` | Owning workspace id — the single source of ownership. Every agent is stamped with one at create time; legacy cwd-only records are backfilled once by `migrations/backfill-workspace-id.migration.ts` (the only place a cwd→id mapping exists). Runtime code never infers ownership or status from cwd: status is computed per `workspaceId`, and same-cwd siblings are independent. | -| `createdAt` | `string` (ISO 8601) | Creation timestamp | -| `updatedAt` | `string` (ISO 8601) | Last update timestamp | -| `lastActivityAt` | `string?` (ISO 8601) | Last activity timestamp | -| `lastUserMessageAt` | `string?` (ISO 8601) | Last user message timestamp | -| `title` | `string?` | User-visible title | -| `labels` | `Record` | Key-value labels (default `{}`). `paseo.parent-agent-id` is set automatically for agent-scoped creation and removed by detach — see [agent-lifecycle.md](./agent-lifecycle.md) | -| `lastStatus` | `AgentStatus` | One of: `"initializing"`, `"idle"`, `"running"`, `"error"`, `"closed"`. `closed` means the record is resumable but has no live provider runtime; archive remains represented separately by `archivedAt`. | -| `lastModeId` | `string?` | Last active mode ID | -| `config` | `SerializableConfig?` | Agent session configuration (see below) | -| `runtimeInfo` | `RuntimeInfo?` | Live runtime state (see below) | -| `features` | `AgentFeature[]?` | Provider-reported features (toggles/selects) | -| `persistence` | `PersistenceHandle?` | Handle for resuming sessions | -| `lastError` | `string?` (nullable) | Last error message, if any | -| `requiresAttention` | `boolean?` | Whether the agent needs user attention | -| `attentionReason` | `"finished" \| "error" \| "permission"?` | Why attention is needed | -| `attentionTimestamp` | `string?` (ISO 8601) | When attention was flagged | -| `internal` | `boolean?` | Whether this is a system-internal agent (loop workers, etc.) | -| `archivedAt` | `string?` (ISO 8601) | Soft-delete timestamp | +| Field | Type | Description | +| ----------------------------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | `string` | UUID, primary key | +| `provider` | `string` | Agent provider (`"claude"`, `"codex"`, `"opencode"`, etc.) | +| `cwd` | `string` | Working directory the agent operates in | +| `workspaceId` | `string?` | Owning workspace id — the single source of ownership. Every agent is stamped with one at create time; legacy cwd-only records are backfilled once by `migrations/backfill-workspace-id.migration.ts` (the only place a cwd→id mapping exists). Runtime code never infers ownership or status from cwd: status is computed per `workspaceId`, and same-cwd siblings are independent. | +| `createdAt` | `string` (ISO 8601) | Creation timestamp | +| `updatedAt` | `string` (ISO 8601) | Last update timestamp | +| `lastActivityAt` | `string?` (ISO 8601) | Last activity timestamp | +| `lastUserMessageAt` | `string?` (ISO 8601) | Last user message timestamp | +| `title` | `string?` | User-visible title | +| `labels` | `Record` | Key-value labels (default `{}`). `paseo.parent-agent-id` is set automatically for agent-scoped creation and removed by detach — see [agent-lifecycle.md](./agent-lifecycle.md) | +| `lastStatus` | `AgentStatus` | One of: `"initializing"`, `"idle"`, `"running"`, `"error"`, `"closed"`. `closed` means the record is resumable but has no live provider runtime; archive remains represented separately by `archivedAt`. | +| `lastModeId` | `string?` | Last active mode ID | +| `config` | `SerializableConfig?` | Agent session configuration (see below) | +| `runtimeInfo` | `RuntimeInfo?` | Live runtime state (see below) | +| `features` | `AgentFeature[]?` | Provider-reported features (toggles/selects) | +| `persistence` | `PersistenceHandle?` | Handle for resuming sessions | +| `lastError` | `string?` (nullable) | Last error message, if any | +| `lastTurnOutcome` | `"completed" \| "failed" \| "canceled"?` | Last terminal turn result. Cleared when a new turn starts. | +| `materialProgressContinuationBoundarySeq` | `number?` (positive integer) | Manager-owned timeline boundary for the latest accepted continuation. This is persisted before a running turn is published, so inspection does not depend on a provider having written a user-message row. Legacy records omit it and use the latest user message. | +| `materialProgress` | `MaterialProgressPayload?` | Last derived material-progress classification, retained when the live timeline is unavailable after archive or daemon restart. | +| `requiresAttention` | `boolean?` | Whether the agent needs user attention | +| `attentionReason` | `"finished" \| "error" \| "permission"?` | Why attention is needed | +| `attentionTimestamp` | `string?` (ISO 8601) | When attention was flagged | +| `internal` | `boolean?` | Whether this is a system-internal agent (loop workers, etc.) | +| `archivedAt` | `string?` (ISO 8601) | Soft-delete timestamp | + +`materialProgress.lastMaterialProgressKind` records completed edits and writes, concrete read/search/ +fetch evidence, successful shell verification with output, explicit decisions, and terminal assistant +deliverables. Repeating the same result does not reset the compaction count. The classification does +not decide whether evidence is relevant, a deliverable is correct, or the user's wider goal succeeded. +Final messages from failed or canceled turns are non-material. ### Nested: SerializableConfig diff --git a/packages/cli/src/commands/agent/inspect.test.ts b/packages/cli/src/commands/agent/inspect.test.ts new file mode 100644 index 0000000000..d46f95c124 --- /dev/null +++ b/packages/cli/src/commands/agent/inspect.test.ts @@ -0,0 +1,46 @@ +import type { AgentSnapshotPayload } from "@getpaseo/protocol/messages"; +import { describe, expect, it } from "vitest"; +import { toInspectData, toInspectRows } from "./inspect.js"; + +describe("agent inspect material progress", () => { + it("preserves the structured signal and renders its state and reason", () => { + const snapshot = { + id: "agent-1", + provider: "opencode", + cwd: "/tmp/work", + model: "test-model", + createdAt: "2026-07-31T00:00:00.000Z", + updatedAt: "2026-07-31T00:01:00.000Z", + lastUserMessageAt: "2026-07-31T00:00:30.000Z", + status: "running", + capabilities: { + supportsStreaming: true, + supportsSessionPersistence: true, + supportsDynamicModes: false, + supportsMcpServers: true, + supportsReasoningStream: true, + supportsToolInvocations: true, + }, + currentModeId: null, + availableModes: [], + pendingPermissions: [], + persistence: null, + title: "Stalled worker", + labels: {}, + materialProgress: { + state: "stalled", + completedCompactionsSinceMaterialProgress: 2, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Two compactions completed without later material progress.", + }, + } as AgentSnapshotPayload; + + const inspect = toInspectData(snapshot); + expect(inspect.MaterialProgress).toEqual(snapshot.materialProgress); + expect(toInspectRows(inspect)).toContainEqual({ + key: "MaterialProgress", + value: "stalled: Two compactions completed without later material progress.", + }); + }); +}); diff --git a/packages/cli/src/commands/agent/inspect.ts b/packages/cli/src/commands/agent/inspect.ts index 93f1da3d4d..4103725a91 100644 --- a/packages/cli/src/commands/agent/inspect.ts +++ b/packages/cli/src/commands/agent/inspect.ts @@ -11,7 +11,7 @@ export function addInspectOptions(cmd: Command): Command { } /** Agent inspect data for display (matches CLI spec format) */ -interface AgentInspect { +export interface AgentInspect { Id: string; Name: string; Provider: string; @@ -46,10 +46,11 @@ interface AgentInspect { }>; Worktree: string | null; ParentAgentId: string | null; + MaterialProgress: AgentSnapshotPayload["materialProgress"] | null; } /** Key-value row for table display */ -interface InspectRow { +export interface InspectRow { key: string; value: string; } @@ -126,7 +127,7 @@ function buildCapabilities(snapshot: AgentSnapshotPayload): AgentInspect["Capabi } /** Convert agent snapshot to inspection data */ -function toInspectData(snapshot: AgentSnapshotPayload): AgentInspect { +export function toInspectData(snapshot: AgentSnapshotPayload): AgentInspect { return { Id: snapshot.id, Name: snapshot.title ?? "-", @@ -151,11 +152,12 @@ function toInspectData(snapshot: AgentSnapshotPayload): AgentInspect { })), Worktree: snapshot.labels?.["paseo.worktree"] ?? null, ParentAgentId: snapshot.labels?.[PARENT_AGENT_ID_LABEL] ?? null, + MaterialProgress: snapshot.materialProgress ?? null, }; } /** Convert agent to key-value rows for table display */ -function toInspectRows(agent: AgentInspect): InspectRow[] { +export function toInspectRows(agent: AgentInspect): InspectRow[] { const rows: InspectRow[] = [ { key: "Id", value: agent.Id }, { key: "Name", value: agent.Name }, @@ -202,6 +204,12 @@ function toInspectRows(agent: AgentInspect): InspectRow[] { rows.push({ key: "Worktree", value: agent.Worktree ?? "null" }); rows.push({ key: "ParentAgentId", value: agent.ParentAgentId ?? "null" }); + rows.push({ + key: "MaterialProgress", + value: agent.MaterialProgress + ? `${agent.MaterialProgress.state}: ${agent.MaterialProgress.reason}` + : "unavailable", + }); return rows; } diff --git a/packages/protocol/src/material-progress-schema.test.ts b/packages/protocol/src/material-progress-schema.test.ts new file mode 100644 index 0000000000..7227f374ba --- /dev/null +++ b/packages/protocol/src/material-progress-schema.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { AgentSnapshotPayloadSchema } from "./messages.js"; + +const baseSnapshot = { + id: "agent-1", + provider: "opencode", + cwd: "/tmp/work", + model: null, + createdAt: "2026-07-31T00:00:00.000Z", + updatedAt: "2026-07-31T00:00:00.000Z", + lastUserMessageAt: null, + status: "running" as const, + capabilities: { + supportsStreaming: true, + supportsSessionPersistence: true, + supportsDynamicModes: false, + supportsMcpServers: true, + supportsReasoningStream: true, + supportsToolInvocations: true, + }, + currentModeId: null, + availableModes: [], + pendingPermissions: [], + persistence: null, + title: null, + labels: {}, +}; + +describe("material progress wire compatibility", () => { + it("accepts legacy snapshots without the optional signal", () => { + expect(AgentSnapshotPayloadSchema.parse(baseSnapshot).materialProgress).toBeUndefined(); + }); + + it("accepts a structured material progress signal", () => { + const parsed = AgentSnapshotPayloadSchema.parse({ + ...baseSnapshot, + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "One compaction completed without later material progress.", + }, + }); + + expect(parsed.materialProgress?.state).toBe("warning"); + }); + + it.each(["evidence", "verification", "decision"] as const)( + "accepts the %s material progress kind", + (lastMaterialProgressKind) => { + const parsed = AgentSnapshotPayloadSchema.parse({ + ...baseSnapshot, + materialProgress: { + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: "2026-07-31T00:00:01.000Z", + lastMaterialProgressKind, + reason: "Material progress followed the latest user message.", + }, + }); + + expect(parsed.materialProgress?.lastMaterialProgressKind).toBe(lastMaterialProgressKind); + }, + ); +}); diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 378cd55762..8e296586cc 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -693,6 +693,31 @@ const AgentRuntimeInfoSchema: z.ZodType = z.object({ extra: z.record(z.string(), z.unknown()).optional(), }); +export interface MaterialProgressPayload { + state: "none" | "progressing" | "warning" | "stalled"; + completedCompactionsSinceMaterialProgress: number; + lastMaterialProgressAt: string | null; + lastMaterialProgressKind: + | "edit" + | "write" + | "evidence" + | "verification" + | "decision" + | "assistant_result" + | null; + reason: string; +} + +export const MaterialProgressPayloadSchema: z.ZodType = z.object({ + state: z.enum(["none", "progressing", "warning", "stalled"]), + completedCompactionsSinceMaterialProgress: z.number().int().nonnegative(), + lastMaterialProgressAt: z.string().nullable(), + lastMaterialProgressKind: z + .enum(["edit", "write", "evidence", "verification", "decision", "assistant_result"]) + .nullable(), + reason: z.string(), +}); + export const AgentSnapshotPayloadSchema = z.object({ id: z.string(), provider: AgentProviderSchema, @@ -721,6 +746,8 @@ export const AgentSnapshotPayloadSchema = z.object({ attentionTimestamp: z.string().nullable().optional(), archivedAt: z.string().nullable().optional(), providerUnavailable: z.boolean().optional(), + // COMPAT(materialProgress): added in v0.2.5, remove optional after 2027-01-31. + materialProgress: MaterialProgressPayloadSchema.optional(), }); export type AgentSnapshotPayload = z.infer; diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 045e243324..b9888b52bb 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -14,6 +14,7 @@ import { type ManagedAgent, } from "./agent-manager.js"; import { AgentStorage } from "./agent-storage.js"; +import { InMemoryAgentTimelineStore } from "./agent-timeline-store.js"; import { toAgentPayload } from "./agent-projections.js"; import { PARENT_AGENT_ID_LABEL } from "@getpaseo/protocol/agent-labels"; import { formatSystemNotificationPrompt } from "./agent-prompt.js"; @@ -40,6 +41,14 @@ import type { } from "./agent-sdk-types.js"; import type { PaseoToolCatalog } from "./tools/types.js"; import type { ProviderDefinition } from "./provider-registry.js"; +import type { + AgentTimelineFetchOptions, + AgentTimelineFetchResult, + AgentTimelineRow, + AgentTimelineStore, +} from "./agent-timeline-store-types.js"; +import { CodexAppServerAgentSession } from "./providers/codex-app-server-agent.js"; +import { createFakeCodexAppServer } from "./providers/codex/test-utils/fake-app-server.js"; interface Deferred { promise: Promise; @@ -57,6 +66,109 @@ function deferred(): Deferred { return { promise, resolve, reject }; } +function timelineRow(seq: number, item: AgentTimelineItem): AgentTimelineRow { + return { + seq, + timestamp: new Date(Date.UTC(2026, 6, 31, 0, 0, 0, seq)).toISOString(), + item, + }; +} + +class TestDurableTimelineStore implements AgentTimelineStore { + private readonly store = new InMemoryAgentTimelineStore(); + afterNextFetch: (() => Promise) | null = null; + + async appendCommitted( + agentId: string, + item: AgentTimelineItem, + options?: { timestamp?: string }, + ): Promise { + this.ensure(agentId); + return this.store.append(agentId, item, options); + } + + async fetchCommitted( + agentId: string, + options?: AgentTimelineFetchOptions, + ): Promise { + this.ensure(agentId); + const result = this.store.fetch(agentId, options); + const afterFetch = this.afterNextFetch; + this.afterNextFetch = null; + await afterFetch?.(); + return result; + } + + async getLatestCommittedSeq(agentId: string): Promise { + this.ensure(agentId); + return this.store.getRows(agentId).at(-1)?.seq ?? 0; + } + + async getCommittedRows(agentId: string): Promise { + this.ensure(agentId); + return this.store.getRows(agentId); + } + + async getLastItem(agentId: string): Promise { + this.ensure(agentId); + return this.store.getLastItem(agentId); + } + + async getLastAssistantMessage(agentId: string): Promise { + this.ensure(agentId); + return this.store.getLastAssistantMessage(agentId); + } + + async deleteAgent(agentId: string): Promise { + if (this.store.has(agentId)) this.store.delete(agentId); + } + + async bulkInsert(agentId: string, rows: readonly AgentTimelineRow[]): Promise { + const existingRows = this.store.has(agentId) ? this.store.getRows(agentId) : []; + const bySeq = new Map(existingRows.map((row) => [row.seq, row])); + for (const row of rows) bySeq.set(row.seq, row); + const mergedRows = [...bySeq.values()].sort((left, right) => left.seq - right.seq); + this.store.initialize(agentId, { + rows: mergedRows, + nextSeq: (mergedRows.at(-1)?.seq ?? 0) + 1, + }); + } + + private ensure(agentId: string): void { + if (!this.store.has(agentId)) this.store.initialize(agentId); + } +} + +class HeldSnapshotStorage extends AgentStorage { + private readonly applyStarted = deferred(); + private readonly applyAllowed = deferred(); + private holdNextApply = false; + + holdNextSnapshot(): void { + this.holdNextApply = true; + } + + waitForHeldSnapshot(): Promise { + return this.applyStarted.promise; + } + + releaseHeldSnapshot(): void { + this.applyAllowed.resolve(); + } + + override async applySnapshot( + agent: ManagedAgent, + options?: Parameters[1], + ): Promise { + if (this.holdNextApply) { + this.holdNextApply = false; + this.applyStarted.resolve(); + await this.applyAllowed.promise; + } + await super.applySnapshot(agent, options); + } +} + function waitForAgentLifecycle( manager: AgentManager, agentId: string, @@ -448,14 +560,17 @@ class ControlledInterruptSession extends TestAgentSession { config: AgentSessionConfig, readonly turnId: string, private readonly interruptBehavior: (session: ControlledInterruptSession) => Promise, + private readonly deferTurnStarted: boolean, ) { super(config); } override async startTurn(): Promise<{ turnId: string }> { - setTimeout(() => { - this.pushEvent({ type: "turn_started", provider: this.provider, turnId: this.turnId }); - }, 0); + if (!this.deferTurnStarted) { + setTimeout(() => { + this.pushEvent({ type: "turn_started", provider: this.provider, turnId: this.turnId }); + }, 0); + } return { turnId: this.turnId }; } @@ -465,12 +580,99 @@ class ControlledInterruptSession extends TestAgentSession { } } +class PendingStartSession extends TestAgentSession { + private readonly firstStartRequested = deferred(); + private readonly firstStartAllowed = deferred(); + private turnCount = 0; + + override async startTurn(): Promise<{ turnId: string }> { + const turnId = `pending-start-turn-${++this.turnCount}`; + if (this.turnCount === 1) { + this.firstStartRequested.resolve(); + await this.firstStartAllowed.promise; + return { turnId }; + } + + setTimeout(() => { + this.pushEvent({ type: "turn_started", provider: this.provider, turnId }); + this.pushEvent({ type: "turn_completed", provider: this.provider, turnId }); + }, 0); + return { turnId }; + } + + waitForPendingStart(): Promise { + return this.firstStartRequested.promise; + } + + resolvePendingStart(): void { + this.firstStartAllowed.resolve(); + } + + override async interrupt(): Promise {} +} + +interface PendingStartFixture { + agentId: string; + manager: AgentManager; + session: PendingStartSession; + startDrain: Promise; + cleanup(): Promise; +} + +async function createPendingStartFixture( + name: string, + agentId: string, +): Promise { + const workdir = mkdtempSync(join(tmpdir(), `agent-manager-${name}-`)); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const session = new PendingStartSession({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + rescueTimeouts: { interruptSessionMs: 10 }, + idFactory: () => agentId, + }); + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + const stream = manager.streamAgent(agent.id, "hold start resolution"); + const startDrain = (async () => { + for await (const _event of stream) { + // Keep the stream subscribed until the pending generation settles. + } + })(); + await session.waitForPendingStart(); + + return { + agentId: agent.id, + manager, + session, + startDrain, + async cleanup() { + session.resolvePendingStart(); + await startDrain.catch(() => undefined); + await manager.flush().catch(() => undefined); + await storage.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + }, + }; +} + interface ControlledInterruptFixture { agentId: string; manager: AgentManager; session: ControlledInterruptSession; + holdNextSnapshotPersist(): void; + waitForSnapshotPersistStart(): Promise; + releaseSnapshotPersist(): void; startForegroundRun(): Promise; - cleanup(): void; + cleanup(): Promise; } async function createControlledInterruptFixture(options: { @@ -478,12 +680,15 @@ async function createControlledInterruptFixture(options: { agentId: string; turnId: string; interrupt: (session: ControlledInterruptSession) => Promise; + deferTurnStarted?: boolean; }): Promise { const workdir = mkdtempSync(join(tmpdir(), `agent-manager-${options.name}-`)); + const storage = new HeldSnapshotStorage(join(workdir, "agents"), logger); const session = new ControlledInterruptSession( { provider: "codex", cwd: workdir }, options.turnId, options.interrupt, + options.deferTurnStarted ?? false, ); const client = new (class extends TestAgentClient { override async createSession(): Promise { @@ -492,7 +697,7 @@ async function createControlledInterruptFixture(options: { })(); const manager = new AgentManager({ clients: { codex: client }, - registry: new AgentStorage(join(workdir, "agents"), logger), + registry: storage, logger, rescueTimeouts: { interruptSessionMs: 10 }, idFactory: () => options.agentId, @@ -505,6 +710,15 @@ async function createControlledInterruptFixture(options: { agentId: agent.id, manager, session, + holdNextSnapshotPersist() { + storage.holdNextSnapshot(); + }, + waitForSnapshotPersistStart() { + return storage.waitForHeldSnapshot(); + }, + releaseSnapshotPersist() { + storage.releaseHeldSnapshot(); + }, async startForegroundRun() { const run = manager.streamAgent(agent.id, "exercise cancellation"); void (async () => { @@ -514,7 +728,12 @@ async function createControlledInterruptFixture(options: { })(); await manager.waitForAgentRunStart(agent.id); }, - cleanup: () => rmSync(workdir, { recursive: true, force: true }), + async cleanup() { + storage.releaseHeldSnapshot(); + await manager.flush().catch(() => undefined); + await storage.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + }, }; } @@ -1530,7 +1749,7 @@ test("cancelAgentRun preserves running state when the provider interrupt hangs", expect(fixture.session.interruptCalled).toBe(true); expect(fixture.manager.getAgent(fixture.agentId)?.lifecycle).toBe("running"); } finally { - fixture.cleanup(); + await fixture.cleanup(); } }); @@ -1561,7 +1780,7 @@ test("cancelAgentRun preserves the active turn when the provider rejects the int turnId: "provider-still-active-turn", }); } finally { - fixture.cleanup(); + await fixture.cleanup(); } }); @@ -1594,15 +1813,16 @@ test("cancelAgentRun succeeds when the foreground turn finishes before the provi activeForegroundTurnId: null, }); } finally { - fixture.cleanup(); + await fixture.cleanup(); } }); -test("cancelAgentRun succeeds when the provider queues completion before rejecting the interrupt", async () => { +test("cancelAgentRun settles a queued completion while turn-start persistence is blocked", async () => { const fixture = await createControlledInterruptFixture({ name: "interrupt-queued-completion", agentId: "00000000-0000-4000-8000-000000000306", turnId: "queued-completion-turn", + deferTurnStarted: true, interrupt: async (session) => { session.pushEvent({ type: "turn_completed", @@ -1615,7 +1835,99 @@ test("cancelAgentRun succeeds when the provider queues completion before rejecti try { await fixture.startForegroundRun(); + fixture.holdNextSnapshotPersist(); + fixture.session.pushEvent({ + type: "turn_started", + provider: "codex", + turnId: "queued-completion-turn", + }); + await fixture.waitForSnapshotPersistStart(); + + await expect(fixture.manager.cancelAgentRun(fixture.agentId)).resolves.toEqual({ + status: "settled", + }); + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + } finally { + fixture.releaseSnapshotPersist(); + await fixture.cleanup(); + } +}); + +test("a terminal emitted before startTurn resolves settles the matching foreground stream", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-eager-terminal-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const session = new (class extends TestAgentSession { + override async startTurn(): Promise<{ turnId: string }> { + const turnId = "eager-terminal-turn"; + this.pushEvent({ type: "turn_started", provider: this.provider, turnId }); + this.pushEvent({ + type: "timeline", + provider: this.provider, + turnId, + item: { type: "assistant_message", text: "finished before start returned" }, + }); + this.pushEvent({ type: "turn_completed", provider: this.provider, turnId }); + await new Promise((resolve) => setImmediate(resolve)); + setTimeout(() => { + this.pushEvent({ + type: "turn_failed", + provider: this.provider, + turnId, + error: "late fallback should be ignored", + }); + }, 0); + return { turnId }; + } + })({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000309", + }); + + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + const events: AgentStreamEvent[] = []; + for await (const event of manager.streamAgent(agent.id, "finish eagerly")) { + events.push(event); + } + await manager.flush(); + + expect(events.map((event) => event.type)).toContain("turn_completed"); + expect(events.map((event) => event.type)).not.toContain("turn_failed"); + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + lastTurnOutcome: "completed", + }); + expect(manager.getTimeline(agent.id)).toContainEqual({ + type: "assistant_message", + text: "finished before start returned", + }); + } finally { + await manager.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("cancelAgentRun force-cancels an acknowledged pending start generation", async () => { + const fixture = await createPendingStartFixture( + "cancel-pending-start", + "00000000-0000-4000-8000-000000000310", + ); + try { await expect(fixture.manager.cancelAgentRun(fixture.agentId)).resolves.toEqual({ status: "settled", }); @@ -1623,8 +1935,128 @@ test("cancelAgentRun succeeds when the provider queues completion before rejecti lifecycle: "idle", activeForegroundTurnId: null, }); + + fixture.session.resolvePendingStart(); + await fixture.startDrain; + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + } finally { + await fixture.cleanup(); + } +}); + +test("replaceAgentRun does not let a late pending start replace the new generation", async () => { + const fixture = await createPendingStartFixture( + "replace-pending-start", + "00000000-0000-4000-8000-000000000311", + ); + + try { + const replacement = await fixture.manager.replaceAgentRun(fixture.agentId, "replacement work"); + fixture.session.resolvePendingStart(); + await fixture.startDrain; + + const replacementEvents: AgentStreamEvent[] = []; + for await (const event of replacement) replacementEvents.push(event); + + expect(replacementEvents.map((event) => event.type)).toContain("turn_completed"); + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + } finally { + await fixture.cleanup(); + } +}); + +test("reloadAgentSession does not restore a force-canceled pending start", async () => { + const fixture = await createPendingStartFixture( + "reload-pending-start", + "00000000-0000-4000-8000-000000000312", + ); + + try { + await expect(fixture.manager.reloadAgentSession(fixture.agentId)).resolves.toMatchObject({ + id: fixture.agentId, + lifecycle: "idle", + activeForegroundTurnId: null, + }); + + fixture.session.resolvePendingStart(); + await fixture.startDrain; + await new Promise((resolve) => setImmediate(resolve)); + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + + await fixture.manager.runAgent(fixture.agentId, "work after reload"); + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + } finally { + await fixture.cleanup(); + } +}); + +test("reloadAgentSession proceeds when the active run completes before interrupt rejection", async () => { + let fixture!: ControlledInterruptFixture; + fixture = await createControlledInterruptFixture({ + name: "reload-after-completion", + agentId: "00000000-0000-4000-8000-000000000307", + turnId: "reload-completed-turn", + interrupt: async (session) => { + const settled = waitForAgentLifecycle(fixture.manager, fixture.agentId, "idle"); + session.pushEvent({ + type: "turn_completed", + provider: session.provider, + turnId: "reload-completed-turn", + }); + await settled; + throw new Error("turn already completed"); + }, + }); + + try { + await fixture.startForegroundRun(); + + await expect(fixture.manager.reloadAgentSession(fixture.agentId)).resolves.toMatchObject({ + id: fixture.agentId, + lifecycle: "idle", + }); + } finally { + await fixture.cleanup(); + } +}); + +test("replaceAgentRun proceeds when the active run completes before interrupt rejection", async () => { + const fixture = await createControlledInterruptFixture({ + name: "replace-after-completion", + agentId: "00000000-0000-4000-8000-000000000308", + turnId: "replace-completed-turn", + interrupt: async (session) => { + session.pushEvent({ + type: "turn_completed", + provider: session.provider, + turnId: "replace-completed-turn", + }); + await new Promise((resolve) => setImmediate(resolve)); + throw new Error("turn already completed"); + }, + }); + + try { + await fixture.startForegroundRun(); + + await expect( + fixture.manager.replaceAgentRun(fixture.agentId, "replacement work"), + ).resolves.toBeDefined(); + expect(fixture.manager.getAgent(fixture.agentId)?.activeForegroundTurnId).toBeNull(); } finally { - fixture.cleanup(); + await fixture.cleanup(); } }); @@ -3837,15 +4269,736 @@ test("getTimelineRows falls back to the in-memory timeline when no durable store }, }, ]); + + await expect(manager.getMaterialProgressSnapshot(snapshot.id, 1)).resolves.toMatchObject({ + rows: [ + { seq: 1, item: { type: "assistant_message", text: "row one" } }, + { seq: 2, item: { type: "assistant_message", text: "row two" } }, + ], + }); + + await manager.closeAgent(snapshot.id); + const retained = await manager.getMaterialProgressSnapshot(snapshot.id); + expect(retained.rows).toHaveLength(2); + expect(retained.turnOutcome).toBeNull(); }); -test("getAgent does not expose committed history internals once manager owns the seam", async () => { - const workdir = mkdtempSync(join(tmpdir(), "agent-manager-timeline-boundary-")); - const storagePath = join(workdir, "agents"); - const storage = new AgentStorage(storagePath, logger); - const manager = new AgentManager({ - clients: { - codex: new TestAgentClient(), +test("an unhydrated resumed agent exposes its persisted material progress", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-resume-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const agentId = "00000000-0000-4000-8000-000000000141"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + await firstManager.appendTimelineItem(created.id, { + type: "user_message", + text: "persist this result", + }); + await firstManager.appendTimelineItem(created.id, { + type: "tool_call", + callId: "write-1", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "result.txt", content: "durable" }, + }); + await firstManager.closeAgent(created.id); + + const record = await storage.get(created.id); + expect(record?.persistence).toBeTruthy(); + + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + record!.persistence!, + { cwd: workdir }, + created.id, + ); + + await expect(resumedManager.getMaterialProgressSnapshot(created.id)).resolves.toMatchObject({ + rows: null, + persisted: { + state: "progressing", + lastMaterialProgressKind: "write", + }, + }); + + await resumedManager.closeAgent(created.id); + rmSync(workdir, { recursive: true, force: true }); +}); + +test("a resumed agent with an empty live timeline derives progress from its durable tail", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-durable-resume-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const durableTimelineStore = new TestDurableTimelineStore(); + const agentId = "00000000-0000-4000-8000-000000000142"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + try { + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + await firstManager.closeAgent(created.id); + await durableTimelineStore.bulkInsert(created.id, [ + timelineRow(1, { type: "user_message", text: "deliver this" }), + timelineRow(2, { + type: "tool_call", + callId: "write-durable-resume", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "durable.txt", content: "done" }, + }), + ]); + + const stored = await storage.get(created.id); + expect(stored?.persistence).toBeTruthy(); + await storage.upsert({ + ...stored!, + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Stale stored classification.", + }, + }); + + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + durableTimelineStore, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + stored!.persistence!, + { cwd: workdir }, + created.id, + ); + + expect(resumedManager.getTimeline(created.id)).toEqual([]); + await expect(resumedManager.getMaterialProgressSnapshot(created.id)).resolves.toMatchObject({ + rows: [ + { seq: 1, item: { type: "user_message", text: "deliver this" } }, + { + seq: 2, + item: { + type: "tool_call", + detail: { type: "write", filePath: "durable.txt" }, + }, + }, + ], + persisted: { + state: "progressing", + lastMaterialProgressKind: "write", + }, + }); + expect((await storage.get(created.id))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + + await resumedManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + await expect(resumedManager.getMaterialProgressSnapshot(created.id)).resolves.toMatchObject({ + rows: [ + { seq: 1, item: { type: "user_message" } }, + { seq: 2, item: { type: "tool_call" } }, + { seq: 3, item: { type: "compaction" } }, + ], + persisted: { + state: "warning", + lastMaterialProgressKind: "write", + }, + }); + + await resumedManager.closeAgent(created.id); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("material progress pages beyond the durable 1000-row tail to the current continuation", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-durable-paging-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const durableTimelineStore = new TestDurableTimelineStore(); + const agentId = "00000000-0000-4000-8000-000000000143"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + try { + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + await firstManager.closeAgent(created.id); + + const durableRows: AgentTimelineRow[] = [ + timelineRow(1, { type: "user_message", text: "previous continuation" }), + timelineRow(2, { type: "compaction", status: "completed" }), + timelineRow(3, { type: "user_message", text: "current continuation" }), + timelineRow(4, { + type: "tool_call", + callId: "write-before-tail", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "paged.txt", content: "done" }, + }), + ]; + for (let seq = 5; seq <= 1_004; seq += 1) { + durableRows.push(timelineRow(seq, { type: "reasoning", text: `reasoning ${seq}` })); + } + await durableTimelineStore.bulkInsert(created.id, durableRows); + + const stored = await storage.get(created.id); + expect(stored?.persistence).toBeTruthy(); + await storage.upsert({ + ...stored!, + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Classification to replace after complete paging.", + }, + }); + + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + durableTimelineStore, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + stored!.persistence!, + { cwd: workdir }, + created.id, + ); + + expect(resumedManager.getTimeline(created.id)).toEqual([]); + const snapshot = await resumedManager.getMaterialProgressSnapshot(created.id); + expect(snapshot.rows).toHaveLength(1_002); + expect(snapshot.rows?.[0]).toMatchObject({ + seq: 3, + item: { type: "user_message", text: "current continuation" }, + }); + expect(snapshot.persisted).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + expect((await storage.get(created.id))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + + await resumedManager.closeAgent(created.id); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("material progress keeps a known continuation boundary across a later live user row", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-known-boundary-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const durableTimelineStore = new TestDurableTimelineStore(); + const agentId = "00000000-0000-4000-8000-000000000171"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + try { + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + await firstManager.closeAgent(created.id); + + const durableRows: AgentTimelineRow[] = [ + timelineRow(1, { type: "user_message", text: "previous continuation" }), + ]; + for (let seq = 2; seq < 10_004; seq += 1) { + durableRows.push(timelineRow(seq, { type: "reasoning", text: `old reasoning ${seq}` })); + } + durableRows.push( + timelineRow(10_004, { + type: "tool_call", + callId: "write-known-boundary", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "boundary.txt", content: "done" }, + }), + timelineRow(10_005, { type: "reasoning", text: "verify the boundary result" }), + ); + await durableTimelineStore.bulkInsert(created.id, durableRows); + + const stored = await storage.get(created.id); + expect(stored?.persistence).toBeTruthy(); + await storage.upsert({ + ...stored!, + materialProgressContinuationBoundarySeq: 10_004, + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Classification to replace from the known continuation boundary.", + }, + }); + + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + durableTimelineStore, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + stored!.persistence!, + { cwd: workdir }, + created.id, + { materialProgressContinuationBoundarySeq: 10_004 }, + ); + await resumedManager.appendTimelineItem(created.id, { + type: "user_message", + text: "provider follow-up within the same continuation", + }); + await resumedManager.appendTimelineItem(created.id, { + type: "reasoning", + text: "continue after the provider follow-up", + }); + + const snapshot = await resumedManager.getMaterialProgressSnapshot(created.id); + expect(snapshot.rows).toMatchObject([ + { seq: 10_004, item: { type: "tool_call", detail: { type: "write" } } }, + { seq: 10_005, item: { type: "reasoning" } }, + { seq: 10_006, item: { type: "user_message" } }, + { seq: 10_007, item: { type: "reasoning" } }, + ]); + expect(snapshot).toMatchObject({ + persisted: { + state: "progressing", + lastMaterialProgressKind: "write", + }, + }); + expect((await storage.get(created.id))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + + await resumedManager.closeAgent(created.id); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("a delayed material progress write cannot overwrite a newer continuation", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-write-order-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const durableTimelineStore = new TestDurableTimelineStore(); + const agentId = "00000000-0000-4000-8000-000000000144"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + try { + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + await firstManager.closeAgent(created.id); + await durableTimelineStore.bulkInsert(created.id, [ + timelineRow(1, { type: "user_message", text: "old continuation" }), + timelineRow(2, { type: "compaction", status: "completed" }), + ]); + + const stored = await storage.get(created.id); + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + durableTimelineStore, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + stored!.persistence!, + { cwd: workdir }, + created.id, + ); + + const delayedFetchStarted = deferred(); + const releaseDelayedFetch = deferred(); + durableTimelineStore.afterNextFetch = async () => { + delayedFetchStarted.resolve(undefined); + await releaseDelayedFetch.promise; + }; + const olderPersist = resumedManager.appendTimelineItem(created.id, { + type: "reasoning", + text: "old continuation still running", + }); + await delayedFetchStarted.promise; + + const userPersist = resumedManager.appendTimelineItem(created.id, { + type: "user_message", + text: "new continuation", + }); + const writePersist = resumedManager.appendTimelineItem(created.id, { + type: "tool_call", + callId: "write-new-continuation", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "new.txt", content: "new" }, + }); + releaseDelayedFetch.resolve(undefined); + await Promise.all([olderPersist, userPersist, writePersist]); + + expect((await storage.get(created.id))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + await resumedManager.closeAgent(created.id); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("persists terminal outcomes and clears the previous outcome when a new turn starts", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-turn-outcome-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + let nextTurn = 0; + const session = new (class extends TestAgentSession { + override async startTurn(): Promise<{ turnId: string }> { + nextTurn += 1; + const turnId = `held-turn-${nextTurn}`; + if (nextTurn === 2) { + // OpenCode can publish accepted-start and timeline events before startTurn resolves. + // The post-resolution manager path must not move the boundary past those events. + this.pushEvent({ type: "turn_started", provider: "codex", turnId }); + this.pushEvent({ + type: "timeline", + provider: "codex", + turnId, + item: { type: "reasoning", text: "provider accepted the continuation" }, + }); + await new Promise((resolve) => setImmediate(resolve)); + } + return { turnId }; + } + })({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000139", + }); + + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + + const firstRunning = waitForAgentLifecycle(manager, agent.id, "running"); + const firstTurn = (async () => { + for await (const _event of manager.streamAgent(agent.id, "first turn")) { + // Consume through the terminal event so the managed run settles. + } + })(); + await firstRunning; + session.pushEvent({ + type: "turn_completed", + provider: "codex", + turnId: "held-turn-1", + }); + await firstTurn; + await manager.flush(); + expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBe("completed"); + expect((await storage.get(agent.id))?.lastTurnOutcome).toBe("completed"); + + await manager.appendTimelineItem(agent.id, { + type: "user_message", + text: "previous continuation", + }); + await manager.appendTimelineItem(agent.id, { + type: "compaction", + status: "completed", + }); + await manager.appendTimelineItem(agent.id, { + type: "compaction", + status: "completed", + }); + + const secondRunning = waitForAgentLifecycle(manager, agent.id, "running"); + const secondTurn = (async () => { + for await (const _event of manager.streamAgent(agent.id, "second turn")) { + // Consume through the terminal event so the managed run settles. + } + })(); + await secondRunning; + await manager.flush(); + const secondTurnSnapshot = await manager.getMaterialProgressSnapshot(agent.id); + expect(secondTurnSnapshot.turnOutcome).toBeNull(); + expect(secondTurnSnapshot.continuationBoundarySeq).toBe(4); + expect(await storage.get(agent.id)).toMatchObject({ + materialProgressContinuationBoundarySeq: 4, + materialProgress: { state: "none" }, + }); + expect((await storage.get(agent.id))?.lastTurnOutcome ?? null).toBeNull(); + + const staleRecord = await storage.get(agent.id); + await storage.upsert({ ...staleRecord!, lastTurnOutcome: "completed" }); + expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBeNull(); + + session.pushEvent({ + type: "turn_canceled", + provider: "codex", + turnId: "held-turn-2", + reason: "test interruption", + }); + await secondTurn; + await manager.flush(); + expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBe("canceled"); + expect((await storage.get(agent.id))?.lastTurnOutcome).toBe("canceled"); + } finally { + await manager.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("an old autonomous terminal cannot end a newer foreground continuation", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-stale-autonomous-terminal-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const session = new (class extends TestAgentSession { + override async startTurn(): Promise<{ turnId: string }> { + return { turnId: "new-foreground-turn" }; + } + })({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000170", + }); + + let foregroundDrain: Promise | null = null; + let unsubscribe: (() => void) | null = null; + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + session.pushEvent({ + type: "turn_started", + provider: "codex", + turnId: "old-autonomous-turn", + }); + session.pushEvent({ + type: "turn_completed", + provider: "codex", + turnId: "old-autonomous-turn", + }); + await manager.flush(); + + const streamedEvents: AgentStreamEvent[] = []; + unsubscribe = manager.subscribe( + (event) => { + if (event.type === "agent_stream") streamedEvents.push(event.event); + }, + { agentId: agent.id, replayState: false }, + ); + + const foreground = manager.streamAgent(agent.id, "new work"); + foregroundDrain = (async () => { + for await (const _event of foreground) { + // Drain through the current turn's terminal event. + } + })(); + await manager.waitForAgentRunStart(agent.id); + + session.pushEvent({ + type: "turn_completed", + provider: "codex", + turnId: "old-autonomous-turn", + }); + await manager.flush(); + + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "running", + activeForegroundTurnId: "new-foreground-turn", + materialProgressContinuationActive: true, + materialProgressContinuationTurnId: "new-foreground-turn", + lastTurnOutcome: null, + }); + expect(streamedEvents).not.toContainEqual( + expect.objectContaining({ type: "turn_completed", turnId: "old-autonomous-turn" }), + ); + + session.pushEvent({ + type: "turn_canceled", + provider: "codex", + turnId: "new-foreground-turn", + reason: "test cleanup", + }); + await foregroundDrain; + } finally { + unsubscribe?.(); + await manager.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("Codex binds only the exact native terminal across consecutive managed turns", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-codex-native-stale-terminal-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const appServer = createFakeCodexAppServer(); + const session = new CodexAppServerAgentSession( + { provider: "codex", cwd: workdir, modeId: "auto", model: "gpt-5.4" }, + null, + logger, + async () => appServer.child, + ); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000172", + }); + + let unsubscribe: (() => void) | null = null; + let unsubscribeProvider: (() => void) | null = null; + let foregroundDrain: Promise | null = null; + let waitForAgent: Promise | null = null; + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + + appServer.startsTurn({ threadId: "thread-1", turnId: "native-old-turn" }); + await vi.waitFor(() => expect(manager.getAgent(agent.id)?.lifecycle).toBe("running")); + appServer.completeTurn({ turnId: "native-old-turn" }); + await vi.waitFor(() => expect(manager.getAgent(agent.id)?.lifecycle).toBe("idle")); + + const providerEvents: AgentStreamEvent[] = []; + unsubscribeProvider = session.subscribe((event) => providerEvents.push(event)); + const streamedEvents: AgentStreamEvent[] = []; + unsubscribe = manager.subscribe( + (event) => { + if (event.type === "agent_stream") streamedEvents.push(event.event); + }, + { agentId: agent.id, replayState: false }, + ); + + let foregroundSettled = false; + foregroundDrain = (async () => { + for await (const _event of manager.streamAgent(agent.id, "new work")) { + // Drain through the current turn's terminal event. + } + foregroundSettled = true; + })(); + await appServer.waitForTurnStart(); + appServer.startsTurn({ threadId: "thread-1", turnId: "native-new-turn" }); + await manager.waitForAgentRunStart(agent.id); + providerEvents.length = 0; + + let waiterSettled = false; + waitForAgent = manager.waitForAgentEvent(agent.id).then((result) => { + waiterSettled = true; + return result; + }); + + appServer.completeTurn({ turnId: "native-old-turn" }); + await vi.waitFor(() => + expect(providerEvents).toContainEqual( + expect.objectContaining({ type: "turn_completed", turnId: "native-old-turn" }), + ), + ); + await manager.flush(); + + const currentForegroundTurnId = manager.getAgent(agent.id)?.activeForegroundTurnId; + expect(currentForegroundTurnId).toEqual(expect.any(String)); + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "running", + activeForegroundTurnId: currentForegroundTurnId, + materialProgressContinuationActive: true, + lastTurnOutcome: null, + }); + expect(streamedEvents).not.toContainEqual( + expect.objectContaining({ type: "turn_completed", turnId: "native-old-turn" }), + ); + expect(foregroundSettled).toBe(false); + expect(waiterSettled).toBe(false); + + appServer.setThreadTurnStatus({ + threadId: "thread-1", + turnId: "native-new-turn", + status: "completed", + }); + appServer.completeTurn({ omitTurnId: true }); + await foregroundDrain; + await waitForAgent; + await manager.flush(); + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + materialProgressContinuationActive: false, + lastTurnOutcome: "completed", + }); + expect(streamedEvents).toContainEqual( + expect.objectContaining({ type: "turn_completed", turnId: currentForegroundTurnId }), + ); + appServer.assertNoErrors(); + } finally { + unsubscribe?.(); + unsubscribeProvider?.(); + await manager.closeAgent("00000000-0000-4000-8000-000000000172").catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("getAgent does not expose committed history internals once manager owns the seam", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-timeline-boundary-")); + const storagePath = join(workdir, "agents"); + const storage = new AgentStorage(storagePath, logger); + const manager = new AgentManager({ + clients: { + codex: new TestAgentClient(), }, registry: storage, logger, @@ -4587,7 +5740,6 @@ test("replaceAgentRun does not emit idle or resolve waiters between interrupted })(); await manager.waitForAgentRunStart(snapshot.id); - const waitPromise = manager.waitForAgentEvent(snapshot.id); const secondRun = await manager.replaceAgentRun(snapshot.id, "second run"); const secondRunDrain = (async () => { diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 72fe1a7ec8..3a7c70b8d8 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -73,9 +73,14 @@ import { type ProviderSubagentDescriptor, type ProviderSubagentStoreEvent, } from "./provider-subagents/store.js"; +import { analyzeMaterialProgress } from "./material-progress.js"; +import { projectTimelineRows } from "./timeline-projection.js"; +import type { MaterialProgressPayload } from "../messages.js"; const RELOAD_SESSION_CLOSE_TIMEOUT_MS = 3_000; const INTERRUPT_SESSION_TIMEOUT_MS = 2_000; +const MATERIAL_PROGRESS_PAGE_SIZE = 1_000; +const MATERIAL_PROGRESS_MAX_SCAN_ROWS = 10_000; const STORED_AGENT_CAPABILITIES: AgentCapabilityFlags = { supportsStreaming: false, supportsSessionPersistence: true, @@ -127,10 +132,92 @@ interface TimeoutOptions { onLateError?: (error: unknown) => void; } +interface PersistSnapshotOptions { + title?: string | null; + internal?: boolean; + includeInternal?: boolean; +} + function formatProviderList(providers: readonly string[]): string { return providers.length > 0 ? providers.join(", ") : "none"; } +type FetchOlderMaterialProgressRows = ( + options: AgentTimelineFetchOptions, +) => Promise; + +function currentContinuationRows(rows: readonly AgentTimelineRow[]): AgentTimelineRow[] { + for (let index = rows.length - 1; index >= 0; index -= 1) { + if (rows[index]?.item.type === "user_message") return rows.slice(index); + } + return [...rows]; +} + +function materialProgressScanReachedStart( + rows: readonly AgentTimelineRow[], + continuationBoundarySeq: number | null, +): boolean { + if (continuationBoundarySeq === null) { + return rows.some((row) => row.item.type === "user_message"); + } + return (rows[0]?.seq ?? Number.POSITIVE_INFINITY) <= continuationBoundarySeq; +} + +function selectMaterialProgressRows( + rows: readonly AgentTimelineRow[], + continuationBoundarySeq: number | null, + reachedStart: boolean, +): AgentTimelineRow[] | null { + if (continuationBoundarySeq !== null) { + if (!reachedStart) return null; + return rows.filter((row) => row.seq >= continuationBoundarySeq); + } + if (!reachedStart && rows[0]?.seq !== 1) return null; + return currentContinuationRows(rows); +} + +async function pageMaterialProgressRows( + initialPage: AgentTimelineFetchResult, + fetchOlder: FetchOlderMaterialProgressRows, + continuationBoundarySeq: number | null, +): Promise { + let page = initialPage; + const pageRows = [page.rows]; + let rowCount = page.rows.length; + if (continuationBoundarySeq !== null) { + if (continuationBoundarySeq > page.window.nextSeq) return null; + if (continuationBoundarySeq === page.window.nextSeq) { + return []; + } + } + let reachedStart = materialProgressScanReachedStart(page.rows, continuationBoundarySeq); + while (page.hasOlder && !reachedStart) { + const firstRow = pageRows[0]?.[0]; + const remainingCapacity = MATERIAL_PROGRESS_MAX_SCAN_ROWS - rowCount; + if (!firstRow || remainingCapacity <= 0) return null; + + const olderPage = await fetchOlder({ + direction: "before", + cursor: { epoch: page.epoch, seq: firstRow.seq }, + limit: Math.min(MATERIAL_PROGRESS_PAGE_SIZE, remainingCapacity), + }); + if ( + olderPage.epoch !== page.epoch || + olderPage.reset || + olderPage.staleCursor || + olderPage.gap || + olderPage.rows.length === 0 + ) { + return null; + } + pageRows.unshift(olderPage.rows); + rowCount += olderPage.rows.length; + reachedStart = materialProgressScanReachedStart(olderPage.rows, continuationBoundarySeq); + page = olderPage; + } + return selectMaterialProgressRows(pageRows.flat(), continuationBoundarySeq, reachedStart); +} + function buildStoredAgentConfig(record: StoredAgentRecord): AgentSessionConfig { const config: AgentSessionConfig = { provider: record.provider, @@ -280,6 +367,15 @@ type AttentionState = attentionTimestamp: Date; }; +type AgentTurnOutcome = "completed" | "failed" | "canceled"; + +export interface MaterialProgressSnapshot { + rows: AgentTimelineRow[] | null; + turnOutcome: AgentTurnOutcome | null; + continuationBoundarySeq: number | null; + persisted: MaterialProgressPayload | null; +} + function resolveInitialAttention(input: AttentionState | undefined): AttentionState { if (input == null || !input.requiresAttention) { return { requiresAttention: false }; @@ -331,6 +427,10 @@ interface ManagedAgentBase { lastUserMessageAt: Date | null; lastUsage?: AgentUsage; lastError?: string; + lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; + materialProgressContinuationActive?: boolean; + materialProgressContinuationTurnId?: string | null; attention: AttentionState; foregroundTurnWaiters: Set; finalizedForegroundTurnIds: Set; @@ -563,6 +663,7 @@ export class AgentManager { private readonly providerSubagents = new ProviderSubagentStore(); private readonly agentsAwaitingInitialSnapshotPersist = new Set(); private readonly sessionEventTails = new Map>(); + private readonly snapshotPersistenceTails = new Map>(); private readonly runs = new AgentRunState(); private readonly subscribers = new Set(); private readonly idFactory: () => string; @@ -971,6 +1072,25 @@ export class AgentManager { return this.timelineStore.getRows(id); } + async getMaterialProgressSnapshot(id: string, limit = 1_000): Promise { + const live = this.agents.get(id); + const record = await this.registry?.get(id); + const continuationBoundarySeq = live + ? (live.materialProgressContinuationBoundarySeq ?? null) + : (record?.materialProgressContinuationBoundarySeq ?? null); + const rows = await this.resolveMaterialProgressRows(id, { + limit, + historyPrimed: live?.historyPrimed, + continuationBoundarySeq, + }); + return { + rows, + turnOutcome: live ? (live.lastTurnOutcome ?? null) : (record?.lastTurnOutcome ?? null), + continuationBoundarySeq, + persisted: record?.materialProgress ?? null, + }; + } + fetchTimeline(id: string, options?: AgentTimelineFetchOptions): AgentTimelineFetchResult { this.requireAgent(id); return this.timelineStore.fetch(id, options); @@ -1061,6 +1181,8 @@ export class AgentManager { labels?: Record; workspaceId?: string; owner?: AgentOwner; + lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -1080,6 +1202,8 @@ export class AgentManager { labels?: Record; workspaceId?: string; owner?: AgentOwner; + lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -1224,6 +1348,9 @@ export class AgentManager { const preservedHistoryPrimed = existing.historyPrimed; const preservedLastUsage = existing.lastUsage; const preservedLastError = existing.lastError; + const preservedLastTurnOutcome = existing.lastTurnOutcome; + const preservedMaterialProgressContinuationBoundarySeq = + existing.materialProgressContinuationBoundarySeq; const preservedAttention = existing.attention; const handle = existing.persistence; const provider = handle?.provider ?? existing.provider; @@ -1275,6 +1402,8 @@ export class AgentManager { historyPrimed: rehydrateFromDisk ? false : preservedHistoryPrimed, lastUsage: preservedLastUsage, lastError: preservedLastError, + lastTurnOutcome: preservedLastTurnOutcome, + materialProgressContinuationBoundarySeq: preservedMaterialProgressContinuationBoundarySeq, attention: preservedAttention, }); } finally { @@ -1423,8 +1552,9 @@ export class AgentManager { throw new Error("Agent storage is not configured"); } - await this.registry.applySnapshot(agent, { + await this.persistSnapshot(agent, { internal: agent.internal, + includeInternal: true, }); const stored = await this.registry.get(agentId); if (!stored) { @@ -1531,6 +1661,8 @@ export class AgentManager { lastUserMessageAt: record.lastUserMessageAt ? new Date(record.lastUserMessageAt) : null, lastUsage: undefined, lastError: record.lastError ?? undefined, + lastTurnOutcome: record.lastTurnOutcome, + materialProgressContinuationBoundarySeq: record.materialProgressContinuationBoundarySeq, attention: { requiresAttention: false }, internal: record.internal, labels: record.labels, @@ -1984,10 +2116,16 @@ export class AgentManager { const streamForwarder = async function* streamForwarder(this: AgentManager) { let turnId: string; let turnStream: ReturnType | null = null; + if (!this.runs.isPendingRun(agentId, pendingRun.token)) { + return; + } try { const result = await agent.session.startTurn(prompt, options); turnId = result.turnId; } catch (error) { + if (!this.runs.isPendingRun(agentId, pendingRun.token)) { + return; + } agent.pendingReplacement = false; const errorMsg = error instanceof Error ? error.message : "Failed to start turn"; await this.handleStreamEvent(agent, { @@ -2000,15 +2138,22 @@ export class AgentManager { throw error; } - pendingRun.started = true; - pendingRun.turnId = turnId; + const pendingTerminalEvents = this.runs.bindPendingRun(agentId, pendingRun.token, turnId); + if (pendingTerminalEvents === null) { + this.runs.rememberFinalizedTurn(agent, turnId); + return; + } if (isReplacement) { agent.pendingReplacement = false; } agent.activeForegroundTurnId = turnId; agent.lifecycle = "running"; + this.beginMaterialProgressContinuation(agent, turnId); + turnStream = this.runs.createTurnStream(turnId); + this.runs.addWaiter(agent, turnStream.waiter); this.touchUpdatedAt(agent); - this.emitState(agent); + this.enqueueBackgroundPersist(agent); + this.emitState(agent, { persist: false }); this.logger.trace( { agentId, @@ -2021,8 +2166,9 @@ export class AgentManager { "agent.manager.stream.start", ); - turnStream = this.runs.createTurnStream(turnId); - this.runs.addWaiter(agent, turnStream.waiter); + for (const terminalEvent of pendingTerminalEvents) { + await this.dispatchSessionEvent(agent, terminalEvent); + } try { for await (const event of turnStream.events(isTurnTerminalEvent)) { @@ -2268,6 +2414,17 @@ export class AgentManager { } const interruptAcknowledged = await this.interruptSession(agent.session, agentId); + if ( + interruptAcknowledged && + run.kind === "foreground" && + run.turnId === null && + this.runs.settleForegroundRun(agentId, run.token) + ) { + agent.lastError = undefined; + agent.lastTurnOutcome = "canceled"; + this.endMaterialProgressContinuation(agent); + this.finalizeForegroundTurn(agent); + } const settlement = await this.waitWithTimeout({ operation: run.settledPromise, timeoutMs: interruptAcknowledged @@ -2671,6 +2828,8 @@ export class AgentManager { historyPrimed?: boolean; lastUsage?: AgentUsage; lastError?: string; + lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; attention?: AttentionState; initialTitle?: string | null; publishWhenReady?: boolean; @@ -2811,6 +2970,8 @@ export class AgentManager { historyPrimed?: boolean; lastUsage?: AgentUsage; lastError?: string; + lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; attention?: AttentionState; persistence?: AgentPersistenceHandle; workspaceId?: string; @@ -2819,19 +2980,20 @@ export class AgentManager { | undefined; }): ActiveManagedAgent { const { resolvedAgentId, session, config, now, durableTimelineHasRows, options } = params; + const resolvedOptions = options ?? {}; return { id: resolvedAgentId, provider: config.provider, cwd: config.cwd, - workspaceId: options?.workspaceId, - owner: options?.owner, + workspaceId: resolvedOptions.workspaceId, + owner: resolvedOptions.owner, session, capabilities: session.capabilities, config, runtimeInfo: undefined, lifecycle: "initializing", - createdAt: options?.createdAt ?? now, - updatedAt: options?.updatedAt ?? now, + createdAt: resolvedOptions.createdAt ?? now, + updatedAt: resolvedOptions.updatedAt ?? now, availableModes: [], currentModeId: null, pendingPermissions: new Map(), @@ -2843,16 +3005,21 @@ export class AgentManager { finalizedForegroundTurnIds: new Set(), unsubscribeSession: null, persistence: attachPersistenceCwd( - options?.persistence ?? session.describePersistence(), + resolvedOptions.persistence ?? session.describePersistence(), config.cwd, ), - historyPrimed: options?.historyPrimed ?? durableTimelineHasRows, - lastUserMessageAt: options?.lastUserMessageAt ?? null, - lastUsage: options?.lastUsage, - lastError: options?.lastError, - attention: resolveInitialAttention(options?.attention), + historyPrimed: resolvedOptions.historyPrimed ?? durableTimelineHasRows, + lastUserMessageAt: resolvedOptions.lastUserMessageAt ?? null, + lastUsage: resolvedOptions.lastUsage, + lastError: resolvedOptions.lastError, + lastTurnOutcome: resolvedOptions.lastTurnOutcome, + materialProgressContinuationBoundarySeq: + resolvedOptions.materialProgressContinuationBoundarySeq, + materialProgressContinuationActive: false, + materialProgressContinuationTurnId: null, + attention: resolveInitialAttention(resolvedOptions.attention), internal: config.internal ?? false, - labels: options?.labels ?? {}, + labels: resolvedOptions.labels ?? {}, } as ActiveManagedAgent; } @@ -3002,6 +3169,9 @@ export class AgentManager { this.dispatch({ type: "provider_subagent", event: update }); return; } + if (isTurnTerminalEvent(event) && this.runs.bufferPendingTerminalEvent(agent.id, event)) { + return; + } const turnId = getAgentStreamEventTurnId(event); const matchingWaiters = this.runs.getMatchingWaiters(agent, turnId); this.logger.trace( @@ -3055,18 +3225,134 @@ export class AgentManager { return explicitTitle ?? fallbackTitle; } - private async persistSnapshot( - agent: ManagedAgent, - options?: { title?: string | null; internal?: boolean }, - ): Promise { + private persistSnapshot(agent: ManagedAgent, options?: PersistSnapshotOptions): Promise { if (!this.registry) { - return; + return Promise.resolve(); } // Don't persist internal agents - they're ephemeral system tasks - if (agent.internal) { - return; + if (agent.internal && !options?.includeInternal) { + return Promise.resolve(); + } + const previous = (this.snapshotPersistenceTails.get(agent.id) ?? Promise.resolve()).catch( + () => undefined, + ); + const next = previous.then(() => this.writeSnapshot(agent, options)); + const tracked = next.finally(() => { + if (this.snapshotPersistenceTails.get(agent.id) === tracked) { + this.snapshotPersistenceTails.delete(agent.id); + } + }); + this.snapshotPersistenceTails.set(agent.id, tracked); + return tracked; + } + + private async writeSnapshot( + agent: ManagedAgent, + options?: PersistSnapshotOptions, + ): Promise { + const registry = this.registry; + if (!registry) return; + + const materialProgress = await this.buildPersistedMaterialProgress(agent); + await registry.applySnapshot(agent, { + ...(options?.title !== undefined ? { title: options.title } : {}), + ...(options?.internal !== undefined ? { internal: options.internal } : {}), + ...(materialProgress ? { materialProgress } : {}), + }); + } + + private async buildPersistedMaterialProgress( + agent: ManagedAgent, + ): Promise { + const rows = await this.resolveMaterialProgressRows(agent.id, { + limit: MATERIAL_PROGRESS_PAGE_SIZE, + historyPrimed: agent.historyPrimed, + continuationBoundarySeq: agent.materialProgressContinuationBoundarySeq ?? null, + }); + // Omitting the override makes AgentStorage preserve the last trustworthy classification. + if (rows === null) return undefined; + return analyzeMaterialProgress({ + entries: projectTimelineRows({ rows, mode: "projected" }), + turnOutcome: agent.lastTurnOutcome ?? null, + continuationBoundarySeq: agent.materialProgressContinuationBoundarySeq ?? null, + }); + } + + private async resolveMaterialProgressRows( + agentId: string, + options: { + limit: number; + historyPrimed?: boolean; + continuationBoundarySeq: number | null; + }, + ): Promise { + const limit = Math.max(1, Math.min(Math.floor(options.limit), MATERIAL_PROGRESS_PAGE_SIZE)); + const { continuationBoundarySeq } = options; + if (this.timelineStore.has(agentId)) { + const livePage = this.timelineStore.fetch(agentId, { direction: "tail", limit }); + if (livePage.rows.length > 0) { + const liveRows = await pageMaterialProgressRows( + livePage, + async (fetchOptions) => this.timelineStore.fetch(agentId, fetchOptions), + continuationBoundarySeq, + ); + if (liveRows !== null) return liveRows; + + const durableRows = await this.resolveDurableMaterialProgressRows( + agentId, + limit, + continuationBoundarySeq, + ); + if (durableRows === null) return null; + const firstLiveSeq = livePage.rows[0]?.seq; + const olderDurableRows = durableRows.filter((row) => row.seq < (firstLiveSeq ?? 0)); + if ( + firstLiveSeq === undefined || + olderDurableRows[olderDurableRows.length - 1]?.seq !== firstLiveSeq - 1 + ) { + return null; + } + const combinedRows = [...olderDurableRows, ...livePage.rows]; + return selectMaterialProgressRows(combinedRows, continuationBoundarySeq, true); + } + + const durableRows = await this.resolveDurableMaterialProgressRows( + agentId, + limit, + continuationBoundarySeq, + ); + if (durableRows && durableRows.length > 0) return durableRows; + if (livePage.window.nextSeq > 1 || options.historyPrimed === false) return null; + return []; + } + + return await this.resolveDurableMaterialProgressRows(agentId, limit, continuationBoundarySeq); + } + + private async resolveDurableMaterialProgressRows( + agentId: string, + limit: number, + continuationBoundarySeq: number | null, + ): Promise { + const durableTimelineStore = this.durableTimelineStore; + if (!durableTimelineStore) return null; + try { + const durablePage = await durableTimelineStore.fetchCommitted(agentId, { + direction: "tail", + limit, + }); + return await pageMaterialProgressRows( + durablePage, + (fetchOptions) => durableTimelineStore.fetchCommitted(agentId, fetchOptions), + continuationBoundarySeq, + ); + } catch (error) { + this.logger.debug( + { err: error, agentId }, + "Durable material progress timeline is unavailable", + ); + return null; } - await this.registry.applySnapshot(agent, options); } private requireRegistry(): AgentStorage { @@ -3306,12 +3592,15 @@ export class AgentManager { const isForegroundEvent = Boolean(eventTurnId && agent.activeForegroundTurnId === eventTurnId); this.traceHandleStreamEventStart(agent, event, eventTurnId, isForegroundEvent); if ( + event.type === "turn_started" && eventTurnId && - isTurnTerminalEvent(event) && this.runs.hasFinalizedTurn(agent, eventTurnId) ) { return false; } + if (this.shouldIgnoreTerminalEvent(agent, event, eventTurnId)) { + return false; + } // Only update timestamp for live events, not history replay if (!options?.fromHistory) { @@ -3353,6 +3642,18 @@ export class AgentManager { return flags.shouldNotifyWaiters; } + private shouldIgnoreTerminalEvent( + agent: ActiveManagedAgent, + event: AgentStreamEvent, + eventTurnId: string | undefined, + ): boolean { + if (!eventTurnId || !isTurnTerminalEvent(event)) return false; + return ( + this.runs.hasFinalizedTurn(agent, eventTurnId) || + !this.terminalEventOwnsCurrentTurn(agent, eventTurnId) + ); + } + private traceHandleStreamEventStart( agent: ActiveManagedAgent, event: AgentStreamEvent, @@ -3479,7 +3780,7 @@ export class AgentManager { this.onStreamTurnCanceled({ agent, event, eventTurnId, isForegroundEvent, options }); return undefined; case "turn_started": - this.onStreamTurnStarted({ agent, eventTurnId, isForegroundEvent }); + this.onStreamTurnStarted({ agent, eventTurnId, isForegroundEvent, options }); return undefined; case "permission_requested": this.onStreamPermissionRequested(agent, event); @@ -3492,6 +3793,18 @@ export class AgentManager { } } + private terminalEventOwnsCurrentTurn( + agent: ActiveManagedAgent, + eventTurnId: string | undefined, + ): boolean { + if (!eventTurnId) return true; + return ( + (!agent.activeForegroundTurnId || agent.activeForegroundTurnId === eventTurnId) && + (!agent.materialProgressContinuationTurnId || + agent.materialProgressContinuationTurnId === eventTurnId) + ); + } + private onStreamThreadStarted(agent: ActiveManagedAgent): void { const previousSessionId = agent.persistence?.sessionId ?? null; const handle = agent.session.describePersistence(); @@ -3564,6 +3877,8 @@ export class AgentManager { // data accumulated during streaming isn't lost when the provider omits // it from the completion event. agent.lastError = undefined; + agent.lastTurnOutcome = "completed"; + this.endMaterialProgressContinuation(agent); if (!isForegroundEvent && agent.lifecycle !== "idle" && !agent.pendingReplacement) { (agent as ActiveManagedAgent).lifecycle = "idle"; this.emitState(agent); @@ -3598,6 +3913,8 @@ export class AgentManager { agent.lifecycle = "error"; } agent.lastError = event.error; + agent.lastTurnOutcome = "failed"; + this.endMaterialProgressContinuation(agent); await this.appendSystemErrorTimelineMessage( agent, event.provider, @@ -3638,6 +3955,8 @@ export class AgentManager { agent.lifecycle = "idle"; } agent.lastError = undefined; + agent.lastTurnOutcome = "canceled"; + this.endMaterialProgressContinuation(agent); this.resolvePendingPermissionsForAgent(agent, event.provider, options, "Interrupted"); if (!isForegroundEvent) { this.emitState(agent); @@ -3648,8 +3967,14 @@ export class AgentManager { agent: ActiveManagedAgent; eventTurnId: string | undefined; isForegroundEvent: boolean; + options: HandleStreamEventOptions | undefined; }): void { - const { agent, eventTurnId, isForegroundEvent } = params; + const { agent, eventTurnId, isForegroundEvent, options } = params; + if (options?.fromHistory) { + agent.lastTurnOutcome = null; + return; + } + this.beginMaterialProgressContinuation(agent, eventTurnId); this.logger.trace( { agentId: agent.id, @@ -3664,8 +3989,37 @@ export class AgentManager { if (!isForegroundEvent) { this.runs.trackAutonomousRun(agent.id, eventTurnId ?? null); agent.lifecycle = "running"; - this.emitState(agent); + this.enqueueBackgroundPersist(agent); + this.emitState(agent, { persist: false }); + return; + } + this.enqueueBackgroundPersist(agent); + } + + private beginMaterialProgressContinuation( + agent: ActiveManagedAgent, + turnId: string | undefined, + ): void { + if (agent.materialProgressContinuationActive) { + if (!agent.materialProgressContinuationTurnId && turnId) { + agent.materialProgressContinuationTurnId = turnId; + } + agent.lastTurnOutcome = null; + return; } + + agent.materialProgressContinuationBoundarySeq = this.timelineStore.fetch(agent.id, { + direction: "tail", + limit: 1, + }).window.nextSeq; + agent.materialProgressContinuationActive = true; + agent.materialProgressContinuationTurnId = turnId ?? null; + agent.lastTurnOutcome = null; + } + + private endMaterialProgressContinuation(agent: ActiveManagedAgent): void { + agent.materialProgressContinuationActive = false; + agent.materialProgressContinuationTurnId = null; } private onStreamPermissionRequested( diff --git a/packages/server/src/server/agent/agent-projections.ts b/packages/server/src/server/agent/agent-projections.ts index a01c67b29d..745c92f8ef 100644 --- a/packages/server/src/server/agent/agent-projections.ts +++ b/packages/server/src/server/agent/agent-projections.ts @@ -1,6 +1,7 @@ import type { AgentListItemPayload, AgentSnapshotPayload, + MaterialProgressPayload, RecentProviderSessionDescriptorPayload, } from "../messages.js"; import type { SerializableAgentConfig, StoredAgentRecord } from "./agent-storage.js"; @@ -26,6 +27,7 @@ interface ProjectionOptions { title?: string | null; createdAt?: string; internal?: boolean; + materialProgress?: MaterialProgressPayload; } interface RecentProviderSessionProjectionOptions { @@ -87,6 +89,13 @@ export function toStoredAgentRecord( features: normalizeFeatures(agent.features), persistence, lastError: agent.lastError ?? undefined, + ...(agent.lastTurnOutcome ? { lastTurnOutcome: agent.lastTurnOutcome } : {}), + ...(agent.materialProgressContinuationBoundarySeq != null + ? { + materialProgressContinuationBoundarySeq: agent.materialProgressContinuationBoundarySeq, + } + : {}), + ...(options?.materialProgress ? { materialProgress: options.materialProgress } : {}), requiresAttention: agent.attention.requiresAttention, attentionReason: agent.attention.requiresAttention ? agent.attention.attentionReason : null, attentionTimestamp: agent.attention.requiresAttention @@ -236,6 +245,7 @@ export function buildStoredAgentPayload( attentionTimestamp: record.attentionTimestamp ?? null, archivedAt: record.archivedAt ?? null, labels: normalizeLabels(record.labels), + ...(record.materialProgress ? { materialProgress: record.materialProgress } : {}), ...(providerAvailable ? {} : { providerUnavailable: true }), }; } diff --git a/packages/server/src/server/agent/agent-run-state.ts b/packages/server/src/server/agent/agent-run-state.ts index e747317181..c04fa42f64 100644 --- a/packages/server/src/server/agent/agent-run-state.ts +++ b/packages/server/src/server/agent/agent-run-state.ts @@ -14,6 +14,7 @@ export interface PendingForegroundRun { token: string; kind: "foreground"; turnId: string | null; + pendingTerminalEvents: AgentStreamEvent[]; started: boolean; settled: boolean; settledPromise: Promise; @@ -59,6 +60,32 @@ export class AgentRunState { return this.runs.get(agentId) ?? null; } + isPendingRun(agentId: string, token: string): boolean { + const run = this.runs.get(agentId); + return run?.kind === "foreground" && run.token === token; + } + + bindPendingRun(agentId: string, token: string, turnId: string): AgentStreamEvent[] | null { + const run = this.runs.get(agentId); + if (run?.kind !== "foreground" || run.token !== token) { + return null; + } + + run.started = true; + run.turnId = turnId; + return run.pendingTerminalEvents.splice(0); + } + + bufferPendingTerminalEvent(agentId: string, event: AgentStreamEvent): boolean { + const run = this.runs.get(agentId); + if (run?.kind !== "foreground" || run.turnId !== null) { + return false; + } + + run.pendingTerminalEvents.push(event); + return true; + } + hasRun(agentId: string): boolean { return this.runs.has(agentId); } @@ -94,13 +121,14 @@ export class AgentRunState { this.clearRun(agentId, run); } - settleForegroundRun(agentId: string, token: string): void { + settleForegroundRun(agentId: string, token: string): boolean { const run = this.runs.get(agentId); if (run?.kind !== "foreground" || run.token !== token) { - return; + return false; } this.clearRun(agentId, run); + return true; } clearAgentRun(agentId: string): void { @@ -259,12 +287,18 @@ export class ForegroundTurnStream { } function createPendingForegroundRun(): PendingForegroundRun { - return createTrackedRun({ kind: "foreground", turnId: null, started: false }); + return createTrackedRun({ + kind: "foreground", + turnId: null, + pendingTerminalEvents: [], + started: false, + }); } function createTrackedRun(input: { kind: "foreground"; turnId: null; + pendingTerminalEvents: AgentStreamEvent[]; started: false; }): PendingForegroundRun; function createTrackedRun(input: { @@ -274,7 +308,12 @@ function createTrackedRun(input: { }): AutonomousAgentRun; function createTrackedRun( input: - | { kind: "foreground"; turnId: null; started: false } + | { + kind: "foreground"; + turnId: null; + pendingTerminalEvents: AgentStreamEvent[]; + started: false; + } | { kind: "autonomous"; turnId: string | null; started: true }, ): TrackedAgentRun { let resolveSettled!: () => void; diff --git a/packages/server/src/server/agent/agent-storage.test.ts b/packages/server/src/server/agent/agent-storage.test.ts index a9b1ab762f..fc2e88d3cc 100644 --- a/packages/server/src/server/agent/agent-storage.test.ts +++ b/packages/server/src/server/agent/agent-storage.test.ts @@ -296,6 +296,34 @@ describe("AgentStorage", () => { expect(recordAfterSnapshot?.archivedAt).toBe(archivedAt); }); + test("concurrent snapshots preserve a newly derived material progress signal", async () => { + const agentId = "agent-material-progress"; + await storage.initialize(); + + const withMaterialProgress = storage.applySnapshot(createManagedAgent({ id: agentId }), { + materialProgress: { + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: "2026-07-31T00:00:00.000Z", + lastMaterialProgressKind: "write", + reason: "Material progress followed the latest user message.", + }, + }); + const genericSnapshot = storage.applySnapshot( + createManagedAgent({ + id: agentId, + updatedAt: new Date("2026-07-31T00:01:00.000Z"), + }), + ); + + await Promise.all([withMaterialProgress, genericSnapshot]); + + expect((await storage.get(agentId))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + }); + test("stores titles independently of snapshots", async () => { await storage.applySnapshot( createManagedAgent({ @@ -314,6 +342,75 @@ describe("AgentStorage", () => { expect(persisted?.title).toBe("Fix Login Bug"); }); + test("setTitle started before a newer snapshot cannot overwrite its material progress", async () => { + const agentId = "agent-title-progress-race"; + await storage.applySnapshot(createManagedAgent({ id: agentId }), { + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "One compaction completed without later material progress.", + }, + }); + + let releasePendingWrite: (() => void) | null = null; + const pendingWrite = new Promise((resolve) => { + releasePendingWrite = resolve; + }); + const storageInternals = storage as unknown as { + pendingWrites: Map>; + }; + storageInternals.pendingWrites.set(agentId, pendingWrite); + + const titleWrite = storage.setTitle(agentId, "Final title"); + const newerSnapshot = storage.applySnapshot( + createManagedAgent({ + id: agentId, + updatedAt: new Date("2026-07-31T00:02:00.000Z"), + }), + { + materialProgress: { + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: "2026-07-31T00:01:00.000Z", + lastMaterialProgressKind: "write", + reason: "Material progress followed the latest user message.", + }, + }, + ); + releasePendingWrite?.(); + + await Promise.all([titleWrite, newerSnapshot]); + expect(await storage.get(agentId)).toMatchObject({ + title: "Final title", + materialProgress: { + state: "progressing", + lastMaterialProgressKind: "write", + }, + }); + }); + + test("setTitle still runs after an earlier queued write fails", async () => { + const agentId = "agent-title-after-failure"; + await storage.applySnapshot(createManagedAgent({ id: agentId })); + + let rejectFailedWrite: ((reason: Error) => void) | null = null; + const failedWrite = new Promise((_resolve, reject) => { + rejectFailedWrite = reject; + }); + const storageInternals = storage as unknown as { + pendingWrites: Map>; + }; + storageInternals.pendingWrites.set(agentId, failedWrite); + + const titleWrite = storage.setTitle(agentId, "Recovered title"); + rejectFailedWrite?.(new Error("simulated earlier write failure")); + + await expect(titleWrite).resolves.toBeUndefined(); + expect((await storage.get(agentId))?.title).toBe("Recovered title"); + }); + test("setTitle throws when the agent record does not exist", async () => { await expect(storage.setTitle("missing-agent", "Impossible")).rejects.toThrow( "Agent missing-agent not found", diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 5e504cee64..f6bb7507c9 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -4,7 +4,12 @@ import { z } from "zod"; import type { Logger } from "pino"; import { writeJsonFileAtomic } from "../atomic-file.js"; -import { AgentFeatureSchema, AgentStatusSchema } from "../messages.js"; +import { + AgentFeatureSchema, + AgentStatusSchema, + MaterialProgressPayloadSchema, + type MaterialProgressPayload, +} from "../messages.js"; import { toStoredAgentRecord } from "./agent-projections.js"; import type { ManagedAgent } from "./agent-manager.js"; import type { AgentSessionConfig } from "./agent-sdk-types.js"; @@ -60,6 +65,9 @@ const STORED_AGENT_SCHEMA = z.object({ features: z.array(AgentFeatureSchema).optional(), persistence: PERSISTENCE_HANDLE_SCHEMA, lastError: z.string().nullable().optional(), + lastTurnOutcome: z.enum(["completed", "failed", "canceled"]).nullable().optional(), + materialProgressContinuationBoundarySeq: z.number().int().positive().nullable().optional(), + materialProgress: MaterialProgressPayloadSchema.optional(), requiresAttention: z.boolean().optional(), attentionReason: z.enum(["finished", "error", "permission"]).nullable().optional(), attentionTimestamp: z.string().nullable().optional(), @@ -128,13 +136,20 @@ export class AgentStorage { } private queueRecordWrite(record: StoredAgentRecord): Promise { - const agentId = record.id; - const prev = this.pendingWrites.get(agentId) ?? Promise.resolve(); + return this.queueRecordMutation(record.id, () => record); + } + + private queueRecordMutation( + agentId: string, + buildRecord: (existing: StoredAgentRecord | null) => StoredAgentRecord, + ): Promise { + const prev = (this.pendingWrites.get(agentId) ?? Promise.resolve()).catch(() => undefined); const next = prev.then(async () => { if (this.deleting.has(agentId)) { return undefined; } + const record = buildRecord(this.cache.get(agentId) ?? null); await this.writeRecord(record); return undefined; }); @@ -204,38 +219,47 @@ export class AgentStorage { async applySnapshot( agent: ManagedAgent, - options?: { title?: string | null; internal?: boolean }, + options?: { + title?: string | null; + internal?: boolean; + materialProgress?: MaterialProgressPayload; + }, ): Promise { await this.load(); - await this.waitForPendingWrite(agent.id); - const existing = (await this.get(agent.id)) ?? null; const hasTitleOverride = options !== undefined && Object.prototype.hasOwnProperty.call(options, "title"); const hasInternalOverride = options !== undefined && Object.prototype.hasOwnProperty.call(options, "internal"); - const record = toStoredAgentRecord(agent, { - title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), - createdAt: existing?.createdAt, - internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), + const hasMaterialProgressOverride = + options !== undefined && Object.prototype.hasOwnProperty.call(options, "materialProgress"); + await this.queueRecordMutation(agent.id, (existing) => { + const record = toStoredAgentRecord(agent, { + title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), + createdAt: existing?.createdAt, + internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), + materialProgress: hasMaterialProgressOverride + ? options?.materialProgress + : existing?.materialProgress, + }); + + // Preserve soft-delete/archive status across snapshot flushes. + // `archivedAt` is not part of the ManagedAgent snapshot, so a naive projection + // would wipe it during normal persistence (including on daemon restart). + if (existing && existing.archivedAt !== undefined) { + record.archivedAt = existing.archivedAt; + } + return record; }); - - // Preserve soft-delete/archive status across snapshot flushes. - // `archivedAt` is not part of the ManagedAgent snapshot, so a naive projection - // would wipe it during normal persistence (including on daemon restart). - if (existing && existing.archivedAt !== undefined) { - record.archivedAt = existing.archivedAt; - } - await this.upsert(record); } async setTitle(agentId: string, title: string): Promise { await this.load(); - await this.waitForPendingWrite(agentId); - const record = await this.get(agentId); - if (!record) { - throw new Error(`Agent ${agentId} not found`); - } - await this.upsert({ ...record, title }); + await this.queueRecordMutation(agentId, (existing) => { + if (!existing) { + throw new Error(`Agent ${agentId} not found`); + } + return { ...existing, title }; + }); } async flush(): Promise { @@ -386,10 +410,6 @@ export class AgentStorage { } this.daemonExecutionKeysByAgentId.delete(agentId); } - - private async waitForPendingWrite(agentId: string): Promise { - await (this.pendingWrites.get(agentId) ?? Promise.resolve()).catch(() => undefined); - } } function projectDirNameFromCwd(cwd: string): string { diff --git a/packages/server/src/server/agent/material-progress.test.ts b/packages/server/src/server/agent/material-progress.test.ts new file mode 100644 index 0000000000..5f61692527 --- /dev/null +++ b/packages/server/src/server/agent/material-progress.test.ts @@ -0,0 +1,512 @@ +import { describe, expect, it } from "vitest"; +import type { AgentTimelineItem } from "./agent-sdk-types.js"; +import type { TimelineProjectionEntry } from "./timeline-projection.js"; +import { analyzeMaterialProgress } from "./material-progress.js"; + +function entry(seq: number, item: AgentTimelineItem): TimelineProjectionEntry { + return { + item, + timestamp: `2026-07-31T00:00:0${seq}.000Z`, + seqStart: seq, + seqEnd: seq, + sourceSeqRanges: [{ startSeq: seq, endSeq: seq }], + collapsed: [], + }; +} + +describe("analyzeMaterialProgress", () => { + it("reports none when history or a current continuation is unavailable", () => { + expect(analyzeMaterialProgress({ entries: null, turnOutcome: null })).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Timeline history is unavailable.", + }); + + expect(analyzeMaterialProgress({ entries: [], turnOutcome: null }).state).toBe("none"); + }); + + it("treats completed edits and writes as material progress", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "implement" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { + type: "tool_call", + callId: "edit-1", + name: "edit", + status: "completed", + error: null, + detail: { type: "edit", filePath: "src/a.ts", unifiedDiff: "+change" }, + }), + entry(4, { type: "compaction", status: "completed" }), + entry(5, { + type: "tool_call", + callId: "write-1", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "src/b.ts", content: "content" }, + }), + ], + turnOutcome: null, + }); + + expect(result).toEqual({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: "2026-07-31T00:00:05.000Z", + lastMaterialProgressKind: "write", + reason: "Material progress followed the latest user message.", + }); + }); + + it("warns after one compaction and stalls after two", () => { + expect( + analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "continue" }), + entry(2, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }), + ).toMatchObject({ + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + }); + + expect( + analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "continue" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { type: "reasoning", text: "still working" }), + entry(4, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }), + ).toMatchObject({ + state: "stalled", + completedCompactionsSinceMaterialProgress: 2, + }); + }); + + it("does not inherit a previous continuation's compactions after a new turn is accepted", () => { + const previousContinuation = [ + entry(1, { type: "user_message", text: "old continuation" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { type: "compaction", status: "completed" }), + ]; + + expect( + analyzeMaterialProgress({ entries: previousContinuation, turnOutcome: null }), + ).toMatchObject({ state: "stalled" }); + + expect( + analyzeMaterialProgress({ + entries: previousContinuation, + turnOutcome: null, + continuationBoundarySeq: 4, + }), + ).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "No material progress has been recorded for the current continuation.", + }); + }); + + it("uses only the latest user continuation and completion sequence", () => { + const completedWrite = entry(2, { + type: "tool_call", + callId: "write-1", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "proof.txt", content: "passed" }, + }); + completedWrite.seqEnd = 5; + + expect( + analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "old work" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { type: "user_message", text: "current work" }), + completedWrite, + entry(4, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }), + ).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "write", + }); + }); + + it("keeps a read-only audit progressing across two compactions", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "audit the authentication flow" }), + entry(2, { + type: "tool_call", + callId: "read-1", + name: "read", + status: "completed", + error: null, + detail: { type: "read", filePath: "src/auth.ts", content: "export function login() {}" }, + }), + entry(3, { type: "compaction", status: "completed" }), + entry(4, { + type: "tool_call", + callId: "search-1", + name: "grep", + status: "completed", + error: null, + detail: { type: "search", query: "login(", numFiles: 2, numMatches: 3 }, + }), + entry(5, { type: "compaction", status: "completed" }), + entry(6, { + type: "tool_call", + callId: "fetch-1", + name: "fetch", + status: "completed", + error: null, + detail: { + type: "fetch", + url: "https://example.com/auth-spec", + result: "Tokens expire after one hour.", + code: 200, + }, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "evidence", + }); + }); + + it("keeps test-only verification progressing across two compactions", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "verify the change" }), + entry(2, { + type: "tool_call", + callId: "test-1", + name: "bash", + status: "completed", + error: null, + detail: { type: "shell", command: "npm test", output: "12 tests passed", exitCode: 0 }, + }), + entry(3, { type: "compaction", status: "completed" }), + entry(4, { + type: "tool_call", + callId: "build-1", + name: "bash", + status: "completed", + error: null, + detail: { + type: "shell", + command: "npm run build", + output: "build complete", + exitCode: 0, + }, + }), + entry(5, { type: "compaction", status: "completed" }), + entry(6, { + type: "tool_call", + callId: "typecheck-1", + name: "bash", + status: "completed", + error: null, + detail: { + type: "shell", + command: "npm run typecheck", + output: "typecheck passed", + exitCode: 0, + }, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "verification", + }); + }); + + it("treats a non-empty explicit plan as a decision", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "decide how to proceed" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { + type: "tool_call", + callId: "plan-1", + name: "plan", + status: "completed", + error: null, + detail: { type: "plan", text: "Use the persisted boundary as the source of truth." }, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "decision", + }); + }); + + it("stalls on repeated irrelevant reads across two compactions", () => { + const repeatedRead = { + type: "read" as const, + filePath: "notes/unrelated.txt", + content: "unchanged unrelated notes", + }; + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "audit the authentication flow" }), + entry(2, { + type: "tool_call", + callId: "read-1", + name: "read", + status: "completed", + error: null, + detail: repeatedRead, + }), + entry(3, { type: "compaction", status: "completed" }), + entry(4, { + type: "tool_call", + callId: "read-2", + name: "read", + status: "completed", + error: null, + detail: repeatedRead, + }), + entry(5, { type: "compaction", status: "completed" }), + entry(6, { + type: "tool_call", + callId: "read-3", + name: "read", + status: "completed", + error: null, + detail: repeatedRead, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "stalled", + completedCompactionsSinceMaterialProgress: 2, + lastMaterialProgressAt: "2026-07-31T00:00:02.000Z", + lastMaterialProgressKind: "evidence", + }); + }); + + it("does not count empty, failed, or no-op tools and commentary as material progress", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "investigate" }), + entry(2, { + type: "tool_call", + callId: "read-1", + name: "read", + status: "completed", + error: null, + detail: { type: "read", filePath: "src/a.ts" }, + }), + entry(3, { + type: "tool_call", + callId: "search-1", + name: "grep", + status: "completed", + error: null, + detail: { type: "search", query: "TODO" }, + }), + entry(4, { + type: "tool_call", + callId: "fetch-1", + name: "fetch", + status: "completed", + error: null, + detail: { type: "fetch", url: "https://example.com" }, + }), + entry(5, { + type: "tool_call", + callId: "shell-1", + name: "bash", + status: "completed", + error: null, + detail: { type: "shell", command: "false", output: "failed", exitCode: 1 }, + }), + entry(6, { + type: "tool_call", + callId: "plan-1", + name: "plan", + status: "completed", + error: null, + detail: { type: "plan", text: " " }, + }), + entry(7, { type: "assistant_message", text: "Still investigating." }), + entry(8, { + type: "tool_call", + callId: "edit-1", + name: "edit", + status: "failed", + error: "permission denied", + detail: { type: "edit", filePath: "src/a.ts", unifiedDiff: "+change" }, + }), + entry(9, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressKind: null, + }); + }); + + it("does not count a completed edit that leaves content unchanged", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "update the file" }), + entry(2, { + type: "tool_call", + callId: "edit-no-op", + name: "edit", + status: "completed", + error: null, + detail: { + type: "edit", + filePath: "src/a.ts", + oldString: "unchanged", + newString: "unchanged", + }, + }), + entry(3, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressKind: null, + }); + }); + + it("counts a concrete multi-edit diff when its first replacement is unchanged", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "update the file" }), + entry(2, { + type: "tool_call", + callId: "multi-edit", + name: "edit", + status: "completed", + error: null, + detail: { + type: "edit", + filePath: "src/a.ts", + oldString: "unchanged first replacement", + newString: "unchanged first replacement", + unifiedDiff: "+changed by a later replacement", + }, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "edit", + }); + }); + + it("counts a completed turn's final message as a deliverable, not semantic success", () => { + const entries = [ + entry(1, { type: "user_message", text: "answer" }), + entry(2, { type: "assistant_message", text: "I am checking." }), + entry(3, { type: "compaction", status: "completed" }), + entry(4, { type: "assistant_message", text: "Delivered result." }), + ]; + + expect(analyzeMaterialProgress({ entries, turnOutcome: null }).state).toBe("warning"); + expect(analyzeMaterialProgress({ entries, turnOutcome: "failed" }).state).toBe("warning"); + expect(analyzeMaterialProgress({ entries, turnOutcome: "canceled" }).state).toBe("warning"); + expect(analyzeMaterialProgress({ entries, turnOutcome: "completed" })).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "assistant_result", + }); + + const emptyFinalMessage = [...entries, entry(5, { type: "assistant_message", text: " " })]; + expect( + analyzeMaterialProgress({ entries: emptyFinalMessage, turnOutcome: "completed" }), + ).toMatchObject({ + state: "warning", + lastMaterialProgressKind: null, + }); + + const failedActivityAfterCommentary = [ + ...entries.slice(0, -1), + entry(4, { + type: "tool_call", + callId: "failed-edit", + name: "edit", + status: "failed", + error: "permission denied", + detail: { type: "edit", filePath: "src/a.ts", unifiedDiff: "+change" }, + }), + ]; + expect( + analyzeMaterialProgress({ + entries: failedActivityAfterCommentary, + turnOutcome: "completed", + }), + ).toMatchObject({ + state: "warning", + lastMaterialProgressKind: null, + }); + + const noOpActivityAfterCommentary = [ + ...entries.slice(0, -1), + entry(4, { + type: "tool_call", + callId: "no-op-edit", + name: "edit", + status: "completed", + error: null, + detail: { + type: "edit", + filePath: "src/a.ts", + oldString: "unchanged", + newString: "unchanged", + }, + }), + ]; + expect( + analyzeMaterialProgress({ + entries: noOpActivityAfterCommentary, + turnOutcome: "completed", + }), + ).toMatchObject({ + state: "warning", + lastMaterialProgressKind: null, + }); + }); +}); diff --git a/packages/server/src/server/agent/material-progress.ts b/packages/server/src/server/agent/material-progress.ts new file mode 100644 index 0000000000..3108c771c9 --- /dev/null +++ b/packages/server/src/server/agent/material-progress.ts @@ -0,0 +1,259 @@ +import { createHash } from "node:crypto"; +import type { MaterialProgressPayload } from "../messages.js"; +import type { AgentTimelineItem, ToolCallDetail, ToolCallTimelineItem } from "./agent-sdk-types.js"; +import type { TimelineProjectionEntry } from "./timeline-projection.js"; + +export interface AnalyzeMaterialProgressInput { + entries: readonly TimelineProjectionEntry[] | null; + turnOutcome: "completed" | "failed" | "canceled" | null; + /** + * Manager-owned sequence at which the currently accepted continuation begins. + * Older persisted agents omit it and retain the latest-user-message fallback. + */ + continuationBoundarySeq?: number | null; +} + +type MaterialProgressKind = NonNullable; + +interface MaterialProgressEvent { + kind: MaterialProgressKind; + fingerprint: string; +} + +function hasConcreteText(value: unknown): boolean { + return typeof value === "string" && value.trim().length > 0; +} + +function progressFingerprint(kind: MaterialProgressKind, result: unknown): string { + const digest = createHash("sha256").update(JSON.stringify(result)).digest("hex"); + return `${kind}:${digest}`; +} + +function progressEvent(kind: MaterialProgressKind, result: unknown): MaterialProgressEvent { + return { kind, fingerprint: progressFingerprint(kind, result) }; +} + +type EditDetail = Extract; +type WriteDetail = Extract; +type EvidenceDetail = Extract; +type SearchDetail = Extract; +type ShellDetail = Extract; +type PlanDetail = Extract; + +function editEvent(detail: EditDetail): MaterialProgressEvent | null { + const { filePath, oldString, newString, unifiedDiff } = detail; + if (!hasConcreteText(unifiedDiff) && oldString !== undefined && oldString === newString) { + return null; + } + if (!hasConcreteText(unifiedDiff) && oldString === undefined && newString === undefined) { + return null; + } + return progressEvent("edit", { filePath, oldString, newString, unifiedDiff }); +} + +function writeEvent(detail: WriteDetail): MaterialProgressEvent | null { + const { filePath, content } = detail; + return content === undefined ? null : progressEvent("write", { filePath, content }); +} + +function hasSearchResult(detail: SearchDetail): boolean { + return ( + hasConcreteText(detail.content) || + detail.numFiles !== undefined || + detail.numMatches !== undefined || + (detail.filePaths?.length ?? 0) > 0 || + (detail.webResults?.length ?? 0) > 0 || + (detail.annotations?.length ?? 0) > 0 + ); +} + +function evidenceEvent(detail: EvidenceDetail): MaterialProgressEvent | null { + switch (detail.type) { + case "read": { + const { filePath, content } = detail; + return hasConcreteText(content) + ? progressEvent("evidence", { type: "read", filePath, content }) + : null; + } + case "search": { + const { query, content, filePaths, webResults, annotations, numFiles, numMatches } = detail; + return hasSearchResult(detail) + ? progressEvent("evidence", { + type: "search", + query, + content, + filePaths, + webResults, + annotations, + numFiles, + numMatches, + }) + : null; + } + case "fetch": { + const { url, result, code, bytes } = detail; + const succeeded = + hasConcreteText(result) && (code === undefined || (code >= 200 && code < 400)); + return succeeded + ? progressEvent("evidence", { type: "fetch", url, result, code, bytes }) + : null; + } + } +} + +function verificationEvent(detail: ShellDetail): MaterialProgressEvent | null { + const { command, cwd, output, exitCode } = detail; + return exitCode === 0 && hasConcreteText(output) + ? progressEvent("verification", { command, cwd, output, exitCode }) + : null; +} + +function decisionEvent(detail: PlanDetail): MaterialProgressEvent | null { + const text = detail.text.trim(); + return text ? progressEvent("decision", text) : null; +} + +function materialToolEvent(item: ToolCallTimelineItem): MaterialProgressEvent | null { + if (item.status !== "completed") return null; + switch (item.detail.type) { + case "edit": + return editEvent(item.detail); + case "write": + return writeEvent(item.detail); + case "read": + case "search": + case "fetch": + return evidenceEvent(item.detail); + case "shell": + return verificationEvent(item.detail); + case "plan": + return decisionEvent(item.detail); + case "worktree_setup": + case "sub_agent": + case "unknown": + case "plain_text": + return null; + } +} + +function validTimestamp(value: string): string | null { + return Number.isNaN(Date.parse(value)) ? null : value; +} + +function orderedEntries(entries: readonly TimelineProjectionEntry[]): TimelineProjectionEntry[] { + return [...entries].sort( + (left, right) => left.seqEnd - right.seqEnd || left.seqStart - right.seqStart, + ); +} + +function currentContinuation( + ordered: readonly TimelineProjectionEntry[], + continuationBoundarySeq: number | null | undefined, +): TimelineProjectionEntry[] | null { + if (continuationBoundarySeq != null) { + return ordered.filter((entry) => entry.seqEnd >= continuationBoundarySeq); + } + + let latestUserIndex = -1; + for (let index = 0; index < ordered.length; index += 1) { + if (ordered[index]?.item.type === "user_message") latestUserIndex = index; + } + return latestUserIndex < 0 ? null : ordered.slice(latestUserIndex + 1); +} + +function noProgress(reason: string): MaterialProgressPayload { + return { + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason, + }; +} + +function materialEvent( + item: AgentTimelineItem, + isFinalTerminalAssistant: boolean, +): MaterialProgressEvent | null { + if (item.type === "tool_call") return materialToolEvent(item); + if (item.type === "assistant_message" && isFinalTerminalAssistant && hasConcreteText(item.text)) { + const text = item.text.trim(); + return { + kind: "assistant_result", + fingerprint: progressFingerprint("assistant_result", text), + }; + } + return null; +} + +export function analyzeMaterialProgress({ + entries, + turnOutcome, + continuationBoundarySeq, +}: AnalyzeMaterialProgressInput): MaterialProgressPayload { + if (entries === null) return noProgress("Timeline history is unavailable."); + + const ordered = orderedEntries(entries); + const continuation = currentContinuation(ordered, continuationBoundarySeq); + if (continuation === null) return noProgress("No current continuation is available."); + const terminalAssistantIndex = + turnOutcome === "completed" && continuation.at(-1)?.item.type === "assistant_message" + ? continuation.length - 1 + : -1; + + let completedCompactions = 0; + let lastMaterialProgressAt: string | null = null; + let lastMaterialProgressKind: MaterialProgressKind | null = null; + const seenProgress = new Set(); + + for (let index = 0; index < continuation.length; index += 1) { + const entry = continuation[index]; + if (!entry) continue; + const event = materialEvent( + entry.item, + turnOutcome === "completed" && index === terminalAssistantIndex, + ); + if (event && !seenProgress.has(event.fingerprint)) { + seenProgress.add(event.fingerprint); + completedCompactions = 0; + lastMaterialProgressAt = validTimestamp(entry.timestamp); + lastMaterialProgressKind = event.kind; + continue; + } + if (entry.item.type === "compaction" && entry.item.status === "completed") { + completedCompactions += 1; + } + } + + if (completedCompactions >= 2) { + return { + state: "stalled", + completedCompactionsSinceMaterialProgress: completedCompactions, + lastMaterialProgressAt, + lastMaterialProgressKind, + reason: + completedCompactions === 2 + ? "Two compactions completed without later material progress." + : `${completedCompactions} compactions completed without later material progress.`, + }; + } + if (completedCompactions === 1) { + return { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt, + lastMaterialProgressKind, + reason: "One compaction completed without later material progress.", + }; + } + if (lastMaterialProgressKind) { + return { + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt, + lastMaterialProgressKind, + reason: "Material progress followed the latest user message.", + }; + } + return noProgress("No material progress has been recorded for the current continuation."); +} diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index 57235d3773..17e0b99efa 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -1094,11 +1094,23 @@ describe("Codex app-server provider", () => { ); await session.startTurn("remember first"); + appServer.startsTurn({ threadId: "thread-1", turnId: "native-first" }); emitCodexUserMessage(appServer, { id: "codex-first", text: "remember first" }); appServer.completeTurn(); + await vi.waitFor(() => + expect(castInternals<{ activeForegroundTurnId: string | null }>(session)).toMatchObject({ + activeForegroundTurnId: null, + }), + ); await session.startTurn("remember second"); + appServer.startsTurn({ threadId: "thread-1", turnId: "native-second" }); emitCodexUserMessage(appServer, { id: "codex-second", text: "remember second" }); appServer.completeTurn(); + await vi.waitFor(() => + expect(castInternals<{ activeForegroundTurnId: string | null }>(session)).toMatchObject({ + activeForegroundTurnId: null, + }), + ); await session.revertConversation({ messageId: "codex-first" }); @@ -2540,10 +2552,22 @@ describe("Codex app-server provider", () => { }); }); - test("never treats an unmapped foreign terminal as the root terminal", () => { + test("never treats an unmapped foreign terminal as the root terminal", async () => { const session = createSession(); const events: AgentStreamEvent[] = []; session.subscribe((event) => events.push(event)); + session.client = createStub({ + request: async (method) => { + if (method === "thread/read") { + return { + thread: { + turns: [{ id: "first-root-turn", status: "completed", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${method}`); + }, + }); asInternals(session).handleNotification("turn/completed", { threadId: "unmapped-child-thread", @@ -2551,10 +2575,15 @@ describe("Codex app-server provider", () => { }); expect(events).toEqual([]); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "first-root-turn" }, + }); asInternals(session).handleNotification("turn/completed", { threadId: "test-thread", turn: { status: "completed" }, }); + await vi.waitFor(() => expect(session.activeForegroundTurnId).toBeNull()); expect(events.filter((event) => event.type === "turn_completed")).toHaveLength(1); asInternals(session).handleNotification("turn/started", { @@ -2577,11 +2606,27 @@ describe("Codex app-server provider", () => { }); }); - test("routes msg-scoped legacy Codex events to their child thread", () => { + test("routes msg-scoped legacy Codex events to their child thread", async () => { const session = createSession(); const events: AgentStreamEvent[] = []; session.subscribe((event) => events.push(event)); + session.client = createStub({ + request: async (method) => { + if (method === "thread/read") { + return { + thread: { + turns: [{ id: "root-turn", status: "completed", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${method}`); + }, + }); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "root-turn" }, + }); asInternals(session).handleNotification("item/completed", { threadId: "test-thread", item: { @@ -2641,6 +2686,7 @@ describe("Codex app-server provider", () => { asInternals(session).handleNotification("codex/event/task_complete", { msg: { type: "task_complete" }, }); + await vi.waitFor(() => expect(session.activeForegroundTurnId).toBeNull()); expect(events.filter((event) => event.type === "turn_completed")).toHaveLength(1); }); @@ -3889,7 +3935,7 @@ describe("Codex app-server provider", () => { ], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-1", status: "completed", error: null }, }); expect( @@ -3952,7 +3998,7 @@ describe("Codex app-server provider", () => { }, }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-thread-item", status: "completed", error: null }, }); expect(events).not.toContainEqual( @@ -3991,7 +4037,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the first plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-first", status: "completed", error: null }, }); asInternals(session).handleNotification("turn/started", { @@ -4001,7 +4047,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the revised plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-second", status: "completed", error: null }, }); expect(session.getPendingPermissions()).toEqual([ @@ -4026,7 +4072,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the original plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-pending", status: "completed", error: null }, }); const pendingPlan = session.getPendingPermissions()[0]; expect(pendingPlan).toBeDefined(); @@ -4066,7 +4112,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the original plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-pending", status: "completed", error: null }, }); const pendingPlan = session.getPendingPermissions()[0]; expect(pendingPlan).toBeDefined(); @@ -4120,7 +4166,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the original plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-pending", status: "completed", error: null }, }); let acceptPrompt: (() => void) | undefined; @@ -4144,11 +4190,14 @@ describe("Codex app-server provider", () => { const startTurn = session.startTurn("Revise the plan"); await promptRequested; + asInternals(session).handleNotification("turn/started", { + turn: { id: "turn-plan-newer" }, + }); asInternals(session).handleNotification("turn/plan/updated", { plan: [{ step: "Implement the newer plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-newer", status: "completed", error: null }, }); acceptPrompt?.(); await startTurn; @@ -4170,7 +4219,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the original plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-pending", status: "completed", error: null }, }); const pendingPlan = session.getPendingPermissions()[0]; expect(pendingPlan).toBeDefined(); @@ -4416,7 +4465,7 @@ describe("Codex app-server provider", () => { }), ).not.toThrow(); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "test-turn", status: "completed", error: null }, }); expect(events).toEqual([ @@ -4446,7 +4495,7 @@ describe("Codex app-server provider", () => { }, }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "test-turn", status: "completed", error: null }, }); expect(events).toContainEqual({ @@ -4475,6 +4524,243 @@ describe("Codex app-server provider", () => { }); }); + test("ignores an uncorrelated terminal while a foreground turn is active", () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/completed", { + turn: { status: "completed", error: null }, + }); + + expect(events).toEqual([]); + }); + + test.each([ + ["turn/completed", { threadId: "test-thread", turn: { status: "completed", error: null } }], + ["codex/event/task_complete", { threadId: "test-thread", msg: { type: "task_complete" } }], + ])("reconciles a current first-turn ID-less terminal from %s", async (method, params) => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + const request = vi.fn(async (requestMethod: string) => { + if (requestMethod === "thread/read") { + return { + thread: { + turns: [{ id: "native-current-turn", status: "completed", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${requestMethod}`); + }); + session.client = createStub({ request }); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-current-turn" }, + }); + asInternals(session).handleNotification(method, params); + + await vi.waitFor(() => expect(session.activeForegroundTurnId).toBeNull()); + expect(request).toHaveBeenCalledWith("thread/read", { + threadId: "test-thread", + includeTurns: true, + }); + expect(session.activeForegroundTurnId).toBeNull(); + expect(events.at(-1)).toEqual({ + type: "turn_completed", + provider: "codex", + turnId: "test-turn", + usage: undefined, + }); + }); + + test("does not bind a delayed ID-less terminal to an in-progress first native turn", async () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + const request = vi.fn(async (requestMethod: string) => { + if (requestMethod === "thread/read") { + return { + thread: { + turns: [{ id: "native-current-turn", status: "inProgress", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${requestMethod}`); + }); + session.client = createStub({ request }); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-current-turn" }, + }); + const eventCountBeforeDelayedTerminal = events.length; + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { status: "completed", error: null }, + }); + + await vi.waitFor(() => expect(request).toHaveBeenCalledTimes(1)); + expect(session.activeForegroundTurnId).toBe("test-turn"); + expect(events).toHaveLength(eventCountBeforeDelayedTerminal); + }); + + test("does not bind a stale ID-less terminal to a newer native turn", async () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-old-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-old-turn", status: "completed", error: null }, + }); + session.client = createStub({ + request: async (method) => { + if (method === "thread/loaded/list") return { data: ["test-thread"] }; + if (method === "turn/start") return {}; + if (method === "thread/read") { + return { + thread: { + turns: [{ id: "native-new-turn", status: "inProgress", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${method}`); + }, + }); + + const { turnId } = await session.startTurn("new work"); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-new-turn" }, + }); + const eventCountBeforeStaleTerminal = events.length; + asInternals(session).handleNotification("codex/event/task_complete", { + threadId: "test-thread", + msg: { type: "task_complete" }, + }); + await new Promise((resolve) => setImmediate(resolve)); + + expect(turnId).toBe("codex-turn-0"); + expect(session.activeForegroundTurnId).toBe("codex-turn-0"); + expect(events).toHaveLength(eventCountBeforeStaleTerminal); + + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-new-turn", status: "completed", error: null }, + }); + expect(session.activeForegroundTurnId).toBeNull(); + }); + + test("reconciles an ID-less terminal for the exact current native turn", async () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-old-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-old-turn", status: "completed", error: null }, + }); + session.client = createStub({ + request: async (method) => { + if (method === "thread/loaded/list") return { data: ["test-thread"] }; + if (method === "turn/start") return {}; + if (method === "thread/read") { + return { + thread: { + turns: [{ id: "native-new-turn", status: "completed", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${method}`); + }, + }); + + const { turnId } = await session.startTurn("new work"); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-new-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { status: "completed", error: null }, + }); + + await vi.waitFor(() => expect(session.activeForegroundTurnId).toBeNull()); + expect(turnId).toBe("codex-turn-0"); + expect(events.at(-1)).toEqual({ + type: "turn_completed", + provider: "codex", + turnId: "codex-turn-0", + usage: undefined, + }); + + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { status: "completed", error: null }, + }); + await new Promise((resolve) => setImmediate(resolve)); + expect( + events.filter((event) => event.type === "turn_completed" && event.turnId === "codex-turn-0"), + ).toHaveLength(1); + }); + + test("ignores a delayed start for a finalized native turn", async () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-old-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-old-turn", status: "completed", error: null }, + }); + session.client = createStub({ + request: async (method) => { + if (method === "thread/loaded/list") return { data: ["test-thread"] }; + if (method === "turn/start") return {}; + throw new Error(`Unexpected request: ${method}`); + }, + }); + + const { turnId } = await session.startTurn("new work"); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-new-turn" }, + }); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-old-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-new-turn", status: "completed", error: null }, + }); + + expect(turnId).toBe("codex-turn-0"); + expect(session.activeForegroundTurnId).toBeNull(); + expect(events.filter((event) => event.type === "turn_started")).toEqual([ + { type: "turn_started", provider: "codex", turnId: "test-turn" }, + { type: "turn_started", provider: "codex", turnId: "codex-turn-0" }, + ]); + expect(events.filter((event) => event.type === "turn_completed")).toEqual([ + { type: "turn_completed", provider: "codex", turnId: "test-turn", usage: undefined }, + { type: "turn_completed", provider: "codex", turnId: "codex-turn-0", usage: undefined }, + ]); + }); + test("streams Codex assistant message deltas and does not replay completed text", () => { const session = createSession(); const events: AgentStreamEvent[] = []; @@ -4696,7 +4982,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the new flow", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-2", status: "completed", error: null }, }); const request = events.find( @@ -4750,7 +5036,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the safe flow", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-3", status: "completed", error: null }, }); const request = events.find( @@ -4818,7 +5104,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the fast flow", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-4", status: "completed", error: null }, }); const permissionRequest = events.find( diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.ts index fe1e83406a..cc6952cb8f 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.ts @@ -1865,6 +1865,15 @@ const CodexThreadReadResponseSchema = z type CodexThreadReadResponse = z.infer; type CodexThreadReadRequest = (threadId: string) => Promise; +function codexThreadHasTurnStatus(response: unknown, turnId: string, status: string): boolean { + const thread = toObjectRecord(toObjectRecord(response)?.thread); + if (!Array.isArray(thread?.turns)) return false; + return thread.turns.some((turn) => { + const record = toObjectRecord(turn); + return record?.id === turnId && record.status === status; + }); +} + async function requestCodexThreadHistory( requestThread: CodexThreadReadRequest, threadId: string, @@ -1988,6 +1997,7 @@ const TurnCompletedNotificationSchema = z threadId: z.string().optional(), turn: z .object({ + id: z.string().optional(), status: z.string(), error: z .object({ @@ -2263,6 +2273,7 @@ type ParsedCodexNotification = | { kind: "turn_started"; turnId: string; threadId: string | null } | { kind: "turn_completed"; + turnId: string | null; status: string; errorMessage: string | null; threadId: string | null; @@ -2413,6 +2424,7 @@ const CodexNotificationSchema = z.union([ .transform( ({ params }): ParsedCodexNotification => ({ kind: "turn_completed", + turnId: params.turn.id ?? null, status: params.turn.status, errorMessage: params.turn.error?.message ?? null, threadId: params.threadId ?? null, @@ -2833,6 +2845,7 @@ const CodexNotificationSchema = z.union([ .transform( ({ params }): ParsedCodexNotification => ({ kind: "turn_completed", + turnId: null, status: "interrupted", errorMessage: null, threadId: getCodexEventThreadId(params), @@ -2853,6 +2866,7 @@ const CodexNotificationSchema = z.union([ .transform( ({ params }): ParsedCodexNotification => ({ kind: "turn_completed", + turnId: null, status: "completed", errorMessage: null, threadId: getCodexEventThreadId(params), @@ -3112,6 +3126,8 @@ export class CodexAppServerAgentSession implements AgentSession { private readonly subscribers = new Set<(event: AgentStreamEvent) => void>(); private nextTurnOrdinal = 0; private activeForegroundTurnId: string | null = null; + private readonly managerTurnIdByNativeTurnId = new Map(); + private readonly finalizedNativeTurnIds = new Set(); private activeClientMessageId: string | null = null; private cachedRuntimeInfo: AgentRuntimeInfo | null = null; private serviceTier: "fast" | null = null; @@ -4285,6 +4301,8 @@ export class CodexAppServerAgentSession implements AgentSession { this.pendingPermissions.clear(); this.resolvedPermissionRequests.clear(); this.pendingSubAgentNotificationsByThreadId.clear(); + this.managerTurnIdByNativeTurnId.clear(); + this.finalizedNativeTurnIds.clear(); this.subscribers.clear(); this.activeForegroundTurnId = null; this.activeClientMessageId = null; @@ -4587,7 +4605,12 @@ export class CodexAppServerAgentSession implements AgentSession { } private notifySubscribers(event: AgentStreamEvent): void { - const turnId = this.activeForegroundTurnId; + const eventTurnId = getAgentStreamEventTurnId(event); + const terminal = + event.type === "turn_completed" || + event.type === "turn_failed" || + event.type === "turn_canceled"; + const turnId = eventTurnId ?? (terminal ? null : this.activeForegroundTurnId); const tagged = turnId ? { ...event, turnId } : event; this.logger.trace( { @@ -5299,9 +5322,12 @@ export class CodexAppServerAgentSession implements AgentSession { this.emitSubAgentActivityUpdate(subAgentCallId, "running"); return; } + if (this.finalizedNativeTurnIds.has(parsed.turnId)) return; this.currentTurnId = parsed.turnId; + const managerTurnId = this.activeForegroundTurnId ?? parsed.turnId; + this.managerTurnIdByNativeTurnId.set(parsed.turnId, managerTurnId); this.resetTurnTrackingState(); - this.emitEvent({ type: "turn_started", provider: CODEX_PROVIDER }); + this.emitEvent({ type: "turn_started", provider: CODEX_PROVIDER, turnId: managerTurnId }); } private handleTurnCompletedNotification( @@ -5318,30 +5344,94 @@ export class CodexAppServerAgentSession implements AgentSession { this.emitSubAgentActivityUpdate(subAgentCallId, status); return; } + if (!parsed.turnId) { + const nativeTurnId = this.currentTurnId; + const managerTurnId = nativeTurnId + ? this.managerTurnIdByNativeTurnId.get(nativeTurnId) + : undefined; + if (!nativeTurnId || !managerTurnId) return; + if (this.finalizedNativeTurnIds.has(nativeTurnId)) return; + void this.reconcileIdlessTurnCompletedNotification(parsed, nativeTurnId, managerTurnId); + return; + } + const managerTurnId = this.managerTurnIdByNativeTurnId.get(parsed.turnId) ?? parsed.turnId; + this.finishTurnCompletedNotification(parsed, parsed.turnId, managerTurnId); + } + + private finishTurnCompletedNotification( + parsed: Extract, + nativeTurnId: string, + managerTurnId: string, + ): void { + this.rememberFinalizedNativeTurn(nativeTurnId); + const completesCurrentTurn = nativeTurnId === this.currentTurnId; if (parsed.status === "failed") { this.emitEvent({ type: "turn_failed", provider: CODEX_PROVIDER, error: parsed.errorMessage ?? "Codex turn failed", + turnId: managerTurnId, }); } else if (parsed.status === "interrupted") { - this.emitEvent({ type: "turn_canceled", provider: CODEX_PROVIDER, reason: "interrupted" }); + this.emitEvent({ + type: "turn_canceled", + provider: CODEX_PROVIDER, + reason: "interrupted", + turnId: managerTurnId, + }); } else { - if (this.planModeEnabled && this.latestPlanResult?.text) { + if (completesCurrentTurn && this.planModeEnabled && this.latestPlanResult?.text) { this.emitSyntheticPlanApprovalRequest(this.latestPlanResult.text); } this.emitEvent({ type: "turn_completed", provider: CODEX_PROVIDER, usage: this.latestUsage, + turnId: managerTurnId, }); } + if (!completesCurrentTurn) return; this.activeForegroundTurnId = null; this.activeClientMessageId = null; this.pendingSubAgentNotificationsByThreadId.clear(); this.resetTurnTrackingState(); } + private rememberFinalizedNativeTurn(turnId: string): void { + this.finalizedNativeTurnIds.add(turnId); + if (this.finalizedNativeTurnIds.size <= 50) return; + const oldestTurnId = this.finalizedNativeTurnIds.values().next().value; + if (!oldestTurnId) return; + this.finalizedNativeTurnIds.delete(oldestTurnId); + this.managerTurnIdByNativeTurnId.delete(oldestTurnId); + } + + private async reconcileIdlessTurnCompletedNotification( + parsed: Extract, + nativeTurnId: string, + managerTurnId: string, + ): Promise { + const client = this.client; + const threadId = this.currentThreadId; + if (!client || !threadId) return; + try { + const response = await readCodexThread(client, threadId); + const stillOwnsActiveTurn = + this.currentTurnId === nativeTurnId && + this.managerTurnIdByNativeTurnId.get(nativeTurnId) === managerTurnId && + !this.finalizedNativeTurnIds.has(nativeTurnId) && + (this.activeForegroundTurnId === null || this.activeForegroundTurnId === managerTurnId); + if (!stillOwnsActiveTurn) return; + if (!codexThreadHasTurnStatus(response, nativeTurnId, parsed.status)) return; + this.finishTurnCompletedNotification(parsed, nativeTurnId, managerTurnId); + } catch (error) { + this.logger.debug( + { err: error, threadId, nativeTurnId, managerTurnId }, + "Failed to reconcile ID-less Codex terminal notification", + ); + } + } + private resetTurnTrackingState(): void { this.latestPlanResult = null; this.emittedItemStartedIds.clear(); diff --git a/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts b/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts index 0e7488e9aa..41084891ee 100644 --- a/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts +++ b/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts @@ -51,7 +51,8 @@ export interface FakeCodexAppServer { waitForTurnStart(): Promise; nextResponse(): Promise; startsTurn(params: { threadId: string; turnId?: string }): void; - completeTurn(params?: { threadId?: string }): void; + setThreadTurnStatus(params: { threadId: string; turnId: string; status: string }): void; + completeTurn(params?: { threadId?: string; turnId?: string; omitTurnId?: boolean }): void; startsSubAgent(params: { callId: string; threadId: string; @@ -114,6 +115,8 @@ export function createFakeCodexAppServer( ): FakeCodexAppServer { const child = createCodexAppServerChildProcess(); const recordedRollbacks: JsonObject[] = []; + const activeTurnIdByThreadId = new Map(); + const turnStatusesByThreadId = new Map>(); const responseHandlers: Record = { initialize: () => ({}), "collaborationMode/list": () => ({ data: [] }), @@ -164,7 +167,16 @@ export function createFakeCodexAppServer( }, }; }, - "thread/read": () => ({ thread: { turns: [] } }), + "thread/read": (params) => { + const threadId = toJsonObject(params).threadId; + const statuses = + typeof threadId === "string" ? turnStatusesByThreadId.get(threadId) : undefined; + return { + thread: { + turns: Array.from(statuses ?? []).map(([id, status]) => ({ id, status, items: [] })), + }, + }; + }, ...handlers, }; const messages: JsonObject[] = []; @@ -309,21 +321,52 @@ export function createFakeCodexAppServer( }); }, startsTurn(params) { + const turnId = params.turnId ?? `turn-${params.threadId}`; + activeTurnIdByThreadId.set(params.threadId, turnId); + const statuses = turnStatusesByThreadId.get(params.threadId) ?? new Map(); + statuses.set(turnId, "inProgress"); + turnStatusesByThreadId.set(params.threadId, statuses); child.stdout.write( `${JSON.stringify({ method: "turn/started", params: { threadId: params.threadId, - turn: { id: params.turnId ?? `turn-${params.threadId}` }, + turn: { id: turnId }, }, })}\n`, ); }, + setThreadTurnStatus(params) { + const statuses = turnStatusesByThreadId.get(params.threadId) ?? new Map(); + statuses.set(params.turnId, params.status); + turnStatusesByThreadId.set(params.threadId, statuses); + }, completeTurn(params = {}) { + const threadId = params.threadId ?? "thread-1"; + let turnId = params.turnId ?? activeTurnIdByThreadId.get(threadId); + if (!turnId && !params.omitTurnId && threadId === "thread-1") { + turnId = `turn-${threadId}`; + activeTurnIdByThreadId.set(threadId, turnId); + const statuses = turnStatusesByThreadId.get(threadId) ?? new Map(); + statuses.set(turnId, "inProgress"); + turnStatusesByThreadId.set(threadId, statuses); + writeNotification("turn/started", { threadId, turn: { id: turnId } }); + } + if (turnId) { + const statuses = turnStatusesByThreadId.get(threadId) ?? new Map(); + statuses.set(turnId, "completed"); + turnStatusesByThreadId.set(threadId, statuses); + } child.stdout.write( `${JSON.stringify({ method: "turn/completed", - params: { threadId: params.threadId ?? "thread-1", turn: { status: "completed" } }, + params: { + threadId, + turn: { + ...(turnId && !params.omitTurnId ? { id: turnId } : {}), + status: "completed", + }, + }, })}\n`, ); }, diff --git a/packages/server/src/server/agent/rewind/rewind.test.ts b/packages/server/src/server/agent/rewind/rewind.test.ts index a321fb79fd..b2a8f0e20b 100644 --- a/packages/server/src/server/agent/rewind/rewind.test.ts +++ b/packages/server/src/server/agent/rewind/rewind.test.ts @@ -129,6 +129,100 @@ describe("AgentManager rewind", () => { }); }); + test("rewinds when the active turn completes before interrupt rejection", async () => { + let manager!: AgentManager; + let agentId!: string; + + class CompletingInterruptSession extends FakeRewindSession { + override async interrupt(): Promise { + const settled = manager.waitForAgentEvent(agentId); + this.completeActiveTurn(); + await settled; + throw new Error("turn already completed"); + } + } + + const session = new CompletingInterruptSession(); + manager = new AgentManager({ + clients: { claude: new FakeRewindClient(session) }, + logger: createTestLogger(), + idFactory: () => "00000000-0000-4000-8000-000000000903", + }); + const agent = await manager.createAgent({ provider: "claude", cwd: process.cwd() }, undefined, { + workspaceId: undefined, + }); + agentId = agent.id; + const run = manager.streamAgent(agentId, "keep working"); + await run.next(); + + await expect(manager.rewind(agentId, "message-1", "files")).resolves.toBeUndefined(); + expect(session.recordedRewinds).toEqual([{ mode: "files", messageId: "message-1" }]); + }); + + test("rewinds after force-canceling an acknowledged pending start generation", async () => { + let resolveStart!: () => void; + let allowTerminal!: () => void; + let startRequested!: () => void; + const startGate = new Promise((resolve) => { + resolveStart = resolve; + }); + const terminalGate = new Promise((resolve) => { + allowTerminal = resolve; + }); + const requested = new Promise((resolve) => { + startRequested = resolve; + }); + + class PendingStartRewindSession extends FakeRewindSession { + override async startTurn(): Promise<{ turnId: string }> { + startRequested(); + await startGate; + const turnId = "pending-rewind-turn"; + this.activeTurnId = turnId; + void terminalGate.then(() => { + this.activeTurnId = null; + this.emit({ type: "turn_completed", provider: this.provider, turnId }); + return undefined; + }); + return { turnId }; + } + + override async interrupt(): Promise { + this.aborted = true; + } + } + + const session = new PendingStartRewindSession(); + const manager = new AgentManager({ + clients: { claude: new FakeRewindClient(session) }, + logger: createTestLogger(), + idFactory: () => "00000000-0000-4000-8000-000000000904", + }); + const agent = await manager.createAgent({ provider: "claude", cwd: process.cwd() }, undefined, { + workspaceId: undefined, + }); + const stream = manager.streamAgent(agent.id, "keep working"); + const drain = (async () => { + for await (const _event of stream) { + // Drain until the canceled pending generation exits. + } + })(); + await requested; + + await manager.rewind(agent.id, "message-1", "files"); + resolveStart(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(session.recordedRewinds).toEqual([{ mode: "files", messageId: "message-1" }]); + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + + allowTerminal(); + await drain; + }); + test("blocks new prompts until the rehydrate epoch broadcasts", async () => { const historyGate = new RewindHistoryGate(); historyGate.hold(); diff --git a/packages/server/src/server/agent/rewind/test-rewind-session.ts b/packages/server/src/server/agent/rewind/test-rewind-session.ts index e717ca051d..dc916468a7 100644 --- a/packages/server/src/server/agent/rewind/test-rewind-session.ts +++ b/packages/server/src/server/agent/rewind/test-rewind-session.ts @@ -37,7 +37,7 @@ export class FakeRewindSession implements AgentSession { history: AgentTimelineItem[] = [{ type: "user_message", text: "before", messageId: "message-1" }]; private subscribers = new Set<(event: AgentStreamEvent) => void>(); - private activeTurnId: string | null = null; + protected activeTurnId: string | null = null; constructor(private readonly waitBeforeHistory?: () => Promise) {} @@ -103,6 +103,13 @@ export class FakeRewindSession implements AgentSession { } } + completeActiveTurn(): void { + if (!this.activeTurnId) return; + const turnId = this.activeTurnId; + this.activeTurnId = null; + this.emit({ type: "turn_completed", provider: this.provider, turnId }); + } + async close(): Promise {} async revertConversation(input: { messageId: string }): Promise { @@ -117,7 +124,7 @@ export class FakeRewindSession implements AgentSession { this.recordedRewinds.push({ mode: "both", messageId: input.messageId }); } - private emit(event: AgentStreamEvent): void { + protected emit(event: AgentStreamEvent): void { for (const subscriber of this.subscribers) { subscriber(event); } diff --git a/packages/server/src/server/material-progress.e2e.test.ts b/packages/server/src/server/material-progress.e2e.test.ts new file mode 100644 index 0000000000..6000dbc7f5 --- /dev/null +++ b/packages/server/src/server/material-progress.e2e.test.ts @@ -0,0 +1,311 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, test } from "vitest"; +import { DaemonClient } from "./test-utils/index.js"; +import { createTestPaseoDaemon, type TestPaseoDaemon } from "./test-utils/paseo-daemon.js"; + +async function recordMaterialWrite(daemon: TestPaseoDaemon, agentId: string): Promise { + await daemon.daemon.agentManager.appendTimelineItem(agentId, { + type: "user_message", + text: "implement", + }); + await daemon.daemon.agentManager.appendTimelineItem(agentId, { + type: "compaction", + status: "completed", + }); + await daemon.daemon.agentManager.appendTimelineItem(agentId, { + type: "tool_call", + callId: "write-1", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "proof.txt", content: "done" }, + }); +} + +function expectMaterialWriteProgress( + materialProgress: NonNullable< + Awaited> + >["agent"]["materialProgress"], +): void { + expect(materialProgress).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "write", + }); +} + +test("fetch agent exposes a backward-compatible material progress signal", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-")); + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Material progress probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + const fetched = await client.fetchAgent({ agentId: created.id }); + + expect(fetched?.agent.materialProgress).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Timeline history is unavailable.", + }); + } finally { + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + +test("fetch agent preserves material progress after the live worker is collected", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-retained-")); + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Retained material progress probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + await recordMaterialWrite(daemon, created.id); + await daemon.daemon.agentManager.closeAgent(created.id); + + const fetched = await client.fetchAgent({ agentId: created.id }); + expectMaterialWriteProgress(fetched?.agent.materialProgress); + } finally { + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + +test("fetch agent does not inherit a previous stall after a new turn is accepted", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-boundary-")); + let agentId: string | null = null; + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Accepted continuation boundary probe", + modeId: "default", + model: "gpt-5.4-mini", + }); + agentId = created.id; + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "user_message", + text: "previous continuation", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + + expect( + (await client.fetchAgent({ agentId: created.id }))?.agent.materialProgress, + ).toMatchObject({ + state: "stalled", + completedCompactionsSinceMaterialProgress: 2, + }); + + // The fake provider stops at a permission request after accepting the turn. It intentionally + // emits no user_message row, reproducing providers that acknowledge a continuation before + // (or without) projecting the prompt into timeline history. + await client.sendMessage(created.id, "read /etc/hosts"); + await expect + .poll(async () => (await client.fetchAgent({ agentId: created.id }))?.agent.status ?? null, { + timeout: 10_000, + interval: 25, + }) + .toBe("running"); + + const fetched = await client.fetchAgent({ agentId: created.id }); + expect(fetched?.agent.materialProgress).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "No material progress has been recorded for the current continuation.", + }); + expect((await daemon.daemon.agentManager.getMaterialProgressSnapshot(created.id)).rows).toEqual( + [], + ); + } finally { + if (agentId) await client.cancelAgent(agentId).catch(() => undefined); + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + +test("fetch agent does not inherit a previous stall when an accepted turn fails before a user row", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-failed-boundary-")); + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Failed continuation boundary probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "user_message", + text: "previous continuation", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + expect( + (await client.fetchAgent({ agentId: created.id }))?.agent.materialProgress, + ).toMatchObject({ state: "stalled" }); + + // The fake provider emits turn_started followed by turn_failed, without a user_message row. + await client.sendMessage(created.id, "emit a turn failure"); + await expect + .poll(async () => (await client.fetchAgent({ agentId: created.id }))?.agent.status ?? null, { + timeout: 10_000, + interval: 25, + }) + .toBe("error"); + + const snapshot = await daemon.daemon.agentManager.getMaterialProgressSnapshot(created.id); + expect(snapshot.turnOutcome).toBe("failed"); + expect(snapshot.continuationBoundarySeq).toBe(4); + expect((await client.fetchAgent({ agentId: created.id }))?.agent.materialProgress).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "No material progress has been recorded for the current continuation.", + }); + } finally { + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + +test("fetch agent preserves material progress after archive discards the live timeline", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-archived-")); + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Archived material progress probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + await recordMaterialWrite(daemon, created.id); + await client.archiveAgent(created.id); + + const fetched = await client.fetchAgent({ agentId: created.id }); + expect(fetched?.agent.archivedAt).toEqual(expect.any(String)); + expectMaterialWriteProgress(fetched?.agent.materialProgress); + } finally { + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + +test("fetch agent restores material progress after daemon restart", async () => { + const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-home-")); + const staticDir = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-static-")); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-restart-")); + let daemon: TestPaseoDaemon | null = null; + let client: DaemonClient | null = null; + + try { + daemon = await createTestPaseoDaemon({ + paseoHomeRoot, + staticDir, + cleanup: false, + }); + client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Restarted material progress probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + await recordMaterialWrite(daemon, created.id); + + await client.close(); + client = null; + await daemon.close(); + daemon = null; + + daemon = await createTestPaseoDaemon({ + paseoHomeRoot, + staticDir, + cleanup: false, + }); + client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + await client.connect(); + + const fetched = await client.fetchAgent({ agentId: created.id }); + expectMaterialWriteProgress(fetched?.agent.materialProgress); + } finally { + await client?.close().catch(() => undefined); + await daemon?.close().catch(() => undefined); + rmSync(cwd, { recursive: true, force: true }); + rmSync(paseoHomeRoot, { recursive: true, force: true }); + rmSync(staticDir, { recursive: true, force: true }); + } +}); diff --git a/packages/server/src/server/persistence-hooks.ts b/packages/server/src/server/persistence-hooks.ts index ba220eefa5..86c7cec56b 100644 --- a/packages/server/src/server/persistence-hooks.ts +++ b/packages/server/src/server/persistence-hooks.ts @@ -111,6 +111,8 @@ export function extractTimestamps(record: StoredAgentRecord): { labels?: Record; workspaceId?: string; owner?: StoredAgentRecord["owner"]; + lastTurnOutcome?: StoredAgentRecord["lastTurnOutcome"]; + materialProgressContinuationBoundarySeq?: StoredAgentRecord["materialProgressContinuationBoundarySeq"]; } { return { createdAt: new Date(record.createdAt), @@ -119,6 +121,8 @@ export function extractTimestamps(record: StoredAgentRecord): { labels: record.labels, workspaceId: record.workspaceId, owner: record.owner, + lastTurnOutcome: record.lastTurnOutcome, + materialProgressContinuationBoundarySeq: record.materialProgressContinuationBoundarySeq, }; } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index a73bb66161..c545f4a620 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -75,6 +75,7 @@ import type { AgentTimelineFetchDirection, AgentTimelineFetchResult, ManagedAgent, + MaterialProgressSnapshot, } from "./agent/agent-manager.js"; import { createAgentCommand } from "./agent/create-agent/create.js"; import { resolveCreateAgentIntent, type CreateAgentIntent } from "./agent/create-agent/intent.js"; @@ -101,6 +102,7 @@ import { type TimelineProjectionEntry, type TimelineProjectionMode, } from "./agent/timeline-projection.js"; +import { analyzeMaterialProgress } from "./agent/material-progress.js"; import { buildAgentForkContextAttachment } from "./agent/activity-curator.js"; import { buildAgentPrompt } from "./agent/prompt-attachments.js"; import type { StructuredGenerationDaemonConfig } from "./agent/structured-generation-providers.js"; @@ -4029,7 +4031,9 @@ export class Session { const live = this.agentManager.getAgent(agentId); if (live) { const payload = await this.buildAgentPayload(live); - return this.isProviderVisibleToClient(payload.provider) ? payload : null; + return this.isProviderVisibleToClient(payload.provider) + ? await this.attachMaterialProgress(payload) + : null; } const record = await this.agentStorage.get(agentId); @@ -4037,7 +4041,44 @@ export class Session { return null; } const payload = this.buildStoredAgentPayload(record); - return this.isProviderVisibleToClient(payload.provider) ? payload : null; + return this.isProviderVisibleToClient(payload.provider) + ? await this.attachMaterialProgress(payload) + : null; + } + + private async attachMaterialProgress( + payload: AgentSnapshotPayload, + ): Promise { + let entries: TimelineProjectionEntry[] | null = null; + let turnOutcome: MaterialProgressSnapshot["turnOutcome"] = null; + let continuationBoundarySeq: number | null = null; + let persisted = payload.materialProgress ?? null; + try { + const snapshot = await this.agentManager.getMaterialProgressSnapshot(payload.id); + entries = + snapshot.rows === null + ? null + : projectTimelineRows({ rows: snapshot.rows, mode: "projected" }); + turnOutcome = snapshot.turnOutcome; + continuationBoundarySeq = snapshot.continuationBoundarySeq; + persisted = snapshot.persisted ?? persisted; + } catch (error) { + this.sessionLogger.debug( + { err: error, agentId: payload.id }, + "Material progress timeline is unavailable", + ); + } + return { + ...payload, + materialProgress: + entries === null && persisted + ? persisted + : analyzeMaterialProgress({ + entries, + turnOutcome, + continuationBoundarySeq, + }), + }; } private async resolveDelegationRootWorkspaceId(agentId: string): Promise {