From 6c410041b669b4cd34ccf1eaadee1fe4e4e54db2 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 04:43:31 +0100 Subject: [PATCH 01/17] feat: add durable material progress inspection the repo-pinned Node 22 runtime is not already available. --- docs/data-model.md | 2 + .../cli/src/commands/agent/inspect.test.ts | 46 +++++ packages/cli/src/commands/agent/inspect.ts | 16 +- .../src/material-progress-schema.test.ts | 48 +++++ packages/protocol/src/messages.ts | 12 ++ .../src/server/agent/agent-manager.test.ts | 136 +++++++++++++ .../server/src/server/agent/agent-manager.ts | 90 ++++++++- .../src/server/agent/agent-projections.ts | 5 + .../src/server/agent/agent-storage.test.ts | 28 +++ .../server/src/server/agent/agent-storage.ts | 57 ++++-- .../server/agent/material-progress.test.ts | 165 ++++++++++++++++ .../src/server/agent/material-progress.ts | 142 +++++++++++++ .../src/server/material-progress.e2e.test.ts | 186 ++++++++++++++++++ .../server/src/server/persistence-hooks.ts | 2 + packages/server/src/server/session.ts | 42 +++- patches/local-patches.json | 24 +++ 16 files changed, 977 insertions(+), 24 deletions(-) create mode 100644 packages/cli/src/commands/agent/inspect.test.ts create mode 100644 packages/protocol/src/material-progress-schema.test.ts create mode 100644 packages/server/src/server/agent/material-progress.test.ts create mode 100644 packages/server/src/server/agent/material-progress.ts create mode 100644 packages/server/src/server/material-progress.e2e.test.ts create mode 100644 patches/local-patches.json diff --git a/docs/data-model.md b/docs/data-model.md index 318e4835d6..f264feff33 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -96,6 +96,8 @@ Each agent is stored as a separate JSON file, grouped by project directory. | `features` | `AgentFeature[]?` | Provider-reported features (toggles/selects) | | `persistence` | `PersistenceHandle?` | Handle for resuming sessions | | `lastError` | `string?` (nullable) | Last error message, if any | +| `lastTurnOutcome` | `"completed" \| "failed" \| "canceled"?` | Last terminal turn result. Cleared when a new turn starts. | +| `materialProgress` | `MaterialProgressPayload?` | Last derived material-progress classification, retained when the live timeline is unavailable after archive or daemon restart. | | `requiresAttention` | `boolean?` | Whether the agent needs user attention | | `attentionReason` | `"finished" \| "error" \| "permission"?` | Why attention is needed | | `attentionTimestamp` | `string?` (ISO 8601) | When attention was flagged | diff --git a/packages/cli/src/commands/agent/inspect.test.ts b/packages/cli/src/commands/agent/inspect.test.ts new file mode 100644 index 0000000000..d46f95c124 --- /dev/null +++ b/packages/cli/src/commands/agent/inspect.test.ts @@ -0,0 +1,46 @@ +import type { AgentSnapshotPayload } from "@getpaseo/protocol/messages"; +import { describe, expect, it } from "vitest"; +import { toInspectData, toInspectRows } from "./inspect.js"; + +describe("agent inspect material progress", () => { + it("preserves the structured signal and renders its state and reason", () => { + const snapshot = { + id: "agent-1", + provider: "opencode", + cwd: "/tmp/work", + model: "test-model", + createdAt: "2026-07-31T00:00:00.000Z", + updatedAt: "2026-07-31T00:01:00.000Z", + lastUserMessageAt: "2026-07-31T00:00:30.000Z", + status: "running", + capabilities: { + supportsStreaming: true, + supportsSessionPersistence: true, + supportsDynamicModes: false, + supportsMcpServers: true, + supportsReasoningStream: true, + supportsToolInvocations: true, + }, + currentModeId: null, + availableModes: [], + pendingPermissions: [], + persistence: null, + title: "Stalled worker", + labels: {}, + materialProgress: { + state: "stalled", + completedCompactionsSinceMaterialProgress: 2, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Two compactions completed without later material progress.", + }, + } as AgentSnapshotPayload; + + const inspect = toInspectData(snapshot); + expect(inspect.MaterialProgress).toEqual(snapshot.materialProgress); + expect(toInspectRows(inspect)).toContainEqual({ + key: "MaterialProgress", + value: "stalled: Two compactions completed without later material progress.", + }); + }); +}); diff --git a/packages/cli/src/commands/agent/inspect.ts b/packages/cli/src/commands/agent/inspect.ts index 93f1da3d4d..4103725a91 100644 --- a/packages/cli/src/commands/agent/inspect.ts +++ b/packages/cli/src/commands/agent/inspect.ts @@ -11,7 +11,7 @@ export function addInspectOptions(cmd: Command): Command { } /** Agent inspect data for display (matches CLI spec format) */ -interface AgentInspect { +export interface AgentInspect { Id: string; Name: string; Provider: string; @@ -46,10 +46,11 @@ interface AgentInspect { }>; Worktree: string | null; ParentAgentId: string | null; + MaterialProgress: AgentSnapshotPayload["materialProgress"] | null; } /** Key-value row for table display */ -interface InspectRow { +export interface InspectRow { key: string; value: string; } @@ -126,7 +127,7 @@ function buildCapabilities(snapshot: AgentSnapshotPayload): AgentInspect["Capabi } /** Convert agent snapshot to inspection data */ -function toInspectData(snapshot: AgentSnapshotPayload): AgentInspect { +export function toInspectData(snapshot: AgentSnapshotPayload): AgentInspect { return { Id: snapshot.id, Name: snapshot.title ?? "-", @@ -151,11 +152,12 @@ function toInspectData(snapshot: AgentSnapshotPayload): AgentInspect { })), Worktree: snapshot.labels?.["paseo.worktree"] ?? null, ParentAgentId: snapshot.labels?.[PARENT_AGENT_ID_LABEL] ?? null, + MaterialProgress: snapshot.materialProgress ?? null, }; } /** Convert agent to key-value rows for table display */ -function toInspectRows(agent: AgentInspect): InspectRow[] { +export function toInspectRows(agent: AgentInspect): InspectRow[] { const rows: InspectRow[] = [ { key: "Id", value: agent.Id }, { key: "Name", value: agent.Name }, @@ -202,6 +204,12 @@ function toInspectRows(agent: AgentInspect): InspectRow[] { rows.push({ key: "Worktree", value: agent.Worktree ?? "null" }); rows.push({ key: "ParentAgentId", value: agent.ParentAgentId ?? "null" }); + rows.push({ + key: "MaterialProgress", + value: agent.MaterialProgress + ? `${agent.MaterialProgress.state}: ${agent.MaterialProgress.reason}` + : "unavailable", + }); return rows; } diff --git a/packages/protocol/src/material-progress-schema.test.ts b/packages/protocol/src/material-progress-schema.test.ts new file mode 100644 index 0000000000..472bf1e45a --- /dev/null +++ b/packages/protocol/src/material-progress-schema.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { AgentSnapshotPayloadSchema } from "./messages.js"; + +const baseSnapshot = { + id: "agent-1", + provider: "opencode", + cwd: "/tmp/work", + model: null, + createdAt: "2026-07-31T00:00:00.000Z", + updatedAt: "2026-07-31T00:00:00.000Z", + lastUserMessageAt: null, + status: "running" as const, + capabilities: { + supportsStreaming: true, + supportsSessionPersistence: true, + supportsDynamicModes: false, + supportsMcpServers: true, + supportsReasoningStream: true, + supportsToolInvocations: true, + }, + currentModeId: null, + availableModes: [], + pendingPermissions: [], + persistence: null, + title: null, + labels: {}, +}; + +describe("material progress wire compatibility", () => { + it("accepts legacy snapshots without the optional signal", () => { + expect(AgentSnapshotPayloadSchema.parse(baseSnapshot).materialProgress).toBeUndefined(); + }); + + it("accepts a structured material progress signal", () => { + const parsed = AgentSnapshotPayloadSchema.parse({ + ...baseSnapshot, + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "One compaction completed without later material progress.", + }, + }); + + expect(parsed.materialProgress?.state).toBe("warning"); + }); +}); diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index af44e09f90..73361ece74 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -685,6 +685,16 @@ const AgentRuntimeInfoSchema: z.ZodType = z.object({ extra: z.record(z.string(), z.unknown()).optional(), }); +export const MaterialProgressPayloadSchema = z.object({ + state: z.enum(["none", "progressing", "warning", "stalled"]), + completedCompactionsSinceMaterialProgress: z.number().int().nonnegative(), + lastMaterialProgressAt: z.string().nullable(), + lastMaterialProgressKind: z.enum(["edit", "write", "assistant_result"]).nullable(), + reason: z.string(), +}); + +export type MaterialProgressPayload = z.infer; + export const AgentSnapshotPayloadSchema = z.object({ id: z.string(), provider: AgentProviderSchema, @@ -713,6 +723,8 @@ export const AgentSnapshotPayloadSchema = z.object({ attentionTimestamp: z.string().nullable().optional(), archivedAt: z.string().nullable().optional(), providerUnavailable: z.boolean().optional(), + // COMPAT(materialProgress): added in v0.2.5, remove optional after 2027-01-31. + materialProgress: MaterialProgressPayloadSchema.optional(), }); export type AgentSnapshotPayload = z.infer; diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 045e243324..13e0f481ab 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -3837,6 +3837,142 @@ test("getTimelineRows falls back to the in-memory timeline when no durable store }, }, ]); + + await expect(manager.getMaterialProgressSnapshot(snapshot.id, 1)).resolves.toMatchObject({ + rows: null, + }); + + await manager.closeAgent(snapshot.id); + const retained = await manager.getMaterialProgressSnapshot(snapshot.id); + expect(retained.rows).toHaveLength(2); + expect(retained.turnOutcome).toBeNull(); +}); + +test("an unhydrated resumed agent exposes its persisted material progress", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-resume-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const agentId = "00000000-0000-4000-8000-000000000141"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + const created = await firstManager.createAgent( + { provider: "codex", cwd: workdir }, + undefined, + { workspaceId: undefined }, + ); + await firstManager.appendTimelineItem(created.id, { + type: "user_message", + text: "persist this result", + }); + await firstManager.appendTimelineItem(created.id, { + type: "tool_call", + callId: "write-1", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "result.txt", content: "durable" }, + }); + await firstManager.closeAgent(created.id); + + const record = await storage.get(created.id); + expect(record?.persistence).toBeTruthy(); + + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + record!.persistence!, + { cwd: workdir }, + created.id, + ); + + await expect(resumedManager.getMaterialProgressSnapshot(created.id)).resolves.toMatchObject({ + rows: null, + persisted: { + state: "progressing", + lastMaterialProgressKind: "write", + }, + }); + + await resumedManager.closeAgent(created.id); + rmSync(workdir, { recursive: true, force: true }); +}); + +test("persists terminal outcomes and clears the previous outcome when a new turn starts", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-turn-outcome-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + let nextTurn = 0; + const session = new (class extends TestAgentSession { + override async startTurn(): Promise<{ turnId: string }> { + nextTurn += 1; + return { turnId: `held-turn-${nextTurn}` }; + } + })({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000139", + }); + + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + + const firstRunning = waitForAgentLifecycle(manager, agent.id, "running"); + const firstTurn = (async () => { + for await (const _event of manager.streamAgent(agent.id, "first turn")) { + // Consume through the terminal event so the managed run settles. + } + })(); + await firstRunning; + session.pushEvent({ + type: "turn_completed", + provider: "codex", + turnId: "held-turn-1", + }); + await firstTurn; + await manager.flush(); + expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBe("completed"); + expect((await storage.get(agent.id))?.lastTurnOutcome).toBe("completed"); + + const secondRunning = waitForAgentLifecycle(manager, agent.id, "running"); + const secondTurn = (async () => { + for await (const _event of manager.streamAgent(agent.id, "second turn")) { + // Consume through the terminal event so the managed run settles. + } + })(); + await secondRunning; + await manager.flush(); + expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBeNull(); + expect((await storage.get(agent.id))?.lastTurnOutcome ?? null).toBeNull(); + + session.pushEvent({ + type: "turn_canceled", + provider: "codex", + turnId: "held-turn-2", + reason: "test interruption", + }); + await secondTurn; + await manager.flush(); + expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBe("canceled"); + expect((await storage.get(agent.id))?.lastTurnOutcome).toBe("canceled"); + } finally { + await manager.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } }); test("getAgent does not expose committed history internals once manager owns the seam", async () => { diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 72fe1a7ec8..3fc2e7006e 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -73,6 +73,9 @@ import { type ProviderSubagentDescriptor, type ProviderSubagentStoreEvent, } from "./provider-subagents/store.js"; +import { analyzeMaterialProgress } from "./material-progress.js"; +import { projectTimelineRows } from "./timeline-projection.js"; +import type { MaterialProgressPayload } from "../messages.js"; const RELOAD_SESSION_CLOSE_TIMEOUT_MS = 3_000; const INTERRUPT_SESSION_TIMEOUT_MS = 2_000; @@ -280,6 +283,14 @@ type AttentionState = attentionTimestamp: Date; }; +type AgentTurnOutcome = "completed" | "failed" | "canceled"; + +export interface MaterialProgressSnapshot { + rows: AgentTimelineRow[] | null; + turnOutcome: AgentTurnOutcome | null; + persisted: MaterialProgressPayload | null; +} + function resolveInitialAttention(input: AttentionState | undefined): AttentionState { if (input == null || !input.requiresAttention) { return { requiresAttention: false }; @@ -331,6 +342,7 @@ interface ManagedAgentBase { lastUserMessageAt: Date | null; lastUsage?: AgentUsage; lastError?: string; + lastTurnOutcome?: AgentTurnOutcome | null; attention: AttentionState; foregroundTurnWaiters: Set; finalizedForegroundTurnIds: Set; @@ -971,6 +983,45 @@ export class AgentManager { return this.timelineStore.getRows(id); } + async getMaterialProgressSnapshot( + id: string, + limit = 1_000, + ): Promise { + const live = this.agents.get(id); + if (live && this.timelineStore.has(id)) { + const timeline = this.timelineStore.fetch(id, { direction: "tail", limit }); + const rowsUnavailable = + (timeline.rows.length === 0 && !live.historyPrimed) || + (timeline.hasOlder && !timeline.rows.some((row) => row.item.type === "user_message")); + const record = rowsUnavailable ? await this.registry?.get(id) : null; + return { + rows: rowsUnavailable ? null : timeline.rows, + turnOutcome: live.lastTurnOutcome ?? null, + persisted: record?.materialProgress ?? null, + }; + } + + let timeline: AgentTimelineFetchResult | null = null; + if (this.durableTimelineStore) { + timeline = await this.durableTimelineStore.fetchCommitted(id, { + direction: "tail", + limit, + }); + } else if (this.timelineStore.has(id)) { + timeline = this.timelineStore.fetch(id, { direction: "tail", limit }); + } + const record = await this.registry?.get(id); + return { + rows: + timeline === null || + (timeline.hasOlder && !timeline.rows.some((row) => row.item.type === "user_message")) + ? null + : timeline.rows, + turnOutcome: record?.lastTurnOutcome ?? null, + persisted: record?.materialProgress ?? null, + }; + } + fetchTimeline(id: string, options?: AgentTimelineFetchOptions): AgentTimelineFetchResult { this.requireAgent(id); return this.timelineStore.fetch(id, options); @@ -1061,6 +1112,7 @@ export class AgentManager { labels?: Record; workspaceId?: string; owner?: AgentOwner; + lastTurnOutcome?: AgentTurnOutcome | null; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -1080,6 +1132,7 @@ export class AgentManager { labels?: Record; workspaceId?: string; owner?: AgentOwner; + lastTurnOutcome?: AgentTurnOutcome | null; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -1224,6 +1277,7 @@ export class AgentManager { const preservedHistoryPrimed = existing.historyPrimed; const preservedLastUsage = existing.lastUsage; const preservedLastError = existing.lastError; + const preservedLastTurnOutcome = existing.lastTurnOutcome; const preservedAttention = existing.attention; const handle = existing.persistence; const provider = handle?.provider ?? existing.provider; @@ -1275,6 +1329,7 @@ export class AgentManager { historyPrimed: rehydrateFromDisk ? false : preservedHistoryPrimed, lastUsage: preservedLastUsage, lastError: preservedLastError, + lastTurnOutcome: preservedLastTurnOutcome, attention: preservedAttention, }); } finally { @@ -1423,8 +1478,10 @@ export class AgentManager { throw new Error("Agent storage is not configured"); } + const materialProgress = this.buildPersistedMaterialProgress(agent); await this.registry.applySnapshot(agent, { internal: agent.internal, + ...(materialProgress ? { materialProgress } : {}), }); const stored = await this.registry.get(agentId); if (!stored) { @@ -1531,6 +1588,7 @@ export class AgentManager { lastUserMessageAt: record.lastUserMessageAt ? new Date(record.lastUserMessageAt) : null, lastUsage: undefined, lastError: record.lastError ?? undefined, + lastTurnOutcome: record.lastTurnOutcome, attention: { requiresAttention: false }, internal: record.internal, labels: record.labels, @@ -2007,6 +2065,7 @@ export class AgentManager { } agent.activeForegroundTurnId = turnId; agent.lifecycle = "running"; + agent.lastTurnOutcome = null; this.touchUpdatedAt(agent); this.emitState(agent); this.logger.trace( @@ -2671,6 +2730,7 @@ export class AgentManager { historyPrimed?: boolean; lastUsage?: AgentUsage; lastError?: string; + lastTurnOutcome?: AgentTurnOutcome | null; attention?: AttentionState; initialTitle?: string | null; publishWhenReady?: boolean; @@ -2811,6 +2871,7 @@ export class AgentManager { historyPrimed?: boolean; lastUsage?: AgentUsage; lastError?: string; + lastTurnOutcome?: AgentTurnOutcome | null; attention?: AttentionState; persistence?: AgentPersistenceHandle; workspaceId?: string; @@ -2850,6 +2911,7 @@ export class AgentManager { lastUserMessageAt: options?.lastUserMessageAt ?? null, lastUsage: options?.lastUsage, lastError: options?.lastError, + lastTurnOutcome: options?.lastTurnOutcome, attention: resolveInitialAttention(options?.attention), internal: config.internal ?? false, labels: options?.labels ?? {}, @@ -3066,7 +3128,29 @@ export class AgentManager { if (agent.internal) { return; } - await this.registry.applySnapshot(agent, options); + const materialProgress = this.buildPersistedMaterialProgress(agent); + await this.registry.applySnapshot(agent, { + ...options, + ...(materialProgress ? { materialProgress } : {}), + }); + } + + private buildPersistedMaterialProgress(agent: ManagedAgent): MaterialProgressPayload | undefined { + const timeline = this.timelineStore.has(agent.id) + ? this.timelineStore.fetch(agent.id, { direction: "tail", limit: 1_000 }) + : null; + if (timeline?.rows.length === 0 && !agent.historyPrimed) { + return undefined; + } + const rows = + timeline === null || + (timeline.hasOlder && !timeline.rows.some((row) => row.item.type === "user_message")) + ? null + : timeline.rows; + return analyzeMaterialProgress({ + entries: rows === null ? null : projectTimelineRows({ rows, mode: "projected" }), + turnOutcome: agent.lastTurnOutcome ?? null, + }); } private requireRegistry(): AgentStorage { @@ -3564,6 +3648,7 @@ export class AgentManager { // data accumulated during streaming isn't lost when the provider omits // it from the completion event. agent.lastError = undefined; + agent.lastTurnOutcome = "completed"; if (!isForegroundEvent && agent.lifecycle !== "idle" && !agent.pendingReplacement) { (agent as ActiveManagedAgent).lifecycle = "idle"; this.emitState(agent); @@ -3598,6 +3683,7 @@ export class AgentManager { agent.lifecycle = "error"; } agent.lastError = event.error; + agent.lastTurnOutcome = "failed"; await this.appendSystemErrorTimelineMessage( agent, event.provider, @@ -3638,6 +3724,7 @@ export class AgentManager { agent.lifecycle = "idle"; } agent.lastError = undefined; + agent.lastTurnOutcome = "canceled"; this.resolvePendingPermissionsForAgent(agent, event.provider, options, "Interrupted"); if (!isForegroundEvent) { this.emitState(agent); @@ -3650,6 +3737,7 @@ export class AgentManager { isForegroundEvent: boolean; }): void { const { agent, eventTurnId, isForegroundEvent } = params; + agent.lastTurnOutcome = null; this.logger.trace( { agentId: agent.id, diff --git a/packages/server/src/server/agent/agent-projections.ts b/packages/server/src/server/agent/agent-projections.ts index a01c67b29d..9c8750e575 100644 --- a/packages/server/src/server/agent/agent-projections.ts +++ b/packages/server/src/server/agent/agent-projections.ts @@ -1,6 +1,7 @@ import type { AgentListItemPayload, AgentSnapshotPayload, + MaterialProgressPayload, RecentProviderSessionDescriptorPayload, } from "../messages.js"; import type { SerializableAgentConfig, StoredAgentRecord } from "./agent-storage.js"; @@ -26,6 +27,7 @@ interface ProjectionOptions { title?: string | null; createdAt?: string; internal?: boolean; + materialProgress?: MaterialProgressPayload; } interface RecentProviderSessionProjectionOptions { @@ -87,6 +89,8 @@ export function toStoredAgentRecord( features: normalizeFeatures(agent.features), persistence, lastError: agent.lastError ?? undefined, + ...(agent.lastTurnOutcome ? { lastTurnOutcome: agent.lastTurnOutcome } : {}), + ...(options?.materialProgress ? { materialProgress: options.materialProgress } : {}), requiresAttention: agent.attention.requiresAttention, attentionReason: agent.attention.requiresAttention ? agent.attention.attentionReason : null, attentionTimestamp: agent.attention.requiresAttention @@ -236,6 +240,7 @@ export function buildStoredAgentPayload( attentionTimestamp: record.attentionTimestamp ?? null, archivedAt: record.archivedAt ?? null, labels: normalizeLabels(record.labels), + ...(record.materialProgress ? { materialProgress: record.materialProgress } : {}), ...(providerAvailable ? {} : { providerUnavailable: true }), }; } diff --git a/packages/server/src/server/agent/agent-storage.test.ts b/packages/server/src/server/agent/agent-storage.test.ts index a9b1ab762f..54fa0f6d60 100644 --- a/packages/server/src/server/agent/agent-storage.test.ts +++ b/packages/server/src/server/agent/agent-storage.test.ts @@ -296,6 +296,34 @@ describe("AgentStorage", () => { expect(recordAfterSnapshot?.archivedAt).toBe(archivedAt); }); + test("concurrent snapshots preserve a newly derived material progress signal", async () => { + const agentId = "agent-material-progress"; + await storage.initialize(); + + const withMaterialProgress = storage.applySnapshot(createManagedAgent({ id: agentId }), { + materialProgress: { + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: "2026-07-31T00:00:00.000Z", + lastMaterialProgressKind: "write", + reason: "Material progress followed the latest user message.", + }, + }); + const genericSnapshot = storage.applySnapshot( + createManagedAgent({ + id: agentId, + updatedAt: new Date("2026-07-31T00:01:00.000Z"), + }), + ); + + await Promise.all([withMaterialProgress, genericSnapshot]); + + expect((await storage.get(agentId))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + }); + test("stores titles independently of snapshots", async () => { await storage.applySnapshot( createManagedAgent({ diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 5e504cee64..1f0be9caed 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -4,7 +4,12 @@ import { z } from "zod"; import type { Logger } from "pino"; import { writeJsonFileAtomic } from "../atomic-file.js"; -import { AgentFeatureSchema, AgentStatusSchema } from "../messages.js"; +import { + AgentFeatureSchema, + AgentStatusSchema, + MaterialProgressPayloadSchema, + type MaterialProgressPayload, +} from "../messages.js"; import { toStoredAgentRecord } from "./agent-projections.js"; import type { ManagedAgent } from "./agent-manager.js"; import type { AgentSessionConfig } from "./agent-sdk-types.js"; @@ -60,6 +65,8 @@ const STORED_AGENT_SCHEMA = z.object({ features: z.array(AgentFeatureSchema).optional(), persistence: PERSISTENCE_HANDLE_SCHEMA, lastError: z.string().nullable().optional(), + lastTurnOutcome: z.enum(["completed", "failed", "canceled"]).nullable().optional(), + materialProgress: MaterialProgressPayloadSchema.optional(), requiresAttention: z.boolean().optional(), attentionReason: z.enum(["finished", "error", "permission"]).nullable().optional(), attentionTimestamp: z.string().nullable().optional(), @@ -128,13 +135,20 @@ export class AgentStorage { } private queueRecordWrite(record: StoredAgentRecord): Promise { - const agentId = record.id; + return this.queueRecordMutation(record.id, () => record); + } + + private queueRecordMutation( + agentId: string, + buildRecord: (existing: StoredAgentRecord | null) => StoredAgentRecord, + ): Promise { const prev = this.pendingWrites.get(agentId) ?? Promise.resolve(); const next = prev.then(async () => { if (this.deleting.has(agentId)) { return undefined; } + const record = buildRecord(this.cache.get(agentId) ?? null); await this.writeRecord(record); return undefined; }); @@ -204,28 +218,37 @@ export class AgentStorage { async applySnapshot( agent: ManagedAgent, - options?: { title?: string | null; internal?: boolean }, + options?: { + title?: string | null; + internal?: boolean; + materialProgress?: MaterialProgressPayload; + }, ): Promise { await this.load(); - await this.waitForPendingWrite(agent.id); - const existing = (await this.get(agent.id)) ?? null; const hasTitleOverride = options !== undefined && Object.prototype.hasOwnProperty.call(options, "title"); const hasInternalOverride = options !== undefined && Object.prototype.hasOwnProperty.call(options, "internal"); - const record = toStoredAgentRecord(agent, { - title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), - createdAt: existing?.createdAt, - internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), + const hasMaterialProgressOverride = + options !== undefined && Object.prototype.hasOwnProperty.call(options, "materialProgress"); + await this.queueRecordMutation(agent.id, (existing) => { + const record = toStoredAgentRecord(agent, { + title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), + createdAt: existing?.createdAt, + internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), + materialProgress: hasMaterialProgressOverride + ? options?.materialProgress + : existing?.materialProgress, + }); + + // Preserve soft-delete/archive status across snapshot flushes. + // `archivedAt` is not part of the ManagedAgent snapshot, so a naive projection + // would wipe it during normal persistence (including on daemon restart). + if (existing && existing.archivedAt !== undefined) { + record.archivedAt = existing.archivedAt; + } + return record; }); - - // Preserve soft-delete/archive status across snapshot flushes. - // `archivedAt` is not part of the ManagedAgent snapshot, so a naive projection - // would wipe it during normal persistence (including on daemon restart). - if (existing && existing.archivedAt !== undefined) { - record.archivedAt = existing.archivedAt; - } - await this.upsert(record); } async setTitle(agentId: string, title: string): Promise { diff --git a/packages/server/src/server/agent/material-progress.test.ts b/packages/server/src/server/agent/material-progress.test.ts new file mode 100644 index 0000000000..dced4baa51 --- /dev/null +++ b/packages/server/src/server/agent/material-progress.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from "vitest"; +import type { AgentTimelineItem } from "./agent-sdk-types.js"; +import type { TimelineProjectionEntry } from "./timeline-projection.js"; +import { analyzeMaterialProgress } from "./material-progress.js"; + +function entry(seq: number, item: AgentTimelineItem): TimelineProjectionEntry { + return { + item, + timestamp: `2026-07-31T00:00:0${seq}.000Z`, + seqStart: seq, + seqEnd: seq, + sourceSeqRanges: [{ startSeq: seq, endSeq: seq }], + collapsed: [], + }; +} + +describe("analyzeMaterialProgress", () => { + it("reports none when history or a current continuation is unavailable", () => { + expect(analyzeMaterialProgress({ entries: null, turnOutcome: null })).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Timeline history is unavailable.", + }); + + expect(analyzeMaterialProgress({ entries: [], turnOutcome: null }).state).toBe("none"); + }); + + it("treats completed edits and writes as material progress", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "implement" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { + type: "tool_call", + callId: "edit-1", + name: "edit", + status: "completed", + error: null, + detail: { type: "edit", filePath: "src/a.ts", unifiedDiff: "+change" }, + }), + entry(4, { type: "compaction", status: "completed" }), + entry(5, { + type: "tool_call", + callId: "write-1", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "src/b.ts", content: "content" }, + }), + ], + turnOutcome: null, + }); + + expect(result).toEqual({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: "2026-07-31T00:00:05.000Z", + lastMaterialProgressKind: "write", + reason: "Material progress followed the latest user message.", + }); + }); + + it("warns after one compaction and stalls after two", () => { + expect( + analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "continue" }), + entry(2, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }), + ).toMatchObject({ + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + }); + + expect( + analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "continue" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { type: "reasoning", text: "still working" }), + entry(4, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }), + ).toMatchObject({ + state: "stalled", + completedCompactionsSinceMaterialProgress: 2, + }); + }); + + it("uses only the latest user continuation and completion sequence", () => { + const completedWrite = entry(2, { + type: "tool_call", + callId: "write-1", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "proof.txt", content: "passed" }, + }); + completedWrite.seqEnd = 5; + + expect( + analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "old work" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { type: "user_message", text: "current work" }), + completedWrite, + entry(4, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }), + ).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "write", + }); + }); + + it("does not count read-only tools or commentary as material progress", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "investigate" }), + entry(2, { + type: "tool_call", + callId: "read-1", + name: "read", + status: "completed", + error: null, + detail: { type: "read", filePath: "src/a.ts", content: "source" }, + }), + entry(3, { type: "assistant_message", text: "Still investigating." }), + entry(4, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressKind: null, + }); + }); + + it("counts only the final assistant message from a completed turn", () => { + const entries = [ + entry(1, { type: "user_message", text: "answer" }), + entry(2, { type: "assistant_message", text: "I am checking." }), + entry(3, { type: "compaction", status: "completed" }), + entry(4, { type: "assistant_message", text: "Delivered result." }), + ]; + + expect(analyzeMaterialProgress({ entries, turnOutcome: null }).state).toBe("warning"); + expect(analyzeMaterialProgress({ entries, turnOutcome: "canceled" }).state).toBe("warning"); + expect(analyzeMaterialProgress({ entries, turnOutcome: "completed" })).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "assistant_result", + }); + }); +}); diff --git a/packages/server/src/server/agent/material-progress.ts b/packages/server/src/server/agent/material-progress.ts new file mode 100644 index 0000000000..47b5c8683b --- /dev/null +++ b/packages/server/src/server/agent/material-progress.ts @@ -0,0 +1,142 @@ +import type { MaterialProgressPayload } from "../messages.js"; +import type { AgentTimelineItem, ToolCallTimelineItem } from "./agent-sdk-types.js"; +import type { TimelineProjectionEntry } from "./timeline-projection.js"; + +export interface AnalyzeMaterialProgressInput { + entries: readonly TimelineProjectionEntry[] | null; + turnOutcome: "completed" | "failed" | "canceled" | null; +} + +type MaterialProgressKind = NonNullable; + +function hasConcreteText(value: unknown): boolean { + return typeof value === "string" && value.trim().length > 0; +} + +function materialToolKind(item: ToolCallTimelineItem): MaterialProgressKind | null { + if (item.status !== "completed") return null; + + switch (item.detail.type) { + case "edit": + return "edit"; + case "write": + return "write"; + case "shell": + case "worktree_setup": + case "sub_agent": + case "unknown": + case "read": + case "search": + case "fetch": + case "plain_text": + case "plan": + return null; + } +} + +function validTimestamp(value: string): string | null { + return Number.isNaN(Date.parse(value)) ? null : value; +} + +function orderedEntries(entries: readonly TimelineProjectionEntry[]): TimelineProjectionEntry[] { + return [...entries].sort( + (left, right) => left.seqEnd - right.seqEnd || left.seqStart - right.seqStart, + ); +} + +function noProgress(reason: string): MaterialProgressPayload { + return { + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason, + }; +} + +function materialKind( + item: AgentTimelineItem, + isFinalTerminalAssistant: boolean, +): MaterialProgressKind | null { + if (item.type === "tool_call") return materialToolKind(item); + if (item.type === "assistant_message" && isFinalTerminalAssistant && hasConcreteText(item.text)) { + return "assistant_result"; + } + return null; +} + +export function analyzeMaterialProgress({ + entries, + turnOutcome, +}: AnalyzeMaterialProgressInput): MaterialProgressPayload { + if (entries === null) return noProgress("Timeline history is unavailable."); + + const ordered = orderedEntries(entries); + let latestUserIndex = -1; + for (let index = 0; index < ordered.length; index += 1) { + if (ordered[index]?.item.type === "user_message") latestUserIndex = index; + } + if (latestUserIndex < 0) return noProgress("No current continuation is available."); + + const continuation = ordered.slice(latestUserIndex + 1); + let finalAssistantIndex = -1; + if (turnOutcome === "completed") { + for (let index = 0; index < continuation.length; index += 1) { + if (continuation[index]?.item.type === "assistant_message") finalAssistantIndex = index; + } + } + + let completedCompactions = 0; + let lastMaterialProgressAt: string | null = null; + let lastMaterialProgressKind: MaterialProgressKind | null = null; + + for (let index = 0; index < continuation.length; index += 1) { + const entry = continuation[index]; + if (!entry) continue; + const kind = materialKind( + entry.item, + turnOutcome === "completed" && index === finalAssistantIndex, + ); + if (kind) { + completedCompactions = 0; + lastMaterialProgressAt = validTimestamp(entry.timestamp); + lastMaterialProgressKind = kind; + continue; + } + if (entry.item.type === "compaction" && entry.item.status === "completed") { + completedCompactions += 1; + } + } + + if (completedCompactions >= 2) { + return { + state: "stalled", + completedCompactionsSinceMaterialProgress: completedCompactions, + lastMaterialProgressAt, + lastMaterialProgressKind, + reason: + completedCompactions === 2 + ? "Two compactions completed without later material progress." + : `${completedCompactions} compactions completed without later material progress.`, + }; + } + if (completedCompactions === 1) { + return { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt, + lastMaterialProgressKind, + reason: "One compaction completed without later material progress.", + }; + } + if (lastMaterialProgressKind) { + return { + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt, + lastMaterialProgressKind, + reason: "Material progress followed the latest user message.", + }; + } + return noProgress("No material progress has been recorded for the current continuation."); +} diff --git a/packages/server/src/server/material-progress.e2e.test.ts b/packages/server/src/server/material-progress.e2e.test.ts new file mode 100644 index 0000000000..5dd6465820 --- /dev/null +++ b/packages/server/src/server/material-progress.e2e.test.ts @@ -0,0 +1,186 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, test } from "vitest"; +import { DaemonClient } from "./test-utils/index.js"; +import { + createTestPaseoDaemon, + type TestPaseoDaemon, +} from "./test-utils/paseo-daemon.js"; + +async function recordMaterialWrite(daemon: TestPaseoDaemon, agentId: string): Promise { + await daemon.daemon.agentManager.appendTimelineItem(agentId, { + type: "user_message", + text: "implement", + }); + await daemon.daemon.agentManager.appendTimelineItem(agentId, { + type: "compaction", + status: "completed", + }); + await daemon.daemon.agentManager.appendTimelineItem(agentId, { + type: "tool_call", + callId: "write-1", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "proof.txt", content: "done" }, + }); +} + +function expectMaterialWriteProgress( + materialProgress: NonNullable< + Awaited> + >["agent"]["materialProgress"], +): void { + expect(materialProgress).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "write", + }); +} + +test("fetch agent exposes a backward-compatible material progress signal", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-")); + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Material progress probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + const fetched = await client.fetchAgent({ agentId: created.id }); + + expect(fetched?.agent.materialProgress).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "No current continuation is available.", + }); + } finally { + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + +test("fetch agent preserves material progress after the live worker is collected", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-retained-")); + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Retained material progress probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + await recordMaterialWrite(daemon, created.id); + await daemon.daemon.agentManager.closeAgent(created.id); + + const fetched = await client.fetchAgent({ agentId: created.id }); + expectMaterialWriteProgress(fetched?.agent.materialProgress); + } finally { + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + +test("fetch agent preserves material progress after archive discards the live timeline", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-archived-")); + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Archived material progress probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + await recordMaterialWrite(daemon, created.id); + await client.archiveAgent(created.id); + + const fetched = await client.fetchAgent({ agentId: created.id }); + expect(fetched?.agent.archivedAt).toEqual(expect.any(String)); + expectMaterialWriteProgress(fetched?.agent.materialProgress); + } finally { + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + +test("fetch agent restores material progress after daemon restart", async () => { + const paseoHomeRoot = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-home-")); + const staticDir = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-static-")); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-restart-")); + let daemon: TestPaseoDaemon | null = null; + let client: DaemonClient | null = null; + + try { + daemon = await createTestPaseoDaemon({ + paseoHomeRoot, + staticDir, + cleanup: false, + }); + client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Restarted material progress probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + await recordMaterialWrite(daemon, created.id); + + await client.close(); + client = null; + await daemon.close(); + daemon = null; + + daemon = await createTestPaseoDaemon({ + paseoHomeRoot, + staticDir, + cleanup: false, + }); + client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + await client.connect(); + + const fetched = await client.fetchAgent({ agentId: created.id }); + expectMaterialWriteProgress(fetched?.agent.materialProgress); + } finally { + await client?.close().catch(() => undefined); + await daemon?.close().catch(() => undefined); + rmSync(cwd, { recursive: true, force: true }); + rmSync(paseoHomeRoot, { recursive: true, force: true }); + rmSync(staticDir, { recursive: true, force: true }); + } +}); diff --git a/packages/server/src/server/persistence-hooks.ts b/packages/server/src/server/persistence-hooks.ts index ba220eefa5..d286c9fb8c 100644 --- a/packages/server/src/server/persistence-hooks.ts +++ b/packages/server/src/server/persistence-hooks.ts @@ -111,6 +111,7 @@ export function extractTimestamps(record: StoredAgentRecord): { labels?: Record; workspaceId?: string; owner?: StoredAgentRecord["owner"]; + lastTurnOutcome?: StoredAgentRecord["lastTurnOutcome"]; } { return { createdAt: new Date(record.createdAt), @@ -119,6 +120,7 @@ export function extractTimestamps(record: StoredAgentRecord): { labels: record.labels, workspaceId: record.workspaceId, owner: record.owner, + lastTurnOutcome: record.lastTurnOutcome, }; } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index a87aa42e6c..cb6877ce93 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -75,6 +75,7 @@ import type { AgentTimelineFetchDirection, AgentTimelineFetchResult, ManagedAgent, + MaterialProgressSnapshot, } from "./agent/agent-manager.js"; import { createAgentCommand } from "./agent/create-agent/create.js"; import { resolveCreateAgentIntent, type CreateAgentIntent } from "./agent/create-agent/intent.js"; @@ -101,6 +102,7 @@ import { type TimelineProjectionEntry, type TimelineProjectionMode, } from "./agent/timeline-projection.js"; +import { analyzeMaterialProgress } from "./agent/material-progress.js"; import { buildAgentForkContextAttachment } from "./agent/activity-curator.js"; import { buildAgentPrompt } from "./agent/prompt-attachments.js"; import type { StructuredGenerationDaemonConfig } from "./agent/structured-generation-providers.js"; @@ -3956,7 +3958,9 @@ export class Session { const live = this.agentManager.getAgent(agentId); if (live) { const payload = await this.buildAgentPayload(live); - return this.isProviderVisibleToClient(payload.provider) ? payload : null; + return this.isProviderVisibleToClient(payload.provider) + ? await this.attachMaterialProgress(payload) + : null; } const record = await this.agentStorage.get(agentId); @@ -3964,7 +3968,41 @@ export class Session { return null; } const payload = this.buildStoredAgentPayload(record); - return this.isProviderVisibleToClient(payload.provider) ? payload : null; + return this.isProviderVisibleToClient(payload.provider) + ? await this.attachMaterialProgress(payload) + : null; + } + + private async attachMaterialProgress( + payload: AgentSnapshotPayload, + ): Promise { + let entries: TimelineProjectionEntry[] | null = null; + let turnOutcome: MaterialProgressSnapshot["turnOutcome"] = null; + let persisted = payload.materialProgress ?? null; + try { + const snapshot = await this.agentManager.getMaterialProgressSnapshot(payload.id); + entries = + snapshot.rows === null + ? null + : projectTimelineRows({ rows: snapshot.rows, mode: "projected" }); + turnOutcome = snapshot.turnOutcome; + persisted = snapshot.persisted ?? persisted; + } catch (error) { + this.sessionLogger.debug( + { err: error, agentId: payload.id }, + "Material progress timeline is unavailable", + ); + } + return { + ...payload, + materialProgress: + entries === null && persisted + ? persisted + : analyzeMaterialProgress({ + entries, + turnOutcome, + }), + }; } private async resolveDelegationRootWorkspaceId(agentId: string): Promise { diff --git a/patches/local-patches.json b/patches/local-patches.json new file mode 100644 index 0000000000..cecb16d264 --- /dev/null +++ b/patches/local-patches.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": 1, + "patches": [ + { + "id": "paseo-v025-material-progress-inspect", + "category": "stable-core-observability", + "base": { + "tag": "v0.2.5", + "commit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b" + }, + "tests": [ + "packages/protocol/src/material-progress-schema.test.ts", + "packages/server/src/server/agent/material-progress.test.ts", + "packages/server/src/server/agent/agent-manager.test.ts", + "packages/server/src/server/material-progress.e2e.test.ts", + "packages/cli/src/commands/agent/inspect.test.ts" + ], + "upstreamable": true, + "dropCondition": "Remove when the official stable release exposes equivalent singular agent material-progress inspection with persisted terminal-turn semantics.", + "deployedReceipt": null, + "deployedAt": null + } + ] +} From e8b4123725d615496d4aaf1a50e279b91f46f73d Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 04:48:37 +0100 Subject: [PATCH 02/17] test: align unhydrated progress readback --- packages/server/src/server/material-progress.e2e.test.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/server/src/server/material-progress.e2e.test.ts b/packages/server/src/server/material-progress.e2e.test.ts index 5dd6465820..f3f6ec971a 100644 --- a/packages/server/src/server/material-progress.e2e.test.ts +++ b/packages/server/src/server/material-progress.e2e.test.ts @@ -3,10 +3,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { expect, test } from "vitest"; import { DaemonClient } from "./test-utils/index.js"; -import { - createTestPaseoDaemon, - type TestPaseoDaemon, -} from "./test-utils/paseo-daemon.js"; +import { createTestPaseoDaemon, type TestPaseoDaemon } from "./test-utils/paseo-daemon.js"; async function recordMaterialWrite(daemon: TestPaseoDaemon, agentId: string): Promise { await daemon.daemon.agentManager.appendTimelineItem(agentId, { @@ -63,7 +60,7 @@ test("fetch agent exposes a backward-compatible material progress signal", async completedCompactionsSinceMaterialProgress: 0, lastMaterialProgressAt: null, lastMaterialProgressKind: null, - reason: "No current continuation is available.", + reason: "Timeline history is unavailable.", }); } finally { await client.close(); From c57281abedc8552c17b486db4aad66edb7bbb635 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 04:49:55 +0100 Subject: [PATCH 03/17] refactor: satisfy stable progress quality gates --- .../src/server/agent/agent-manager.test.ts | 8 ++--- .../server/src/server/agent/agent-manager.ts | 31 ++++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 13e0f481ab..154c1c0353 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -3859,11 +3859,9 @@ test("an unhydrated resumed agent exposes its persisted material progress", asyn idFactory: () => agentId, }); - const created = await firstManager.createAgent( - { provider: "codex", cwd: workdir }, - undefined, - { workspaceId: undefined }, - ); + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); await firstManager.appendTimelineItem(created.id, { type: "user_message", text: "persist this result", diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 3fc2e7006e..e3e874555c 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -134,6 +134,18 @@ function formatProviderList(providers: readonly string[]): string { return providers.length > 0 ? providers.join(", ") : "none"; } +function materialProgressRows( + timeline: AgentTimelineFetchResult | null, + historyPrimed = true, +): AgentTimelineRow[] | null { + if (timeline === null) return null; + if (timeline.rows.length === 0 && !historyPrimed) return null; + if (timeline.hasOlder && !timeline.rows.some((row) => row.item.type === "user_message")) { + return null; + } + return timeline.rows; +} + function buildStoredAgentConfig(record: StoredAgentRecord): AgentSessionConfig { const config: AgentSessionConfig = { provider: record.provider, @@ -983,19 +995,14 @@ export class AgentManager { return this.timelineStore.getRows(id); } - async getMaterialProgressSnapshot( - id: string, - limit = 1_000, - ): Promise { + async getMaterialProgressSnapshot(id: string, limit = 1_000): Promise { const live = this.agents.get(id); if (live && this.timelineStore.has(id)) { const timeline = this.timelineStore.fetch(id, { direction: "tail", limit }); - const rowsUnavailable = - (timeline.rows.length === 0 && !live.historyPrimed) || - (timeline.hasOlder && !timeline.rows.some((row) => row.item.type === "user_message")); - const record = rowsUnavailable ? await this.registry?.get(id) : null; + const rows = materialProgressRows(timeline, live.historyPrimed); + const record = rows === null ? await this.registry?.get(id) : null; return { - rows: rowsUnavailable ? null : timeline.rows, + rows, turnOutcome: live.lastTurnOutcome ?? null, persisted: record?.materialProgress ?? null, }; @@ -1012,11 +1019,7 @@ export class AgentManager { } const record = await this.registry?.get(id); return { - rows: - timeline === null || - (timeline.hasOlder && !timeline.rows.some((row) => row.item.type === "user_message")) - ? null - : timeline.rows, + rows: materialProgressRows(timeline), turnOutcome: record?.lastTurnOutcome ?? null, persisted: record?.materialProgress ?? null, }; From 9dbb76b0d75f628f5d5167447928451c958bcafc Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 05:33:27 +0100 Subject: [PATCH 04/17] fix: harden material progress persistence --- docs/data-model.md | 5 + .../src/server/agent/agent-manager.test.ts | 344 +++++++++++++++++- .../server/src/server/agent/agent-manager.ts | 212 ++++++++--- .../src/server/agent/agent-storage.test.ts | 69 ++++ .../server/src/server/agent/agent-storage.ts | 17 +- .../server/agent/material-progress.test.ts | 11 +- 6 files changed, 590 insertions(+), 68 deletions(-) diff --git a/docs/data-model.md b/docs/data-model.md index f264feff33..69e214606b 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -104,6 +104,11 @@ Each agent is stored as a separate JSON file, grouped by project directory. | `internal` | `boolean?` | Whether this is a system-internal agent (loop workers, etc.) | | `archivedAt` | `string?` (ISO 8601) | Soft-delete timestamp | +`materialProgress.lastMaterialProgressKind: "assistant_result"` means a completed turn ended +with a non-empty assistant message. It records a terminal deliverable, not whether the answer was +semantically correct or the user's wider goal succeeded. Final messages from failed or canceled +turns are non-material; completed edits and writes remain material independently of that boundary. + ### Nested: SerializableConfig | Field | Type | Description | diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 154c1c0353..481a7682b3 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -14,6 +14,7 @@ import { type ManagedAgent, } from "./agent-manager.js"; import { AgentStorage } from "./agent-storage.js"; +import { InMemoryAgentTimelineStore } from "./agent-timeline-store.js"; import { toAgentPayload } from "./agent-projections.js"; import { PARENT_AGENT_ID_LABEL } from "@getpaseo/protocol/agent-labels"; import { formatSystemNotificationPrompt } from "./agent-prompt.js"; @@ -40,6 +41,12 @@ import type { } from "./agent-sdk-types.js"; import type { PaseoToolCatalog } from "./tools/types.js"; import type { ProviderDefinition } from "./provider-registry.js"; +import type { + AgentTimelineFetchOptions, + AgentTimelineFetchResult, + AgentTimelineRow, + AgentTimelineStore, +} from "./agent-timeline-store-types.js"; interface Deferred { promise: Promise; @@ -57,6 +64,79 @@ function deferred(): Deferred { return { promise, resolve, reject }; } +function timelineRow(seq: number, item: AgentTimelineItem): AgentTimelineRow { + return { + seq, + timestamp: new Date(Date.UTC(2026, 6, 31, 0, 0, 0, seq)).toISOString(), + item, + }; +} + +class TestDurableTimelineStore implements AgentTimelineStore { + private readonly store = new InMemoryAgentTimelineStore(); + afterNextFetch: (() => Promise) | null = null; + + async appendCommitted( + agentId: string, + item: AgentTimelineItem, + options?: { timestamp?: string }, + ): Promise { + this.ensure(agentId); + return this.store.append(agentId, item, options); + } + + async fetchCommitted( + agentId: string, + options?: AgentTimelineFetchOptions, + ): Promise { + this.ensure(agentId); + const result = this.store.fetch(agentId, options); + const afterFetch = this.afterNextFetch; + this.afterNextFetch = null; + await afterFetch?.(); + return result; + } + + async getLatestCommittedSeq(agentId: string): Promise { + this.ensure(agentId); + return this.store.getRows(agentId).at(-1)?.seq ?? 0; + } + + async getCommittedRows(agentId: string): Promise { + this.ensure(agentId); + return this.store.getRows(agentId); + } + + async getLastItem(agentId: string): Promise { + this.ensure(agentId); + return this.store.getLastItem(agentId); + } + + async getLastAssistantMessage(agentId: string): Promise { + this.ensure(agentId); + return this.store.getLastAssistantMessage(agentId); + } + + async deleteAgent(agentId: string): Promise { + if (this.store.has(agentId)) this.store.delete(agentId); + } + + async bulkInsert(agentId: string, rows: readonly AgentTimelineRow[]): Promise { + const existingRows = this.store.has(agentId) ? this.store.getRows(agentId) : []; + const bySeq = new Map(existingRows.map((row) => [row.seq, row])); + for (const row of rows) bySeq.set(row.seq, row); + const mergedRows = [...bySeq.values()].sort((left, right) => left.seq - right.seq); + this.store.initialize(agentId, { + rows: mergedRows, + nextSeq: (mergedRows.at(-1)?.seq ?? 0) + 1, + }); + } + + private ensure(agentId: string): void { + if (!this.store.has(agentId)) this.store.initialize(agentId); + } +} + function waitForAgentLifecycle( manager: AgentManager, agentId: string, @@ -3839,7 +3919,10 @@ test("getTimelineRows falls back to the in-memory timeline when no durable store ]); await expect(manager.getMaterialProgressSnapshot(snapshot.id, 1)).resolves.toMatchObject({ - rows: null, + rows: [ + { seq: 1, item: { type: "assistant_message", text: "row one" } }, + { seq: 2, item: { type: "assistant_message", text: "row two" } }, + ], }); await manager.closeAgent(snapshot.id); @@ -3902,6 +3985,265 @@ test("an unhydrated resumed agent exposes its persisted material progress", asyn rmSync(workdir, { recursive: true, force: true }); }); +test("a resumed agent with an empty live timeline derives progress from its durable tail", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-durable-resume-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const durableTimelineStore = new TestDurableTimelineStore(); + const agentId = "00000000-0000-4000-8000-000000000142"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + try { + const created = await firstManager.createAgent( + { provider: "codex", cwd: workdir }, + undefined, + { workspaceId: undefined }, + ); + await firstManager.closeAgent(created.id); + await durableTimelineStore.bulkInsert(created.id, [ + timelineRow(1, { type: "user_message", text: "deliver this" }), + timelineRow(2, { + type: "tool_call", + callId: "write-durable-resume", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "durable.txt", content: "done" }, + }), + ]); + + const stored = await storage.get(created.id); + expect(stored?.persistence).toBeTruthy(); + await storage.upsert({ + ...stored!, + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Stale stored classification.", + }, + }); + + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + durableTimelineStore, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + stored!.persistence!, + { cwd: workdir }, + created.id, + ); + + expect(resumedManager.getTimeline(created.id)).toEqual([]); + await expect(resumedManager.getMaterialProgressSnapshot(created.id)).resolves.toMatchObject({ + rows: [ + { seq: 1, item: { type: "user_message", text: "deliver this" } }, + { + seq: 2, + item: { + type: "tool_call", + detail: { type: "write", filePath: "durable.txt" }, + }, + }, + ], + persisted: { + state: "progressing", + lastMaterialProgressKind: "write", + }, + }); + expect((await storage.get(created.id))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + + await resumedManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + await expect(resumedManager.getMaterialProgressSnapshot(created.id)).resolves.toMatchObject({ + rows: [ + { seq: 1, item: { type: "user_message" } }, + { seq: 2, item: { type: "tool_call" } }, + { seq: 3, item: { type: "compaction" } }, + ], + persisted: { + state: "warning", + lastMaterialProgressKind: "write", + }, + }); + + await resumedManager.closeAgent(created.id); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("material progress pages beyond the durable 1000-row tail to the current continuation", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-durable-paging-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const durableTimelineStore = new TestDurableTimelineStore(); + const agentId = "00000000-0000-4000-8000-000000000143"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + try { + const created = await firstManager.createAgent( + { provider: "codex", cwd: workdir }, + undefined, + { workspaceId: undefined }, + ); + await firstManager.closeAgent(created.id); + + const durableRows: AgentTimelineRow[] = [ + timelineRow(1, { type: "user_message", text: "previous continuation" }), + timelineRow(2, { type: "compaction", status: "completed" }), + timelineRow(3, { type: "user_message", text: "current continuation" }), + timelineRow(4, { + type: "tool_call", + callId: "write-before-tail", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "paged.txt", content: "done" }, + }), + ]; + for (let seq = 5; seq <= 1_004; seq += 1) { + durableRows.push(timelineRow(seq, { type: "reasoning", text: `reasoning ${seq}` })); + } + await durableTimelineStore.bulkInsert(created.id, durableRows); + + const stored = await storage.get(created.id); + expect(stored?.persistence).toBeTruthy(); + await storage.upsert({ + ...stored!, + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Classification to replace after complete paging.", + }, + }); + + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + durableTimelineStore, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + stored!.persistence!, + { cwd: workdir }, + created.id, + ); + + expect(resumedManager.getTimeline(created.id)).toEqual([]); + const snapshot = await resumedManager.getMaterialProgressSnapshot(created.id); + expect(snapshot.rows).toHaveLength(1_002); + expect(snapshot.rows?.[0]).toMatchObject({ + seq: 3, + item: { type: "user_message", text: "current continuation" }, + }); + expect(snapshot.persisted).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + expect((await storage.get(created.id))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + + await resumedManager.closeAgent(created.id); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("a delayed material progress write cannot overwrite a newer continuation", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-write-order-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const durableTimelineStore = new TestDurableTimelineStore(); + const agentId = "00000000-0000-4000-8000-000000000144"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + try { + const created = await firstManager.createAgent( + { provider: "codex", cwd: workdir }, + undefined, + { workspaceId: undefined }, + ); + await firstManager.closeAgent(created.id); + await durableTimelineStore.bulkInsert(created.id, [ + timelineRow(1, { type: "user_message", text: "old continuation" }), + timelineRow(2, { type: "compaction", status: "completed" }), + ]); + + const stored = await storage.get(created.id); + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + durableTimelineStore, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + stored!.persistence!, + { cwd: workdir }, + created.id, + ); + + const delayedFetchStarted = deferred(); + const releaseDelayedFetch = deferred(); + durableTimelineStore.afterNextFetch = async () => { + delayedFetchStarted.resolve(undefined); + await releaseDelayedFetch.promise; + }; + const olderPersist = resumedManager.appendTimelineItem(created.id, { + type: "reasoning", + text: "old continuation still running", + }); + await delayedFetchStarted.promise; + + const userPersist = resumedManager.appendTimelineItem(created.id, { + type: "user_message", + text: "new continuation", + }); + const writePersist = resumedManager.appendTimelineItem(created.id, { + type: "tool_call", + callId: "write-new-continuation", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "new.txt", content: "new" }, + }); + releaseDelayedFetch.resolve(undefined); + await Promise.all([olderPersist, userPersist, writePersist]); + + expect((await storage.get(created.id))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + await resumedManager.closeAgent(created.id); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("persists terminal outcomes and clears the previous outcome when a new turn starts", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-turn-outcome-")); const storage = new AgentStorage(join(workdir, "agents"), logger); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index e3e874555c..df6baa91d9 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -79,6 +79,8 @@ import type { MaterialProgressPayload } from "../messages.js"; const RELOAD_SESSION_CLOSE_TIMEOUT_MS = 3_000; const INTERRUPT_SESSION_TIMEOUT_MS = 2_000; +const MATERIAL_PROGRESS_PAGE_SIZE = 1_000; +const MATERIAL_PROGRESS_MAX_SCAN_ROWS = 10_000; const STORED_AGENT_CAPABILITIES: AgentCapabilityFlags = { supportsStreaming: false, supportsSessionPersistence: true, @@ -130,20 +132,59 @@ interface TimeoutOptions { onLateError?: (error: unknown) => void; } +interface PersistSnapshotOptions { + title?: string | null; + internal?: boolean; + includeInternal?: boolean; +} + function formatProviderList(providers: readonly string[]): string { return providers.length > 0 ? providers.join(", ") : "none"; } -function materialProgressRows( - timeline: AgentTimelineFetchResult | null, - historyPrimed = true, -): AgentTimelineRow[] | null { - if (timeline === null) return null; - if (timeline.rows.length === 0 && !historyPrimed) return null; - if (timeline.hasOlder && !timeline.rows.some((row) => row.item.type === "user_message")) { +type FetchOlderMaterialProgressRows = ( + options: AgentTimelineFetchOptions, +) => Promise; + +function currentContinuationRows(rows: readonly AgentTimelineRow[]): AgentTimelineRow[] { + for (let index = rows.length - 1; index >= 0; index -= 1) { + if (rows[index]?.item.type === "user_message") return rows.slice(index); + } + return [...rows]; +} + +async function pageMaterialProgressRows( + initialPage: AgentTimelineFetchResult, + fetchOlder: FetchOlderMaterialProgressRows, +): Promise { + let page = initialPage; + let rows = [...page.rows]; + while (page.hasOlder && !rows.some((row) => row.item.type === "user_message")) { + const firstRow = rows[0]; + const remainingCapacity = MATERIAL_PROGRESS_MAX_SCAN_ROWS - rows.length; + if (!firstRow || remainingCapacity <= 0) return null; + + const olderPage = await fetchOlder({ + direction: "before", + cursor: { epoch: page.epoch, seq: firstRow.seq }, + limit: Math.min(MATERIAL_PROGRESS_PAGE_SIZE, remainingCapacity), + }); + if ( + olderPage.epoch !== page.epoch || + olderPage.reset || + olderPage.staleCursor || + olderPage.gap || + olderPage.rows.length === 0 + ) { + return null; + } + rows = [...olderPage.rows, ...rows]; + page = olderPage; + } + if (!rows.some((row) => row.item.type === "user_message") && rows[0]?.seq !== 1) { return null; } - return timeline.rows; + return currentContinuationRows(rows); } function buildStoredAgentConfig(record: StoredAgentRecord): AgentSessionConfig { @@ -587,6 +628,7 @@ export class AgentManager { private readonly providerSubagents = new ProviderSubagentStore(); private readonly agentsAwaitingInitialSnapshotPersist = new Set(); private readonly sessionEventTails = new Map>(); + private readonly snapshotPersistenceTails = new Map>(); private readonly runs = new AgentRunState(); private readonly subscribers = new Set(); private readonly idFactory: () => string; @@ -997,30 +1039,16 @@ export class AgentManager { async getMaterialProgressSnapshot(id: string, limit = 1_000): Promise { const live = this.agents.get(id); - if (live && this.timelineStore.has(id)) { - const timeline = this.timelineStore.fetch(id, { direction: "tail", limit }); - const rows = materialProgressRows(timeline, live.historyPrimed); - const record = rows === null ? await this.registry?.get(id) : null; - return { - rows, - turnOutcome: live.lastTurnOutcome ?? null, - persisted: record?.materialProgress ?? null, - }; - } - - let timeline: AgentTimelineFetchResult | null = null; - if (this.durableTimelineStore) { - timeline = await this.durableTimelineStore.fetchCommitted(id, { - direction: "tail", + const [rows, record] = await Promise.all([ + this.resolveMaterialProgressRows(id, { limit, - }); - } else if (this.timelineStore.has(id)) { - timeline = this.timelineStore.fetch(id, { direction: "tail", limit }); - } - const record = await this.registry?.get(id); + historyPrimed: live?.historyPrimed, + }), + this.registry?.get(id), + ]); return { - rows: materialProgressRows(timeline), - turnOutcome: record?.lastTurnOutcome ?? null, + rows, + turnOutcome: live?.lastTurnOutcome ?? record?.lastTurnOutcome ?? null, persisted: record?.materialProgress ?? null, }; } @@ -1481,10 +1509,9 @@ export class AgentManager { throw new Error("Agent storage is not configured"); } - const materialProgress = this.buildPersistedMaterialProgress(agent); - await this.registry.applySnapshot(agent, { + await this.persistSnapshot(agent, { internal: agent.internal, - ...(materialProgress ? { materialProgress } : {}), + includeInternal: true, }); const stored = await this.registry.get(agentId); if (!stored) { @@ -3120,42 +3147,115 @@ export class AgentManager { return explicitTitle ?? fallbackTitle; } - private async persistSnapshot( - agent: ManagedAgent, - options?: { title?: string | null; internal?: boolean }, - ): Promise { + private persistSnapshot(agent: ManagedAgent, options?: PersistSnapshotOptions): Promise { if (!this.registry) { - return; + return Promise.resolve(); } // Don't persist internal agents - they're ephemeral system tasks - if (agent.internal) { - return; + if (agent.internal && !options?.includeInternal) { + return Promise.resolve(); } - const materialProgress = this.buildPersistedMaterialProgress(agent); - await this.registry.applySnapshot(agent, { - ...options, + const previous = (this.snapshotPersistenceTails.get(agent.id) ?? Promise.resolve()).catch( + () => undefined, + ); + const next = previous.then(() => this.writeSnapshot(agent, options)); + const tracked = next.finally(() => { + if (this.snapshotPersistenceTails.get(agent.id) === tracked) { + this.snapshotPersistenceTails.delete(agent.id); + } + }); + this.snapshotPersistenceTails.set(agent.id, tracked); + return tracked; + } + + private async writeSnapshot( + agent: ManagedAgent, + options?: PersistSnapshotOptions, + ): Promise { + const registry = this.registry; + if (!registry) return; + + const materialProgress = await this.buildPersistedMaterialProgress(agent); + await registry.applySnapshot(agent, { + ...(options?.title !== undefined ? { title: options.title } : {}), + ...(options?.internal !== undefined ? { internal: options.internal } : {}), ...(materialProgress ? { materialProgress } : {}), }); } - private buildPersistedMaterialProgress(agent: ManagedAgent): MaterialProgressPayload | undefined { - const timeline = this.timelineStore.has(agent.id) - ? this.timelineStore.fetch(agent.id, { direction: "tail", limit: 1_000 }) - : null; - if (timeline?.rows.length === 0 && !agent.historyPrimed) { - return undefined; - } - const rows = - timeline === null || - (timeline.hasOlder && !timeline.rows.some((row) => row.item.type === "user_message")) - ? null - : timeline.rows; + private async buildPersistedMaterialProgress( + agent: ManagedAgent, + ): Promise { + const rows = await this.resolveMaterialProgressRows(agent.id, { + limit: MATERIAL_PROGRESS_PAGE_SIZE, + historyPrimed: agent.historyPrimed, + }); + // Omitting the override makes AgentStorage preserve the last trustworthy classification. + if (rows === null) return undefined; return analyzeMaterialProgress({ - entries: rows === null ? null : projectTimelineRows({ rows, mode: "projected" }), + entries: projectTimelineRows({ rows, mode: "projected" }), turnOutcome: agent.lastTurnOutcome ?? null, }); } + private async resolveMaterialProgressRows( + agentId: string, + options: { limit: number; historyPrimed?: boolean }, + ): Promise { + const limit = Math.max(1, Math.min(Math.floor(options.limit), MATERIAL_PROGRESS_PAGE_SIZE)); + if (this.timelineStore.has(agentId)) { + const livePage = this.timelineStore.fetch(agentId, { direction: "tail", limit }); + if (livePage.rows.length > 0) { + const liveRows = await pageMaterialProgressRows(livePage, async (fetchOptions) => + this.timelineStore.fetch(agentId, fetchOptions), + ); + if (liveRows !== null) return liveRows; + + const durableRows = await this.resolveDurableMaterialProgressRows(agentId, limit); + if (durableRows === null) return null; + const firstLiveSeq = livePage.rows[0]?.seq; + const olderDurableRows = durableRows.filter((row) => row.seq < (firstLiveSeq ?? 0)); + if ( + firstLiveSeq === undefined || + olderDurableRows[olderDurableRows.length - 1]?.seq !== firstLiveSeq - 1 + ) { + return null; + } + return currentContinuationRows([...olderDurableRows, ...livePage.rows]); + } + + const durableRows = await this.resolveDurableMaterialProgressRows(agentId, limit); + if (durableRows && durableRows.length > 0) return durableRows; + if (livePage.window.nextSeq > 1 || options.historyPrimed === false) return null; + return []; + } + + return await this.resolveDurableMaterialProgressRows(agentId, limit); + } + + private async resolveDurableMaterialProgressRows( + agentId: string, + limit: number, + ): Promise { + const durableTimelineStore = this.durableTimelineStore; + if (!durableTimelineStore) return null; + try { + const durablePage = await durableTimelineStore.fetchCommitted(agentId, { + direction: "tail", + limit, + }); + return await pageMaterialProgressRows(durablePage, (fetchOptions) => + durableTimelineStore.fetchCommitted(agentId, fetchOptions), + ); + } catch (error) { + this.logger.debug( + { err: error, agentId }, + "Durable material progress timeline is unavailable", + ); + return null; + } + } + private requireRegistry(): AgentStorage { if (!this.registry) { throw new Error("Agent storage unavailable"); diff --git a/packages/server/src/server/agent/agent-storage.test.ts b/packages/server/src/server/agent/agent-storage.test.ts index 54fa0f6d60..fc2e88d3cc 100644 --- a/packages/server/src/server/agent/agent-storage.test.ts +++ b/packages/server/src/server/agent/agent-storage.test.ts @@ -342,6 +342,75 @@ describe("AgentStorage", () => { expect(persisted?.title).toBe("Fix Login Bug"); }); + test("setTitle started before a newer snapshot cannot overwrite its material progress", async () => { + const agentId = "agent-title-progress-race"; + await storage.applySnapshot(createManagedAgent({ id: agentId }), { + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "One compaction completed without later material progress.", + }, + }); + + let releasePendingWrite: (() => void) | null = null; + const pendingWrite = new Promise((resolve) => { + releasePendingWrite = resolve; + }); + const storageInternals = storage as unknown as { + pendingWrites: Map>; + }; + storageInternals.pendingWrites.set(agentId, pendingWrite); + + const titleWrite = storage.setTitle(agentId, "Final title"); + const newerSnapshot = storage.applySnapshot( + createManagedAgent({ + id: agentId, + updatedAt: new Date("2026-07-31T00:02:00.000Z"), + }), + { + materialProgress: { + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: "2026-07-31T00:01:00.000Z", + lastMaterialProgressKind: "write", + reason: "Material progress followed the latest user message.", + }, + }, + ); + releasePendingWrite?.(); + + await Promise.all([titleWrite, newerSnapshot]); + expect(await storage.get(agentId)).toMatchObject({ + title: "Final title", + materialProgress: { + state: "progressing", + lastMaterialProgressKind: "write", + }, + }); + }); + + test("setTitle still runs after an earlier queued write fails", async () => { + const agentId = "agent-title-after-failure"; + await storage.applySnapshot(createManagedAgent({ id: agentId })); + + let rejectFailedWrite: ((reason: Error) => void) | null = null; + const failedWrite = new Promise((_resolve, reject) => { + rejectFailedWrite = reject; + }); + const storageInternals = storage as unknown as { + pendingWrites: Map>; + }; + storageInternals.pendingWrites.set(agentId, failedWrite); + + const titleWrite = storage.setTitle(agentId, "Recovered title"); + rejectFailedWrite?.(new Error("simulated earlier write failure")); + + await expect(titleWrite).resolves.toBeUndefined(); + expect((await storage.get(agentId))?.title).toBe("Recovered title"); + }); + test("setTitle throws when the agent record does not exist", async () => { await expect(storage.setTitle("missing-agent", "Impossible")).rejects.toThrow( "Agent missing-agent not found", diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 1f0be9caed..cc9f7e9c18 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -142,7 +142,7 @@ export class AgentStorage { agentId: string, buildRecord: (existing: StoredAgentRecord | null) => StoredAgentRecord, ): Promise { - const prev = this.pendingWrites.get(agentId) ?? Promise.resolve(); + const prev = (this.pendingWrites.get(agentId) ?? Promise.resolve()).catch(() => undefined); const next = prev.then(async () => { if (this.deleting.has(agentId)) { return undefined; @@ -253,12 +253,12 @@ export class AgentStorage { async setTitle(agentId: string, title: string): Promise { await this.load(); - await this.waitForPendingWrite(agentId); - const record = await this.get(agentId); - if (!record) { - throw new Error(`Agent ${agentId} not found`); - } - await this.upsert({ ...record, title }); + await this.queueRecordMutation(agentId, (existing) => { + if (!existing) { + throw new Error(`Agent ${agentId} not found`); + } + return { ...existing, title }; + }); } async flush(): Promise { @@ -410,9 +410,6 @@ export class AgentStorage { this.daemonExecutionKeysByAgentId.delete(agentId); } - private async waitForPendingWrite(agentId: string): Promise { - await (this.pendingWrites.get(agentId) ?? Promise.resolve()).catch(() => undefined); - } } function projectDirNameFromCwd(cwd: string): string { diff --git a/packages/server/src/server/agent/material-progress.test.ts b/packages/server/src/server/agent/material-progress.test.ts index dced4baa51..94687a08ce 100644 --- a/packages/server/src/server/agent/material-progress.test.ts +++ b/packages/server/src/server/agent/material-progress.test.ts @@ -146,7 +146,7 @@ describe("analyzeMaterialProgress", () => { }); }); - it("counts only the final assistant message from a completed turn", () => { + it("counts a completed turn's final message as a deliverable, not semantic success", () => { const entries = [ entry(1, { type: "user_message", text: "answer" }), entry(2, { type: "assistant_message", text: "I am checking." }), @@ -155,11 +155,20 @@ describe("analyzeMaterialProgress", () => { ]; expect(analyzeMaterialProgress({ entries, turnOutcome: null }).state).toBe("warning"); + expect(analyzeMaterialProgress({ entries, turnOutcome: "failed" }).state).toBe("warning"); expect(analyzeMaterialProgress({ entries, turnOutcome: "canceled" }).state).toBe("warning"); expect(analyzeMaterialProgress({ entries, turnOutcome: "completed" })).toMatchObject({ state: "progressing", completedCompactionsSinceMaterialProgress: 0, lastMaterialProgressKind: "assistant_result", }); + + const emptyFinalMessage = [...entries, entry(5, { type: "assistant_message", text: " " })]; + expect( + analyzeMaterialProgress({ entries: emptyFinalMessage, turnOutcome: "completed" }), + ).toMatchObject({ + state: "warning", + lastMaterialProgressKind: null, + }); }); }); From 80a6f595a81982674b5d9ea126a715e6a778be00 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 05:38:46 +0100 Subject: [PATCH 05/17] style: satisfy material progress quality gates --- docs/data-model.md | 2 +- .../src/server/agent/agent-manager.test.ts | 24 +++++++------------ .../server/src/server/agent/agent-manager.ts | 15 ++++++++---- .../server/src/server/agent/agent-storage.ts | 1 - 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/docs/data-model.md b/docs/data-model.md index 69e214606b..1f75424510 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -97,7 +97,7 @@ Each agent is stored as a separate JSON file, grouped by project directory. | `persistence` | `PersistenceHandle?` | Handle for resuming sessions | | `lastError` | `string?` (nullable) | Last error message, if any | | `lastTurnOutcome` | `"completed" \| "failed" \| "canceled"?` | Last terminal turn result. Cleared when a new turn starts. | -| `materialProgress` | `MaterialProgressPayload?` | Last derived material-progress classification, retained when the live timeline is unavailable after archive or daemon restart. | +| `materialProgress` | `MaterialProgressPayload?` | Last derived material-progress classification, retained when the live timeline is unavailable after archive or daemon restart. | | `requiresAttention` | `boolean?` | Whether the agent needs user attention | | `attentionReason` | `"finished" \| "error" \| "permission"?` | Why attention is needed | | `attentionTimestamp` | `string?` (ISO 8601) | When attention was flagged | diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 481a7682b3..f9399b8335 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -3998,11 +3998,9 @@ test("a resumed agent with an empty live timeline derives progress from its dura }); try { - const created = await firstManager.createAgent( - { provider: "codex", cwd: workdir }, - undefined, - { workspaceId: undefined }, - ); + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); await firstManager.closeAgent(created.id); await durableTimelineStore.bulkInsert(created.id, [ timelineRow(1, { type: "user_message", text: "deliver this" }), @@ -4098,11 +4096,9 @@ test("material progress pages beyond the durable 1000-row tail to the current co }); try { - const created = await firstManager.createAgent( - { provider: "codex", cwd: workdir }, - undefined, - { workspaceId: undefined }, - ); + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); await firstManager.closeAgent(created.id); const durableRows: AgentTimelineRow[] = [ @@ -4183,11 +4179,9 @@ test("a delayed material progress write cannot overwrite a newer continuation", }); try { - const created = await firstManager.createAgent( - { provider: "codex", cwd: workdir }, - undefined, - { workspaceId: undefined }, - ); + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); await firstManager.closeAgent(created.id); await durableTimelineStore.bulkInsert(created.id, [ timelineRow(1, { type: "user_message", text: "old continuation" }), diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index df6baa91d9..b47c43ec19 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -158,10 +158,12 @@ async function pageMaterialProgressRows( fetchOlder: FetchOlderMaterialProgressRows, ): Promise { let page = initialPage; - let rows = [...page.rows]; - while (page.hasOlder && !rows.some((row) => row.item.type === "user_message")) { - const firstRow = rows[0]; - const remainingCapacity = MATERIAL_PROGRESS_MAX_SCAN_ROWS - rows.length; + const pageRows = [page.rows]; + let rowCount = page.rows.length; + let hasUserMessage = page.rows.some((row) => row.item.type === "user_message"); + while (page.hasOlder && !hasUserMessage) { + const firstRow = pageRows[0]?.[0]; + const remainingCapacity = MATERIAL_PROGRESS_MAX_SCAN_ROWS - rowCount; if (!firstRow || remainingCapacity <= 0) return null; const olderPage = await fetchOlder({ @@ -178,9 +180,12 @@ async function pageMaterialProgressRows( ) { return null; } - rows = [...olderPage.rows, ...rows]; + pageRows.unshift(olderPage.rows); + rowCount += olderPage.rows.length; + hasUserMessage = olderPage.rows.some((row) => row.item.type === "user_message"); page = olderPage; } + const rows = pageRows.flat(); if (!rows.some((row) => row.item.type === "user_message") && rows[0]?.seq !== 1) { return null; } diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index cc9f7e9c18..d76f7fb567 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -409,7 +409,6 @@ export class AgentStorage { } this.daemonExecutionKeysByAgentId.delete(agentId); } - } function projectDirNameFromCwd(cwd: string): string { From 3894220e0c40d0e90f8630baca62a257a73dd4bc Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 20:54:01 -0400 Subject: [PATCH 06/17] fix: bound material progress schema type --- packages/protocol/src/messages.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 73361ece74..83ae6a93f9 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -685,7 +685,15 @@ const AgentRuntimeInfoSchema: z.ZodType = z.object({ extra: z.record(z.string(), z.unknown()).optional(), }); -export const MaterialProgressPayloadSchema = z.object({ +export interface MaterialProgressPayload { + state: "none" | "progressing" | "warning" | "stalled"; + completedCompactionsSinceMaterialProgress: number; + lastMaterialProgressAt: string | null; + lastMaterialProgressKind: "edit" | "write" | "assistant_result" | null; + reason: string; +} + +export const MaterialProgressPayloadSchema: z.ZodType = z.object({ state: z.enum(["none", "progressing", "warning", "stalled"]), completedCompactionsSinceMaterialProgress: z.number().int().nonnegative(), lastMaterialProgressAt: z.string().nullable(), @@ -693,8 +701,6 @@ export const MaterialProgressPayloadSchema = z.object({ reason: z.string(), }); -export type MaterialProgressPayload = z.infer; - export const AgentSnapshotPayloadSchema = z.object({ id: z.string(), provider: AgentProviderSchema, From a30fc062127a273340be59759c2dd5a134eb645f Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 20:54:12 -0400 Subject: [PATCH 07/17] docs: scope material progress patch inventory --- patches/local-patches.json | 42 +++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index cecb16d264..8bfa3d6b4f 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -9,13 +9,49 @@ "commit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b" }, "tests": [ + "packages/cli/src/commands/agent/inspect.test.ts", "packages/protocol/src/material-progress-schema.test.ts", - "packages/server/src/server/agent/material-progress.test.ts", "packages/server/src/server/agent/agent-manager.test.ts", - "packages/server/src/server/material-progress.e2e.test.ts", - "packages/cli/src/commands/agent/inspect.test.ts" + "packages/server/src/server/agent/agent-storage.test.ts", + "packages/server/src/server/agent/material-progress.test.ts", + "packages/server/src/server/material-progress.e2e.test.ts" + ], + "sources": [ + "packages/cli/src/commands/agent/inspect.ts", + "packages/protocol/src/messages.ts", + "packages/server/src/server/agent/agent-manager.ts", + "packages/server/src/server/agent/agent-projections.ts", + "packages/server/src/server/agent/agent-storage.ts", + "packages/server/src/server/agent/material-progress.ts", + "packages/server/src/server/persistence-hooks.ts", + "packages/server/src/server/session.ts" + ], + "docs": ["docs/data-model.md"], + "testCases": [ + "preserves the structured signal and renders its state and reason", + "accepts legacy snapshots without the optional signal", + "accepts a structured material progress signal", + "an unhydrated resumed agent exposes its persisted material progress", + "a resumed agent with an empty live timeline derives progress from its durable tail", + "material progress pages beyond the durable 1000-row tail to the current continuation", + "a delayed material progress write cannot overwrite a newer continuation", + "persists terminal outcomes and clears the previous outcome when a new turn starts", + "concurrent snapshots preserve a newly derived material progress signal", + "setTitle started before a newer snapshot cannot overwrite its material progress", + "setTitle still runs after an earlier queued write fails", + "reports none when history or a current continuation is unavailable", + "treats completed edits and writes as material progress", + "warns after one compaction and stalls after two", + "uses only the latest user continuation and completion sequence", + "does not count read-only tools or commentary as material progress", + "counts a completed turn's final message as a deliverable, not semantic success", + "fetch agent exposes a backward-compatible material progress signal", + "fetch agent preserves material progress after the live worker is collected", + "fetch agent preserves material progress after archive discards the live timeline", + "fetch agent restores material progress after daemon restart" ], "upstreamable": true, + "upstreamCondition": "Submit independently when upstream still lacks singular material-progress inspection and persisted terminal-turn semantics on the latest stable base.", "dropCondition": "Remove when the official stable release exposes equivalent singular agent material-progress inspection with persisted terminal-turn semantics.", "deployedReceipt": null, "deployedAt": null From 6c71167c459c8abae6b6e0e1a38df82fb2bdfe22 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:25:32 -0400 Subject: [PATCH 08/17] fix: prefer live turn outcome state --- packages/server/src/server/agent/agent-manager.test.ts | 4 ++++ packages/server/src/server/agent/agent-manager.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index f9399b8335..62e03985d4 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -4293,6 +4293,10 @@ test("persists terminal outcomes and clears the previous outcome when a new turn expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBeNull(); expect((await storage.get(agent.id))?.lastTurnOutcome ?? null).toBeNull(); + const staleRecord = await storage.get(agent.id); + await storage.upsert({ ...staleRecord!, lastTurnOutcome: "completed" }); + expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBeNull(); + session.pushEvent({ type: "turn_canceled", provider: "codex", diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index b47c43ec19..0232533f46 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -1053,7 +1053,7 @@ export class AgentManager { ]); return { rows, - turnOutcome: live?.lastTurnOutcome ?? record?.lastTurnOutcome ?? null, + turnOutcome: live ? (live.lastTurnOutcome ?? null) : (record?.lastTurnOutcome ?? null), persisted: record?.materialProgress ?? null, }; } From 409757195caa57b5bbd896c11d822cdddf461aad Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:26:06 -0400 Subject: [PATCH 09/17] docs: record live outcome precedence proof --- patches/local-patches.json | 1 + 1 file changed, 1 insertion(+) diff --git a/patches/local-patches.json b/patches/local-patches.json index 8bfa3d6b4f..901bd44f17 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -36,6 +36,7 @@ "material progress pages beyond the durable 1000-row tail to the current continuation", "a delayed material progress write cannot overwrite a newer continuation", "persists terminal outcomes and clears the previous outcome when a new turn starts", + "a live null turn outcome overrides stale persisted completion while a new turn runs", "concurrent snapshots preserve a newly derived material progress signal", "setTitle started before a newer snapshot cannot overwrite its material progress", "setTitle still runs after an earlier queued write fails", From a045df727f6a161c17998714fcd1322f90bba032 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 23:04:04 -0400 Subject: [PATCH 10/17] fix: bind progress to accepted continuation --- docs/data-model.md | 53 ++++----- .../src/server/agent/agent-manager.test.ts | 36 +++++- .../server/src/server/agent/agent-manager.ts | 105 ++++++++++++++---- .../src/server/agent/agent-projections.ts | 5 + .../server/src/server/agent/agent-storage.ts | 1 + .../server/agent/material-progress.test.ts | 26 +++++ .../src/server/agent/material-progress.ts | 30 +++-- .../src/server/material-progress.e2e.test.ts | 69 ++++++++++++ .../server/src/server/persistence-hooks.ts | 2 + packages/server/src/server/session.ts | 3 + patches/local-patches.json | 2 + 11 files changed, 273 insertions(+), 59 deletions(-) diff --git a/docs/data-model.md b/docs/data-model.md index 65ae6a1e50..f9fd691966 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -78,32 +78,33 @@ The `agents/{sanitized-cwd}/` directory name is derived from the agent's `cwd` b Each agent is stored as a separate JSON file, grouped by project directory. -| Field | Type | Description | -| -------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `id` | `string` | UUID, primary key | -| `provider` | `string` | Agent provider (`"claude"`, `"codex"`, `"opencode"`, etc.) | -| `cwd` | `string` | Working directory the agent operates in | -| `workspaceId` | `string?` | Owning workspace id — the single source of ownership. Every agent is stamped with one at create time; legacy cwd-only records are backfilled once by `migrations/backfill-workspace-id.migration.ts` (the only place a cwd→id mapping exists). Runtime code never infers ownership or status from cwd: status is computed per `workspaceId`, and same-cwd siblings are independent. | -| `createdAt` | `string` (ISO 8601) | Creation timestamp | -| `updatedAt` | `string` (ISO 8601) | Last update timestamp | -| `lastActivityAt` | `string?` (ISO 8601) | Last activity timestamp | -| `lastUserMessageAt` | `string?` (ISO 8601) | Last user message timestamp | -| `title` | `string?` | User-visible title | -| `labels` | `Record` | Key-value labels (default `{}`). `paseo.parent-agent-id` is set automatically for agent-scoped creation and removed by detach — see [agent-lifecycle.md](./agent-lifecycle.md) | -| `lastStatus` | `AgentStatus` | One of: `"initializing"`, `"idle"`, `"running"`, `"error"`, `"closed"`. `closed` means the record is resumable but has no live provider runtime; archive remains represented separately by `archivedAt`. | -| `lastModeId` | `string?` | Last active mode ID | -| `config` | `SerializableConfig?` | Agent session configuration (see below) | -| `runtimeInfo` | `RuntimeInfo?` | Live runtime state (see below) | -| `features` | `AgentFeature[]?` | Provider-reported features (toggles/selects) | -| `persistence` | `PersistenceHandle?` | Handle for resuming sessions | -| `lastError` | `string?` (nullable) | Last error message, if any | -| `lastTurnOutcome` | `"completed" \| "failed" \| "canceled"?` | Last terminal turn result. Cleared when a new turn starts. | -| `materialProgress` | `MaterialProgressPayload?` | Last derived material-progress classification, retained when the live timeline is unavailable after archive or daemon restart. | -| `requiresAttention` | `boolean?` | Whether the agent needs user attention | -| `attentionReason` | `"finished" \| "error" \| "permission"?` | Why attention is needed | -| `attentionTimestamp` | `string?` (ISO 8601) | When attention was flagged | -| `internal` | `boolean?` | Whether this is a system-internal agent (loop workers, etc.) | -| `archivedAt` | `string?` (ISO 8601) | Soft-delete timestamp | +| Field | Type | Description | +| ----------------------------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | `string` | UUID, primary key | +| `provider` | `string` | Agent provider (`"claude"`, `"codex"`, `"opencode"`, etc.) | +| `cwd` | `string` | Working directory the agent operates in | +| `workspaceId` | `string?` | Owning workspace id — the single source of ownership. Every agent is stamped with one at create time; legacy cwd-only records are backfilled once by `migrations/backfill-workspace-id.migration.ts` (the only place a cwd→id mapping exists). Runtime code never infers ownership or status from cwd: status is computed per `workspaceId`, and same-cwd siblings are independent. | +| `createdAt` | `string` (ISO 8601) | Creation timestamp | +| `updatedAt` | `string` (ISO 8601) | Last update timestamp | +| `lastActivityAt` | `string?` (ISO 8601) | Last activity timestamp | +| `lastUserMessageAt` | `string?` (ISO 8601) | Last user message timestamp | +| `title` | `string?` | User-visible title | +| `labels` | `Record` | Key-value labels (default `{}`). `paseo.parent-agent-id` is set automatically for agent-scoped creation and removed by detach — see [agent-lifecycle.md](./agent-lifecycle.md) | +| `lastStatus` | `AgentStatus` | One of: `"initializing"`, `"idle"`, `"running"`, `"error"`, `"closed"`. `closed` means the record is resumable but has no live provider runtime; archive remains represented separately by `archivedAt`. | +| `lastModeId` | `string?` | Last active mode ID | +| `config` | `SerializableConfig?` | Agent session configuration (see below) | +| `runtimeInfo` | `RuntimeInfo?` | Live runtime state (see below) | +| `features` | `AgentFeature[]?` | Provider-reported features (toggles/selects) | +| `persistence` | `PersistenceHandle?` | Handle for resuming sessions | +| `lastError` | `string?` (nullable) | Last error message, if any | +| `lastTurnOutcome` | `"completed" \| "failed" \| "canceled"?` | Last terminal turn result. Cleared when a new turn starts. | +| `materialProgressContinuationBoundarySeq` | `number?` (positive integer) | Manager-owned timeline boundary for the latest accepted continuation. This is persisted before a running turn is published, so inspection does not depend on a provider having written a user-message row. Legacy records omit it and use the latest user message. | +| `materialProgress` | `MaterialProgressPayload?` | Last derived material-progress classification, retained when the live timeline is unavailable after archive or daemon restart. | +| `requiresAttention` | `boolean?` | Whether the agent needs user attention | +| `attentionReason` | `"finished" \| "error" \| "permission"?` | Why attention is needed | +| `attentionTimestamp` | `string?` (ISO 8601) | When attention was flagged | +| `internal` | `boolean?` | Whether this is a system-internal agent (loop workers, etc.) | +| `archivedAt` | `string?` (ISO 8601) | Soft-delete timestamp | `materialProgress.lastMaterialProgressKind: "assistant_result"` means a completed turn ended with a non-empty assistant message. It records a terminal deliverable, not whether the answer was diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 62e03985d4..3a3439023a 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -4245,7 +4245,20 @@ test("persists terminal outcomes and clears the previous outcome when a new turn const session = new (class extends TestAgentSession { override async startTurn(): Promise<{ turnId: string }> { nextTurn += 1; - return { turnId: `held-turn-${nextTurn}` }; + const turnId = `held-turn-${nextTurn}`; + if (nextTurn === 2) { + // OpenCode can publish accepted-start and timeline events before startTurn resolves. + // The post-resolution manager path must not move the boundary past those events. + this.pushEvent({ type: "turn_started", provider: "codex", turnId }); + this.pushEvent({ + type: "timeline", + provider: "codex", + turnId, + item: { type: "reasoning", text: "provider accepted the continuation" }, + }); + await new Promise((resolve) => setImmediate(resolve)); + } + return { turnId }; } })({ provider: "codex", cwd: workdir }); const client = new (class extends TestAgentClient { @@ -4282,6 +4295,19 @@ test("persists terminal outcomes and clears the previous outcome when a new turn expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBe("completed"); expect((await storage.get(agent.id))?.lastTurnOutcome).toBe("completed"); + await manager.appendTimelineItem(agent.id, { + type: "user_message", + text: "previous continuation", + }); + await manager.appendTimelineItem(agent.id, { + type: "compaction", + status: "completed", + }); + await manager.appendTimelineItem(agent.id, { + type: "compaction", + status: "completed", + }); + const secondRunning = waitForAgentLifecycle(manager, agent.id, "running"); const secondTurn = (async () => { for await (const _event of manager.streamAgent(agent.id, "second turn")) { @@ -4290,7 +4316,13 @@ test("persists terminal outcomes and clears the previous outcome when a new turn })(); await secondRunning; await manager.flush(); - expect((await manager.getMaterialProgressSnapshot(agent.id)).turnOutcome).toBeNull(); + const secondTurnSnapshot = await manager.getMaterialProgressSnapshot(agent.id); + expect(secondTurnSnapshot.turnOutcome).toBeNull(); + expect(secondTurnSnapshot.continuationBoundarySeq).toBe(4); + expect(await storage.get(agent.id)).toMatchObject({ + materialProgressContinuationBoundarySeq: 4, + materialProgress: { state: "none" }, + }); expect((await storage.get(agent.id))?.lastTurnOutcome ?? null).toBeNull(); const staleRecord = await storage.get(agent.id); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 0232533f46..2d1554693b 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -346,6 +346,7 @@ type AgentTurnOutcome = "completed" | "failed" | "canceled"; export interface MaterialProgressSnapshot { rows: AgentTimelineRow[] | null; turnOutcome: AgentTurnOutcome | null; + continuationBoundarySeq: number | null; persisted: MaterialProgressPayload | null; } @@ -401,6 +402,9 @@ interface ManagedAgentBase { lastUsage?: AgentUsage; lastError?: string; lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; + materialProgressContinuationActive?: boolean; + materialProgressContinuationTurnId?: string | null; attention: AttentionState; foregroundTurnWaiters: Set; finalizedForegroundTurnIds: Set; @@ -1054,6 +1058,9 @@ export class AgentManager { return { rows, turnOutcome: live ? (live.lastTurnOutcome ?? null) : (record?.lastTurnOutcome ?? null), + continuationBoundarySeq: live + ? (live.materialProgressContinuationBoundarySeq ?? null) + : (record?.materialProgressContinuationBoundarySeq ?? null), persisted: record?.materialProgress ?? null, }; } @@ -1149,6 +1156,7 @@ export class AgentManager { workspaceId?: string; owner?: AgentOwner; lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -1169,6 +1177,7 @@ export class AgentManager { workspaceId?: string; owner?: AgentOwner; lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -1314,6 +1323,8 @@ export class AgentManager { const preservedLastUsage = existing.lastUsage; const preservedLastError = existing.lastError; const preservedLastTurnOutcome = existing.lastTurnOutcome; + const preservedMaterialProgressContinuationBoundarySeq = + existing.materialProgressContinuationBoundarySeq; const preservedAttention = existing.attention; const handle = existing.persistence; const provider = handle?.provider ?? existing.provider; @@ -1366,6 +1377,7 @@ export class AgentManager { lastUsage: preservedLastUsage, lastError: preservedLastError, lastTurnOutcome: preservedLastTurnOutcome, + materialProgressContinuationBoundarySeq: preservedMaterialProgressContinuationBoundarySeq, attention: preservedAttention, }); } finally { @@ -1624,6 +1636,7 @@ export class AgentManager { lastUsage: undefined, lastError: record.lastError ?? undefined, lastTurnOutcome: record.lastTurnOutcome, + materialProgressContinuationBoundarySeq: record.materialProgressContinuationBoundarySeq, attention: { requiresAttention: false }, internal: record.internal, labels: record.labels, @@ -2100,9 +2113,12 @@ export class AgentManager { } agent.activeForegroundTurnId = turnId; agent.lifecycle = "running"; - agent.lastTurnOutcome = null; + this.beginMaterialProgressContinuation(agent, turnId); + turnStream = this.runs.createTurnStream(turnId); + this.runs.addWaiter(agent, turnStream.waiter); this.touchUpdatedAt(agent); - this.emitState(agent); + await this.persistSnapshot(agent); + this.emitState(agent, { persist: false }); this.logger.trace( { agentId, @@ -2115,9 +2131,6 @@ export class AgentManager { "agent.manager.stream.start", ); - turnStream = this.runs.createTurnStream(turnId); - this.runs.addWaiter(agent, turnStream.waiter); - try { for await (const event of turnStream.events(isTurnTerminalEvent)) { yield event; @@ -2766,6 +2779,7 @@ export class AgentManager { lastUsage?: AgentUsage; lastError?: string; lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; attention?: AttentionState; initialTitle?: string | null; publishWhenReady?: boolean; @@ -2907,6 +2921,7 @@ export class AgentManager { lastUsage?: AgentUsage; lastError?: string; lastTurnOutcome?: AgentTurnOutcome | null; + materialProgressContinuationBoundarySeq?: number | null; attention?: AttentionState; persistence?: AgentPersistenceHandle; workspaceId?: string; @@ -2915,19 +2930,20 @@ export class AgentManager { | undefined; }): ActiveManagedAgent { const { resolvedAgentId, session, config, now, durableTimelineHasRows, options } = params; + const resolvedOptions = options ?? {}; return { id: resolvedAgentId, provider: config.provider, cwd: config.cwd, - workspaceId: options?.workspaceId, - owner: options?.owner, + workspaceId: resolvedOptions.workspaceId, + owner: resolvedOptions.owner, session, capabilities: session.capabilities, config, runtimeInfo: undefined, lifecycle: "initializing", - createdAt: options?.createdAt ?? now, - updatedAt: options?.updatedAt ?? now, + createdAt: resolvedOptions.createdAt ?? now, + updatedAt: resolvedOptions.updatedAt ?? now, availableModes: [], currentModeId: null, pendingPermissions: new Map(), @@ -2939,17 +2955,21 @@ export class AgentManager { finalizedForegroundTurnIds: new Set(), unsubscribeSession: null, persistence: attachPersistenceCwd( - options?.persistence ?? session.describePersistence(), + resolvedOptions.persistence ?? session.describePersistence(), config.cwd, ), - historyPrimed: options?.historyPrimed ?? durableTimelineHasRows, - lastUserMessageAt: options?.lastUserMessageAt ?? null, - lastUsage: options?.lastUsage, - lastError: options?.lastError, - lastTurnOutcome: options?.lastTurnOutcome, - attention: resolveInitialAttention(options?.attention), + historyPrimed: resolvedOptions.historyPrimed ?? durableTimelineHasRows, + lastUserMessageAt: resolvedOptions.lastUserMessageAt ?? null, + lastUsage: resolvedOptions.lastUsage, + lastError: resolvedOptions.lastError, + lastTurnOutcome: resolvedOptions.lastTurnOutcome, + materialProgressContinuationBoundarySeq: + resolvedOptions.materialProgressContinuationBoundarySeq, + materialProgressContinuationActive: false, + materialProgressContinuationTurnId: null, + attention: resolveInitialAttention(resolvedOptions.attention), internal: config.internal ?? false, - labels: options?.labels ?? {}, + labels: resolvedOptions.labels ?? {}, } as ActiveManagedAgent; } @@ -3200,6 +3220,7 @@ export class AgentManager { return analyzeMaterialProgress({ entries: projectTimelineRows({ rows, mode: "projected" }), turnOutcome: agent.lastTurnOutcome ?? null, + continuationBoundarySeq: agent.materialProgressContinuationBoundarySeq ?? null, }); } @@ -3671,8 +3692,7 @@ export class AgentManager { this.onStreamTurnCanceled({ agent, event, eventTurnId, isForegroundEvent, options }); return undefined; case "turn_started": - this.onStreamTurnStarted({ agent, eventTurnId, isForegroundEvent }); - return undefined; + return this.onStreamTurnStarted({ agent, eventTurnId, isForegroundEvent, options }); case "permission_requested": this.onStreamPermissionRequested(agent, event); return undefined; @@ -3757,6 +3777,7 @@ export class AgentManager { // it from the completion event. agent.lastError = undefined; agent.lastTurnOutcome = "completed"; + this.endMaterialProgressContinuation(agent); if (!isForegroundEvent && agent.lifecycle !== "idle" && !agent.pendingReplacement) { (agent as ActiveManagedAgent).lifecycle = "idle"; this.emitState(agent); @@ -3792,6 +3813,7 @@ export class AgentManager { } agent.lastError = event.error; agent.lastTurnOutcome = "failed"; + this.endMaterialProgressContinuation(agent); await this.appendSystemErrorTimelineMessage( agent, event.provider, @@ -3833,19 +3855,25 @@ export class AgentManager { } agent.lastError = undefined; agent.lastTurnOutcome = "canceled"; + this.endMaterialProgressContinuation(agent); this.resolvePendingPermissionsForAgent(agent, event.provider, options, "Interrupted"); if (!isForegroundEvent) { this.emitState(agent); } } - private onStreamTurnStarted(params: { + private async onStreamTurnStarted(params: { agent: ActiveManagedAgent; eventTurnId: string | undefined; isForegroundEvent: boolean; - }): void { - const { agent, eventTurnId, isForegroundEvent } = params; - agent.lastTurnOutcome = null; + options: HandleStreamEventOptions | undefined; + }): Promise { + const { agent, eventTurnId, isForegroundEvent, options } = params; + if (options?.fromHistory) { + agent.lastTurnOutcome = null; + return; + } + this.beginMaterialProgressContinuation(agent, eventTurnId); this.logger.trace( { agentId: agent.id, @@ -3860,8 +3888,37 @@ export class AgentManager { if (!isForegroundEvent) { this.runs.trackAutonomousRun(agent.id, eventTurnId ?? null); agent.lifecycle = "running"; - this.emitState(agent); + await this.persistSnapshot(agent); + this.emitState(agent, { persist: false }); + return; + } + await this.persistSnapshot(agent); + } + + private beginMaterialProgressContinuation( + agent: ActiveManagedAgent, + turnId: string | undefined, + ): void { + if (agent.materialProgressContinuationActive) { + if (!agent.materialProgressContinuationTurnId && turnId) { + agent.materialProgressContinuationTurnId = turnId; + } + agent.lastTurnOutcome = null; + return; } + + agent.materialProgressContinuationBoundarySeq = this.timelineStore.fetch(agent.id, { + direction: "tail", + limit: 1, + }).window.nextSeq; + agent.materialProgressContinuationActive = true; + agent.materialProgressContinuationTurnId = turnId ?? null; + agent.lastTurnOutcome = null; + } + + private endMaterialProgressContinuation(agent: ActiveManagedAgent): void { + agent.materialProgressContinuationActive = false; + agent.materialProgressContinuationTurnId = null; } private onStreamPermissionRequested( diff --git a/packages/server/src/server/agent/agent-projections.ts b/packages/server/src/server/agent/agent-projections.ts index 9c8750e575..745c92f8ef 100644 --- a/packages/server/src/server/agent/agent-projections.ts +++ b/packages/server/src/server/agent/agent-projections.ts @@ -90,6 +90,11 @@ export function toStoredAgentRecord( persistence, lastError: agent.lastError ?? undefined, ...(agent.lastTurnOutcome ? { lastTurnOutcome: agent.lastTurnOutcome } : {}), + ...(agent.materialProgressContinuationBoundarySeq != null + ? { + materialProgressContinuationBoundarySeq: agent.materialProgressContinuationBoundarySeq, + } + : {}), ...(options?.materialProgress ? { materialProgress: options.materialProgress } : {}), requiresAttention: agent.attention.requiresAttention, attentionReason: agent.attention.requiresAttention ? agent.attention.attentionReason : null, diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index d76f7fb567..f6bb7507c9 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -66,6 +66,7 @@ const STORED_AGENT_SCHEMA = z.object({ persistence: PERSISTENCE_HANDLE_SCHEMA, lastError: z.string().nullable().optional(), lastTurnOutcome: z.enum(["completed", "failed", "canceled"]).nullable().optional(), + materialProgressContinuationBoundarySeq: z.number().int().positive().nullable().optional(), materialProgress: MaterialProgressPayloadSchema.optional(), requiresAttention: z.boolean().optional(), attentionReason: z.enum(["finished", "error", "permission"]).nullable().optional(), diff --git a/packages/server/src/server/agent/material-progress.test.ts b/packages/server/src/server/agent/material-progress.test.ts index 94687a08ce..f9f8379bb4 100644 --- a/packages/server/src/server/agent/material-progress.test.ts +++ b/packages/server/src/server/agent/material-progress.test.ts @@ -92,6 +92,32 @@ describe("analyzeMaterialProgress", () => { }); }); + it("does not inherit a previous continuation's compactions after a new turn is accepted", () => { + const previousContinuation = [ + entry(1, { type: "user_message", text: "old continuation" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { type: "compaction", status: "completed" }), + ]; + + expect( + analyzeMaterialProgress({ entries: previousContinuation, turnOutcome: null }), + ).toMatchObject({ state: "stalled" }); + + expect( + analyzeMaterialProgress({ + entries: previousContinuation, + turnOutcome: null, + continuationBoundarySeq: 4, + }), + ).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "No material progress has been recorded for the current continuation.", + }); + }); + it("uses only the latest user continuation and completion sequence", () => { const completedWrite = entry(2, { type: "tool_call", diff --git a/packages/server/src/server/agent/material-progress.ts b/packages/server/src/server/agent/material-progress.ts index 47b5c8683b..6d127c2009 100644 --- a/packages/server/src/server/agent/material-progress.ts +++ b/packages/server/src/server/agent/material-progress.ts @@ -5,6 +5,11 @@ import type { TimelineProjectionEntry } from "./timeline-projection.js"; export interface AnalyzeMaterialProgressInput { entries: readonly TimelineProjectionEntry[] | null; turnOutcome: "completed" | "failed" | "canceled" | null; + /** + * Manager-owned sequence at which the currently accepted continuation begins. + * Older persisted agents omit it and retain the latest-user-message fallback. + */ + continuationBoundarySeq?: number | null; } type MaterialProgressKind = NonNullable; @@ -44,6 +49,21 @@ function orderedEntries(entries: readonly TimelineProjectionEntry[]): TimelinePr ); } +function currentContinuation( + ordered: readonly TimelineProjectionEntry[], + continuationBoundarySeq: number | null | undefined, +): TimelineProjectionEntry[] | null { + if (continuationBoundarySeq != null) { + return ordered.filter((entry) => entry.seqEnd >= continuationBoundarySeq); + } + + let latestUserIndex = -1; + for (let index = 0; index < ordered.length; index += 1) { + if (ordered[index]?.item.type === "user_message") latestUserIndex = index; + } + return latestUserIndex < 0 ? null : ordered.slice(latestUserIndex + 1); +} + function noProgress(reason: string): MaterialProgressPayload { return { state: "none", @@ -68,17 +88,13 @@ function materialKind( export function analyzeMaterialProgress({ entries, turnOutcome, + continuationBoundarySeq, }: AnalyzeMaterialProgressInput): MaterialProgressPayload { if (entries === null) return noProgress("Timeline history is unavailable."); const ordered = orderedEntries(entries); - let latestUserIndex = -1; - for (let index = 0; index < ordered.length; index += 1) { - if (ordered[index]?.item.type === "user_message") latestUserIndex = index; - } - if (latestUserIndex < 0) return noProgress("No current continuation is available."); - - const continuation = ordered.slice(latestUserIndex + 1); + const continuation = currentContinuation(ordered, continuationBoundarySeq); + if (continuation === null) return noProgress("No current continuation is available."); let finalAssistantIndex = -1; if (turnOutcome === "completed") { for (let index = 0; index < continuation.length; index += 1) { diff --git a/packages/server/src/server/material-progress.e2e.test.ts b/packages/server/src/server/material-progress.e2e.test.ts index f3f6ec971a..b49f88d558 100644 --- a/packages/server/src/server/material-progress.e2e.test.ts +++ b/packages/server/src/server/material-progress.e2e.test.ts @@ -98,6 +98,75 @@ test("fetch agent preserves material progress after the live worker is collected } }); +test("fetch agent does not inherit a previous stall after a new turn is accepted", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-boundary-")); + let agentId: string | null = null; + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Accepted continuation boundary probe", + modeId: "default", + model: "gpt-5.4-mini", + }); + agentId = created.id; + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "user_message", + text: "previous continuation", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + + expect( + (await client.fetchAgent({ agentId: created.id }))?.agent.materialProgress, + ).toMatchObject({ + state: "stalled", + completedCompactionsSinceMaterialProgress: 2, + }); + + // The fake provider stops at a permission request after accepting the turn. It intentionally + // emits no user_message row, reproducing providers that acknowledge a continuation before + // (or without) projecting the prompt into timeline history. + await client.sendMessage(created.id, "read /etc/hosts"); + await expect + .poll(async () => (await client.fetchAgent({ agentId: created.id }))?.agent.status ?? null, { + timeout: 10_000, + interval: 25, + }) + .toBe("running"); + + const fetched = await client.fetchAgent({ agentId: created.id }); + expect(fetched?.agent.materialProgress).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "No material progress has been recorded for the current continuation.", + }); + expect( + (await daemon.daemon.agentManager.getMaterialProgressSnapshot(created.id)).rows, + ).toHaveLength(3); + } finally { + if (agentId) await client.cancelAgent(agentId).catch(() => undefined); + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + test("fetch agent preserves material progress after archive discards the live timeline", async () => { const daemon = await createTestPaseoDaemon(); const client = new DaemonClient({ diff --git a/packages/server/src/server/persistence-hooks.ts b/packages/server/src/server/persistence-hooks.ts index d286c9fb8c..86c7cec56b 100644 --- a/packages/server/src/server/persistence-hooks.ts +++ b/packages/server/src/server/persistence-hooks.ts @@ -112,6 +112,7 @@ export function extractTimestamps(record: StoredAgentRecord): { workspaceId?: string; owner?: StoredAgentRecord["owner"]; lastTurnOutcome?: StoredAgentRecord["lastTurnOutcome"]; + materialProgressContinuationBoundarySeq?: StoredAgentRecord["materialProgressContinuationBoundarySeq"]; } { return { createdAt: new Date(record.createdAt), @@ -121,6 +122,7 @@ export function extractTimestamps(record: StoredAgentRecord): { workspaceId: record.workspaceId, owner: record.owner, lastTurnOutcome: record.lastTurnOutcome, + materialProgressContinuationBoundarySeq: record.materialProgressContinuationBoundarySeq, }; } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index fcd1204f02..c545f4a620 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -4051,6 +4051,7 @@ export class Session { ): Promise { let entries: TimelineProjectionEntry[] | null = null; let turnOutcome: MaterialProgressSnapshot["turnOutcome"] = null; + let continuationBoundarySeq: number | null = null; let persisted = payload.materialProgress ?? null; try { const snapshot = await this.agentManager.getMaterialProgressSnapshot(payload.id); @@ -4059,6 +4060,7 @@ export class Session { ? null : projectTimelineRows({ rows: snapshot.rows, mode: "projected" }); turnOutcome = snapshot.turnOutcome; + continuationBoundarySeq = snapshot.continuationBoundarySeq; persisted = snapshot.persisted ?? persisted; } catch (error) { this.sessionLogger.debug( @@ -4074,6 +4076,7 @@ export class Session { : analyzeMaterialProgress({ entries, turnOutcome, + continuationBoundarySeq, }), }; } diff --git a/patches/local-patches.json b/patches/local-patches.json index 901bd44f17..3a2ade3919 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -43,11 +43,13 @@ "reports none when history or a current continuation is unavailable", "treats completed edits and writes as material progress", "warns after one compaction and stalls after two", + "does not inherit a previous continuation's compactions after a new turn is accepted", "uses only the latest user continuation and completion sequence", "does not count read-only tools or commentary as material progress", "counts a completed turn's final message as a deliverable, not semantic success", "fetch agent exposes a backward-compatible material progress signal", "fetch agent preserves material progress after the live worker is collected", + "fetch agent does not inherit a previous stall after a new turn is accepted", "fetch agent preserves material progress after archive discards the live timeline", "fetch agent restores material progress after daemon restart" ], From c1dcbffdabaf4cb520d7ccdbd56a295d49324033 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 23:28:38 -0400 Subject: [PATCH 11/17] test: cover failed continuation boundary --- .../src/server/material-progress.e2e.test.ts | 59 +++++++++++++++++++ patches/local-patches.json | 1 + 2 files changed, 60 insertions(+) diff --git a/packages/server/src/server/material-progress.e2e.test.ts b/packages/server/src/server/material-progress.e2e.test.ts index b49f88d558..a534e88e16 100644 --- a/packages/server/src/server/material-progress.e2e.test.ts +++ b/packages/server/src/server/material-progress.e2e.test.ts @@ -167,6 +167,65 @@ test("fetch agent does not inherit a previous stall after a new turn is accepted } }); +test("fetch agent does not inherit a previous stall when an accepted turn fails before a user row", async () => { + const daemon = await createTestPaseoDaemon(); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemon.port}/ws`, + appVersion: "0.2.5", + }); + const cwd = mkdtempSync(path.join(tmpdir(), "paseo-material-progress-failed-boundary-")); + + try { + await client.connect(); + const created = await client.createAgent({ + provider: "codex", + cwd, + title: "Failed continuation boundary probe", + modeId: "full-access", + model: "gpt-5.4-mini", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "user_message", + text: "previous continuation", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + await daemon.daemon.agentManager.appendTimelineItem(created.id, { + type: "compaction", + status: "completed", + }); + expect( + (await client.fetchAgent({ agentId: created.id }))?.agent.materialProgress, + ).toMatchObject({ state: "stalled" }); + + // The fake provider emits turn_started followed by turn_failed, without a user_message row. + await client.sendMessage(created.id, "emit a turn failure"); + await expect + .poll(async () => (await client.fetchAgent({ agentId: created.id }))?.agent.status ?? null, { + timeout: 10_000, + interval: 25, + }) + .toBe("error"); + + const snapshot = await daemon.daemon.agentManager.getMaterialProgressSnapshot(created.id); + expect(snapshot.turnOutcome).toBe("failed"); + expect(snapshot.continuationBoundarySeq).toBe(4); + expect((await client.fetchAgent({ agentId: created.id }))?.agent.materialProgress).toEqual({ + state: "none", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "No material progress has been recorded for the current continuation.", + }); + } finally { + await client.close(); + await daemon.close(); + rmSync(cwd, { recursive: true, force: true }); + } +}); + test("fetch agent preserves material progress after archive discards the live timeline", async () => { const daemon = await createTestPaseoDaemon(); const client = new DaemonClient({ diff --git a/patches/local-patches.json b/patches/local-patches.json index 3a2ade3919..bcbb7a45df 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -50,6 +50,7 @@ "fetch agent exposes a backward-compatible material progress signal", "fetch agent preserves material progress after the live worker is collected", "fetch agent does not inherit a previous stall after a new turn is accepted", + "fetch agent does not inherit a previous stall when an accepted turn fails before a user row", "fetch agent preserves material progress after archive discards the live timeline", "fetch agent restores material progress after daemon restart" ], From 65c98a30a74f8639c0827f7deec50c75b9546a5b Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 03:40:56 -0400 Subject: [PATCH 12/17] fix: classify material progress by results --- docs/data-model.md | 9 +- .../src/material-progress-schema.test.ts | 18 ++ packages/protocol/src/messages.ts | 13 +- .../server/agent/material-progress.test.ts | 216 +++++++++++++++++- .../src/server/agent/material-progress.ts | 132 +++++++++-- patches/local-patches.json | 64 ------ 6 files changed, 362 insertions(+), 90 deletions(-) delete mode 100644 patches/local-patches.json diff --git a/docs/data-model.md b/docs/data-model.md index f9fd691966..387f678428 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -106,10 +106,11 @@ Each agent is stored as a separate JSON file, grouped by project directory. | `internal` | `boolean?` | Whether this is a system-internal agent (loop workers, etc.) | | `archivedAt` | `string?` (ISO 8601) | Soft-delete timestamp | -`materialProgress.lastMaterialProgressKind: "assistant_result"` means a completed turn ended -with a non-empty assistant message. It records a terminal deliverable, not whether the answer was -semantically correct or the user's wider goal succeeded. Final messages from failed or canceled -turns are non-material; completed edits and writes remain material independently of that boundary. +`materialProgress.lastMaterialProgressKind` records completed edits and writes, concrete read/search/ +fetch evidence, successful shell verification with output, explicit decisions, and terminal assistant +deliverables. Repeating the same result does not reset the compaction count. The classification does +not decide whether evidence is relevant, a deliverable is correct, or the user's wider goal succeeded. +Final messages from failed or canceled turns are non-material. ### Nested: SerializableConfig diff --git a/packages/protocol/src/material-progress-schema.test.ts b/packages/protocol/src/material-progress-schema.test.ts index 472bf1e45a..7227f374ba 100644 --- a/packages/protocol/src/material-progress-schema.test.ts +++ b/packages/protocol/src/material-progress-schema.test.ts @@ -45,4 +45,22 @@ describe("material progress wire compatibility", () => { expect(parsed.materialProgress?.state).toBe("warning"); }); + + it.each(["evidence", "verification", "decision"] as const)( + "accepts the %s material progress kind", + (lastMaterialProgressKind) => { + const parsed = AgentSnapshotPayloadSchema.parse({ + ...baseSnapshot, + materialProgress: { + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressAt: "2026-07-31T00:00:01.000Z", + lastMaterialProgressKind, + reason: "Material progress followed the latest user message.", + }, + }); + + expect(parsed.materialProgress?.lastMaterialProgressKind).toBe(lastMaterialProgressKind); + }, + ); }); diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 70cd4de11f..8e296586cc 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -697,7 +697,14 @@ export interface MaterialProgressPayload { state: "none" | "progressing" | "warning" | "stalled"; completedCompactionsSinceMaterialProgress: number; lastMaterialProgressAt: string | null; - lastMaterialProgressKind: "edit" | "write" | "assistant_result" | null; + lastMaterialProgressKind: + | "edit" + | "write" + | "evidence" + | "verification" + | "decision" + | "assistant_result" + | null; reason: string; } @@ -705,7 +712,9 @@ export const MaterialProgressPayloadSchema: z.ZodType = state: z.enum(["none", "progressing", "warning", "stalled"]), completedCompactionsSinceMaterialProgress: z.number().int().nonnegative(), lastMaterialProgressAt: z.string().nullable(), - lastMaterialProgressKind: z.enum(["edit", "write", "assistant_result"]).nullable(), + lastMaterialProgressKind: z + .enum(["edit", "write", "evidence", "verification", "decision", "assistant_result"]) + .nullable(), reason: z.string(), }); diff --git a/packages/server/src/server/agent/material-progress.test.ts b/packages/server/src/server/agent/material-progress.test.ts index f9f8379bb4..0b541ad660 100644 --- a/packages/server/src/server/agent/material-progress.test.ts +++ b/packages/server/src/server/agent/material-progress.test.ts @@ -147,7 +147,175 @@ describe("analyzeMaterialProgress", () => { }); }); - it("does not count read-only tools or commentary as material progress", () => { + it("keeps a read-only audit progressing across two compactions", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "audit the authentication flow" }), + entry(2, { + type: "tool_call", + callId: "read-1", + name: "read", + status: "completed", + error: null, + detail: { type: "read", filePath: "src/auth.ts", content: "export function login() {}" }, + }), + entry(3, { type: "compaction", status: "completed" }), + entry(4, { + type: "tool_call", + callId: "search-1", + name: "grep", + status: "completed", + error: null, + detail: { type: "search", query: "login(", numFiles: 2, numMatches: 3 }, + }), + entry(5, { type: "compaction", status: "completed" }), + entry(6, { + type: "tool_call", + callId: "fetch-1", + name: "fetch", + status: "completed", + error: null, + detail: { + type: "fetch", + url: "https://example.com/auth-spec", + result: "Tokens expire after one hour.", + code: 200, + }, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "evidence", + }); + }); + + it("keeps test-only verification progressing across two compactions", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "verify the change" }), + entry(2, { + type: "tool_call", + callId: "test-1", + name: "bash", + status: "completed", + error: null, + detail: { type: "shell", command: "npm test", output: "12 tests passed", exitCode: 0 }, + }), + entry(3, { type: "compaction", status: "completed" }), + entry(4, { + type: "tool_call", + callId: "build-1", + name: "bash", + status: "completed", + error: null, + detail: { + type: "shell", + command: "npm run build", + output: "build complete", + exitCode: 0, + }, + }), + entry(5, { type: "compaction", status: "completed" }), + entry(6, { + type: "tool_call", + callId: "typecheck-1", + name: "bash", + status: "completed", + error: null, + detail: { + type: "shell", + command: "npm run typecheck", + output: "typecheck passed", + exitCode: 0, + }, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "verification", + }); + }); + + it("treats a non-empty explicit plan as a decision", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "decide how to proceed" }), + entry(2, { type: "compaction", status: "completed" }), + entry(3, { + type: "tool_call", + callId: "plan-1", + name: "plan", + status: "completed", + error: null, + detail: { type: "plan", text: "Use the persisted boundary as the source of truth." }, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "progressing", + completedCompactionsSinceMaterialProgress: 0, + lastMaterialProgressKind: "decision", + }); + }); + + it("stalls on repeated irrelevant reads across two compactions", () => { + const repeatedRead = { + type: "read" as const, + filePath: "notes/unrelated.txt", + content: "unchanged unrelated notes", + }; + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "audit the authentication flow" }), + entry(2, { + type: "tool_call", + callId: "read-1", + name: "read", + status: "completed", + error: null, + detail: repeatedRead, + }), + entry(3, { type: "compaction", status: "completed" }), + entry(4, { + type: "tool_call", + callId: "read-2", + name: "read", + status: "completed", + error: null, + detail: repeatedRead, + }), + entry(5, { type: "compaction", status: "completed" }), + entry(6, { + type: "tool_call", + callId: "read-3", + name: "read", + status: "completed", + error: null, + detail: repeatedRead, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "stalled", + completedCompactionsSinceMaterialProgress: 2, + lastMaterialProgressAt: "2026-07-31T00:00:02.000Z", + lastMaterialProgressKind: "evidence", + }); + }); + + it("does not count empty, failed, or no-op tools and commentary as material progress", () => { const result = analyzeMaterialProgress({ entries: [ entry(1, { type: "user_message", text: "investigate" }), @@ -157,10 +325,50 @@ describe("analyzeMaterialProgress", () => { name: "read", status: "completed", error: null, - detail: { type: "read", filePath: "src/a.ts", content: "source" }, + detail: { type: "read", filePath: "src/a.ts" }, }), - entry(3, { type: "assistant_message", text: "Still investigating." }), - entry(4, { type: "compaction", status: "completed" }), + entry(3, { + type: "tool_call", + callId: "search-1", + name: "grep", + status: "completed", + error: null, + detail: { type: "search", query: "TODO" }, + }), + entry(4, { + type: "tool_call", + callId: "fetch-1", + name: "fetch", + status: "completed", + error: null, + detail: { type: "fetch", url: "https://example.com" }, + }), + entry(5, { + type: "tool_call", + callId: "shell-1", + name: "bash", + status: "completed", + error: null, + detail: { type: "shell", command: "false", output: "failed", exitCode: 1 }, + }), + entry(6, { + type: "tool_call", + callId: "plan-1", + name: "plan", + status: "completed", + error: null, + detail: { type: "plan", text: " " }, + }), + entry(7, { type: "assistant_message", text: "Still investigating." }), + entry(8, { + type: "tool_call", + callId: "edit-1", + name: "edit", + status: "failed", + error: "permission denied", + detail: { type: "edit", filePath: "src/a.ts", unifiedDiff: "+change" }, + }), + entry(9, { type: "compaction", status: "completed" }), ], turnOutcome: null, }); diff --git a/packages/server/src/server/agent/material-progress.ts b/packages/server/src/server/agent/material-progress.ts index 6d127c2009..50ed64a6e4 100644 --- a/packages/server/src/server/agent/material-progress.ts +++ b/packages/server/src/server/agent/material-progress.ts @@ -1,5 +1,6 @@ +import { createHash } from "node:crypto"; import type { MaterialProgressPayload } from "../messages.js"; -import type { AgentTimelineItem, ToolCallTimelineItem } from "./agent-sdk-types.js"; +import type { AgentTimelineItem, ToolCallDetail, ToolCallTimelineItem } from "./agent-sdk-types.js"; import type { TimelineProjectionEntry } from "./timeline-projection.js"; export interface AnalyzeMaterialProgressInput { @@ -14,27 +15,120 @@ export interface AnalyzeMaterialProgressInput { type MaterialProgressKind = NonNullable; +interface MaterialProgressEvent { + kind: MaterialProgressKind; + fingerprint: string; +} + function hasConcreteText(value: unknown): boolean { return typeof value === "string" && value.trim().length > 0; } -function materialToolKind(item: ToolCallTimelineItem): MaterialProgressKind | null { - if (item.status !== "completed") return null; +function progressFingerprint(kind: MaterialProgressKind, result: unknown): string { + const digest = createHash("sha256").update(JSON.stringify(result)).digest("hex"); + return `${kind}:${digest}`; +} + +function progressEvent(kind: MaterialProgressKind, result: unknown): MaterialProgressEvent { + return { kind, fingerprint: progressFingerprint(kind, result) }; +} + +type EditDetail = Extract; +type WriteDetail = Extract; +type EvidenceDetail = Extract; +type SearchDetail = Extract; +type ShellDetail = Extract; +type PlanDetail = Extract; + +function editEvent(detail: EditDetail): MaterialProgressEvent | null { + const { filePath, oldString, newString, unifiedDiff } = detail; + if (!hasConcreteText(unifiedDiff) && oldString === undefined && newString === undefined) { + return null; + } + return progressEvent("edit", { filePath, oldString, newString, unifiedDiff }); +} + +function writeEvent(detail: WriteDetail): MaterialProgressEvent | null { + const { filePath, content } = detail; + return content === undefined ? null : progressEvent("write", { filePath, content }); +} + +function hasSearchResult(detail: SearchDetail): boolean { + return ( + hasConcreteText(detail.content) || + detail.numFiles !== undefined || + detail.numMatches !== undefined || + (detail.filePaths?.length ?? 0) > 0 || + (detail.webResults?.length ?? 0) > 0 || + (detail.annotations?.length ?? 0) > 0 + ); +} + +function evidenceEvent(detail: EvidenceDetail): MaterialProgressEvent | null { + switch (detail.type) { + case "read": { + const { filePath, content } = detail; + return hasConcreteText(content) + ? progressEvent("evidence", { type: "read", filePath, content }) + : null; + } + case "search": { + const { query, content, filePaths, webResults, annotations, numFiles, numMatches } = detail; + return hasSearchResult(detail) + ? progressEvent("evidence", { + type: "search", + query, + content, + filePaths, + webResults, + annotations, + numFiles, + numMatches, + }) + : null; + } + case "fetch": { + const { url, result, code, bytes } = detail; + const succeeded = + hasConcreteText(result) && (code === undefined || (code >= 200 && code < 400)); + return succeeded + ? progressEvent("evidence", { type: "fetch", url, result, code, bytes }) + : null; + } + } +} + +function verificationEvent(detail: ShellDetail): MaterialProgressEvent | null { + const { command, cwd, output, exitCode } = detail; + return exitCode === 0 && hasConcreteText(output) + ? progressEvent("verification", { command, cwd, output, exitCode }) + : null; +} +function decisionEvent(detail: PlanDetail): MaterialProgressEvent | null { + const text = detail.text.trim(); + return text ? progressEvent("decision", text) : null; +} + +function materialToolEvent(item: ToolCallTimelineItem): MaterialProgressEvent | null { + if (item.status !== "completed") return null; switch (item.detail.type) { case "edit": - return "edit"; + return editEvent(item.detail); case "write": - return "write"; + return writeEvent(item.detail); + case "read": + case "search": + case "fetch": + return evidenceEvent(item.detail); case "shell": + return verificationEvent(item.detail); + case "plan": + return decisionEvent(item.detail); case "worktree_setup": case "sub_agent": case "unknown": - case "read": - case "search": - case "fetch": case "plain_text": - case "plan": return null; } } @@ -74,13 +168,17 @@ function noProgress(reason: string): MaterialProgressPayload { }; } -function materialKind( +function materialEvent( item: AgentTimelineItem, isFinalTerminalAssistant: boolean, -): MaterialProgressKind | null { - if (item.type === "tool_call") return materialToolKind(item); +): MaterialProgressEvent | null { + if (item.type === "tool_call") return materialToolEvent(item); if (item.type === "assistant_message" && isFinalTerminalAssistant && hasConcreteText(item.text)) { - return "assistant_result"; + const text = item.text.trim(); + return { + kind: "assistant_result", + fingerprint: progressFingerprint("assistant_result", text), + }; } return null; } @@ -105,18 +203,20 @@ export function analyzeMaterialProgress({ let completedCompactions = 0; let lastMaterialProgressAt: string | null = null; let lastMaterialProgressKind: MaterialProgressKind | null = null; + const seenProgress = new Set(); for (let index = 0; index < continuation.length; index += 1) { const entry = continuation[index]; if (!entry) continue; - const kind = materialKind( + const event = materialEvent( entry.item, turnOutcome === "completed" && index === finalAssistantIndex, ); - if (kind) { + if (event && !seenProgress.has(event.fingerprint)) { + seenProgress.add(event.fingerprint); completedCompactions = 0; lastMaterialProgressAt = validTimestamp(entry.timestamp); - lastMaterialProgressKind = kind; + lastMaterialProgressKind = event.kind; continue; } if (entry.item.type === "compaction" && entry.item.status === "completed") { diff --git a/patches/local-patches.json b/patches/local-patches.json deleted file mode 100644 index bcbb7a45df..0000000000 --- a/patches/local-patches.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "schemaVersion": 1, - "patches": [ - { - "id": "paseo-v025-material-progress-inspect", - "category": "stable-core-observability", - "base": { - "tag": "v0.2.5", - "commit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b" - }, - "tests": [ - "packages/cli/src/commands/agent/inspect.test.ts", - "packages/protocol/src/material-progress-schema.test.ts", - "packages/server/src/server/agent/agent-manager.test.ts", - "packages/server/src/server/agent/agent-storage.test.ts", - "packages/server/src/server/agent/material-progress.test.ts", - "packages/server/src/server/material-progress.e2e.test.ts" - ], - "sources": [ - "packages/cli/src/commands/agent/inspect.ts", - "packages/protocol/src/messages.ts", - "packages/server/src/server/agent/agent-manager.ts", - "packages/server/src/server/agent/agent-projections.ts", - "packages/server/src/server/agent/agent-storage.ts", - "packages/server/src/server/agent/material-progress.ts", - "packages/server/src/server/persistence-hooks.ts", - "packages/server/src/server/session.ts" - ], - "docs": ["docs/data-model.md"], - "testCases": [ - "preserves the structured signal and renders its state and reason", - "accepts legacy snapshots without the optional signal", - "accepts a structured material progress signal", - "an unhydrated resumed agent exposes its persisted material progress", - "a resumed agent with an empty live timeline derives progress from its durable tail", - "material progress pages beyond the durable 1000-row tail to the current continuation", - "a delayed material progress write cannot overwrite a newer continuation", - "persists terminal outcomes and clears the previous outcome when a new turn starts", - "a live null turn outcome overrides stale persisted completion while a new turn runs", - "concurrent snapshots preserve a newly derived material progress signal", - "setTitle started before a newer snapshot cannot overwrite its material progress", - "setTitle still runs after an earlier queued write fails", - "reports none when history or a current continuation is unavailable", - "treats completed edits and writes as material progress", - "warns after one compaction and stalls after two", - "does not inherit a previous continuation's compactions after a new turn is accepted", - "uses only the latest user continuation and completion sequence", - "does not count read-only tools or commentary as material progress", - "counts a completed turn's final message as a deliverable, not semantic success", - "fetch agent exposes a backward-compatible material progress signal", - "fetch agent preserves material progress after the live worker is collected", - "fetch agent does not inherit a previous stall after a new turn is accepted", - "fetch agent does not inherit a previous stall when an accepted turn fails before a user row", - "fetch agent preserves material progress after archive discards the live timeline", - "fetch agent restores material progress after daemon restart" - ], - "upstreamable": true, - "upstreamCondition": "Submit independently when upstream still lacks singular material-progress inspection and persisted terminal-turn semantics on the latest stable base.", - "dropCondition": "Remove when the official stable release exposes equivalent singular agent material-progress inspection with persisted terminal-turn semantics.", - "deployedReceipt": null, - "deployedAt": null - } - ] -} From 93327203df45edcf9beb4435972b6973fc2ba2e0 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 04:56:52 -0400 Subject: [PATCH 13/17] fix: harden material progress ownership --- .../src/server/agent/agent-manager.test.ts | 86 +++++++++++++++ .../server/src/server/agent/agent-manager.ts | 30 ++++- .../server/agent/material-progress.test.ts | 104 ++++++++++++++++++ .../src/server/agent/material-progress.ts | 15 +-- 4 files changed, 223 insertions(+), 12 deletions(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 3a3439023a..0f258927a7 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -4345,6 +4345,92 @@ test("persists terminal outcomes and clears the previous outcome when a new turn } }); +test("an old autonomous terminal cannot end a newer foreground continuation", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-stale-autonomous-terminal-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const session = new (class extends TestAgentSession { + override async startTurn(): Promise<{ turnId: string }> { + return { turnId: "new-foreground-turn" }; + } + })({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000170", + }); + + let foregroundDrain: Promise | null = null; + let unsubscribe: (() => void) | null = null; + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + session.pushEvent({ + type: "turn_started", + provider: "codex", + turnId: "old-autonomous-turn", + }); + session.pushEvent({ + type: "turn_completed", + provider: "codex", + turnId: "old-autonomous-turn", + }); + await manager.flush(); + + const streamedEvents: AgentStreamEvent[] = []; + unsubscribe = manager.subscribe( + (event) => { + if (event.type === "agent_stream") streamedEvents.push(event.event); + }, + { agentId: agent.id, replayState: false }, + ); + + const foreground = manager.streamAgent(agent.id, "new work"); + foregroundDrain = (async () => { + for await (const _event of foreground) { + // Drain through the current turn's terminal event. + } + })(); + await manager.waitForAgentRunStart(agent.id); + + session.pushEvent({ + type: "turn_completed", + provider: "codex", + turnId: "old-autonomous-turn", + }); + await manager.flush(); + + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "running", + activeForegroundTurnId: "new-foreground-turn", + materialProgressContinuationActive: true, + materialProgressContinuationTurnId: "new-foreground-turn", + lastTurnOutcome: null, + }); + expect(streamedEvents).not.toContainEqual( + expect.objectContaining({ type: "turn_completed", turnId: "old-autonomous-turn" }), + ); + + session.pushEvent({ + type: "turn_canceled", + provider: "codex", + turnId: "new-foreground-turn", + reason: "test cleanup", + }); + await foregroundDrain; + } finally { + unsubscribe?.(); + await manager.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("getAgent does not expose committed history internals once manager owns the seam", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-timeline-boundary-")); const storagePath = join(workdir, "agents"); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 2d1554693b..ae4d16e9df 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -3518,11 +3518,7 @@ export class AgentManager { const eventTurnId = getAgentStreamEventTurnId(event); const isForegroundEvent = Boolean(eventTurnId && agent.activeForegroundTurnId === eventTurnId); this.traceHandleStreamEventStart(agent, event, eventTurnId, isForegroundEvent); - if ( - eventTurnId && - isTurnTerminalEvent(event) && - this.runs.hasFinalizedTurn(agent, eventTurnId) - ) { + if (this.shouldIgnoreTerminalEvent(agent, event, eventTurnId)) { return false; } @@ -3566,6 +3562,18 @@ export class AgentManager { return flags.shouldNotifyWaiters; } + private shouldIgnoreTerminalEvent( + agent: ActiveManagedAgent, + event: AgentStreamEvent, + eventTurnId: string | undefined, + ): boolean { + if (!eventTurnId || !isTurnTerminalEvent(event)) return false; + return ( + this.runs.hasFinalizedTurn(agent, eventTurnId) || + !this.terminalEventOwnsCurrentTurn(agent, eventTurnId) + ); + } + private traceHandleStreamEventStart( agent: ActiveManagedAgent, event: AgentStreamEvent, @@ -3704,6 +3712,18 @@ export class AgentManager { } } + private terminalEventOwnsCurrentTurn( + agent: ActiveManagedAgent, + eventTurnId: string | undefined, + ): boolean { + if (!eventTurnId) return true; + return ( + (!agent.activeForegroundTurnId || agent.activeForegroundTurnId === eventTurnId) && + (!agent.materialProgressContinuationTurnId || + agent.materialProgressContinuationTurnId === eventTurnId) + ); + } + private onStreamThreadStarted(agent: ActiveManagedAgent): void { const previousSessionId = agent.persistence?.sessionId ?? null; const handle = agent.session.describePersistence(); diff --git a/packages/server/src/server/agent/material-progress.test.ts b/packages/server/src/server/agent/material-progress.test.ts index 0b541ad660..5f61692527 100644 --- a/packages/server/src/server/agent/material-progress.test.ts +++ b/packages/server/src/server/agent/material-progress.test.ts @@ -380,6 +380,63 @@ describe("analyzeMaterialProgress", () => { }); }); + it("does not count a completed edit that leaves content unchanged", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "update the file" }), + entry(2, { + type: "tool_call", + callId: "edit-no-op", + name: "edit", + status: "completed", + error: null, + detail: { + type: "edit", + filePath: "src/a.ts", + oldString: "unchanged", + newString: "unchanged", + }, + }), + entry(3, { type: "compaction", status: "completed" }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressKind: null, + }); + }); + + it("counts a concrete multi-edit diff when its first replacement is unchanged", () => { + const result = analyzeMaterialProgress({ + entries: [ + entry(1, { type: "user_message", text: "update the file" }), + entry(2, { + type: "tool_call", + callId: "multi-edit", + name: "edit", + status: "completed", + error: null, + detail: { + type: "edit", + filePath: "src/a.ts", + oldString: "unchanged first replacement", + newString: "unchanged first replacement", + unifiedDiff: "+changed by a later replacement", + }, + }), + ], + turnOutcome: null, + }); + + expect(result).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "edit", + }); + }); + it("counts a completed turn's final message as a deliverable, not semantic success", () => { const entries = [ entry(1, { type: "user_message", text: "answer" }), @@ -404,5 +461,52 @@ describe("analyzeMaterialProgress", () => { state: "warning", lastMaterialProgressKind: null, }); + + const failedActivityAfterCommentary = [ + ...entries.slice(0, -1), + entry(4, { + type: "tool_call", + callId: "failed-edit", + name: "edit", + status: "failed", + error: "permission denied", + detail: { type: "edit", filePath: "src/a.ts", unifiedDiff: "+change" }, + }), + ]; + expect( + analyzeMaterialProgress({ + entries: failedActivityAfterCommentary, + turnOutcome: "completed", + }), + ).toMatchObject({ + state: "warning", + lastMaterialProgressKind: null, + }); + + const noOpActivityAfterCommentary = [ + ...entries.slice(0, -1), + entry(4, { + type: "tool_call", + callId: "no-op-edit", + name: "edit", + status: "completed", + error: null, + detail: { + type: "edit", + filePath: "src/a.ts", + oldString: "unchanged", + newString: "unchanged", + }, + }), + ]; + expect( + analyzeMaterialProgress({ + entries: noOpActivityAfterCommentary, + turnOutcome: "completed", + }), + ).toMatchObject({ + state: "warning", + lastMaterialProgressKind: null, + }); }); }); diff --git a/packages/server/src/server/agent/material-progress.ts b/packages/server/src/server/agent/material-progress.ts index 50ed64a6e4..3108c771c9 100644 --- a/packages/server/src/server/agent/material-progress.ts +++ b/packages/server/src/server/agent/material-progress.ts @@ -42,6 +42,9 @@ type PlanDetail = Extract; function editEvent(detail: EditDetail): MaterialProgressEvent | null { const { filePath, oldString, newString, unifiedDiff } = detail; + if (!hasConcreteText(unifiedDiff) && oldString !== undefined && oldString === newString) { + return null; + } if (!hasConcreteText(unifiedDiff) && oldString === undefined && newString === undefined) { return null; } @@ -193,12 +196,10 @@ export function analyzeMaterialProgress({ const ordered = orderedEntries(entries); const continuation = currentContinuation(ordered, continuationBoundarySeq); if (continuation === null) return noProgress("No current continuation is available."); - let finalAssistantIndex = -1; - if (turnOutcome === "completed") { - for (let index = 0; index < continuation.length; index += 1) { - if (continuation[index]?.item.type === "assistant_message") finalAssistantIndex = index; - } - } + const terminalAssistantIndex = + turnOutcome === "completed" && continuation.at(-1)?.item.type === "assistant_message" + ? continuation.length - 1 + : -1; let completedCompactions = 0; let lastMaterialProgressAt: string | null = null; @@ -210,7 +211,7 @@ export function analyzeMaterialProgress({ if (!entry) continue; const event = materialEvent( entry.item, - turnOutcome === "completed" && index === finalAssistantIndex, + turnOutcome === "completed" && index === terminalAssistantIndex, ); if (event && !seenProgress.has(event.fingerprint)) { seenProgress.add(event.fingerprint); From cac07ecc61c5cfa379e1f0e68afd86553ac00980 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 05:41:01 -0400 Subject: [PATCH 14/17] fix: preserve turn and progress boundaries --- .../src/server/agent/agent-manager.test.ts | 184 ++++++++++++++++++ .../server/src/server/agent/agent-manager.ts | 100 +++++++--- .../providers/codex-app-server-agent.test.ts | 24 ++- .../agent/providers/codex-app-server-agent.ts | 35 +++- .../codex/test-utils/fake-app-server.ts | 17 +- 5 files changed, 323 insertions(+), 37 deletions(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 0f258927a7..09bbf5c6bd 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -47,6 +47,8 @@ import type { AgentTimelineRow, AgentTimelineStore, } from "./agent-timeline-store-types.js"; +import { CodexAppServerAgentSession } from "./providers/codex-app-server-agent.js"; +import { createFakeCodexAppServer } from "./providers/codex/test-utils/fake-app-server.js"; interface Deferred { promise: Promise; @@ -4166,6 +4168,92 @@ test("material progress pages beyond the durable 1000-row tail to the current co } }); +test("material progress stops at a known continuation boundary in a tail without a new user row", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-known-boundary-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const durableTimelineStore = new TestDurableTimelineStore(); + const agentId = "00000000-0000-4000-8000-000000000171"; + const firstManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + logger, + idFactory: () => agentId, + }); + + try { + const created = await firstManager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + await firstManager.closeAgent(created.id); + + const durableRows: AgentTimelineRow[] = [ + timelineRow(1, { type: "user_message", text: "previous continuation" }), + ]; + for (let seq = 2; seq < 10_004; seq += 1) { + durableRows.push(timelineRow(seq, { type: "reasoning", text: `old reasoning ${seq}` })); + } + durableRows.push( + timelineRow(10_004, { + type: "tool_call", + callId: "write-known-boundary", + name: "write", + status: "completed", + error: null, + detail: { type: "write", filePath: "boundary.txt", content: "done" }, + }), + timelineRow(10_005, { type: "reasoning", text: "verify the boundary result" }), + ); + await durableTimelineStore.bulkInsert(created.id, durableRows); + + const stored = await storage.get(created.id); + expect(stored?.persistence).toBeTruthy(); + await storage.upsert({ + ...stored!, + materialProgressContinuationBoundarySeq: 10_004, + materialProgress: { + state: "warning", + completedCompactionsSinceMaterialProgress: 1, + lastMaterialProgressAt: null, + lastMaterialProgressKind: null, + reason: "Classification to replace from the known continuation boundary.", + }, + }); + + const resumedManager = new AgentManager({ + clients: { codex: new TestAgentClient() }, + registry: storage, + durableTimelineStore, + logger, + }); + await resumedManager.resumeAgentFromPersistence( + stored!.persistence!, + { cwd: workdir }, + created.id, + { materialProgressContinuationBoundarySeq: 10_004 }, + ); + + const snapshot = await resumedManager.getMaterialProgressSnapshot(created.id); + expect(snapshot.rows?.slice(-2)).toMatchObject([ + { seq: 10_004, item: { type: "tool_call", detail: { type: "write" } } }, + { seq: 10_005, item: { type: "reasoning" } }, + ]); + expect(snapshot).toMatchObject({ + persisted: { + state: "progressing", + lastMaterialProgressKind: "write", + }, + }); + expect((await storage.get(created.id))?.materialProgress).toMatchObject({ + state: "progressing", + lastMaterialProgressKind: "write", + }); + + await resumedManager.closeAgent(created.id); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("a delayed material progress write cannot overwrite a newer continuation", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-write-order-")); const storage = new AgentStorage(join(workdir, "agents"), logger); @@ -4431,6 +4519,102 @@ test("an old autonomous terminal cannot end a newer foreground continuation", as } }); +test("a stale native Codex completion cannot cross the provider boundary into a newer turn", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-codex-native-stale-terminal-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const appServer = createFakeCodexAppServer(); + const session = new CodexAppServerAgentSession( + { provider: "codex", cwd: workdir, modeId: "auto", model: "gpt-5.4" }, + null, + logger, + async () => appServer.child, + ); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000172", + }); + + let unsubscribe: (() => void) | null = null; + let unsubscribeProvider: (() => void) | null = null; + let foregroundDrain: Promise | null = null; + let waitForAgent: Promise | null = null; + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + + appServer.startsTurn({ threadId: "thread-1", turnId: "native-old-turn" }); + await vi.waitFor(() => expect(manager.getAgent(agent.id)?.lifecycle).toBe("running")); + appServer.completeTurn({ turnId: "native-old-turn" }); + await vi.waitFor(() => expect(manager.getAgent(agent.id)?.lifecycle).toBe("idle")); + + const providerEvents: AgentStreamEvent[] = []; + unsubscribeProvider = session.subscribe((event) => providerEvents.push(event)); + const streamedEvents: AgentStreamEvent[] = []; + unsubscribe = manager.subscribe( + (event) => { + if (event.type === "agent_stream") streamedEvents.push(event.event); + }, + { agentId: agent.id, replayState: false }, + ); + + let foregroundSettled = false; + foregroundDrain = (async () => { + for await (const _event of manager.streamAgent(agent.id, "new work")) { + // Drain through the current turn's terminal event. + } + foregroundSettled = true; + })(); + await appServer.waitForTurnStart(); + appServer.startsTurn({ threadId: "thread-1", turnId: "native-new-turn" }); + await manager.waitForAgentRunStart(agent.id); + providerEvents.length = 0; + + let waiterSettled = false; + waitForAgent = manager.waitForAgentEvent(agent.id).then((result) => { + waiterSettled = true; + return result; + }); + + appServer.completeTurn({ turnId: "native-old-turn" }); + await vi.waitFor(() => + expect(providerEvents).toContainEqual( + expect.objectContaining({ type: "turn_completed", turnId: "native-old-turn" }), + ), + ); + await manager.flush(); + + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "running", + activeForegroundTurnId: expect.any(String), + materialProgressContinuationActive: true, + lastTurnOutcome: null, + }); + expect(streamedEvents).not.toContainEqual( + expect.objectContaining({ type: "turn_completed", turnId: "native-old-turn" }), + ); + expect(foregroundSettled).toBe(false); + expect(waiterSettled).toBe(false); + + appServer.completeTurn({ turnId: "native-new-turn" }); + await foregroundDrain; + await waitForAgent; + appServer.assertNoErrors(); + } finally { + unsubscribe?.(); + unsubscribeProvider?.(); + await manager.closeAgent("00000000-0000-4000-8000-000000000172").catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("getAgent does not expose committed history internals once manager owns the seam", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-timeline-boundary-")); const storagePath = join(workdir, "agents"); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index ae4d16e9df..16c45f3c5d 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -153,15 +153,45 @@ function currentContinuationRows(rows: readonly AgentTimelineRow[]): AgentTimeli return [...rows]; } +function materialProgressScanReachedStart( + rows: readonly AgentTimelineRow[], + continuationBoundarySeq: number | null, +): boolean { + if (continuationBoundarySeq === null) { + return rows.some((row) => row.item.type === "user_message"); + } + return (rows[0]?.seq ?? Number.POSITIVE_INFINITY) <= continuationBoundarySeq; +} + +function selectMaterialProgressRows( + rows: readonly AgentTimelineRow[], + continuationBoundarySeq: number | null, + reachedStart: boolean, +): AgentTimelineRow[] | null { + if (continuationBoundarySeq !== null) { + if (!reachedStart) return null; + return currentContinuationRows(rows); + } + if (!reachedStart && rows[0]?.seq !== 1) return null; + return currentContinuationRows(rows); +} + async function pageMaterialProgressRows( initialPage: AgentTimelineFetchResult, fetchOlder: FetchOlderMaterialProgressRows, + continuationBoundarySeq: number | null, ): Promise { let page = initialPage; const pageRows = [page.rows]; let rowCount = page.rows.length; - let hasUserMessage = page.rows.some((row) => row.item.type === "user_message"); - while (page.hasOlder && !hasUserMessage) { + if (continuationBoundarySeq !== null) { + if (continuationBoundarySeq > page.window.nextSeq) return null; + if (continuationBoundarySeq === page.window.nextSeq) { + return currentContinuationRows(page.rows); + } + } + let reachedStart = materialProgressScanReachedStart(page.rows, continuationBoundarySeq); + while (page.hasOlder && !reachedStart) { const firstRow = pageRows[0]?.[0]; const remainingCapacity = MATERIAL_PROGRESS_MAX_SCAN_ROWS - rowCount; if (!firstRow || remainingCapacity <= 0) return null; @@ -182,14 +212,10 @@ async function pageMaterialProgressRows( } pageRows.unshift(olderPage.rows); rowCount += olderPage.rows.length; - hasUserMessage = olderPage.rows.some((row) => row.item.type === "user_message"); + reachedStart = materialProgressScanReachedStart(olderPage.rows, continuationBoundarySeq); page = olderPage; } - const rows = pageRows.flat(); - if (!rows.some((row) => row.item.type === "user_message") && rows[0]?.seq !== 1) { - return null; - } - return currentContinuationRows(rows); + return selectMaterialProgressRows(pageRows.flat(), continuationBoundarySeq, reachedStart); } function buildStoredAgentConfig(record: StoredAgentRecord): AgentSessionConfig { @@ -1048,19 +1074,19 @@ export class AgentManager { async getMaterialProgressSnapshot(id: string, limit = 1_000): Promise { const live = this.agents.get(id); - const [rows, record] = await Promise.all([ - this.resolveMaterialProgressRows(id, { - limit, - historyPrimed: live?.historyPrimed, - }), - this.registry?.get(id), - ]); + const record = await this.registry?.get(id); + const continuationBoundarySeq = live + ? (live.materialProgressContinuationBoundarySeq ?? null) + : (record?.materialProgressContinuationBoundarySeq ?? null); + const rows = await this.resolveMaterialProgressRows(id, { + limit, + historyPrimed: live?.historyPrimed, + continuationBoundarySeq, + }); return { rows, turnOutcome: live ? (live.lastTurnOutcome ?? null) : (record?.lastTurnOutcome ?? null), - continuationBoundarySeq: live - ? (live.materialProgressContinuationBoundarySeq ?? null) - : (record?.materialProgressContinuationBoundarySeq ?? null), + continuationBoundarySeq, persisted: record?.materialProgress ?? null, }; } @@ -3214,6 +3240,7 @@ export class AgentManager { const rows = await this.resolveMaterialProgressRows(agent.id, { limit: MATERIAL_PROGRESS_PAGE_SIZE, historyPrimed: agent.historyPrimed, + continuationBoundarySeq: agent.materialProgressContinuationBoundarySeq ?? null, }); // Omitting the override makes AgentStorage preserve the last trustworthy classification. if (rows === null) return undefined; @@ -3226,18 +3253,29 @@ export class AgentManager { private async resolveMaterialProgressRows( agentId: string, - options: { limit: number; historyPrimed?: boolean }, + options: { + limit: number; + historyPrimed?: boolean; + continuationBoundarySeq: number | null; + }, ): Promise { const limit = Math.max(1, Math.min(Math.floor(options.limit), MATERIAL_PROGRESS_PAGE_SIZE)); + const { continuationBoundarySeq } = options; if (this.timelineStore.has(agentId)) { const livePage = this.timelineStore.fetch(agentId, { direction: "tail", limit }); if (livePage.rows.length > 0) { - const liveRows = await pageMaterialProgressRows(livePage, async (fetchOptions) => - this.timelineStore.fetch(agentId, fetchOptions), + const liveRows = await pageMaterialProgressRows( + livePage, + async (fetchOptions) => this.timelineStore.fetch(agentId, fetchOptions), + continuationBoundarySeq, ); if (liveRows !== null) return liveRows; - const durableRows = await this.resolveDurableMaterialProgressRows(agentId, limit); + const durableRows = await this.resolveDurableMaterialProgressRows( + agentId, + limit, + continuationBoundarySeq, + ); if (durableRows === null) return null; const firstLiveSeq = livePage.rows[0]?.seq; const olderDurableRows = durableRows.filter((row) => row.seq < (firstLiveSeq ?? 0)); @@ -3247,21 +3285,27 @@ export class AgentManager { ) { return null; } - return currentContinuationRows([...olderDurableRows, ...livePage.rows]); + const combinedRows = [...olderDurableRows, ...livePage.rows]; + return currentContinuationRows(combinedRows); } - const durableRows = await this.resolveDurableMaterialProgressRows(agentId, limit); + const durableRows = await this.resolveDurableMaterialProgressRows( + agentId, + limit, + continuationBoundarySeq, + ); if (durableRows && durableRows.length > 0) return durableRows; if (livePage.window.nextSeq > 1 || options.historyPrimed === false) return null; return []; } - return await this.resolveDurableMaterialProgressRows(agentId, limit); + return await this.resolveDurableMaterialProgressRows(agentId, limit, continuationBoundarySeq); } private async resolveDurableMaterialProgressRows( agentId: string, limit: number, + continuationBoundarySeq: number | null, ): Promise { const durableTimelineStore = this.durableTimelineStore; if (!durableTimelineStore) return null; @@ -3270,8 +3314,10 @@ export class AgentManager { direction: "tail", limit, }); - return await pageMaterialProgressRows(durablePage, (fetchOptions) => - durableTimelineStore.fetchCommitted(agentId, fetchOptions), + return await pageMaterialProgressRows( + durablePage, + (fetchOptions) => durableTimelineStore.fetchCommitted(agentId, fetchOptions), + continuationBoundarySeq, ); } catch (error) { this.logger.debug( diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index 57235d3773..50810979dd 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -3889,7 +3889,7 @@ describe("Codex app-server provider", () => { ], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-1", status: "completed", error: null }, }); expect( @@ -4416,7 +4416,7 @@ describe("Codex app-server provider", () => { }), ).not.toThrow(); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "test-turn", status: "completed", error: null }, }); expect(events).toEqual([ @@ -4446,7 +4446,7 @@ describe("Codex app-server provider", () => { }, }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "test-turn", status: "completed", error: null }, }); expect(events).toContainEqual({ @@ -4475,6 +4475,24 @@ describe("Codex app-server provider", () => { }); }); + test("does not tag an uncorrelated terminal with the active foreground turn", () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/completed", { + turn: { status: "completed", error: null }, + }); + + expect(events).toEqual([ + { + type: "turn_completed", + provider: "codex", + usage: undefined, + }, + ]); + }); + test("streams Codex assistant message deltas and does not replay completed text", () => { const session = createSession(); const events: AgentStreamEvent[] = []; diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.ts index fe1e83406a..07c0facf21 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.ts @@ -1988,6 +1988,7 @@ const TurnCompletedNotificationSchema = z threadId: z.string().optional(), turn: z .object({ + id: z.string().optional(), status: z.string(), error: z .object({ @@ -2263,6 +2264,7 @@ type ParsedCodexNotification = | { kind: "turn_started"; turnId: string; threadId: string | null } | { kind: "turn_completed"; + turnId: string | null; status: string; errorMessage: string | null; threadId: string | null; @@ -2413,6 +2415,7 @@ const CodexNotificationSchema = z.union([ .transform( ({ params }): ParsedCodexNotification => ({ kind: "turn_completed", + turnId: params.turn.id ?? null, status: params.turn.status, errorMessage: params.turn.error?.message ?? null, threadId: params.threadId ?? null, @@ -2833,6 +2836,7 @@ const CodexNotificationSchema = z.union([ .transform( ({ params }): ParsedCodexNotification => ({ kind: "turn_completed", + turnId: null, status: "interrupted", errorMessage: null, threadId: getCodexEventThreadId(params), @@ -2853,6 +2857,7 @@ const CodexNotificationSchema = z.union([ .transform( ({ params }): ParsedCodexNotification => ({ kind: "turn_completed", + turnId: null, status: "completed", errorMessage: null, threadId: getCodexEventThreadId(params), @@ -3112,6 +3117,7 @@ export class CodexAppServerAgentSession implements AgentSession { private readonly subscribers = new Set<(event: AgentStreamEvent) => void>(); private nextTurnOrdinal = 0; private activeForegroundTurnId: string | null = null; + private readonly managerTurnIdByNativeTurnId = new Map(); private activeClientMessageId: string | null = null; private cachedRuntimeInfo: AgentRuntimeInfo | null = null; private serviceTier: "fast" | null = null; @@ -4285,6 +4291,7 @@ export class CodexAppServerAgentSession implements AgentSession { this.pendingPermissions.clear(); this.resolvedPermissionRequests.clear(); this.pendingSubAgentNotificationsByThreadId.clear(); + this.managerTurnIdByNativeTurnId.clear(); this.subscribers.clear(); this.activeForegroundTurnId = null; this.activeClientMessageId = null; @@ -4587,7 +4594,12 @@ export class CodexAppServerAgentSession implements AgentSession { } private notifySubscribers(event: AgentStreamEvent): void { - const turnId = this.activeForegroundTurnId; + const eventTurnId = getAgentStreamEventTurnId(event); + const terminal = + event.type === "turn_completed" || + event.type === "turn_failed" || + event.type === "turn_canceled"; + const turnId = eventTurnId ?? (terminal ? null : this.activeForegroundTurnId); const tagged = turnId ? { ...event, turnId } : event; this.logger.trace( { @@ -5300,8 +5312,10 @@ export class CodexAppServerAgentSession implements AgentSession { return; } this.currentTurnId = parsed.turnId; + const managerTurnId = this.activeForegroundTurnId ?? parsed.turnId; + this.managerTurnIdByNativeTurnId.set(parsed.turnId, managerTurnId); this.resetTurnTrackingState(); - this.emitEvent({ type: "turn_started", provider: CODEX_PROVIDER }); + this.emitEvent({ type: "turn_started", provider: CODEX_PROVIDER, turnId: managerTurnId }); } private handleTurnCompletedNotification( @@ -5318,24 +5332,37 @@ export class CodexAppServerAgentSession implements AgentSession { this.emitSubAgentActivityUpdate(subAgentCallId, status); return; } + const managerTurnId = parsed.turnId + ? (this.managerTurnIdByNativeTurnId.get(parsed.turnId) ?? parsed.turnId) + : null; + const turnIdentity = managerTurnId ? { turnId: managerTurnId } : {}; + const completesCurrentTurn = parsed.turnId === null || parsed.turnId === this.currentTurnId; if (parsed.status === "failed") { this.emitEvent({ type: "turn_failed", provider: CODEX_PROVIDER, error: parsed.errorMessage ?? "Codex turn failed", + ...turnIdentity, }); } else if (parsed.status === "interrupted") { - this.emitEvent({ type: "turn_canceled", provider: CODEX_PROVIDER, reason: "interrupted" }); + this.emitEvent({ + type: "turn_canceled", + provider: CODEX_PROVIDER, + reason: "interrupted", + ...turnIdentity, + }); } else { - if (this.planModeEnabled && this.latestPlanResult?.text) { + if (completesCurrentTurn && this.planModeEnabled && this.latestPlanResult?.text) { this.emitSyntheticPlanApprovalRequest(this.latestPlanResult.text); } this.emitEvent({ type: "turn_completed", provider: CODEX_PROVIDER, usage: this.latestUsage, + ...turnIdentity, }); } + if (!completesCurrentTurn) return; this.activeForegroundTurnId = null; this.activeClientMessageId = null; this.pendingSubAgentNotificationsByThreadId.clear(); diff --git a/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts b/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts index 0e7488e9aa..ef6d5a9390 100644 --- a/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts +++ b/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts @@ -51,7 +51,7 @@ export interface FakeCodexAppServer { waitForTurnStart(): Promise; nextResponse(): Promise; startsTurn(params: { threadId: string; turnId?: string }): void; - completeTurn(params?: { threadId?: string }): void; + completeTurn(params?: { threadId?: string; turnId?: string }): void; startsSubAgent(params: { callId: string; threadId: string; @@ -114,6 +114,7 @@ export function createFakeCodexAppServer( ): FakeCodexAppServer { const child = createCodexAppServerChildProcess(); const recordedRollbacks: JsonObject[] = []; + const activeTurnIdByThreadId = new Map(); const responseHandlers: Record = { initialize: () => ({}), "collaborationMode/list": () => ({ data: [] }), @@ -309,21 +310,31 @@ export function createFakeCodexAppServer( }); }, startsTurn(params) { + const turnId = params.turnId ?? `turn-${params.threadId}`; + activeTurnIdByThreadId.set(params.threadId, turnId); child.stdout.write( `${JSON.stringify({ method: "turn/started", params: { threadId: params.threadId, - turn: { id: params.turnId ?? `turn-${params.threadId}` }, + turn: { id: turnId }, }, })}\n`, ); }, completeTurn(params = {}) { + const threadId = params.threadId ?? "thread-1"; + const turnId = params.turnId ?? activeTurnIdByThreadId.get(threadId); child.stdout.write( `${JSON.stringify({ method: "turn/completed", - params: { threadId: params.threadId ?? "thread-1", turn: { status: "completed" } }, + params: { + threadId, + turn: { + ...(turnId ? { id: turnId } : {}), + status: "completed", + }, + }, })}\n`, ); }, From 662a9968e8264e8a8d381b1ecc1cdc4a50d386b1 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 06:42:03 -0400 Subject: [PATCH 15/17] fix: enforce exact continuation boundaries --- .../src/server/agent/agent-manager.test.ts | 25 +- .../server/src/server/agent/agent-manager.ts | 4 +- .../providers/codex-app-server-agent.test.ts | 219 +++++++++++++++++- .../agent/providers/codex-app-server-agent.ts | 83 ++++++- .../codex/test-utils/fake-app-server.ts | 40 +++- .../src/server/material-progress.e2e.test.ts | 6 +- 6 files changed, 344 insertions(+), 33 deletions(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 09bbf5c6bd..252286d53a 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -4233,7 +4233,7 @@ test("material progress stops at a known continuation boundary in a tail without ); const snapshot = await resumedManager.getMaterialProgressSnapshot(created.id); - expect(snapshot.rows?.slice(-2)).toMatchObject([ + expect(snapshot.rows).toMatchObject([ { seq: 10_004, item: { type: "tool_call", detail: { type: "write" } } }, { seq: 10_005, item: { type: "reasoning" } }, ]); @@ -4519,7 +4519,7 @@ test("an old autonomous terminal cannot end a newer foreground continuation", as } }); -test("a stale native Codex completion cannot cross the provider boundary into a newer turn", async () => { +test("Codex binds only the exact native terminal across consecutive managed turns", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-codex-native-stale-terminal-")); const storage = new AgentStorage(join(workdir, "agents"), logger); const appServer = createFakeCodexAppServer(); @@ -4591,9 +4591,11 @@ test("a stale native Codex completion cannot cross the provider boundary into a ); await manager.flush(); + const currentForegroundTurnId = manager.getAgent(agent.id)?.activeForegroundTurnId; + expect(currentForegroundTurnId).toEqual(expect.any(String)); expect(manager.getAgent(agent.id)).toMatchObject({ lifecycle: "running", - activeForegroundTurnId: expect.any(String), + activeForegroundTurnId: currentForegroundTurnId, materialProgressContinuationActive: true, lastTurnOutcome: null, }); @@ -4603,9 +4605,24 @@ test("a stale native Codex completion cannot cross the provider boundary into a expect(foregroundSettled).toBe(false); expect(waiterSettled).toBe(false); - appServer.completeTurn({ turnId: "native-new-turn" }); + appServer.setThreadTurnStatus({ + threadId: "thread-1", + turnId: "native-new-turn", + status: "completed", + }); + appServer.completeTurn({ omitTurnId: true }); await foregroundDrain; await waitForAgent; + await manager.flush(); + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + materialProgressContinuationActive: false, + lastTurnOutcome: "completed", + }); + expect(streamedEvents).toContainEqual( + expect.objectContaining({ type: "turn_completed", turnId: currentForegroundTurnId }), + ); appServer.assertNoErrors(); } finally { unsubscribe?.(); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 16c45f3c5d..d41caf9b64 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -170,7 +170,7 @@ function selectMaterialProgressRows( ): AgentTimelineRow[] | null { if (continuationBoundarySeq !== null) { if (!reachedStart) return null; - return currentContinuationRows(rows); + return rows.filter((row) => row.seq >= continuationBoundarySeq); } if (!reachedStart && rows[0]?.seq !== 1) return null; return currentContinuationRows(rows); @@ -187,7 +187,7 @@ async function pageMaterialProgressRows( if (continuationBoundarySeq !== null) { if (continuationBoundarySeq > page.window.nextSeq) return null; if (continuationBoundarySeq === page.window.nextSeq) { - return currentContinuationRows(page.rows); + return []; } } let reachedStart = materialProgressScanReachedStart(page.rows, continuationBoundarySeq); diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index 50810979dd..d3feca59cf 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -1094,11 +1094,23 @@ describe("Codex app-server provider", () => { ); await session.startTurn("remember first"); + appServer.startsTurn({ threadId: "thread-1", turnId: "native-first" }); emitCodexUserMessage(appServer, { id: "codex-first", text: "remember first" }); appServer.completeTurn(); + await vi.waitFor(() => + expect(castInternals<{ activeForegroundTurnId: string | null }>(session)).toMatchObject({ + activeForegroundTurnId: null, + }), + ); await session.startTurn("remember second"); + appServer.startsTurn({ threadId: "thread-1", turnId: "native-second" }); emitCodexUserMessage(appServer, { id: "codex-second", text: "remember second" }); appServer.completeTurn(); + await vi.waitFor(() => + expect(castInternals<{ activeForegroundTurnId: string | null }>(session)).toMatchObject({ + activeForegroundTurnId: null, + }), + ); await session.revertConversation({ messageId: "codex-first" }); @@ -2551,6 +2563,10 @@ describe("Codex app-server provider", () => { }); expect(events).toEqual([]); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "first-root-turn" }, + }); asInternals(session).handleNotification("turn/completed", { threadId: "test-thread", turn: { status: "completed" }, @@ -2582,6 +2598,10 @@ describe("Codex app-server provider", () => { const events: AgentStreamEvent[] = []; session.subscribe((event) => events.push(event)); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "root-turn" }, + }); asInternals(session).handleNotification("item/completed", { threadId: "test-thread", item: { @@ -3952,7 +3972,7 @@ describe("Codex app-server provider", () => { }, }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-thread-item", status: "completed", error: null }, }); expect(events).not.toContainEqual( @@ -3991,7 +4011,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the first plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-first", status: "completed", error: null }, }); asInternals(session).handleNotification("turn/started", { @@ -4001,7 +4021,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the revised plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-second", status: "completed", error: null }, }); expect(session.getPendingPermissions()).toEqual([ @@ -4026,7 +4046,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the original plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-pending", status: "completed", error: null }, }); const pendingPlan = session.getPendingPermissions()[0]; expect(pendingPlan).toBeDefined(); @@ -4120,7 +4140,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the original plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-pending", status: "completed", error: null }, }); let acceptPrompt: (() => void) | undefined; @@ -4144,11 +4164,14 @@ describe("Codex app-server provider", () => { const startTurn = session.startTurn("Revise the plan"); await promptRequested; + asInternals(session).handleNotification("turn/started", { + turn: { id: "turn-plan-newer" }, + }); asInternals(session).handleNotification("turn/plan/updated", { plan: [{ step: "Implement the newer plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-newer", status: "completed", error: null }, }); acceptPrompt?.(); await startTurn; @@ -4475,7 +4498,7 @@ describe("Codex app-server provider", () => { }); }); - test("does not tag an uncorrelated terminal with the active foreground turn", () => { + test("ignores an uncorrelated terminal while a foreground turn is active", () => { const session = createSession(); const events: AgentStreamEvent[] = []; session.subscribe((event) => events.push(event)); @@ -4484,12 +4507,184 @@ describe("Codex app-server provider", () => { turn: { status: "completed", error: null }, }); - expect(events).toEqual([ - { - type: "turn_completed", - provider: "codex", - usage: undefined, + expect(events).toEqual([]); + }); + + test.each([ + ["turn/completed", { threadId: "test-thread", turn: { status: "completed", error: null } }], + ["codex/event/task_complete", { threadId: "test-thread", msg: { type: "task_complete" } }], + ])("binds a current ID-less terminal from %s to the exact active run", (method, params) => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-current-turn" }, + }); + asInternals(session).handleNotification(method, params); + + expect(session.activeForegroundTurnId).toBeNull(); + expect(events.at(-1)).toEqual({ + type: "turn_completed", + provider: "codex", + turnId: "test-turn", + usage: undefined, + }); + }); + + test("does not bind a stale ID-less terminal to a newer native turn", async () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-old-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-old-turn", status: "completed", error: null }, + }); + session.client = createStub({ + request: async (method) => { + if (method === "thread/loaded/list") return { data: ["test-thread"] }; + if (method === "turn/start") return {}; + if (method === "thread/read") { + return { + thread: { + turns: [{ id: "native-new-turn", status: "inProgress", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${method}`); }, + }); + + const { turnId } = await session.startTurn("new work"); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-new-turn" }, + }); + const eventCountBeforeStaleTerminal = events.length; + asInternals(session).handleNotification("codex/event/task_complete", { + threadId: "test-thread", + msg: { type: "task_complete" }, + }); + await new Promise((resolve) => setImmediate(resolve)); + + expect(turnId).toBe("codex-turn-0"); + expect(session.activeForegroundTurnId).toBe("codex-turn-0"); + expect(events).toHaveLength(eventCountBeforeStaleTerminal); + + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-new-turn", status: "completed", error: null }, + }); + expect(session.activeForegroundTurnId).toBeNull(); + }); + + test("reconciles an ID-less terminal for the exact current native turn", async () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-old-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-old-turn", status: "completed", error: null }, + }); + session.client = createStub({ + request: async (method) => { + if (method === "thread/loaded/list") return { data: ["test-thread"] }; + if (method === "turn/start") return {}; + if (method === "thread/read") { + return { + thread: { + turns: [{ id: "native-new-turn", status: "completed", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${method}`); + }, + }); + + const { turnId } = await session.startTurn("new work"); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-new-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { status: "completed", error: null }, + }); + + await vi.waitFor(() => expect(session.activeForegroundTurnId).toBeNull()); + expect(turnId).toBe("codex-turn-0"); + expect(events.at(-1)).toEqual({ + type: "turn_completed", + provider: "codex", + turnId: "codex-turn-0", + usage: undefined, + }); + + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { status: "completed", error: null }, + }); + await new Promise((resolve) => setImmediate(resolve)); + expect( + events.filter((event) => event.type === "turn_completed" && event.turnId === "codex-turn-0"), + ).toHaveLength(1); + }); + + test("ignores a delayed start for a finalized native turn", async () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-old-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-old-turn", status: "completed", error: null }, + }); + session.client = createStub({ + request: async (method) => { + if (method === "thread/loaded/list") return { data: ["test-thread"] }; + if (method === "turn/start") return {}; + throw new Error(`Unexpected request: ${method}`); + }, + }); + + const { turnId } = await session.startTurn("new work"); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-new-turn" }, + }); + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-old-turn" }, + }); + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { id: "native-new-turn", status: "completed", error: null }, + }); + + expect(turnId).toBe("codex-turn-0"); + expect(session.activeForegroundTurnId).toBeNull(); + expect(events.filter((event) => event.type === "turn_started")).toEqual([ + { type: "turn_started", provider: "codex", turnId: "test-turn" }, + { type: "turn_started", provider: "codex", turnId: "codex-turn-0" }, + ]); + expect(events.filter((event) => event.type === "turn_completed")).toEqual([ + { type: "turn_completed", provider: "codex", turnId: "test-turn", usage: undefined }, + { type: "turn_completed", provider: "codex", turnId: "codex-turn-0", usage: undefined }, ]); }); diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.ts index 07c0facf21..8462ef13d3 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.ts @@ -1865,6 +1865,15 @@ const CodexThreadReadResponseSchema = z type CodexThreadReadResponse = z.infer; type CodexThreadReadRequest = (threadId: string) => Promise; +function codexThreadHasTurnStatus(response: unknown, turnId: string, status: string): boolean { + const thread = toObjectRecord(toObjectRecord(response)?.thread); + if (!Array.isArray(thread?.turns)) return false; + return thread.turns.some((turn) => { + const record = toObjectRecord(turn); + return record?.id === turnId && record.status === status; + }); +} + async function requestCodexThreadHistory( requestThread: CodexThreadReadRequest, threadId: string, @@ -3118,6 +3127,7 @@ export class CodexAppServerAgentSession implements AgentSession { private nextTurnOrdinal = 0; private activeForegroundTurnId: string | null = null; private readonly managerTurnIdByNativeTurnId = new Map(); + private readonly finalizedNativeTurnIds = new Set(); private activeClientMessageId: string | null = null; private cachedRuntimeInfo: AgentRuntimeInfo | null = null; private serviceTier: "fast" | null = null; @@ -4292,6 +4302,7 @@ export class CodexAppServerAgentSession implements AgentSession { this.resolvedPermissionRequests.clear(); this.pendingSubAgentNotificationsByThreadId.clear(); this.managerTurnIdByNativeTurnId.clear(); + this.finalizedNativeTurnIds.clear(); this.subscribers.clear(); this.activeForegroundTurnId = null; this.activeClientMessageId = null; @@ -5311,6 +5322,7 @@ export class CodexAppServerAgentSession implements AgentSession { this.emitSubAgentActivityUpdate(subAgentCallId, "running"); return; } + if (this.finalizedNativeTurnIds.has(parsed.turnId)) return; this.currentTurnId = parsed.turnId; const managerTurnId = this.activeForegroundTurnId ?? parsed.turnId; this.managerTurnIdByNativeTurnId.set(parsed.turnId, managerTurnId); @@ -5332,24 +5344,44 @@ export class CodexAppServerAgentSession implements AgentSession { this.emitSubAgentActivityUpdate(subAgentCallId, status); return; } - const managerTurnId = parsed.turnId - ? (this.managerTurnIdByNativeTurnId.get(parsed.turnId) ?? parsed.turnId) - : null; - const turnIdentity = managerTurnId ? { turnId: managerTurnId } : {}; - const completesCurrentTurn = parsed.turnId === null || parsed.turnId === this.currentTurnId; + if (!parsed.turnId) { + const nativeTurnId = this.currentTurnId; + const managerTurnId = nativeTurnId + ? this.managerTurnIdByNativeTurnId.get(nativeTurnId) + : undefined; + if (!nativeTurnId || !managerTurnId) return; + if (this.finalizedNativeTurnIds.has(nativeTurnId)) return; + if (this.finalizedNativeTurnIds.size === 0) { + this.finishTurnCompletedNotification(parsed, nativeTurnId, managerTurnId); + return; + } + void this.reconcileIdlessTurnCompletedNotification(parsed, nativeTurnId, managerTurnId); + return; + } + const managerTurnId = this.managerTurnIdByNativeTurnId.get(parsed.turnId) ?? parsed.turnId; + this.finishTurnCompletedNotification(parsed, parsed.turnId, managerTurnId); + } + + private finishTurnCompletedNotification( + parsed: Extract, + nativeTurnId: string, + managerTurnId: string, + ): void { + this.rememberFinalizedNativeTurn(nativeTurnId); + const completesCurrentTurn = nativeTurnId === this.currentTurnId; if (parsed.status === "failed") { this.emitEvent({ type: "turn_failed", provider: CODEX_PROVIDER, error: parsed.errorMessage ?? "Codex turn failed", - ...turnIdentity, + turnId: managerTurnId, }); } else if (parsed.status === "interrupted") { this.emitEvent({ type: "turn_canceled", provider: CODEX_PROVIDER, reason: "interrupted", - ...turnIdentity, + turnId: managerTurnId, }); } else { if (completesCurrentTurn && this.planModeEnabled && this.latestPlanResult?.text) { @@ -5359,7 +5391,7 @@ export class CodexAppServerAgentSession implements AgentSession { type: "turn_completed", provider: CODEX_PROVIDER, usage: this.latestUsage, - ...turnIdentity, + turnId: managerTurnId, }); } if (!completesCurrentTurn) return; @@ -5369,6 +5401,41 @@ export class CodexAppServerAgentSession implements AgentSession { this.resetTurnTrackingState(); } + private rememberFinalizedNativeTurn(turnId: string): void { + this.finalizedNativeTurnIds.add(turnId); + if (this.finalizedNativeTurnIds.size <= 50) return; + const oldestTurnId = this.finalizedNativeTurnIds.values().next().value; + if (!oldestTurnId) return; + this.finalizedNativeTurnIds.delete(oldestTurnId); + this.managerTurnIdByNativeTurnId.delete(oldestTurnId); + } + + private async reconcileIdlessTurnCompletedNotification( + parsed: Extract, + nativeTurnId: string, + managerTurnId: string, + ): Promise { + const client = this.client; + const threadId = this.currentThreadId; + if (!client || !threadId) return; + try { + const response = await readCodexThread(client, threadId); + const stillOwnsActiveTurn = + this.currentTurnId === nativeTurnId && + this.managerTurnIdByNativeTurnId.get(nativeTurnId) === managerTurnId && + !this.finalizedNativeTurnIds.has(nativeTurnId) && + (this.activeForegroundTurnId === null || this.activeForegroundTurnId === managerTurnId); + if (!stillOwnsActiveTurn) return; + if (!codexThreadHasTurnStatus(response, nativeTurnId, parsed.status)) return; + this.finishTurnCompletedNotification(parsed, nativeTurnId, managerTurnId); + } catch (error) { + this.logger.debug( + { err: error, threadId, nativeTurnId, managerTurnId }, + "Failed to reconcile ID-less Codex terminal notification", + ); + } + } + private resetTurnTrackingState(): void { this.latestPlanResult = null; this.emittedItemStartedIds.clear(); diff --git a/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts b/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts index ef6d5a9390..41084891ee 100644 --- a/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts +++ b/packages/server/src/server/agent/providers/codex/test-utils/fake-app-server.ts @@ -51,7 +51,8 @@ export interface FakeCodexAppServer { waitForTurnStart(): Promise; nextResponse(): Promise; startsTurn(params: { threadId: string; turnId?: string }): void; - completeTurn(params?: { threadId?: string; turnId?: string }): void; + setThreadTurnStatus(params: { threadId: string; turnId: string; status: string }): void; + completeTurn(params?: { threadId?: string; turnId?: string; omitTurnId?: boolean }): void; startsSubAgent(params: { callId: string; threadId: string; @@ -115,6 +116,7 @@ export function createFakeCodexAppServer( const child = createCodexAppServerChildProcess(); const recordedRollbacks: JsonObject[] = []; const activeTurnIdByThreadId = new Map(); + const turnStatusesByThreadId = new Map>(); const responseHandlers: Record = { initialize: () => ({}), "collaborationMode/list": () => ({ data: [] }), @@ -165,7 +167,16 @@ export function createFakeCodexAppServer( }, }; }, - "thread/read": () => ({ thread: { turns: [] } }), + "thread/read": (params) => { + const threadId = toJsonObject(params).threadId; + const statuses = + typeof threadId === "string" ? turnStatusesByThreadId.get(threadId) : undefined; + return { + thread: { + turns: Array.from(statuses ?? []).map(([id, status]) => ({ id, status, items: [] })), + }, + }; + }, ...handlers, }; const messages: JsonObject[] = []; @@ -312,6 +323,9 @@ export function createFakeCodexAppServer( startsTurn(params) { const turnId = params.turnId ?? `turn-${params.threadId}`; activeTurnIdByThreadId.set(params.threadId, turnId); + const statuses = turnStatusesByThreadId.get(params.threadId) ?? new Map(); + statuses.set(turnId, "inProgress"); + turnStatusesByThreadId.set(params.threadId, statuses); child.stdout.write( `${JSON.stringify({ method: "turn/started", @@ -322,16 +336,34 @@ export function createFakeCodexAppServer( })}\n`, ); }, + setThreadTurnStatus(params) { + const statuses = turnStatusesByThreadId.get(params.threadId) ?? new Map(); + statuses.set(params.turnId, params.status); + turnStatusesByThreadId.set(params.threadId, statuses); + }, completeTurn(params = {}) { const threadId = params.threadId ?? "thread-1"; - const turnId = params.turnId ?? activeTurnIdByThreadId.get(threadId); + let turnId = params.turnId ?? activeTurnIdByThreadId.get(threadId); + if (!turnId && !params.omitTurnId && threadId === "thread-1") { + turnId = `turn-${threadId}`; + activeTurnIdByThreadId.set(threadId, turnId); + const statuses = turnStatusesByThreadId.get(threadId) ?? new Map(); + statuses.set(turnId, "inProgress"); + turnStatusesByThreadId.set(threadId, statuses); + writeNotification("turn/started", { threadId, turn: { id: turnId } }); + } + if (turnId) { + const statuses = turnStatusesByThreadId.get(threadId) ?? new Map(); + statuses.set(turnId, "completed"); + turnStatusesByThreadId.set(threadId, statuses); + } child.stdout.write( `${JSON.stringify({ method: "turn/completed", params: { threadId, turn: { - ...(turnId ? { id: turnId } : {}), + ...(turnId && !params.omitTurnId ? { id: turnId } : {}), status: "completed", }, }, diff --git a/packages/server/src/server/material-progress.e2e.test.ts b/packages/server/src/server/material-progress.e2e.test.ts index a534e88e16..6000dbc7f5 100644 --- a/packages/server/src/server/material-progress.e2e.test.ts +++ b/packages/server/src/server/material-progress.e2e.test.ts @@ -156,9 +156,9 @@ test("fetch agent does not inherit a previous stall after a new turn is accepted lastMaterialProgressKind: null, reason: "No material progress has been recorded for the current continuation.", }); - expect( - (await daemon.daemon.agentManager.getMaterialProgressSnapshot(created.id)).rows, - ).toHaveLength(3); + expect((await daemon.daemon.agentManager.getMaterialProgressSnapshot(created.id)).rows).toEqual( + [], + ); } finally { if (agentId) await client.cancelAgent(agentId).catch(() => undefined); await client.close(); From c79fc439675a45f7a46b952e38b306978a99e4fa Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 07:39:01 -0400 Subject: [PATCH 16/17] fix: settle terminal lifecycle races --- .../src/server/agent/agent-manager.test.ts | 143 ++++++++++++++++-- .../server/src/server/agent/agent-manager.ts | 20 ++- .../providers/codex-app-server-agent.test.ts | 89 ++++++++++- .../agent/providers/codex-app-server-agent.ts | 4 - .../src/server/agent/rewind/rewind.test.ts | 30 ++++ .../agent/rewind/test-rewind-session.ts | 7 + 6 files changed, 263 insertions(+), 30 deletions(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 252286d53a..ddaf67d89d 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -139,6 +139,36 @@ class TestDurableTimelineStore implements AgentTimelineStore { } } +class HeldSnapshotStorage extends AgentStorage { + private readonly applyStarted = deferred(); + private readonly applyAllowed = deferred(); + private holdNextApply = false; + + holdNextSnapshot(): void { + this.holdNextApply = true; + } + + waitForHeldSnapshot(): Promise { + return this.applyStarted.promise; + } + + releaseHeldSnapshot(): void { + this.applyAllowed.resolve(); + } + + override async applySnapshot( + agent: ManagedAgent, + options?: Parameters[1], + ): Promise { + if (this.holdNextApply) { + this.holdNextApply = false; + this.applyStarted.resolve(); + await this.applyAllowed.promise; + } + await super.applySnapshot(agent, options); + } +} + function waitForAgentLifecycle( manager: AgentManager, agentId: string, @@ -530,14 +560,17 @@ class ControlledInterruptSession extends TestAgentSession { config: AgentSessionConfig, readonly turnId: string, private readonly interruptBehavior: (session: ControlledInterruptSession) => Promise, + private readonly deferTurnStarted: boolean, ) { super(config); } override async startTurn(): Promise<{ turnId: string }> { - setTimeout(() => { - this.pushEvent({ type: "turn_started", provider: this.provider, turnId: this.turnId }); - }, 0); + if (!this.deferTurnStarted) { + setTimeout(() => { + this.pushEvent({ type: "turn_started", provider: this.provider, turnId: this.turnId }); + }, 0); + } return { turnId: this.turnId }; } @@ -551,8 +584,11 @@ interface ControlledInterruptFixture { agentId: string; manager: AgentManager; session: ControlledInterruptSession; + holdNextSnapshotPersist(): void; + waitForSnapshotPersistStart(): Promise; + releaseSnapshotPersist(): void; startForegroundRun(): Promise; - cleanup(): void; + cleanup(): Promise; } async function createControlledInterruptFixture(options: { @@ -560,12 +596,15 @@ async function createControlledInterruptFixture(options: { agentId: string; turnId: string; interrupt: (session: ControlledInterruptSession) => Promise; + deferTurnStarted?: boolean; }): Promise { const workdir = mkdtempSync(join(tmpdir(), `agent-manager-${options.name}-`)); + const storage = new HeldSnapshotStorage(join(workdir, "agents"), logger); const session = new ControlledInterruptSession( { provider: "codex", cwd: workdir }, options.turnId, options.interrupt, + options.deferTurnStarted ?? false, ); const client = new (class extends TestAgentClient { override async createSession(): Promise { @@ -574,7 +613,7 @@ async function createControlledInterruptFixture(options: { })(); const manager = new AgentManager({ clients: { codex: client }, - registry: new AgentStorage(join(workdir, "agents"), logger), + registry: storage, logger, rescueTimeouts: { interruptSessionMs: 10 }, idFactory: () => options.agentId, @@ -587,6 +626,15 @@ async function createControlledInterruptFixture(options: { agentId: agent.id, manager, session, + holdNextSnapshotPersist() { + storage.holdNextSnapshot(); + }, + waitForSnapshotPersistStart() { + return storage.waitForHeldSnapshot(); + }, + releaseSnapshotPersist() { + storage.releaseHeldSnapshot(); + }, async startForegroundRun() { const run = manager.streamAgent(agent.id, "exercise cancellation"); void (async () => { @@ -596,7 +644,12 @@ async function createControlledInterruptFixture(options: { })(); await manager.waitForAgentRunStart(agent.id); }, - cleanup: () => rmSync(workdir, { recursive: true, force: true }), + async cleanup() { + storage.releaseHeldSnapshot(); + await manager.flush().catch(() => undefined); + await storage.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + }, }; } @@ -1612,7 +1665,7 @@ test("cancelAgentRun preserves running state when the provider interrupt hangs", expect(fixture.session.interruptCalled).toBe(true); expect(fixture.manager.getAgent(fixture.agentId)?.lifecycle).toBe("running"); } finally { - fixture.cleanup(); + await fixture.cleanup(); } }); @@ -1643,7 +1696,7 @@ test("cancelAgentRun preserves the active turn when the provider rejects the int turnId: "provider-still-active-turn", }); } finally { - fixture.cleanup(); + await fixture.cleanup(); } }); @@ -1676,15 +1729,16 @@ test("cancelAgentRun succeeds when the foreground turn finishes before the provi activeForegroundTurnId: null, }); } finally { - fixture.cleanup(); + await fixture.cleanup(); } }); -test("cancelAgentRun succeeds when the provider queues completion before rejecting the interrupt", async () => { +test("cancelAgentRun settles a queued completion while turn-start persistence is blocked", async () => { const fixture = await createControlledInterruptFixture({ name: "interrupt-queued-completion", agentId: "00000000-0000-4000-8000-000000000306", turnId: "queued-completion-turn", + deferTurnStarted: true, interrupt: async (session) => { session.pushEvent({ type: "turn_completed", @@ -1697,6 +1751,13 @@ test("cancelAgentRun succeeds when the provider queues completion before rejecti try { await fixture.startForegroundRun(); + fixture.holdNextSnapshotPersist(); + fixture.session.pushEvent({ + type: "turn_started", + provider: "codex", + turnId: "queued-completion-turn", + }); + await fixture.waitForSnapshotPersistStart(); await expect(fixture.manager.cancelAgentRun(fixture.agentId)).resolves.toEqual({ status: "settled", @@ -1706,7 +1767,66 @@ test("cancelAgentRun succeeds when the provider queues completion before rejecti activeForegroundTurnId: null, }); } finally { - fixture.cleanup(); + fixture.releaseSnapshotPersist(); + await fixture.cleanup(); + } +}); + +test("reloadAgentSession proceeds when the active run completes before interrupt rejection", async () => { + let fixture!: ControlledInterruptFixture; + fixture = await createControlledInterruptFixture({ + name: "reload-after-completion", + agentId: "00000000-0000-4000-8000-000000000307", + turnId: "reload-completed-turn", + interrupt: async (session) => { + const settled = waitForAgentLifecycle(fixture.manager, fixture.agentId, "idle"); + session.pushEvent({ + type: "turn_completed", + provider: session.provider, + turnId: "reload-completed-turn", + }); + await settled; + throw new Error("turn already completed"); + }, + }); + + try { + await fixture.startForegroundRun(); + + await expect(fixture.manager.reloadAgentSession(fixture.agentId)).resolves.toMatchObject({ + id: fixture.agentId, + lifecycle: "idle", + }); + } finally { + await fixture.cleanup(); + } +}); + +test("replaceAgentRun proceeds when the active run completes before interrupt rejection", async () => { + const fixture = await createControlledInterruptFixture({ + name: "replace-after-completion", + agentId: "00000000-0000-4000-8000-000000000308", + turnId: "replace-completed-turn", + interrupt: async (session) => { + session.pushEvent({ + type: "turn_completed", + provider: session.provider, + turnId: "replace-completed-turn", + }); + await new Promise((resolve) => setImmediate(resolve)); + throw new Error("turn already completed"); + }, + }); + + try { + await fixture.startForegroundRun(); + + await expect( + fixture.manager.replaceAgentRun(fixture.agentId, "replacement work"), + ).resolves.toBeDefined(); + expect(fixture.manager.getAgent(fixture.agentId)?.activeForegroundTurnId).toBeNull(); + } finally { + await fixture.cleanup(); } }); @@ -5380,7 +5500,6 @@ test("replaceAgentRun does not emit idle or resolve waiters between interrupted })(); await manager.waitForAgentRunStart(snapshot.id); - const waitPromise = manager.waitForAgentEvent(snapshot.id); const secondRun = await manager.replaceAgentRun(snapshot.id, "second run"); const secondRunDrain = (async () => { diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index d41caf9b64..1b88cda01d 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -2143,7 +2143,7 @@ export class AgentManager { turnStream = this.runs.createTurnStream(turnId); this.runs.addWaiter(agent, turnStream.waiter); this.touchUpdatedAt(agent); - await this.persistSnapshot(agent); + this.enqueueBackgroundPersist(agent); this.emitState(agent, { persist: false }); this.logger.trace( { @@ -3564,6 +3564,13 @@ export class AgentManager { const eventTurnId = getAgentStreamEventTurnId(event); const isForegroundEvent = Boolean(eventTurnId && agent.activeForegroundTurnId === eventTurnId); this.traceHandleStreamEventStart(agent, event, eventTurnId, isForegroundEvent); + if ( + event.type === "turn_started" && + eventTurnId && + this.runs.hasFinalizedTurn(agent, eventTurnId) + ) { + return false; + } if (this.shouldIgnoreTerminalEvent(agent, event, eventTurnId)) { return false; } @@ -3746,7 +3753,8 @@ export class AgentManager { this.onStreamTurnCanceled({ agent, event, eventTurnId, isForegroundEvent, options }); return undefined; case "turn_started": - return this.onStreamTurnStarted({ agent, eventTurnId, isForegroundEvent, options }); + this.onStreamTurnStarted({ agent, eventTurnId, isForegroundEvent, options }); + return undefined; case "permission_requested": this.onStreamPermissionRequested(agent, event); return undefined; @@ -3928,12 +3936,12 @@ export class AgentManager { } } - private async onStreamTurnStarted(params: { + private onStreamTurnStarted(params: { agent: ActiveManagedAgent; eventTurnId: string | undefined; isForegroundEvent: boolean; options: HandleStreamEventOptions | undefined; - }): Promise { + }): void { const { agent, eventTurnId, isForegroundEvent, options } = params; if (options?.fromHistory) { agent.lastTurnOutcome = null; @@ -3954,11 +3962,11 @@ export class AgentManager { if (!isForegroundEvent) { this.runs.trackAutonomousRun(agent.id, eventTurnId ?? null); agent.lifecycle = "running"; - await this.persistSnapshot(agent); + this.enqueueBackgroundPersist(agent); this.emitState(agent, { persist: false }); return; } - await this.persistSnapshot(agent); + this.enqueueBackgroundPersist(agent); } private beginMaterialProgressContinuation( diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts index d3feca59cf..17e0b99efa 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.test.ts @@ -2552,10 +2552,22 @@ describe("Codex app-server provider", () => { }); }); - test("never treats an unmapped foreign terminal as the root terminal", () => { + test("never treats an unmapped foreign terminal as the root terminal", async () => { const session = createSession(); const events: AgentStreamEvent[] = []; session.subscribe((event) => events.push(event)); + session.client = createStub({ + request: async (method) => { + if (method === "thread/read") { + return { + thread: { + turns: [{ id: "first-root-turn", status: "completed", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${method}`); + }, + }); asInternals(session).handleNotification("turn/completed", { threadId: "unmapped-child-thread", @@ -2571,6 +2583,7 @@ describe("Codex app-server provider", () => { threadId: "test-thread", turn: { status: "completed" }, }); + await vi.waitFor(() => expect(session.activeForegroundTurnId).toBeNull()); expect(events.filter((event) => event.type === "turn_completed")).toHaveLength(1); asInternals(session).handleNotification("turn/started", { @@ -2593,10 +2606,22 @@ describe("Codex app-server provider", () => { }); }); - test("routes msg-scoped legacy Codex events to their child thread", () => { + test("routes msg-scoped legacy Codex events to their child thread", async () => { const session = createSession(); const events: AgentStreamEvent[] = []; session.subscribe((event) => events.push(event)); + session.client = createStub({ + request: async (method) => { + if (method === "thread/read") { + return { + thread: { + turns: [{ id: "root-turn", status: "completed", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${method}`); + }, + }); asInternals(session).handleNotification("turn/started", { threadId: "test-thread", @@ -2661,6 +2686,7 @@ describe("Codex app-server provider", () => { asInternals(session).handleNotification("codex/event/task_complete", { msg: { type: "task_complete" }, }); + await vi.waitFor(() => expect(session.activeForegroundTurnId).toBeNull()); expect(events.filter((event) => event.type === "turn_completed")).toHaveLength(1); }); @@ -4086,7 +4112,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the original plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-pending", status: "completed", error: null }, }); const pendingPlan = session.getPendingPermissions()[0]; expect(pendingPlan).toBeDefined(); @@ -4193,7 +4219,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the original plan", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-pending", status: "completed", error: null }, }); const pendingPlan = session.getPendingPermissions()[0]; expect(pendingPlan).toBeDefined(); @@ -4513,10 +4539,21 @@ describe("Codex app-server provider", () => { test.each([ ["turn/completed", { threadId: "test-thread", turn: { status: "completed", error: null } }], ["codex/event/task_complete", { threadId: "test-thread", msg: { type: "task_complete" } }], - ])("binds a current ID-less terminal from %s to the exact active run", (method, params) => { + ])("reconciles a current first-turn ID-less terminal from %s", async (method, params) => { const session = createSession(); const events: AgentStreamEvent[] = []; session.subscribe((event) => events.push(event)); + const request = vi.fn(async (requestMethod: string) => { + if (requestMethod === "thread/read") { + return { + thread: { + turns: [{ id: "native-current-turn", status: "completed", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${requestMethod}`); + }); + session.client = createStub({ request }); asInternals(session).handleNotification("turn/started", { threadId: "test-thread", @@ -4524,6 +4561,11 @@ describe("Codex app-server provider", () => { }); asInternals(session).handleNotification(method, params); + await vi.waitFor(() => expect(session.activeForegroundTurnId).toBeNull()); + expect(request).toHaveBeenCalledWith("thread/read", { + threadId: "test-thread", + includeTurns: true, + }); expect(session.activeForegroundTurnId).toBeNull(); expect(events.at(-1)).toEqual({ type: "turn_completed", @@ -4533,6 +4575,37 @@ describe("Codex app-server provider", () => { }); }); + test("does not bind a delayed ID-less terminal to an in-progress first native turn", async () => { + const session = createSession(); + const events: AgentStreamEvent[] = []; + session.subscribe((event) => events.push(event)); + const request = vi.fn(async (requestMethod: string) => { + if (requestMethod === "thread/read") { + return { + thread: { + turns: [{ id: "native-current-turn", status: "inProgress", items: [] }], + }, + }; + } + throw new Error(`Unexpected request: ${requestMethod}`); + }); + session.client = createStub({ request }); + + asInternals(session).handleNotification("turn/started", { + threadId: "test-thread", + turn: { id: "native-current-turn" }, + }); + const eventCountBeforeDelayedTerminal = events.length; + asInternals(session).handleNotification("turn/completed", { + threadId: "test-thread", + turn: { status: "completed", error: null }, + }); + + await vi.waitFor(() => expect(request).toHaveBeenCalledTimes(1)); + expect(session.activeForegroundTurnId).toBe("test-turn"); + expect(events).toHaveLength(eventCountBeforeDelayedTerminal); + }); + test("does not bind a stale ID-less terminal to a newer native turn", async () => { const session = createSession(); const events: AgentStreamEvent[] = []; @@ -4909,7 +4982,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the new flow", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-2", status: "completed", error: null }, }); const request = events.find( @@ -4963,7 +5036,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the safe flow", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-3", status: "completed", error: null }, }); const request = events.find( @@ -5031,7 +5104,7 @@ describe("Codex app-server provider", () => { plan: [{ step: "Implement the fast flow", status: "pending" }], }); asInternals(session).handleNotification("turn/completed", { - turn: { status: "completed", error: null }, + turn: { id: "turn-plan-4", status: "completed", error: null }, }); const permissionRequest = events.find( diff --git a/packages/server/src/server/agent/providers/codex-app-server-agent.ts b/packages/server/src/server/agent/providers/codex-app-server-agent.ts index 8462ef13d3..cc6952cb8f 100644 --- a/packages/server/src/server/agent/providers/codex-app-server-agent.ts +++ b/packages/server/src/server/agent/providers/codex-app-server-agent.ts @@ -5351,10 +5351,6 @@ export class CodexAppServerAgentSession implements AgentSession { : undefined; if (!nativeTurnId || !managerTurnId) return; if (this.finalizedNativeTurnIds.has(nativeTurnId)) return; - if (this.finalizedNativeTurnIds.size === 0) { - this.finishTurnCompletedNotification(parsed, nativeTurnId, managerTurnId); - return; - } void this.reconcileIdlessTurnCompletedNotification(parsed, nativeTurnId, managerTurnId); return; } diff --git a/packages/server/src/server/agent/rewind/rewind.test.ts b/packages/server/src/server/agent/rewind/rewind.test.ts index a321fb79fd..425f1ba457 100644 --- a/packages/server/src/server/agent/rewind/rewind.test.ts +++ b/packages/server/src/server/agent/rewind/rewind.test.ts @@ -129,6 +129,36 @@ describe("AgentManager rewind", () => { }); }); + test("rewinds when the active turn completes before interrupt rejection", async () => { + let manager!: AgentManager; + let agentId!: string; + + class CompletingInterruptSession extends FakeRewindSession { + override async interrupt(): Promise { + const settled = manager.waitForAgentEvent(agentId); + this.completeActiveTurn(); + await settled; + throw new Error("turn already completed"); + } + } + + const session = new CompletingInterruptSession(); + manager = new AgentManager({ + clients: { claude: new FakeRewindClient(session) }, + logger: createTestLogger(), + idFactory: () => "00000000-0000-4000-8000-000000000903", + }); + const agent = await manager.createAgent({ provider: "claude", cwd: process.cwd() }, undefined, { + workspaceId: undefined, + }); + agentId = agent.id; + const run = manager.streamAgent(agentId, "keep working"); + await run.next(); + + await expect(manager.rewind(agentId, "message-1", "files")).resolves.toBeUndefined(); + expect(session.recordedRewinds).toEqual([{ mode: "files", messageId: "message-1" }]); + }); + test("blocks new prompts until the rehydrate epoch broadcasts", async () => { const historyGate = new RewindHistoryGate(); historyGate.hold(); diff --git a/packages/server/src/server/agent/rewind/test-rewind-session.ts b/packages/server/src/server/agent/rewind/test-rewind-session.ts index e717ca051d..d6b69776e5 100644 --- a/packages/server/src/server/agent/rewind/test-rewind-session.ts +++ b/packages/server/src/server/agent/rewind/test-rewind-session.ts @@ -103,6 +103,13 @@ export class FakeRewindSession implements AgentSession { } } + completeActiveTurn(): void { + if (!this.activeTurnId) return; + const turnId = this.activeTurnId; + this.activeTurnId = null; + this.emit({ type: "turn_completed", provider: this.provider, turnId }); + } + async close(): Promise {} async revertConversation(input: { messageId: string }): Promise { From 751fd6574730e21516171f9fb65ca58f1537bd6d Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 08:32:11 -0400 Subject: [PATCH 17/17] fix: settle pending start boundaries --- .../src/server/agent/agent-manager.test.ts | 242 +++++++++++++++++- .../server/src/server/agent/agent-manager.ts | 33 ++- .../src/server/agent/agent-run-state.ts | 47 +++- .../src/server/agent/rewind/rewind.test.ts | 64 +++++ .../agent/rewind/test-rewind-session.ts | 4 +- 5 files changed, 380 insertions(+), 10 deletions(-) diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index ddaf67d89d..b9888b52bb 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -580,6 +580,90 @@ class ControlledInterruptSession extends TestAgentSession { } } +class PendingStartSession extends TestAgentSession { + private readonly firstStartRequested = deferred(); + private readonly firstStartAllowed = deferred(); + private turnCount = 0; + + override async startTurn(): Promise<{ turnId: string }> { + const turnId = `pending-start-turn-${++this.turnCount}`; + if (this.turnCount === 1) { + this.firstStartRequested.resolve(); + await this.firstStartAllowed.promise; + return { turnId }; + } + + setTimeout(() => { + this.pushEvent({ type: "turn_started", provider: this.provider, turnId }); + this.pushEvent({ type: "turn_completed", provider: this.provider, turnId }); + }, 0); + return { turnId }; + } + + waitForPendingStart(): Promise { + return this.firstStartRequested.promise; + } + + resolvePendingStart(): void { + this.firstStartAllowed.resolve(); + } + + override async interrupt(): Promise {} +} + +interface PendingStartFixture { + agentId: string; + manager: AgentManager; + session: PendingStartSession; + startDrain: Promise; + cleanup(): Promise; +} + +async function createPendingStartFixture( + name: string, + agentId: string, +): Promise { + const workdir = mkdtempSync(join(tmpdir(), `agent-manager-${name}-`)); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const session = new PendingStartSession({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + rescueTimeouts: { interruptSessionMs: 10 }, + idFactory: () => agentId, + }); + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + const stream = manager.streamAgent(agent.id, "hold start resolution"); + const startDrain = (async () => { + for await (const _event of stream) { + // Keep the stream subscribed until the pending generation settles. + } + })(); + await session.waitForPendingStart(); + + return { + agentId: agent.id, + manager, + session, + startDrain, + async cleanup() { + session.resolvePendingStart(); + await startDrain.catch(() => undefined); + await manager.flush().catch(() => undefined); + await storage.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + }, + }; +} + interface ControlledInterruptFixture { agentId: string; manager: AgentManager; @@ -1772,6 +1856,152 @@ test("cancelAgentRun settles a queued completion while turn-start persistence is } }); +test("a terminal emitted before startTurn resolves settles the matching foreground stream", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-eager-terminal-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const session = new (class extends TestAgentSession { + override async startTurn(): Promise<{ turnId: string }> { + const turnId = "eager-terminal-turn"; + this.pushEvent({ type: "turn_started", provider: this.provider, turnId }); + this.pushEvent({ + type: "timeline", + provider: this.provider, + turnId, + item: { type: "assistant_message", text: "finished before start returned" }, + }); + this.pushEvent({ type: "turn_completed", provider: this.provider, turnId }); + await new Promise((resolve) => setImmediate(resolve)); + setTimeout(() => { + this.pushEvent({ + type: "turn_failed", + provider: this.provider, + turnId, + error: "late fallback should be ignored", + }); + }, 0); + return { turnId }; + } + })({ provider: "codex", cwd: workdir }); + const client = new (class extends TestAgentClient { + override async createSession(): Promise { + return session; + } + })(); + const manager = new AgentManager({ + clients: { codex: client }, + registry: storage, + logger, + idFactory: () => "00000000-0000-4000-8000-000000000309", + }); + + try { + const agent = await manager.createAgent({ provider: "codex", cwd: workdir }, undefined, { + workspaceId: undefined, + }); + const events: AgentStreamEvent[] = []; + for await (const event of manager.streamAgent(agent.id, "finish eagerly")) { + events.push(event); + } + await manager.flush(); + + expect(events.map((event) => event.type)).toContain("turn_completed"); + expect(events.map((event) => event.type)).not.toContain("turn_failed"); + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + lastTurnOutcome: "completed", + }); + expect(manager.getTimeline(agent.id)).toContainEqual({ + type: "assistant_message", + text: "finished before start returned", + }); + } finally { + await manager.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + +test("cancelAgentRun force-cancels an acknowledged pending start generation", async () => { + const fixture = await createPendingStartFixture( + "cancel-pending-start", + "00000000-0000-4000-8000-000000000310", + ); + + try { + await expect(fixture.manager.cancelAgentRun(fixture.agentId)).resolves.toEqual({ + status: "settled", + }); + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + + fixture.session.resolvePendingStart(); + await fixture.startDrain; + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + } finally { + await fixture.cleanup(); + } +}); + +test("replaceAgentRun does not let a late pending start replace the new generation", async () => { + const fixture = await createPendingStartFixture( + "replace-pending-start", + "00000000-0000-4000-8000-000000000311", + ); + + try { + const replacement = await fixture.manager.replaceAgentRun(fixture.agentId, "replacement work"); + fixture.session.resolvePendingStart(); + await fixture.startDrain; + + const replacementEvents: AgentStreamEvent[] = []; + for await (const event of replacement) replacementEvents.push(event); + + expect(replacementEvents.map((event) => event.type)).toContain("turn_completed"); + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + } finally { + await fixture.cleanup(); + } +}); + +test("reloadAgentSession does not restore a force-canceled pending start", async () => { + const fixture = await createPendingStartFixture( + "reload-pending-start", + "00000000-0000-4000-8000-000000000312", + ); + + try { + await expect(fixture.manager.reloadAgentSession(fixture.agentId)).resolves.toMatchObject({ + id: fixture.agentId, + lifecycle: "idle", + activeForegroundTurnId: null, + }); + + fixture.session.resolvePendingStart(); + await fixture.startDrain; + await new Promise((resolve) => setImmediate(resolve)); + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + + await fixture.manager.runAgent(fixture.agentId, "work after reload"); + expect(fixture.manager.getAgent(fixture.agentId)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + } finally { + await fixture.cleanup(); + } +}); + test("reloadAgentSession proceeds when the active run completes before interrupt rejection", async () => { let fixture!: ControlledInterruptFixture; fixture = await createControlledInterruptFixture({ @@ -4288,7 +4518,7 @@ test("material progress pages beyond the durable 1000-row tail to the current co } }); -test("material progress stops at a known continuation boundary in a tail without a new user row", async () => { +test("material progress keeps a known continuation boundary across a later live user row", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-material-progress-known-boundary-")); const storage = new AgentStorage(join(workdir, "agents"), logger); const durableTimelineStore = new TestDurableTimelineStore(); @@ -4351,11 +4581,21 @@ test("material progress stops at a known continuation boundary in a tail without created.id, { materialProgressContinuationBoundarySeq: 10_004 }, ); + await resumedManager.appendTimelineItem(created.id, { + type: "user_message", + text: "provider follow-up within the same continuation", + }); + await resumedManager.appendTimelineItem(created.id, { + type: "reasoning", + text: "continue after the provider follow-up", + }); const snapshot = await resumedManager.getMaterialProgressSnapshot(created.id); expect(snapshot.rows).toMatchObject([ { seq: 10_004, item: { type: "tool_call", detail: { type: "write" } } }, { seq: 10_005, item: { type: "reasoning" } }, + { seq: 10_006, item: { type: "user_message" } }, + { seq: 10_007, item: { type: "reasoning" } }, ]); expect(snapshot).toMatchObject({ persisted: { diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 1b88cda01d..3a7c70b8d8 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -2116,10 +2116,16 @@ export class AgentManager { const streamForwarder = async function* streamForwarder(this: AgentManager) { let turnId: string; let turnStream: ReturnType | null = null; + if (!this.runs.isPendingRun(agentId, pendingRun.token)) { + return; + } try { const result = await agent.session.startTurn(prompt, options); turnId = result.turnId; } catch (error) { + if (!this.runs.isPendingRun(agentId, pendingRun.token)) { + return; + } agent.pendingReplacement = false; const errorMsg = error instanceof Error ? error.message : "Failed to start turn"; await this.handleStreamEvent(agent, { @@ -2132,8 +2138,11 @@ export class AgentManager { throw error; } - pendingRun.started = true; - pendingRun.turnId = turnId; + const pendingTerminalEvents = this.runs.bindPendingRun(agentId, pendingRun.token, turnId); + if (pendingTerminalEvents === null) { + this.runs.rememberFinalizedTurn(agent, turnId); + return; + } if (isReplacement) { agent.pendingReplacement = false; } @@ -2157,6 +2166,10 @@ export class AgentManager { "agent.manager.stream.start", ); + for (const terminalEvent of pendingTerminalEvents) { + await this.dispatchSessionEvent(agent, terminalEvent); + } + try { for await (const event of turnStream.events(isTurnTerminalEvent)) { yield event; @@ -2401,6 +2414,17 @@ export class AgentManager { } const interruptAcknowledged = await this.interruptSession(agent.session, agentId); + if ( + interruptAcknowledged && + run.kind === "foreground" && + run.turnId === null && + this.runs.settleForegroundRun(agentId, run.token) + ) { + agent.lastError = undefined; + agent.lastTurnOutcome = "canceled"; + this.endMaterialProgressContinuation(agent); + this.finalizeForegroundTurn(agent); + } const settlement = await this.waitWithTimeout({ operation: run.settledPromise, timeoutMs: interruptAcknowledged @@ -3145,6 +3169,9 @@ export class AgentManager { this.dispatch({ type: "provider_subagent", event: update }); return; } + if (isTurnTerminalEvent(event) && this.runs.bufferPendingTerminalEvent(agent.id, event)) { + return; + } const turnId = getAgentStreamEventTurnId(event); const matchingWaiters = this.runs.getMatchingWaiters(agent, turnId); this.logger.trace( @@ -3286,7 +3313,7 @@ export class AgentManager { return null; } const combinedRows = [...olderDurableRows, ...livePage.rows]; - return currentContinuationRows(combinedRows); + return selectMaterialProgressRows(combinedRows, continuationBoundarySeq, true); } const durableRows = await this.resolveDurableMaterialProgressRows( diff --git a/packages/server/src/server/agent/agent-run-state.ts b/packages/server/src/server/agent/agent-run-state.ts index e747317181..c04fa42f64 100644 --- a/packages/server/src/server/agent/agent-run-state.ts +++ b/packages/server/src/server/agent/agent-run-state.ts @@ -14,6 +14,7 @@ export interface PendingForegroundRun { token: string; kind: "foreground"; turnId: string | null; + pendingTerminalEvents: AgentStreamEvent[]; started: boolean; settled: boolean; settledPromise: Promise; @@ -59,6 +60,32 @@ export class AgentRunState { return this.runs.get(agentId) ?? null; } + isPendingRun(agentId: string, token: string): boolean { + const run = this.runs.get(agentId); + return run?.kind === "foreground" && run.token === token; + } + + bindPendingRun(agentId: string, token: string, turnId: string): AgentStreamEvent[] | null { + const run = this.runs.get(agentId); + if (run?.kind !== "foreground" || run.token !== token) { + return null; + } + + run.started = true; + run.turnId = turnId; + return run.pendingTerminalEvents.splice(0); + } + + bufferPendingTerminalEvent(agentId: string, event: AgentStreamEvent): boolean { + const run = this.runs.get(agentId); + if (run?.kind !== "foreground" || run.turnId !== null) { + return false; + } + + run.pendingTerminalEvents.push(event); + return true; + } + hasRun(agentId: string): boolean { return this.runs.has(agentId); } @@ -94,13 +121,14 @@ export class AgentRunState { this.clearRun(agentId, run); } - settleForegroundRun(agentId: string, token: string): void { + settleForegroundRun(agentId: string, token: string): boolean { const run = this.runs.get(agentId); if (run?.kind !== "foreground" || run.token !== token) { - return; + return false; } this.clearRun(agentId, run); + return true; } clearAgentRun(agentId: string): void { @@ -259,12 +287,18 @@ export class ForegroundTurnStream { } function createPendingForegroundRun(): PendingForegroundRun { - return createTrackedRun({ kind: "foreground", turnId: null, started: false }); + return createTrackedRun({ + kind: "foreground", + turnId: null, + pendingTerminalEvents: [], + started: false, + }); } function createTrackedRun(input: { kind: "foreground"; turnId: null; + pendingTerminalEvents: AgentStreamEvent[]; started: false; }): PendingForegroundRun; function createTrackedRun(input: { @@ -274,7 +308,12 @@ function createTrackedRun(input: { }): AutonomousAgentRun; function createTrackedRun( input: - | { kind: "foreground"; turnId: null; started: false } + | { + kind: "foreground"; + turnId: null; + pendingTerminalEvents: AgentStreamEvent[]; + started: false; + } | { kind: "autonomous"; turnId: string | null; started: true }, ): TrackedAgentRun { let resolveSettled!: () => void; diff --git a/packages/server/src/server/agent/rewind/rewind.test.ts b/packages/server/src/server/agent/rewind/rewind.test.ts index 425f1ba457..b2a8f0e20b 100644 --- a/packages/server/src/server/agent/rewind/rewind.test.ts +++ b/packages/server/src/server/agent/rewind/rewind.test.ts @@ -159,6 +159,70 @@ describe("AgentManager rewind", () => { expect(session.recordedRewinds).toEqual([{ mode: "files", messageId: "message-1" }]); }); + test("rewinds after force-canceling an acknowledged pending start generation", async () => { + let resolveStart!: () => void; + let allowTerminal!: () => void; + let startRequested!: () => void; + const startGate = new Promise((resolve) => { + resolveStart = resolve; + }); + const terminalGate = new Promise((resolve) => { + allowTerminal = resolve; + }); + const requested = new Promise((resolve) => { + startRequested = resolve; + }); + + class PendingStartRewindSession extends FakeRewindSession { + override async startTurn(): Promise<{ turnId: string }> { + startRequested(); + await startGate; + const turnId = "pending-rewind-turn"; + this.activeTurnId = turnId; + void terminalGate.then(() => { + this.activeTurnId = null; + this.emit({ type: "turn_completed", provider: this.provider, turnId }); + return undefined; + }); + return { turnId }; + } + + override async interrupt(): Promise { + this.aborted = true; + } + } + + const session = new PendingStartRewindSession(); + const manager = new AgentManager({ + clients: { claude: new FakeRewindClient(session) }, + logger: createTestLogger(), + idFactory: () => "00000000-0000-4000-8000-000000000904", + }); + const agent = await manager.createAgent({ provider: "claude", cwd: process.cwd() }, undefined, { + workspaceId: undefined, + }); + const stream = manager.streamAgent(agent.id, "keep working"); + const drain = (async () => { + for await (const _event of stream) { + // Drain until the canceled pending generation exits. + } + })(); + await requested; + + await manager.rewind(agent.id, "message-1", "files"); + resolveStart(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(session.recordedRewinds).toEqual([{ mode: "files", messageId: "message-1" }]); + expect(manager.getAgent(agent.id)).toMatchObject({ + lifecycle: "idle", + activeForegroundTurnId: null, + }); + + allowTerminal(); + await drain; + }); + test("blocks new prompts until the rehydrate epoch broadcasts", async () => { const historyGate = new RewindHistoryGate(); historyGate.hold(); diff --git a/packages/server/src/server/agent/rewind/test-rewind-session.ts b/packages/server/src/server/agent/rewind/test-rewind-session.ts index d6b69776e5..dc916468a7 100644 --- a/packages/server/src/server/agent/rewind/test-rewind-session.ts +++ b/packages/server/src/server/agent/rewind/test-rewind-session.ts @@ -37,7 +37,7 @@ export class FakeRewindSession implements AgentSession { history: AgentTimelineItem[] = [{ type: "user_message", text: "before", messageId: "message-1" }]; private subscribers = new Set<(event: AgentStreamEvent) => void>(); - private activeTurnId: string | null = null; + protected activeTurnId: string | null = null; constructor(private readonly waitBeforeHistory?: () => Promise) {} @@ -124,7 +124,7 @@ export class FakeRewindSession implements AgentSession { this.recordedRewinds.push({ mode: "both", messageId: input.messageId }); } - private emit(event: AgentStreamEvent): void { + protected emit(event: AgentStreamEvent): void { for (const subscriber of this.subscribers) { subscriber(event); }