diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index bc7e7a2b26..d60820af03 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -4735,6 +4735,8 @@ export const PaseoWorktreeArchiveResponseSchema = z.object({ payload: z.object({ success: z.boolean(), removedAgents: z.array(z.string()).optional(), + removedDirectory: z.boolean().optional(), + cleanupPending: z.boolean().optional(), error: CheckoutErrorSchema.nullable(), requestId: z.string(), }), diff --git a/packages/server/src/server/agent/agent-loading.ts b/packages/server/src/server/agent/agent-loading.ts index f87f38911c..103f36ebe8 100644 --- a/packages/server/src/server/agent/agent-loading.ts +++ b/packages/server/src/server/agent/agent-loading.ts @@ -22,6 +22,7 @@ export type AgentLoaderManager = Pick< AgentManager, | "createAgent" | "getAgent" + | "getAgentInitializationState" | "getRegisteredProviderIds" | "hydrateTimelineFromProvider" | "resumeAgentFromPersistence" @@ -63,28 +64,26 @@ export async function ensureAgentLoaded( agentId: string, deps: EnsureAgentLoadedDeps, ): Promise { - await deps.agentManager.waitForAgentClose?.(agentId); + while (true) { + await deps.agentManager.waitForAgentClose?.(agentId); - const inflight = pendingAgentInitializations.get(agentId); - if (inflight) { - inflight.options.broadcastTimeline ||= deps.broadcastTimeline === true; - return inflight.promise; - } - - const existing = deps.agentManager.getAgent(agentId); - if (existing) { - return existing; - } - - // A close may have started after the first barrier observed no in-flight - // work. Once the live lookup is empty, this second barrier closes that gap - // before storage-backed resume begins. - await deps.agentManager.waitForAgentClose?.(agentId); + const inflight = pendingAgentInitializations.get(agentId); + if (inflight) { + inflight.options.broadcastTimeline ||= deps.broadcastTimeline === true; + return inflight.promise; + } - const laterInflight = pendingAgentInitializations.get(agentId); - if (laterInflight) { - laterInflight.options.broadcastTimeline ||= deps.broadcastTimeline === true; - return laterInflight.promise; + // The close barrier and the runtime lookup cannot be made atomic across an + // await. Re-read both pieces of state synchronously so a close that starts + // in that gap is joined on the next pass instead of returning its runtime. + const initializationState = deps.agentManager.getAgentInitializationState(agentId); + if (initializationState.closeInFlight) { + continue; + } + if (initializationState.agent) { + return initializationState.agent; + } + break; } const pendingOptions = { @@ -109,7 +108,10 @@ export async function ensureAgentLoaded( handle, buildConfigOverrides(record), agentId, - extractTimestamps(record), + { + ...extractTimestamps(record), + autoArchiveObligation: record.autoArchiveObligation, + }, record.archivedAt ? { purpose: "history" } : undefined, ); deps.logger.info({ agentId, provider: record.provider }, "Agent resumed from persistence"); @@ -124,6 +126,7 @@ export async function ensureAgentLoaded( labels: record.labels, workspaceId: record.workspaceId, owner: record.owner, + autoArchiveObligation: record.autoArchiveObligation, }); deps.logger.info({ agentId, provider: record.provider }, "Agent created from stored config"); } diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 06fd21a59b..4b9c6575c2 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -248,6 +248,7 @@ class HeldReloadCloseClient extends TestAgentClient { private readonly closeAllowed = deferred(); originalSessionClosed = false; replacementSessionClosed = false; + resumeCount = 0; override async createSession(config: AgentSessionConfig): Promise { const signalCloseStarted = () => this.closeStarted.resolve(); @@ -268,6 +269,7 @@ class HeldReloadCloseClient extends TestAgentClient { _handle: AgentPersistenceHandle, config?: Partial, ): Promise { + this.resumeCount += 1; const recordReplacementClosed = () => { this.replacementSessionClosed = true; }; @@ -7916,6 +7918,58 @@ test("load waits for an in-flight explicit close and creates one resumed runtime } }); +test("load joins a close that starts after its barrier but before runtime lookup", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-close-start-gap-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const client = new HeldReloadCloseClient(); + const manager = new AgentManager({ clients: { codex: client }, registry: storage, logger }); + const agentId = "00000000-0000-4000-8000-000000000218"; + let closeTask: Promise | null = null; + + try { + await manager.createAgent({ provider: "codex", cwd: workdir }, agentId, { + workspaceId: undefined, + }); + const waitForAgentClose = manager.waitForAgentClose.bind(manager); + vi.spyOn(manager, "waitForAgentClose") + .mockImplementation(async (id) => waitForAgentClose(id)) + .mockImplementationOnce(async () => { + queueMicrotask(() => { + closeTask = manager.closeAgent(agentId); + }); + }); + + let loadSettled = false; + const load = ensureAgentLoaded(agentId, { + agentManager: manager, + agentStorage: storage, + logger, + }).then((agent) => { + loadSettled = true; + return agent; + }); + + await client.waitForCloseToStart(); + await Promise.resolve(); + expect(loadSettled).toBe(false); + expect(client.resumeCount).toBe(0); + + client.finishClosing(); + const resumed = await load; + await closeTask; + + expect(resumed).toMatchObject({ id: agentId, lifecycle: "idle" }); + expect(client.originalSessionClosed).toBe(true); + expect(client.resumeCount).toBe(1); + } finally { + client.finishClosing(); + await closeTask?.catch(() => undefined); + await manager.closeAgent(agentId).catch(() => undefined); + await storage.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("concurrent explicit closes tear down the runtime once", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-concurrent-close-")); const closeStarted = deferred(); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 2ebe7794f4..70a1577029 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -44,7 +44,7 @@ import { type ListImportableSessionsOptions, } from "./agent-sdk-types.js"; import { buildArchivedAgentRecord, type ArchivedStoredAgentRecord } from "./agent-archive.js"; -import type { StoredAgentRecord, AgentStorage } from "./agent-storage.js"; +import type { AutoArchiveObligation, StoredAgentRecord, AgentStorage } from "./agent-storage.js"; import type { AgentOwner } from "./agent-owner.js"; import { InMemoryAgentTimelineStore, @@ -246,6 +246,7 @@ export interface CreateAgentOptions { // undefined is an explicit decision: the agent never appears in the sidebar. workspaceId: string | undefined; owner?: AgentOwner; + autoArchiveObligation?: AutoArchiveObligation; } export interface AgentManagerOptions { @@ -1000,6 +1001,17 @@ export class AgentManager { return agent ? { ...agent } : null; } + getAgentInitializationState(id: string): { + agent: ManagedAgent | null; + closeInFlight: boolean; + } { + const closeInFlight = this.inFlightAgentCloses.has(id); + return { + agent: closeInFlight ? null : this.getAgent(id), + closeInFlight, + }; + } + async waitForAgentClose(agentId: string): Promise { await this.inFlightAgentCloses?.get(agentId)?.catch(() => undefined); } @@ -1052,6 +1064,10 @@ export class AgentManager { return this.trackAgentRegistrationOperation(this.createAgentInternal(config, agentId, options)); } + allocateAgentId(): string { + return validateAgentId(this.idFactory(), "allocateAgentId"); + } + private async createAgentInternal( config: AgentSessionConfig, agentId: string | undefined, @@ -1083,6 +1099,7 @@ export class AgentManager { initialTitle: options.initialTitle, workspaceId: options.workspaceId, owner: options.owner, + autoArchiveObligation: options.autoArchiveObligation, }); } @@ -1107,6 +1124,7 @@ export class AgentManager { labels?: Record; workspaceId?: string; owner?: AgentOwner; + autoArchiveObligation?: AutoArchiveObligation; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -1126,6 +1144,7 @@ export class AgentManager { labels?: Record; workspaceId?: string; owner?: AgentOwner; + autoArchiveObligation?: AutoArchiveObligation; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -2801,6 +2820,7 @@ export class AgentManager { publishWhenReady?: boolean; workspaceId?: string; owner?: AgentOwner; + autoArchiveObligation?: AutoArchiveObligation; }, ): Promise { let registered = false; @@ -2841,6 +2861,7 @@ export class AgentManager { this.assertAgentRegistrationActive(managed); await this.persistSnapshot(managed, { title: initialPersistedTitle, + autoArchiveObligation: options?.autoArchiveObligation, }); this.assertAgentRegistrationActive(managed); if (!options?.publishWhenReady) { @@ -3191,7 +3212,11 @@ export class AgentManager { private async persistSnapshot( agent: ManagedAgent, - options?: { title?: string | null; internal?: boolean }, + options?: { + title?: string | null; + internal?: boolean; + autoArchiveObligation?: AutoArchiveObligation; + }, ): Promise { if (!this.registry) { return; diff --git a/packages/server/src/server/agent/agent-storage.test.ts b/packages/server/src/server/agent/agent-storage.test.ts index a9b1ab762f..9b3bf76e97 100644 --- a/packages/server/src/server/agent/agent-storage.test.ts +++ b/packages/server/src/server/agent/agent-storage.test.ts @@ -192,6 +192,114 @@ describe("AgentStorage", () => { expect(persisted.config?.extra?.claude).toMatchObject({ maxThinkingTokens: 1024 }); }); + test("persists auto-archive obligations across restart", async () => { + await storage.applySnapshot(createManagedAgent({ id: "agent-auto-archive" }), { + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-auto-archive" }, + }, + }); + + const reloaded = new AgentStorage(storagePath, logger); + await expect(reloaded.get("agent-auto-archive")).resolves.toMatchObject({ + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-auto-archive" }, + }, + }); + }); + + test("persists pending creation intent separately from visible agents", async () => { + await storage.beginPendingAgentCreation("agent-pending-create"); + await storage.planPendingAgentCreationWorktree( + "agent-pending-create", + "/tmp/paseo/worktrees/project/feature-2", + { + worktreeIncarnationId: "4d2ce498-4c27-4ea2-8ed3-46720de7194e", + metadataBaseRefName: "main", + }, + ); + await expect(storage.listPendingAgentCreations()).resolves.toMatchObject([ + { + cleanupTarget: { + kind: "worktree", + targetPath: "/tmp/paseo/worktrees/project/feature-2", + worktreeIncarnationId: "4d2ce498-4c27-4ea2-8ed3-46720de7194e", + directoryIdentity: null, + }, + }, + ]); + await storage.identifyPendingAgentCreationWorktree( + "agent-pending-create", + "/tmp/paseo/worktrees/project/feature-2", + { + worktreeIncarnationId: "4d2ce498-4c27-4ea2-8ed3-46720de7194e", + directoryIdentity: { device: "7", inode: "42" }, + metadataBaseRefName: "main", + }, + ); + + const reloaded = new AgentStorage(storagePath, logger); + await expect(reloaded.list()).resolves.toEqual([]); + await expect(reloaded.listPendingAgentCreations()).resolves.toEqual([ + { + agentId: "agent-pending-create", + createdAt: expect.any(String), + ownerKind: "agent", + cleanupTarget: { + kind: "worktree", + targetPath: "/tmp/paseo/worktrees/project/feature-2", + worktreeIncarnationId: "4d2ce498-4c27-4ea2-8ed3-46720de7194e", + directoryIdentity: { device: "7", inode: "42" }, + metadataBaseRefName: "main", + }, + }, + ]); + + await reloaded.removePendingAgentCreation("agent-pending-create"); + await expect(reloaded.listPendingAgentCreations()).resolves.toEqual([]); + }); + + test.each(["armed", "pending"] as const)( + "explicit undefined preserves a persisted %s auto-archive obligation", + async (phase) => { + const agent = createManagedAgent({ id: `agent-auto-archive-${phase}` }); + await storage.applySnapshot(agent, { + autoArchiveObligation: { phase, target: { kind: "agent" } }, + }); + + await storage.applySnapshot( + { ...agent, updatedAt: new Date("2025-02-01") }, + { + autoArchiveObligation: undefined, + }, + ); + + const reloaded = new AgentStorage(storagePath, logger); + await expect(reloaded.get(agent.id)).resolves.toMatchObject({ + autoArchiveObligation: { phase, target: { kind: "agent" } }, + }); + }, + ); + + test("serialized snapshots cannot overwrite a pending auto-archive obligation", async () => { + const agent = createManagedAgent({ id: "agent-auto-archive-race" }); + await storage.applySnapshot(agent, { + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + }); + + const pending = storage.update(agent.id, (record) => ({ + ...record, + autoArchiveObligation: { phase: "pending", target: { kind: "agent" } }, + })); + const snapshot = storage.applySnapshot({ ...agent, updatedAt: new Date("2025-02-01") }); + await Promise.all([pending, snapshot]); + + await expect(storage.get(agent.id)).resolves.toMatchObject({ + autoArchiveObligation: { phase: "pending", target: { kind: "agent" } }, + }); + }); + test("applySnapshot stores and reloads featureValues when present", async () => { await storage.applySnapshot( createManagedAgent({ diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 5e504cee64..4046399f2a 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -33,6 +33,39 @@ const PERSISTENCE_HANDLE_SCHEMA = z .nullable() .optional(); +const AUTO_ARCHIVE_TARGET_SCHEMA = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("agent") }), + z.object({ kind: z.literal("workspace"), workspaceId: z.string() }), +]); + +const AUTO_ARCHIVE_OBLIGATION_SCHEMA = z.object({ + phase: z.enum(["armed", "pending"]), + target: AUTO_ARCHIVE_TARGET_SCHEMA, +}); + +const PENDING_AGENT_CREATION_SCHEMA = z.object({ + agentId: z.string(), + createdAt: z.string(), + ownerKind: z.enum(["agent", "standalone-worktree"]).optional().default("agent"), + cleanupTarget: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("agent") }), + z.object({ + kind: z.literal("worktree"), + targetPath: z.string(), + worktreeIncarnationId: z.string().uuid(), + directoryIdentity: z + .object({ + device: z.string(), + inode: z.string(), + }) + .nullable() + .optional() + .default(null), + metadataBaseRefName: z.string().min(1), + }), + ]), +}); + const STORED_AGENT_SCHEMA = z.object({ id: z.string(), provider: z.string(), @@ -66,6 +99,7 @@ const STORED_AGENT_SCHEMA = z.object({ internal: z.boolean().optional(), archivedAt: z.string().nullable().optional(), owner: AgentOwnerSchema.optional(), + autoArchiveObligation: AUTO_ARCHIVE_OBLIGATION_SCHEMA.optional(), }); export type SerializableAgentConfig = Pick< @@ -80,6 +114,8 @@ export type SerializableAgentConfig = Pick< >; export type StoredAgentRecord = z.infer; +export type AutoArchiveObligation = z.infer; +export type PendingAgentCreation = z.infer; export function parseStoredAgentRecord(value: unknown): StoredAgentRecord { return STORED_AGENT_SCHEMA.parse(value); } @@ -127,6 +163,124 @@ export class AgentStorage { await this.queueRecordWrite(record); } + async beginPendingAgentCreation( + agentId: string, + options?: { ownerKind?: PendingAgentCreation["ownerKind"] }, + ): Promise { + const record: PendingAgentCreation = { + agentId, + createdAt: new Date().toISOString(), + ownerKind: options?.ownerKind ?? "agent", + cleanupTarget: { kind: "agent" }, + }; + await writeJsonFileAtomic(this.pendingCreationPath(agentId), record); + return record; + } + + async planPendingAgentCreationWorktree( + agentId: string, + targetPath: string, + plan: { + worktreeIncarnationId: string; + metadataBaseRefName: string; + }, + ): Promise { + const record = await this.readPendingAgentCreation(agentId); + if (!record) { + throw new Error(`Pending agent creation ${agentId} not found`); + } + await writeJsonFileAtomic(this.pendingCreationPath(agentId), { + ...record, + cleanupTarget: { + kind: "worktree", + targetPath, + ...plan, + directoryIdentity: null, + }, + } satisfies PendingAgentCreation); + } + + async identifyPendingAgentCreationWorktree( + agentId: string, + targetPath: string, + reservation: { + worktreeIncarnationId: string; + directoryIdentity: { device: string; inode: string }; + metadataBaseRefName: string; + }, + ): Promise { + const record = await this.readPendingAgentCreation(agentId); + if ( + !record || + record.cleanupTarget.kind !== "worktree" || + record.cleanupTarget.targetPath !== targetPath || + record.cleanupTarget.worktreeIncarnationId !== reservation.worktreeIncarnationId + ) { + const recordedTarget = + record?.cleanupTarget.kind === "worktree" + ? `${record.cleanupTarget.targetPath} (${record.cleanupTarget.worktreeIncarnationId})` + : (record?.cleanupTarget.kind ?? "missing"); + throw new Error( + `Pending agent creation ${agentId} does not own ${targetPath} (${reservation.worktreeIncarnationId}); recorded ${recordedTarget}`, + ); + } + await writeJsonFileAtomic(this.pendingCreationPath(agentId), { + ...record, + cleanupTarget: { kind: "worktree", targetPath, ...reservation }, + } satisfies PendingAgentCreation); + } + + async listPendingAgentCreations(): Promise { + let entries: Dirent[]; + try { + entries = await fs.readdir(this.pendingCreationDir(), { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + const records = await Promise.all( + entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".intent")) + .map((entry) => + this.readPendingAgentCreationFile(path.join(this.pendingCreationDir(), entry.name)), + ), + ); + return records.filter((record): record is PendingAgentCreation => record !== null); + } + + async removePendingAgentCreation(agentId: string): Promise { + await fs.rm(this.pendingCreationPath(agentId), { force: true }); + } + + async update( + agentId: string, + mutation: (record: StoredAgentRecord) => StoredAgentRecord, + ): Promise { + await this.load(); + const previousWrite = this.pendingWrites.get(agentId) ?? Promise.resolve(); + const operation = (async () => { + await Promise.allSettled([previousWrite]); + const current = this.cache.get(agentId); + if (!current) { + throw new Error(`Agent ${agentId} not found`); + } + const next = mutation(current); + if (!this.deleting.has(agentId)) { + await this.writeRecord(next); + } + return next; + })(); + const tracked = operation.then( + () => undefined, + () => undefined, + ); + void tracked.finally(() => { + if (this.pendingWrites.get(agentId) === tracked) this.pendingWrites.delete(agentId); + }); + this.pendingWrites.set(agentId, tracked); + return operation; + } + private queueRecordWrite(record: StoredAgentRecord): Promise { const agentId = record.id; const prev = this.pendingWrites.get(agentId) ?? Promise.resolve(); @@ -204,38 +358,46 @@ export class AgentStorage { async applySnapshot( agent: ManagedAgent, - options?: { title?: string | null; internal?: boolean }, + options?: { + title?: string | null; + internal?: boolean; + autoArchiveObligation?: AutoArchiveObligation; + }, ): Promise { await this.load(); - await this.waitForPendingWrite(agent.id); - const existing = (await this.get(agent.id)) ?? null; - const hasTitleOverride = - options !== undefined && Object.prototype.hasOwnProperty.call(options, "title"); - const hasInternalOverride = - options !== undefined && Object.prototype.hasOwnProperty.call(options, "internal"); - const record = toStoredAgentRecord(agent, { - title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), - createdAt: existing?.createdAt, - internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), + const previousWrite = this.pendingWrites.get(agent.id) ?? Promise.resolve(); + const operation = (async () => { + await Promise.allSettled([previousWrite]); + const existing = this.cache.get(agent.id) ?? null; + const hasTitleOverride = + options !== undefined && Object.prototype.hasOwnProperty.call(options, "title"); + const hasInternalOverride = + options !== undefined && Object.prototype.hasOwnProperty.call(options, "internal"); + const record = toStoredAgentRecord(agent, { + title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), + createdAt: existing?.createdAt, + internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), + }); + record.archivedAt = existing?.archivedAt; + record.autoArchiveObligation = + options?.autoArchiveObligation ?? existing?.autoArchiveObligation; + if (!this.deleting.has(agent.id)) { + await this.writeRecord(record); + } + })(); + const tracked = operation.then( + () => undefined, + () => undefined, + ); + void tracked.finally(() => { + if (this.pendingWrites.get(agent.id) === tracked) this.pendingWrites.delete(agent.id); }); - - // Preserve soft-delete/archive status across snapshot flushes. - // `archivedAt` is not part of the ManagedAgent snapshot, so a naive projection - // would wipe it during normal persistence (including on daemon restart). - if (existing && existing.archivedAt !== undefined) { - record.archivedAt = existing.archivedAt; - } - await this.upsert(record); + this.pendingWrites.set(agent.id, tracked); + await operation; } async setTitle(agentId: string, title: string): Promise { - await this.load(); - await this.waitForPendingWrite(agentId); - const record = await this.get(agentId); - if (!record) { - throw new Error(`Agent ${agentId} not found`); - } - await this.upsert({ ...record, title }); + await this.update(agentId, (record) => ({ ...record, title })); } async flush(): Promise { @@ -332,6 +494,31 @@ export class AgentStorage { return records; } + private pendingCreationDir(): string { + return path.join(this.baseDir, ".creating"); + } + + private pendingCreationPath(agentId: string): string { + return path.join(this.pendingCreationDir(), `${agentId}.intent`); + } + + private async readPendingAgentCreation(agentId: string): Promise { + return this.readPendingAgentCreationFile(this.pendingCreationPath(agentId)); + } + + private async readPendingAgentCreationFile( + filePath: string, + ): Promise { + try { + const value: unknown = JSON.parse(await fs.readFile(filePath, "utf8")); + return PENDING_AGENT_CREATION_SCHEMA.parse(value); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + this.logger.warn({ err: error, filePath }, "Ignoring invalid pending agent creation"); + return null; + } + } + private async readRecordFile(filePath: string): Promise { try { const content = await fs.readFile(filePath, "utf8"); @@ -386,10 +573,6 @@ export class AgentStorage { } this.daemonExecutionKeysByAgentId.delete(agentId); } - - private async waitForPendingWrite(agentId: string): Promise { - await (this.pendingWrites.get(agentId) ?? Promise.resolve()).catch(() => undefined); - } } function projectDirNameFromCwd(cwd: string): string { diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index bdb9030435..f3c95020c5 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -1,7 +1,40 @@ -import { expect, test } from "vitest"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve as resolvePath } from "node:path"; +import { expect, test, vi } from "vitest"; import type { AgentManagerEvent, AgentSubscriber } from "./agent-manager.js"; -import { registerAgentAutoArchive } from "./create-agent-lifecycle-dispatch.js"; +import { createTestLogger } from "../../test-utils/test-logger.js"; +import { + CreateAgentLifecycleDispatch, + registerAgentAutoArchive, +} from "./create-agent-lifecycle-dispatch.js"; +import { + requireArchiveCleanupComplete, + WorkspaceCleanupPendingError, +} from "../workspace-archive-service.js"; +import type { PendingAgentCreation, StoredAgentRecord } from "./agent-storage.js"; +import type { PersistedWorkspaceRecord } from "../workspace-registry.js"; +import type { CreatePaseoWorktreeWorkflowResult } from "../worktree-session.js"; +import { readPaseoWorktreeMetadata } from "../../utils/worktree-metadata.js"; + +const PENDING_WORKTREE_INCARNATION_ID = "4d2ce498-4c27-4ea2-8ed3-46720de7194e"; + +function pendingWorktreeTarget( + targetPath: string, +): Extract { + const directoryStat = statSync(targetPath, { bigint: true }); + return { + kind: "worktree", + targetPath, + worktreeIncarnationId: PENDING_WORKTREE_INCARNATION_ID, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, + metadataBaseRefName: "main", + }; +} class AgentLifecycleEvents { private readonly listeners = new Set(); @@ -25,6 +58,488 @@ class AgentLifecycleEvents { } } +class LifecycleRearmEvents { + private readonly listeners = new Set<(workspaceId: string) => void>(); + + subscribe(listener: (workspaceId: string) => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + publish(workspaceId = "ws-cleanup-pending"): void { + for (const listener of this.listeners) listener(workspaceId); + } + + listenerCount(): number { + return this.listeners.size; + } +} + +function createLifecycleDispatch( + agents: AgentLifecycleEvents, + archiveAgentForClose: (agentId: string) => Promise, + options?: { + records?: Map; + archiveWorkspaceForClose?: (workspaceId: string, signal?: AbortSignal) => Promise; + drainWorkspaceLifecycleOperations?: () => Promise; + getWorkspace?: (workspaceId: string) => Promise; + pendingCreations?: Map; + workspaces?: PersistedWorkspaceRecord[]; + }, +): CreateAgentLifecycleDispatch { + const records = options?.records ?? new Map(); + const pendingCreations = options?.pendingCreations ?? new Map(); + const dependencies = { + paseoHome: "/tmp/paseo", + agentManager: agents, + agentStorage: { + list: async () => Array.from(records.values()), + get: async (agentId: string) => records.get(agentId) ?? null, + listPendingAgentCreations: async () => Array.from(pendingCreations.values()), + removePendingAgentCreation: async (agentId: string) => { + pendingCreations.delete(agentId); + }, + update: async ( + agentId: string, + mutation: (record: StoredAgentRecord) => StoredAgentRecord, + ) => { + const current = records.get(agentId) ?? ({ id: agentId } as StoredAgentRecord); + const next = mutation(current); + records.set(agentId, next); + return next; + }, + }, + github: {}, + workspaceGitService: {}, + archiveAgentForClose, + archiveWorkspaceForClose: options?.archiveWorkspaceForClose ?? (async () => undefined), + drainWorkspaceLifecycleOperations: + options?.drainWorkspaceLifecycleOperations ?? (async () => undefined), + findWorkspaceIdForCwd: async () => null, + listActiveWorkspaces: async () => [], + archiveWorkspaceRecord: async () => undefined, + workspaceRegistry: { + get: + options?.getWorkspace ?? + (async (workspaceId: string) => + ({ + workspaceId, + archivedAt: "2025-01-01T00:00:00.000Z", + cleanupPending: null, + }) as PersistedWorkspaceRecord), + list: async () => options?.workspaces ?? [], + }, + emit: () => undefined, + emitAgentRemove: async () => undefined, + emitWorkspaceUpdatesForWorkspaceIds: async () => undefined, + markWorkspaceArchiving: () => undefined, + clearWorkspaceArchiving: () => undefined, + killTerminalsForWorkspace: async () => undefined, + logger: createTestLogger(), + } as unknown as ConstructorParameters[0]; + return new CreateAgentLifecycleDispatch(dependencies); +} + +test("startup removes a side-effect-free pending creation reservation", async () => { + const agentId = "agent-pending-no-side-effects"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: { kind: "agent" }, + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + + await dispatch.recoverPendingAgentCreations(); + + expect(pendingCreations.size).toBe(0); +}); + +test("startup adopts a published agent without cleaning its pending worktree", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-published-agent-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const agentId = "agent-pending-now-persisted"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const records = new Map([ + [ + agentId, + { + id: agentId, + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-published-agent" }, + }, + } as StoredAgentRecord, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + records, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + expect(records.get(agentId)?.autoArchiveObligation).toEqual({ + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-published-agent" }, + }); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup adopts a same-path successful retry instead of cleaning its stale pending worktree", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-retried-agent-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const staleAgentId = "agent-failed-before-publish"; + const retryAgentId = "agent-retry-published"; + const workspaceId = "ws-retry-published"; + const pendingCreations = new Map([ + [ + staleAgentId, + { + agentId: staleAgentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const records = new Map([ + [ + retryAgentId, + { + id: retryAgentId, + workspaceId, + } as StoredAgentRecord, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + records, + workspaces: [ + { + workspaceId, + cwd: worktreePath, + worktreeRoot: worktreePath, + archivedAt: null, + } as PersistedWorkspaceRecord, + ], + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup removes an exact pre-git worktree reservation without archiving", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-pre-git-reservation-")); + const agentId = "agent-pending-pre-git-reservation"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(false); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup retains the journal and directory after a crash between mkdir and identity", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-pre-identity-crash-")); + const agentId = "agent-pending-pre-identity-crash"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: { + kind: "worktree", + targetPath: worktreePath, + worktreeIncarnationId: PENDING_WORKTREE_INCARNATION_ID, + directoryIdentity: null, + metadataBaseRefName: "main", + }, + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.has(agentId)).toBe(true); + expect(existsSync(worktreePath)).toBe(true); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup removes its marked directory after a crash before identity persistence", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-marked-pre-identity-crash-")); + writeFileSync( + join(worktreePath, ".paseo-worktree-creation"), + `${PENDING_WORKTREE_INCARNATION_ID}\n`, + ); + const agentId = "agent-pending-marked-pre-identity-crash"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: { + kind: "worktree", + targetPath: worktreePath, + worktreeIncarnationId: PENDING_WORKTREE_INCARNATION_ID, + directoryIdentity: null, + metadataBaseRefName: "main", + }, + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + + await dispatch.recoverPendingAgentCreations(); + + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(false); +}); + +test("startup adopts a published standalone worktree instead of archiving it", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-published-standalone-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const creationId = "standalone-pending-now-persisted"; + const workspaceId = "ws-published-standalone"; + const pendingCreations = new Map([ + [ + creationId, + { + agentId: creationId, + createdAt: "2025-01-01T00:00:00.000Z", + ownerKind: "standalone-worktree", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const workspaces = [ + { + workspaceId, + cwd: worktreePath, + worktreeRoot: worktreePath, + archivedAt: null, + } as PersistedWorkspaceRecord, + ]; + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + workspaces, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup retains and retries an orphan worktree pending creation", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-orphan-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const agentId = "agent-pending-orphan-worktree"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi + .spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ) + .mockRejectedValueOnce(new Error("physical archive failed")) + .mockResolvedValue(undefined); + + try { + await dispatch.recoverPendingAgentCreations(); + expect(pendingCreations.has(agentId)).toBe(true); + + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).toHaveBeenCalledTimes(2); + expect(archiveWorktreePath).toHaveBeenLastCalledWith(resolvePath(worktreePath)); + expect(pendingCreations.size).toBe(0); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup does not archive a replacement at a reused pending-creation path", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-replaced-worktree-")); + const staleTarget = pendingWorktreeTarget(worktreePath); + rmSync(worktreePath, { recursive: true, force: true }); + mkdirSync(worktreePath); + const agentId = "agent-pending-replaced-worktree"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: staleTarget, + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(statSync(worktreePath).isDirectory()).toBe(true); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup repairs metadata after git creation and before archive recovery", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-unrecorded-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const agentId = "agent-pending-metadata-gap"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi + .spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ) + .mockImplementation(async () => { + expect(readPaseoWorktreeMetadata(worktreePath)).toMatchObject({ + baseRefName: "main", + incarnationId: PENDING_WORKTREE_INCARNATION_ID, + }); + }); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).toHaveBeenCalledWith(resolvePath(worktreePath)); + expect(pendingCreations.size).toBe(0); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + test("auto-archive self-releases once and later cancellation waits harmlessly", async () => { const agentId = "4a7e2521-286d-4ad5-af35-e091c55302e3"; const agents = new AgentLifecycleEvents(); @@ -45,3 +560,539 @@ test("auto-archive self-releases once and later cancellation waits harmlessly", expect(archiveCount).toBe(1); expect(agents.listenerCount()).toBe(0); }); + +test("auto-archive retries autonomously after a transient failure", async () => { + const agentId = "agent-auto-archive-retry"; + const agents = new AgentLifecycleEvents(); + const archive = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("transient archive failure")) + .mockResolvedValueOnce(undefined); + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive, + retryBaseMs: 0, + retryMaxMs: 0, + }); + + agents.completeTurn(agentId); + await vi.waitFor(() => expect(archive).toHaveBeenCalledTimes(2)); + await registration.cancel(); + expect(archive).toHaveBeenCalledTimes(2); + expect(agents.listenerCount()).toBe(0); +}); + +test("cleanup pending waits for a lifecycle rearm without a retry loop", async () => { + vi.useFakeTimers(); + const agentId = "agent-auto-archive-cleanup-pending"; + const agents = new AgentLifecycleEvents(); + const rearmEvents = new LifecycleRearmEvents(); + const archive = vi.fn(async () => { + const cleanupPendingWorkspaceIds = + archive.mock.calls.length === 1 ? ["ws-cleanup-pending"] : []; + requireArchiveCleanupComplete( + { + archivedAgentIds: [], + archivedWorkspaceIds: ["ws-cleanup-pending"], + removedDirectory: cleanupPendingWorkspaceIds.length === 0, + cleanupPendingWorkspaceIds, + }, + "Auto-created worktree archive", + ); + }); + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive, + shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), + subscribeToRearm: (rearm) => rearmEvents.subscribe(rearm), + rearmKeysForError: (error) => + error instanceof WorkspaceCleanupPendingError ? error.workspaceIds : [], + }); + + try { + agents.completeTurn(agentId); + await vi.advanceTimersByTimeAsync(60_000); + + expect(archive).toHaveBeenCalledTimes(1); + expect(rearmEvents.listenerCount()).toBe(1); + + rearmEvents.publish(); + await expect(registration.settled).resolves.toBe("completed"); + + expect(archive).toHaveBeenCalledTimes(2); + expect(agents.listenerCount()).toBe(0); + expect(rearmEvents.listenerCount()).toBe(0); + } finally { + await registration.cancel().catch(() => undefined); + vi.useRealTimers(); + } +}); + +test("a registry mutation during cleanup failure cannot be lost before rearm waiting", async () => { + const agentId = "agent-auto-archive-concurrent-rearm"; + const agents = new AgentLifecycleEvents(); + const rearmEvents = new LifecycleRearmEvents(); + let archiveCount = 0; + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive: async () => { + archiveCount += 1; + if (archiveCount === 1) { + rearmEvents.publish(); + throw new WorkspaceCleanupPendingError("Auto-created worktree archive", [ + "ws-cleanup-pending", + ]); + } + }, + shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), + subscribeToRearm: (rearm) => rearmEvents.subscribe(rearm), + rearmKeysForError: (error) => + error instanceof WorkspaceCleanupPendingError ? error.workspaceIds : [], + }); + + agents.completeTurn(agentId); + await expect(registration.settled).resolves.toBe("completed"); + + expect(archiveCount).toBe(2); + expect(agents.listenerCount()).toBe(0); + expect(rearmEvents.listenerCount()).toBe(0); +}); + +test("cleanup pending ignores unrelated workspace mutations", async () => { + const agentId = "agent-auto-archive-filtered-rearm"; + const agents = new AgentLifecycleEvents(); + const rearmEvents = new LifecycleRearmEvents(); + const archive = vi + .fn<() => Promise>() + .mockRejectedValueOnce( + new WorkspaceCleanupPendingError("Auto-created worktree archive", ["ws-relevant"]), + ) + .mockResolvedValueOnce(undefined); + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive, + shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), + subscribeToRearm: (rearm) => rearmEvents.subscribe(rearm), + rearmKeysForError: (error) => + error instanceof WorkspaceCleanupPendingError ? error.workspaceIds : [], + }); + + agents.completeTurn(agentId); + await vi.waitFor(() => expect(archive).toHaveBeenCalledTimes(1)); + + rearmEvents.publish("ws-unrelated"); + await Promise.resolve(); + expect(archive).toHaveBeenCalledTimes(1); + + rearmEvents.publish("ws-relevant"); + await expect(registration.settled).resolves.toBe("completed"); + expect(archive).toHaveBeenCalledTimes(2); +}); + +test("lifecycle shutdown cancels listeners and truthfully awaits an active archive", async () => { + const agentId = "agent-auto-archive-shutdown-wait"; + const agents = new AgentLifecycleEvents(); + let markArchiveStarted!: () => void; + let releaseArchive!: () => void; + const archiveStarted = new Promise((resolve) => { + markArchiveStarted = resolve; + }); + const archiveFinished = new Promise((resolve) => { + releaseArchive = resolve; + }); + const dispatch = createLifecycleDispatch(agents, async () => { + markArchiveStarted(); + await archiveFinished; + }); + const registration = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + agents.completeTurn(agentId); + await archiveStarted; + + let shutdownSettled = false; + const shutdown = dispatch.shutdown({ timeoutMs: 5_000 }).then((result) => { + shutdownSettled = true; + return result; + }); + await Promise.resolve(); + + expect(shutdownSettled).toBe(false); + expect(agents.listenerCount()).toBe(0); + + releaseArchive(); + await expect(shutdown).resolves.toEqual({ completed: true, pendingAgentIds: [] }); + await expect(registration.settled).resolves.toBe("completed"); +}); + +test("lifecycle shutdown joins failed-create cleanup that starts during drain", async () => { + const agents = new AgentLifecycleEvents(); + let markDrainStarted!: () => void; + let releaseDrain!: () => void; + const drainStarted = new Promise((resolve) => { + markDrainStarted = resolve; + }); + const drainFinished = new Promise((resolve) => { + releaseDrain = resolve; + }); + let drainCount = 0; + const dispatch = createLifecycleDispatch(agents, async () => undefined, { + drainWorkspaceLifecycleOperations: async () => { + drainCount += 1; + if (drainCount === 1) { + markDrainStarted(); + await drainFinished; + } + }, + }); + let markCleanupStarted!: () => void; + let releaseCleanup!: () => void; + const cleanupStarted = new Promise((resolve) => { + markCleanupStarted = resolve; + }); + const cleanupFinished = new Promise((resolve) => { + releaseCleanup = resolve; + }); + vi.spyOn( + dispatch as unknown as { + archiveAutoCreatedWorktree(options: unknown): Promise; + }, + "archiveAutoCreatedWorktree", + ).mockImplementation(async () => { + markCleanupStarted(); + await cleanupFinished; + }); + + let shutdownSettled = false; + const shutdown = dispatch.shutdown({ timeoutMs: 5_000 }).then((result) => { + shutdownSettled = true; + return result; + }); + await drainStarted; + const cleanup = dispatch.cleanupCreatedWorktreeAfterFailedAgentCreate({ + createdAgentId: null, + createdWorktree: { + created: true, + worktree: { worktreePath: "/tmp/paseo/worktrees/repo/failed-create" }, + } as CreatePaseoWorktreeWorkflowResult, + }); + await cleanupStarted; + releaseDrain(); + await Promise.resolve(); + + expect(shutdownSettled).toBe(false); + + releaseCleanup(); + await expect(cleanup).resolves.toBeUndefined(); + await expect(shutdown).resolves.toEqual({ completed: true, pendingAgentIds: [] }); + expect(drainCount).toBe(2); +}); + +test("one dispatcher keeps one lifecycle registration across repeated callers", async () => { + const agentId = "agent-shared-lifecycle-registration"; + const agents = new AgentLifecycleEvents(); + const dispatch = createLifecycleDispatch(agents, async () => undefined); + + const first = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + const second = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + + expect(second).toBe(first); + expect(agents.listenerCount()).toBe(1); + + await first.cancel(); + expect(agents.listenerCount()).toBe(0); +}); + +test("lifecycle shutdown reports its active archive at the deadline", async () => { + vi.useFakeTimers(); + const agentId = "agent-auto-archive-shutdown-deadline"; + const agents = new AgentLifecycleEvents(); + let markArchiveStarted!: () => void; + let releaseArchive!: () => void; + const archiveStarted = new Promise((resolve) => { + markArchiveStarted = resolve; + }); + const archiveFinished = new Promise((resolve) => { + releaseArchive = resolve; + }); + const dispatch = createLifecycleDispatch(agents, async () => { + markArchiveStarted(); + await archiveFinished; + }); + const registration = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + + try { + agents.completeTurn(agentId); + await archiveStarted; + + const shutdown = dispatch.shutdown({ timeoutMs: 100 }); + await vi.advanceTimersByTimeAsync(100); + + await expect(shutdown).resolves.toEqual({ completed: false, pendingAgentIds: [agentId] }); + expect(agents.listenerCount()).toBe(0); + + releaseArchive(); + await expect(registration.settled).resolves.toBe("completed"); + await expect(dispatch.shutdown({ timeoutMs: 100 })).resolves.toEqual({ + completed: true, + pendingAgentIds: [], + }); + } finally { + releaseArchive(); + vi.useRealTimers(); + } +}); + +test("lifecycle shutdown aborts an active workspace archive", async () => { + const agentId = "agent-auto-archive-workspace-shutdown"; + const agents = new AgentLifecycleEvents(); + let markArchiveStarted = () => {}; + const archiveStarted = new Promise((resolve) => { + markArchiveStarted = resolve; + }); + let receivedSignal: AbortSignal | undefined; + const dispatch = createLifecycleDispatch(agents, async () => undefined, { + archiveWorkspaceForClose: async (_workspaceId, signal) => { + receivedSignal = signal; + markArchiveStarted(); + await new Promise((_resolve, reject) => { + signal?.addEventListener("abort", () => reject(new Error("workspace archive canceled")), { + once: true, + }); + }); + }, + }); + dispatch.registerAutoArchive({ + agentId, + obligation: { phase: "armed", target: { kind: "workspace", workspaceId: "ws-shutdown" } }, + }); + agents.completeTurn(agentId); + await archiveStarted; + + await expect(dispatch.shutdown()).resolves.toEqual({ + completed: false, + pendingAgentIds: [agentId], + }); + expect(receivedSignal?.aborted).toBe(true); +}); + +test("auto-archive cancellation observes an in-flight archive rejection", async () => { + const agentId = "agent-auto-archive-cancel-failure"; + const agents = new AgentLifecycleEvents(); + const failure = new Error("archive failed during cancellation"); + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive: async () => { + throw failure; + }, + }); + + agents.completeTurn(agentId); + await expect(registration.cancel()).rejects.toBe(failure); + await expect(registration.settled).resolves.toBe("unresolved"); + expect(agents.listenerCount()).toBe(0); +}); + +test("lifecycle shutdown reports cleanup-pending cancellation as unresolved", async () => { + const agentId = "agent-auto-archive-shutdown-unresolved"; + const agents = new AgentLifecycleEvents(); + const failure = new WorkspaceCleanupPendingError("Auto-created worktree archive", [ + "ws-cleanup-pending", + ]); + const dispatch = createLifecycleDispatch(agents, async () => { + throw failure; + }); + const registration = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + + agents.completeTurn(agentId); + await vi.waitFor(() => expect(agents.listenerCount()).toBe(1)); + + await expect(dispatch.shutdown()).resolves.toEqual({ + completed: false, + pendingAgentIds: [agentId], + }); + await expect(registration.settled).resolves.toBe("unresolved"); + expect(agents.listenerCount()).toBe(0); +}); + +test("lifecycle dispatch does not permanently suppress an agent after archive failure", async () => { + const agentId = "agent-dispatch-auto-archive-retry"; + const agents = new AgentLifecycleEvents(); + const archiveAgentForClose = vi + .fn<(agentId: string) => Promise>() + .mockRejectedValueOnce(new Error("transient close failure")) + .mockResolvedValueOnce(undefined); + const dispatch = createLifecycleDispatch(agents, archiveAgentForClose); + const registration = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + + agents.completeTurn(agentId); + await new Promise((resolve) => setTimeout(resolve, 0)); + agents.completeTurn(agentId); + await registration.cancel(); + + expect(archiveAgentForClose).toHaveBeenCalledTimes(2); + expect(agents.listenerCount()).toBe(0); +}); + +test("startup completes an armed agent-only auto-archive obligation", async () => { + const agentId = "agent-restart-armed"; + const records = new Map([ + [ + agentId, + { + id: agentId, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + } as StoredAgentRecord, + ], + ]); + const archiveAgentForClose = vi.fn(async () => undefined); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), archiveAgentForClose, { + records, + }); + + await dispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => expect(archiveAgentForClose).toHaveBeenCalledWith(agentId)); + + await vi.waitFor(() => expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined()); +}); + +test("startup completes a persisted workspace auto-archive obligation", async () => { + const agentId = "agent-restart-workspace"; + const records = new Map([ + [ + agentId, + { + id: agentId, + autoArchiveObligation: { + phase: "pending", + target: { kind: "workspace", workspaceId: "ws-restart" }, + }, + } as StoredAgentRecord, + ], + ]); + const archiveWorkspaceForClose = vi.fn(async () => undefined); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + records, + archiveWorkspaceForClose, + }); + + await dispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => + expect(archiveWorkspaceForClose).toHaveBeenCalledWith("ws-restart", expect.any(AbortSignal)), + ); + + await vi.waitFor(() => expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined()); +}); + +test.each([ + ["missing", async () => null], + [ + "still active", + async (workspaceId: string) => + ({ workspaceId, archivedAt: null, cleanupPending: null }) as PersistedWorkspaceRecord, + ], + [ + "cleanup pending", + async (workspaceId: string) => + ({ + workspaceId, + archivedAt: "2025-01-01T00:00:00.000Z", + cleanupPending: { directoryPath: "/tmp/worktree" }, + }) as PersistedWorkspaceRecord, + ], +])( + "startup retains a workspace obligation when archive readback is %s", + async (_name, getWorkspace) => { + const agentId = "agent-restart-workspace-unverified"; + const obligation = { + phase: "pending", + target: { kind: "workspace", workspaceId: "ws-unverified" }, + } as const; + const records = new Map([ + [agentId, { id: agentId, autoArchiveObligation: obligation } as StoredAgentRecord], + ]); + const getWorkspaceSpy = vi.fn(getWorkspace); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + records, + getWorkspace: getWorkspaceSpy, + }); + + await dispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => expect(getWorkspaceSpy).toHaveBeenCalledWith("ws-unverified")); + expect(records.get(agentId)?.autoArchiveObligation).toEqual(obligation); + }, +); + +test.each([ + ["workspace cleanup failure", new WorkspaceCleanupPendingError("archive", ["ws-restart"])], + ["failure before cleanup-pending persistence", new Error("git removal failed early")], +])("startup retains and retries a workspace obligation after %s", async (_name, failure) => { + const agentId = "agent-restart-workspace-failure"; + const records = new Map([ + [ + agentId, + { + id: agentId, + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-restart" }, + }, + } as StoredAgentRecord, + ], + ]); + const firstArchive = vi.fn(async () => { + throw failure; + }); + const firstDispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + records, + archiveWorkspaceForClose: firstArchive, + }); + + await firstDispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => expect(firstArchive).toHaveBeenCalledOnce()); + await vi.waitFor(() => + expect(records.get(agentId)?.autoArchiveObligation).toEqual({ + phase: "pending", + target: { kind: "workspace", workspaceId: "ws-restart" }, + }), + ); + + const retryArchive = vi.fn(async () => undefined); + const restartedDispatch = createLifecycleDispatch( + new AgentLifecycleEvents(), + async () => undefined, + { records, archiveWorkspaceForClose: retryArchive }, + ); + await restartedDispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => + expect(retryArchive).toHaveBeenCalledWith("ws-restart", expect.any(AbortSignal)), + ); + await vi.waitFor(() => expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined()); +}); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 53a2177272..04d3b6ae62 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -1,21 +1,28 @@ import { randomUUID } from "node:crypto"; +import { existsSync, statSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { resolve as resolvePath } from "node:path"; import type pino from "pino"; import type { ForgeService } from "../../services/forge-service.js"; -import { isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; -import { archiveByScope, type ActiveWorkspaceRef } from "../workspace-archive-service.js"; -import type { - CreatePaseoWorktreeWorkflowFn, - CreatePaseoWorktreeWorkflowResult, -} from "../worktree-session.js"; +import { withTimeout } from "../../utils/promise-timeout.js"; +import { isPaseoOwnedWorktreeCwd, readWorktreeCreationMarker } from "../../utils/worktree.js"; +import { + readPaseoWorktreeMetadata, + writePaseoWorktreeMetadata, +} from "../../utils/worktree-metadata.js"; +import { + archiveByScope, + requireArchiveCleanupComplete, + type ActiveWorkspaceRef, + WorkspaceCleanupPendingError, +} from "../workspace-archive-service.js"; +import type { CreatePaseoWorktreeWorkflowResult } from "../worktree-session.js"; import type { WorkspaceGitService } from "../workspace-git-service.js"; -import type { - CreateAgentWorktreeTarget, - FirstAgentContext, - SessionOutboundMessage, -} from "../messages.js"; +import type { WorkspaceRegistry } from "../workspace-registry.js"; +import type { SessionOutboundMessage } from "../messages.js"; import type { AgentManager, AgentSubscriber, SubscribeOptions } from "./agent-manager.js"; -import type { AgentStorage } from "./agent-storage.js"; +import type { AgentStorage, AutoArchiveObligation, PendingAgentCreation } from "./agent-storage.js"; interface CreateAgentLifecycleDispatchDependencies { paseoHome: string; @@ -24,11 +31,13 @@ interface CreateAgentLifecycleDispatchDependencies { agentStorage: AgentStorage; github: ForgeService; workspaceGitService: WorkspaceGitService; - createPaseoWorktreeWorkflow: CreatePaseoWorktreeWorkflowFn; archiveAgentForClose: (agentId: string) => Promise; + archiveWorkspaceForClose: (workspaceId: string, signal?: AbortSignal) => Promise; + drainWorkspaceLifecycleOperations: () => Promise; findWorkspaceIdForCwd: (cwd: string) => Promise; listActiveWorkspaces: () => Promise; archiveWorkspaceRecord: (workspaceId: string) => Promise; + workspaceRegistry: Pick; emit: (message: SessionOutboundMessage) => void; emitAgentRemove: (agentId: string) => Promise; emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds: Iterable) => Promise; @@ -39,6 +48,7 @@ interface CreateAgentLifecycleDispatchDependencies { } export interface LifecycleRegistration { + readonly settled: Promise<"completed" | "cancelled" | "unresolved">; cancel(): Promise; } @@ -46,33 +56,25 @@ interface AgentLifecycleEvents { subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void; } -const inactiveRegistration: LifecycleRegistration = { cancel: async () => undefined }; +const inactiveRegistration: LifecycleRegistration = { + settled: Promise.resolve("cancelled"), + cancel: async () => undefined, +}; -type AutoArchiveTarget = - | { kind: "agent-only" } - | { kind: "created-worktree"; result: CreatePaseoWorktreeWorkflowResult }; +export interface LifecycleDispatchShutdownResult { + completed: boolean; + pendingAgentIds: string[]; +} export class CreateAgentLifecycleDispatch { - private readonly autoArchiveAgentIds = new Set(); + private readonly autoArchiveTasks = new Map>(); + private readonly autoArchiveRegistrations = new Map(); + private readonly failedCreateCleanupTasks = new Map>(); + private readonly shutdownController = new AbortController(); + private shuttingDown = false; constructor(private readonly dependencies: CreateAgentLifecycleDispatchDependencies) {} - async createWorktreeForRequest(input: { - cwd: string; - target: CreateAgentWorktreeTarget | undefined; - firstAgentContext: FirstAgentContext; - hasLegacyGitOptions: boolean; - }): Promise { - if (input.target && input.hasLegacyGitOptions) { - throw new Error("create_agent_request worktree cannot be combined with git options"); - } - if (!input.target) { - return null; - } - - return this.createWorktreeForTarget(input.cwd, input.target, input.firstAgentContext); - } - registerAutoArchiveIfRequested(input: { autoArchive: boolean | undefined; agentId: string; @@ -82,10 +84,200 @@ export class CreateAgentLifecycleDispatch { return inactiveRegistration; } - return this.registerAutoArchiveOnTerminalState( - input.agentId, - toAutoArchiveTarget(input.createdWorktree), + return this.registerAutoArchive({ + agentId: input.agentId, + obligation: { + phase: "armed", + target: input.createdWorktree + ? { kind: "workspace", workspaceId: input.createdWorktree.workspace.workspaceId } + : { kind: "agent" }, + }, + }); + } + + registerAutoArchive(input: { + agentId: string; + obligation: AutoArchiveObligation; + }): LifecycleRegistration { + if (input.obligation.phase === "pending") { + void this.autoArchiveAgentOnce(input.agentId, input.obligation.target).catch(() => undefined); + return inactiveRegistration; + } + return this.registerAutoArchiveOnTerminalState(input.agentId, input.obligation.target); + } + + async recoverPersistedAutoArchives(): Promise { + const records = await this.dependencies.agentStorage.list(); + for (const record of records) { + if (!record.autoArchiveObligation) continue; + void this.autoArchiveAgentOnce(record.id, record.autoArchiveObligation.target).catch( + () => undefined, + ); + } + } + + async recoverPendingAgentCreations(): Promise { + const records = await this.dependencies.agentStorage.listPendingAgentCreations(); + for (const record of records) { + await this.recoverPendingAgentCreation(record.agentId); + } + } + + async recoverPendingAgentCreation(agentId: string): Promise { + const pending = (await this.dependencies.agentStorage.listPendingAgentCreations()).find( + (record) => record.agentId === agentId, ); + if (!pending) return; + + if (pending.cleanupTarget.kind === "agent") { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + if (await this.pendingAgentWasPublished(pending)) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + + const pendingWorktree = { ...pending, cleanupTarget: pending.cleanupTarget }; + const targetPath = resolvePath(pendingWorktree.cleanupTarget.targetPath); + try { + if (!existsSync(targetPath) && !(await this.activeWorkspaceOwnsPath(targetPath))) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + + if ( + existsSync(targetPath) && + (await this.recoverExistingPendingDirectory(pendingWorktree, targetPath)) + ) { + return; + } + + await this.archiveWorktreePath(targetPath); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + } catch (error) { + this.dependencies.logger.warn( + { err: error, agentId, worktreePath: targetPath }, + "Failed to recover pending agent creation", + ); + } + } + + private async pendingAgentWasPublished(pending: PendingAgentCreation): Promise { + if ((pending.ownerKind ?? "agent") !== "agent") return false; + return Boolean(await this.dependencies.agentStorage.get(pending.agentId)); + } + + private async activeWorkspaceOwnsPath(targetPath: string): Promise { + return (await this.dependencies.workspaceRegistry.list()).some( + (workspace) => + !workspace.archivedAt && + resolvePath(workspace.worktreeRoot ?? workspace.cwd) === targetPath, + ); + } + + private async activeWorkspaceWithPublishedAgentOwnsPath(targetPath: string): Promise { + const owningWorkspaceIds = new Set( + (await this.dependencies.workspaceRegistry.list()) + .filter( + (workspace) => + !workspace.archivedAt && + resolvePath(workspace.worktreeRoot ?? workspace.cwd) === targetPath, + ) + .map((workspace) => workspace.workspaceId), + ); + if (owningWorkspaceIds.size === 0) return false; + return (await this.dependencies.agentStorage.list()).some( + (record) => record.workspaceId && owningWorkspaceIds.has(record.workspaceId), + ); + } + + private classifyPendingDirectoryOwnership( + pending: PendingAgentCreation & { + cleanupTarget: Extract; + }, + targetPath: string, + ): "owned" | "replaced" | "ambiguous" { + const expectedDirectoryIdentity = pending.cleanupTarget.directoryIdentity; + if (!expectedDirectoryIdentity) { + return readWorktreeCreationMarker(targetPath) === pending.cleanupTarget.worktreeIncarnationId + ? "owned" + : "ambiguous"; + } + const targetStat = statSync(targetPath, { bigint: true }); + return targetStat.dev.toString() === expectedDirectoryIdentity.device && + targetStat.ino.toString() === expectedDirectoryIdentity.inode + ? "owned" + : "replaced"; + } + + private async recoverExistingPendingDirectory( + pending: PendingAgentCreation & { + cleanupTarget: Extract; + }, + targetPath: string, + ): Promise { + const agentId = pending.agentId; + if (await this.activeWorkspaceWithPublishedAgentOwnsPath(targetPath)) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + const directoryOwnership = this.classifyPendingDirectoryOwnership(pending, targetPath); + if (directoryOwnership === "replaced") { + this.dependencies.logger.warn( + { agentId, worktreePath: targetPath }, + "Pending agent creation no longer owns its worktree path", + ); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + if (directoryOwnership === "ambiguous") { + // The process may have stopped after mkdir and before persisting the + // directory identity. Without the matching in-directory marker we + // cannot distinguish our directory from a replacement, so retain the + // durable cleanup obligation and fail closed. + this.dependencies.logger.warn( + { agentId, worktreePath: targetPath }, + "Pending agent creation has ambiguous pre-identity worktree ownership", + ); + return true; + } + + if (!existsSync(resolvePath(targetPath, ".git"))) { + await rm(targetPath, { recursive: true, force: true }); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + const metadata = readPaseoWorktreeMetadata(targetPath); + if ( + metadata?.incarnationId && + metadata.incarnationId !== pending.cleanupTarget.worktreeIncarnationId + ) { + this.dependencies.logger.warn( + { agentId, worktreePath: targetPath }, + "Pending agent creation found a replacement worktree incarnation", + ); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + if (!metadata?.incarnationId) { + writePaseoWorktreeMetadata(targetPath, { + baseRefName: metadata?.baseRefName ?? pending.cleanupTarget.metadataBaseRefName, + incarnationId: pending.cleanupTarget.worktreeIncarnationId, + ...(metadata?.changeRequestLookupTarget + ? { changeRequestLookupTarget: metadata.changeRequestLookupTarget } + : {}), + }); + } + + if ( + pending.ownerKind === "standalone-worktree" && + (await this.activeWorkspaceOwnsPath(targetPath)) + ) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + return false; } async cleanupCreatedWorktreeAfterFailedAgentCreate(input: { @@ -93,95 +285,158 @@ export class CreateAgentLifecycleDispatch { createdAgentId: string | null; }): Promise { const { createdWorktree, createdAgentId } = input; - if (!createdWorktree || createdAgentId) { + if (!createdWorktree?.created || createdAgentId) { return; } - await this.archiveAutoCreatedWorktree({ + const worktreePath = createdWorktree.worktree.worktreePath; + const existingTask = this.failedCreateCleanupTasks.get(worktreePath); + if (existingTask) return existingTask; + + const cleanupTask = this.archiveAutoCreatedWorktree({ agentId: null, createdWorktree, }).catch((archiveError) => { this.dependencies.logger.warn( - { - err: archiveError, - worktreePath: createdWorktree.worktree.worktreePath, - }, + { err: archiveError, worktreePath }, "Failed to clean up worktree after create_agent_request failed", ); }); - } - - private async createWorktreeForTarget( - cwd: string, - target: CreateAgentWorktreeTarget, - firstAgentContext: FirstAgentContext, - ): Promise { - const baseInput = { - cwd, - firstAgentContext, - runSetup: false, - paseoHome: this.dependencies.paseoHome, - worktreesRoot: this.dependencies.worktreesRoot, - } as const; - - switch (target.mode) { - case "branch-off": - return this.dependencies.createPaseoWorktreeWorkflow( - { - ...baseInput, - worktreeSlug: target.newBranch, - action: "branch-off", - ...(target.base ? { refName: target.base } : {}), - }, - target.base ? { resolveDefaultBranch: async () => target.base! } : undefined, - ); - case "checkout-branch": - return this.dependencies.createPaseoWorktreeWorkflow({ - ...baseInput, - action: "checkout", - refName: target.branch, - }); - case "checkout-pr": - return this.dependencies.createPaseoWorktreeWorkflow({ - ...baseInput, - action: "checkout", - githubPrNumber: target.prNumber, - }); - default: - throw new Error("Unsupported create_agent_request worktree target"); + this.failedCreateCleanupTasks.set(worktreePath, cleanupTask); + try { + await cleanupTask; + } finally { + if (this.failedCreateCleanupTasks.get(worktreePath) === cleanupTask) { + this.failedCreateCleanupTasks.delete(worktreePath); + } } } private registerAutoArchiveOnTerminalState( agentId: string, - target: AutoArchiveTarget, + target: AutoArchiveObligation["target"], ): LifecycleRegistration { - return registerAgentAutoArchive({ + if (this.shuttingDown) return inactiveRegistration; + const existing = this.autoArchiveRegistrations.get(agentId); + if (existing) return existing; + + const subscribeToMutations = this.dependencies.workspaceRegistry.subscribeToMutations?.bind( + this.dependencies.workspaceRegistry, + ); + const registration = registerAgentAutoArchive({ agentManager: this.dependencies.agentManager, agentId, archive: () => this.autoArchiveAgentOnce(agentId, target), + shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), + subscribeToRearm: subscribeToMutations + ? (rearm) => subscribeToMutations((mutation) => rearm(mutation.workspaceId)) + : undefined, + rearmKeysForError: (error) => + error instanceof WorkspaceCleanupPendingError ? error.workspaceIds : [], + }); + this.autoArchiveRegistrations.set(agentId, registration); + void registration.settled.then(() => { + if (this.autoArchiveRegistrations.get(agentId) === registration) { + this.autoArchiveRegistrations.delete(agentId); + } + return undefined; }); + return registration; } - private async autoArchiveAgentOnce(agentId: string, target: AutoArchiveTarget): Promise { - if (this.autoArchiveAgentIds.has(agentId)) { - return; + async shutdown(options?: { timeoutMs?: number }): Promise { + this.shuttingDown = true; + this.shutdownController.abort(); + const registrations = Array.from(this.autoArchiveRegistrations.entries()); + const cancellation = Promise.allSettled( + registrations.map(([, registration]) => registration.cancel()), + ).then(async (results) => { + await Promise.allSettled(Array.from(this.autoArchiveTasks.values())); + await this.dependencies.drainWorkspaceLifecycleOperations(); + await Promise.allSettled(Array.from(this.failedCreateCleanupTasks.values())); + await this.dependencies.drainWorkspaceLifecycleOperations(); + return results; + }); + let cancellationResults: PromiseSettledResult[] | null = null; + try { + cancellationResults = await withTimeout( + cancellation, + options?.timeoutMs ?? 10_000, + "Timed out shutting down create-agent lifecycle registrations", + ); + } catch (error) { + this.dependencies.logger.error( + { err: error, agentIds: registrations.map(([registeredAgentId]) => registeredAgentId) }, + "Create-agent lifecycle shutdown remains incomplete", + ); } - this.autoArchiveAgentIds.add(agentId); + const unresolvedAgentIds = cancellationResults + ? registrations.flatMap(([agentId], index) => + cancellationResults[index]?.status === "rejected" ? [agentId] : [], + ) + : registrations.map(([agentId]) => agentId); + const pendingAgentIds = Array.from( + new Set([ + ...unresolvedAgentIds, + ...this.autoArchiveRegistrations.keys(), + ...this.autoArchiveTasks.keys(), + ]), + ); + return { + completed: + cancellationResults !== null && + pendingAgentIds.length === 0 && + this.failedCreateCleanupTasks.size === 0, + pendingAgentIds, + }; + } + private async autoArchiveAgentOnce( + agentId: string, + target: AutoArchiveObligation["target"], + ): Promise { + const existingTask = this.autoArchiveTasks.get(agentId); + if (existingTask) return existingTask; + + const archiveTask = this.runAutoArchiveAgent(agentId, target); + this.autoArchiveTasks.set(agentId, archiveTask); try { - if (target.kind === "created-worktree") { - await this.archiveAutoCreatedWorktree({ - agentId, - createdWorktree: target.result, - }); - return; + await archiveTask; + } catch (error) { + this.dependencies.logger.warn({ err: error, agentId }, "Failed to auto-archive agent"); + throw error; + } finally { + if (this.autoArchiveTasks.get(agentId) === archiveTask) { + this.autoArchiveTasks.delete(agentId); } + } + } + private async runAutoArchiveAgent( + agentId: string, + target: AutoArchiveObligation["target"], + ): Promise { + await this.dependencies.agentStorage.update(agentId, (record) => ({ + ...record, + autoArchiveObligation: { phase: "pending", target }, + })); + if (target.kind === "workspace") { + await this.dependencies.archiveWorkspaceForClose( + target.workspaceId, + this.shutdownController.signal, + ); + const workspace = await this.dependencies.workspaceRegistry.get(target.workspaceId); + if (!workspace?.archivedAt || workspace.cleanupPending) { + throw new WorkspaceCleanupPendingError("Workspace auto-archive verification", [ + target.workspaceId, + ]); + } + } else { await this.dependencies.archiveAgentForClose(agentId); - } catch (error) { - this.dependencies.logger.warn({ err: error, agentId }, "Failed to auto-archive agent"); } + await this.dependencies.agentStorage.update(agentId, (record) => { + return { ...record, autoArchiveObligation: undefined }; + }); } private async archiveAutoCreatedWorktree(options: { @@ -198,7 +453,15 @@ export class CreateAgentLifecycleDispatch { throw new Error("Auto-created worktree is not a Paseo-owned worktree"); } - await archiveByScope( + await this.archiveWorktreePath(worktreePath, createdWorktree.workspace.workspaceId); + + if (options.agentId) { + await this.dependencies.emitAgentRemove(options.agentId); + } + } + + private async archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise { + const archiveResult = await archiveByScope( { paseoHome: this.dependencies.paseoHome, paseoWorktreesBaseRoot: this.dependencies.worktreesRoot, @@ -209,6 +472,7 @@ export class CreateAgentLifecycleDispatch { findWorkspaceIdForCwd: this.dependencies.findWorkspaceIdForCwd, listActiveWorkspaces: this.dependencies.listActiveWorkspaces, archiveWorkspaceRecord: this.dependencies.archiveWorkspaceRecord, + workspaceRegistry: this.dependencies.workspaceRegistry, emitWorkspaceUpdatesForWorkspaceIds: this.dependencies.emitWorkspaceUpdatesForWorkspaceIds, markWorkspaceArchiving: this.dependencies.markWorkspaceArchiving, clearWorkspaceArchiving: this.dependencies.clearWorkspaceArchiving, @@ -216,14 +480,13 @@ export class CreateAgentLifecycleDispatch { sessionLogger: this.dependencies.logger, }, { - scope: { kind: "workspace", workspaceId: createdWorktree.workspace.workspaceId }, + scope: workspaceId + ? { kind: "workspace", workspaceId } + : { kind: "worktree", targetPath: worktreePath }, requestId: randomUUID(), }, ); - - if (options.agentId) { - await this.dependencies.emitAgentRemove(options.agentId); - } + requireArchiveCleanupComplete(archiveResult, "Auto-created worktree archive"); } } @@ -231,19 +494,138 @@ export function registerAgentAutoArchive(input: { agentManager: AgentLifecycleEvents; agentId: string; archive: () => Promise; + retryBaseMs?: number; + retryMaxMs?: number; + shouldRetry?: (error: unknown) => boolean; + subscribeToRearm?: (rearm: (key: string) => void) => () => void; + rearmKeysForError?: (error: unknown) => readonly string[]; }): LifecycleRegistration { let unsubscribe: (() => void) | null = null; + let unsubscribeRearm: (() => void) | null = null; let archiveTask: Promise | null = null; + let retryTimer: ReturnType | null = null; + let lastArchiveFailure: unknown | null = null; + let consecutiveFailures = 0; + let mutationVersion = 0; + const mutationVersionByKey = new Map(); + let awaitingRearm = false; + let terminalObserved = false; + let canceled = false; + let settleRegistration!: (result: "completed" | "cancelled" | "unresolved") => void; + let registrationSettled = false; + const settled = new Promise<"completed" | "cancelled" | "unresolved">((resolve) => { + settleRegistration = resolve; + }); + const retryBaseMs = input.retryBaseMs ?? 1_000; + const retryMaxMs = input.retryMaxMs ?? 60_000; + const settle = (result: "completed" | "cancelled" | "unresolved") => { + if (registrationSettled) return; + registrationSettled = true; + settleRegistration(result); + }; const release = () => { - if (!unsubscribe) return; - const subscribed = unsubscribe; + const subscribedToAgent = unsubscribe; unsubscribe = null; - subscribed(); + subscribedToAgent?.(); + const subscribedToRearm = unsubscribeRearm; + unsubscribeRearm = null; + subscribedToRearm?.(); }; + const scheduleRetry = () => { + if (canceled || retryTimer) return; + const delay = Math.min(retryMaxMs, retryBaseMs * 2 ** Math.max(0, consecutiveFailures - 1)); + retryTimer = setTimeout(() => { + retryTimer = null; + attemptArchive(); + }, delay); + (retryTimer as unknown as { unref?: () => void }).unref?.(); + }; + if (input.subscribeToRearm) { + unsubscribeRearm = input.subscribeToRearm((key) => { + mutationVersion += 1; + mutationVersionByKey.set(key, mutationVersion); + const rearmKeys = lastArchiveFailure + ? (input.rearmKeysForError?.(lastArchiveFailure) ?? null) + : null; + if ( + !canceled && + terminalObserved && + awaitingRearm && + !archiveTask && + (rearmKeys === null || rearmKeys.includes(key)) + ) { + awaitingRearm = false; + consecutiveFailures = 0; + attemptArchive(); + } + }); + } + function attemptArchive(): void { + if (canceled || archiveTask) return; + const attemptMutationVersion = mutationVersion; + const task = Promise.resolve().then(input.archive); + archiveTask = task; + void task.then( + () => { + if (archiveTask !== task) return; + archiveTask = null; + lastArchiveFailure = null; + consecutiveFailures = 0; + if (retryTimer) { + clearTimeout(retryTimer); + retryTimer = null; + } + release(); + settle("completed"); + return undefined; + }, + (error: unknown) => { + if (archiveTask !== task) return; + archiveTask = null; + lastArchiveFailure = error; + consecutiveFailures += 1; + const retryAllowed = input.shouldRetry?.(error) ?? true; + if (!retryAllowed) { + awaitingRearm = true; + const rearmKeys = input.rearmKeysForError?.(error) ?? null; + const relevantMutationObserved = rearmKeys + ? rearmKeys.some((key) => (mutationVersionByKey.get(key) ?? 0) > attemptMutationVersion) + : mutationVersion !== attemptMutationVersion; + if (!canceled && terminalObserved && relevantMutationObserved) { + awaitingRearm = false; + consecutiveFailures = 0; + attemptArchive(); + } + } else { + scheduleRetry(); + } + return undefined; + }, + ); + } const registration: LifecycleRegistration = { + settled, async cancel() { + canceled = true; + awaitingRearm = false; + if (retryTimer) { + clearTimeout(retryTimer); + retryTimer = null; + } release(); - await archiveTask; + try { + const pendingArchive = archiveTask; + if (pendingArchive) { + await pendingArchive.catch(() => undefined); + } + if (lastArchiveFailure) { + throw lastArchiveFailure; + } + settle("cancelled"); + } catch (error) { + settle("unresolved"); + throw error; + } }, }; unsubscribe = input.agentManager.subscribe( @@ -256,18 +638,10 @@ export function registerAgentAutoArchive(input: { ) { return; } - release(); - archiveTask = input.archive(); + terminalObserved = true; + attemptArchive(); }, { agentId: input.agentId, replayState: false }, ); return registration; } - -function toAutoArchiveTarget( - createdWorktree: CreatePaseoWorktreeWorkflowResult | null, -): AutoArchiveTarget { - return createdWorktree - ? { kind: "created-worktree", result: createdWorktree } - : { kind: "agent-only" }; -} diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index 89cf7bab6c..9937b9f727 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -1,4 +1,5 @@ -import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { expect, test, vi } from "vitest"; @@ -9,8 +10,10 @@ import { createProviderSnapshotManagerStub } from "../../test-utils/session-stub import { AgentManager } from "../agent-manager.js"; import { AgentStorage } from "../agent-storage.js"; import type { CreatePaseoWorktreeWorkflowResult } from "../../worktree-session.js"; +import type { CreatePaseoWorktreeInput } from "../../paseo-worktree-service.js"; import { createAgentCommand } from "./create.js"; import type { ManagedAgent } from "../agent-manager.js"; +import { WorkspaceLifecycleCoordinator } from "../../workspace-lifecycle-coordinator.js"; const logger = createTestLogger(); @@ -36,7 +39,11 @@ function fakeWorktreeCreator(args: { repoRoot: string; createdWorkspaceId: strin workspace: { workspaceId: args.createdWorkspaceId, cwd: workspaceCwd }, repoRoot: args.repoRoot, created: true, - setupContinuation: { kind: "agent" as const, startAfterAgentCreate: () => {} }, + setupContinuation: { + kind: "agent" as const, + startAfterAgentCreate: () => {}, + releaseWithoutStarting: () => {}, + }, }) as unknown as CreatePaseoWorktreeWorkflowResult; } @@ -79,6 +86,214 @@ test("session create forwards clientMessageId to the initial prompt run options" }); }); +test("session create persists and arms auto-archive before starting the initial prompt", async () => { + const order: string[] = []; + const snapshot = { + id: "agent-auto-archive-gap", + provider: "codex", + cwd: "/tmp/paseo-create-test", + runtimeInfo: null, + } as ManagedAgent; + const createAgent = vi.fn(async (_config, _agentId, options) => { + expect(options.autoArchiveObligation).toEqual({ + phase: "armed", + target: { kind: "agent" }, + }); + order.push("persisted"); + return snapshot; + }); + const streamAgent = vi.fn(() => { + order.push("prompt"); + return (async function* noop() {})(); + }); + + await createAgentCommand( + { + agentManager: { + createAgent, + getAgent: vi.fn(() => snapshot), + tryRunOutOfBand: vi.fn(() => false), + hasInFlightRun: vi.fn(() => false), + streamAgent, + waitForAgentRunStart: vi.fn(async () => undefined), + } as unknown as AgentManager, + agentStorage: {} as AgentStorage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + }, + { + kind: "session", + config: { provider: "codex", cwd: "/tmp/paseo-create-test" }, + workspaceId: "ws-create-test", + initialPrompt: "finish immediately", + labels: {}, + provisionalTitle: null, + firstAgentContext: {}, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + onCreated: () => order.push("armed"), + buildSessionConfig: async (config) => ({ sessionConfig: config }), + }, + ); + + expect(order).toEqual(["persisted", "armed", "prompt"]); +}); + +test("session create reports a durable agent when initial prompt start is not confirmed", async () => { + const agentId = "00000000-0000-4000-8000-000000000401"; + const promptStartTimeout = new Error("initial prompt start timed out"); + const snapshot = { + id: agentId, + provider: "codex", + cwd: "/tmp/paseo-create-test", + runtimeInfo: null, + } as ManagedAgent; + const liveSnapshot = { ...snapshot, lifecycle: "running" as const }; + const removePendingAgentCreation = vi.fn(async () => undefined); + const onCreated = vi.fn(); + + const result = await createAgentCommand( + { + agentManager: { + createAgent: vi.fn(async () => snapshot), + getAgent: vi.fn(() => liveSnapshot), + tryRunOutOfBand: vi.fn(() => false), + hasInFlightRun: vi.fn(() => false), + streamAgent: vi.fn(() => (async function* noop() {})()), + waitForAgentRunStart: vi.fn(async () => { + throw promptStartTimeout; + }), + } as unknown as AgentManager, + agentStorage: { + removePendingAgentCreation, + } as unknown as AgentStorage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + }, + { + kind: "session", + agentId, + config: { provider: "codex", cwd: "/tmp/paseo-create-test" }, + workspaceId: "ws-create-test", + initialPrompt: "start the work", + labels: {}, + provisionalTitle: null, + firstAgentContext: {}, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + onCreated, + buildSessionConfig: async (config) => ({ sessionConfig: config }), + }, + ); + + expect(result).toMatchObject({ + snapshot, + liveSnapshot, + initialPromptStarted: false, + initialPromptError: promptStartTimeout, + }); + expect(removePendingAgentCreation).toHaveBeenCalledWith(agentId); + expect(onCreated).toHaveBeenCalledWith({ + agentId, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + }); +}); + +test("legacy worktree create keeps its journal through durable agent registration", async () => { + const workdir = mkdtempSync(join(tmpdir(), "create-agent-journal-test-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const agentManager = createRealAgentManager(storage); + const agentId = agentManager.allocateAgentId(); + const resolvedWorktreePath = join(workdir, "worktrees", "feature-2"); + const removalOrder: string[] = []; + const removePendingAgentCreation = storage.removePendingAgentCreation.bind(storage); + + try { + await storage.beginPendingAgentCreation(agentId); + vi.spyOn(storage, "removePendingAgentCreation").mockImplementation(async (removingAgentId) => { + expect(removingAgentId).toBe(agentId); + expect(await storage.get(agentId)).toMatchObject({ + id: agentId, + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-created-worktree" }, + }, + }); + removalOrder.push("registered"); + await removePendingAgentCreation(removingAgentId); + }); + + const { snapshot } = await createAgentCommand( + { + agentManager, + agentStorage: storage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + }, + { + kind: "session", + agentId, + config: { provider: "codex", cwd: workdir }, + workspaceId: "ws-source", + labels: {}, + provisionalTitle: null, + firstAgentContext: { attachments: [] }, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + buildSessionConfig: async (config, _git, _worktreeName, _context, journal) => { + const plan = { + worktreeIncarnationId: "30704df3-6339-4c9c-8277-1416544ed7cc", + metadataBaseRefName: "main", + }; + await journal?.onWorktreePathPlanned(resolvedWorktreePath, plan); + expect(await storage.listPendingAgentCreations()).toMatchObject([ + { cleanupTarget: { kind: "worktree", directoryIdentity: null } }, + ]); + mkdirSync(resolvedWorktreePath, { recursive: true }); + const directoryStat = statSync(resolvedWorktreePath, { + bigint: true, + }); + await journal?.onWorktreePathResolved(resolvedWorktreePath, { + ...plan, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, + }); + expect(await storage.listPendingAgentCreations()).toEqual([ + expect.objectContaining({ + agentId, + cleanupTarget: { + kind: "worktree", + targetPath: resolvedWorktreePath, + worktreeIncarnationId: "30704df3-6339-4c9c-8277-1416544ed7cc", + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, + metadataBaseRefName: "main", + }, + }), + ]); + removalOrder.push("worktree-resolved"); + return { + sessionConfig: { ...config, cwd: resolvedWorktreePath }, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: () => undefined, + releaseWithoutStarting: () => undefined, + }, + createdWorkspaceId: "ws-created-worktree", + }; + }, + }, + ); + + expect(snapshot.id).toBe(agentId); + expect(removalOrder).toEqual(["worktree-resolved", "registered"]); + await expect(storage.listPendingAgentCreations()).resolves.toEqual([]); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("session create validates the requested mode against the provider's modes", async () => { const snapshot = { id: "agent-1", @@ -122,6 +337,266 @@ test("session create validates the requested mode against the provider's modes", expect(createAgent).not.toHaveBeenCalled(); }); +test.each([ + ["validation failure", new Error("invalid provider mode")], + [ + "validation abort", + Object.assign(new Error("provider resolution aborted"), { name: "AbortError" }), + ], +])("session create releases setup reservation after %s", async (_caseName, failure) => { + const releaseWithoutStarting = vi.fn(); + const startAfterAgentCreate = vi.fn(); + const stub = createProviderSnapshotManagerStub(); + stub.resolveCreateConfig.mockRejectedValue(failure); + const dependencies: Parameters[0] = { + agentManager: { + createAgent: vi.fn(), + } as unknown as Parameters[0]["agentManager"], + agentStorage: {} as Parameters[0]["agentStorage"], + logger: createTestLogger(), + providerSnapshotManager: stub.manager, + }; + + await expect( + createAgentCommand(dependencies, { + kind: "session", + config: { provider: "opencode", cwd: "/tmp/paseo-create-test" }, + workspaceId: "ws-source", + labels: {}, + provisionalTitle: null, + firstAgentContext: { attachments: [] }, + buildSessionConfig: async (config) => ({ + sessionConfig: config, + setupContinuation: { + kind: "agent", + startAfterAgentCreate, + releaseWithoutStarting, + }, + createdWorkspaceId: "ws-created", + }), + }), + ).rejects.toThrow(failure.message); + + expect(releaseWithoutStarting).toHaveBeenCalledOnce(); + expect(startAfterAgentCreate).not.toHaveBeenCalled(); +}); + +test("session create releases setup reservation when agent creation fails", async () => { + const releaseWithoutStarting = vi.fn(); + const startAfterAgentCreate = vi.fn(); + const createFailure = new Error("agent creation failed"); + const dependencies: Parameters[0] = { + agentManager: { + createAgent: vi.fn(async () => { + throw createFailure; + }), + } as unknown as Parameters[0]["agentManager"], + agentStorage: {} as Parameters[0]["agentStorage"], + logger: createTestLogger(), + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + }; + + await expect( + createAgentCommand(dependencies, { + kind: "session", + config: { provider: "codex", cwd: "/tmp/paseo-create-test" }, + workspaceId: "ws-source", + labels: {}, + provisionalTitle: null, + firstAgentContext: { attachments: [] }, + buildSessionConfig: async (config) => ({ + sessionConfig: config, + setupContinuation: { + kind: "agent", + startAfterAgentCreate, + releaseWithoutStarting, + }, + createdWorkspaceId: "ws-created", + }), + }), + ).rejects.toThrow(createFailure.message); + + expect(releaseWithoutStarting).toHaveBeenCalledOnce(); + expect(startAfterAgentCreate).not.toHaveBeenCalled(); +}); + +test.each(["worktree callback", "provider resolution"] as const)( + "MCP create releases a reserved setup continuation after %s fails", + async (failurePoint) => { + const pendingAgentId = "00000000-0000-4000-8000-000000000560"; + const releaseWithoutStarting = vi.fn(); + const startAfterAgentCreate = vi.fn(); + const createAgent = vi.fn(); + const providerSnapshotManager = createProviderSnapshotManagerStub(); + if (failurePoint === "provider resolution") { + providerSnapshotManager.resolveCreateConfig.mockRejectedValue( + new Error("provider resolution failed"), + ); + } + const createdWorktree = { + worktree: { worktreePath: "/tmp/paseo-mcp-create/worktree" }, + intent: {}, + workspace: { + workspaceId: "ws-mcp-created", + cwd: "/tmp/paseo-mcp-create/worktree", + }, + repoRoot: "/tmp/paseo-mcp-create", + created: true, + setupContinuation: { + kind: "agent" as const, + startAfterAgentCreate, + releaseWithoutStarting, + }, + } as unknown as CreatePaseoWorktreeWorkflowResult; + const beginPendingAgentCreation = vi.fn(async () => ({ + agentId: pendingAgentId, + createdAt: "2026-08-01T00:00:00.000Z", + cleanupTarget: { kind: "agent" as const }, + })); + const removePendingAgentCreation = vi.fn(async () => undefined); + + await expect( + createAgentCommand( + { + agentManager: { + allocateAgentId: () => pendingAgentId, + createAgent, + } as unknown as AgentManager, + agentStorage: { + beginPendingAgentCreation, + listPendingAgentCreations: async () => [ + { + agentId: pendingAgentId, + createdAt: "2026-08-01T00:00:00.000Z", + cleanupTarget: { kind: "agent" as const }, + }, + ], + removePendingAgentCreation, + } as unknown as AgentStorage, + logger, + providerSnapshotManager: providerSnapshotManager.manager, + createPaseoWorktree: vi.fn(async () => createdWorktree), + }, + { + kind: "mcp", + provider: "opencode/test-model", + title: "MCP worktree child", + initialPrompt: "do the work", + background: true, + notifyOnFinish: false, + worktree: { worktreeName: "worktree", baseBranch: "main" }, + ...(failurePoint === "worktree callback" + ? { + onWorktreeCreated: () => { + throw new Error("worktree callback failed"); + }, + } + : {}), + }, + ), + ).rejects.toThrow(failurePoint === "worktree callback" ? "callback" : "provider resolution"); + + expect(releaseWithoutStarting).toHaveBeenCalledOnce(); + expect(startAfterAgentCreate).not.toHaveBeenCalled(); + expect(createAgent).not.toHaveBeenCalled(); + expect(beginPendingAgentCreation).toHaveBeenCalledWith(pendingAgentId); + expect(removePendingAgentCreation).toHaveBeenCalledWith(pendingAgentId); + }, +); + +test("agent creation holds workspace ownership until the agent is attached", async () => { + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + let releaseCreate: (() => void) | undefined; + let markCreateStarted: (() => void) | undefined; + const createStarted = new Promise((resolve) => { + markCreateStarted = resolve; + }); + const createGate = new Promise((resolve) => { + releaseCreate = resolve; + }); + const snapshot = { + id: "agent-owned", + provider: "codex", + cwd: "/tmp/paseo-create-owned", + workspaceId: "ws-owned", + runtimeInfo: null, + } as ManagedAgent; + const createTask = createAgentCommand( + { + agentManager: { + createAgent: vi.fn(async () => { + markCreateStarted?.(); + await createGate; + return snapshot; + }), + } as unknown as AgentManager, + agentStorage: {} as AgentStorage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + lifecycleCoordinator, + }, + { + kind: "session", + config: { provider: "codex", cwd: snapshot.cwd }, + workspaceId: "ws-owned", + labels: {}, + provisionalTitle: null, + firstAgentContext: {}, + buildSessionConfig: async (config) => ({ sessionConfig: config }), + }, + ); + + await createStarted; + const archiveReservation = lifecycleCoordinator.reserveWorkspaceArchive(["ws-owned"]); + let ownershipReleased = false; + const waitTask = lifecycleCoordinator + .waitForWorkspaceOwnershipMutations(["ws-owned"]) + .then(() => { + ownershipReleased = true; + return undefined; + }); + await Promise.resolve(); + expect(ownershipReleased).toBe(false); + + releaseCreate?.(); + await createTask; + await waitTask; + expect(ownershipReleased).toBe(true); + archiveReservation.release(); +}); + +test("agent creation rejects an ownership attach after workspace archive", async () => { + const createAgent = vi.fn(); + const requireActiveWorkspaceForOwnership = vi.fn(async () => { + throw new Error("Workspace not found: ws-archived"); + }); + + await expect( + createAgentCommand( + { + agentManager: { createAgent } as unknown as AgentManager, + agentStorage: {} as AgentStorage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + lifecycleCoordinator: new WorkspaceLifecycleCoordinator(), + requireActiveWorkspaceForOwnership, + }, + { + kind: "session", + config: { provider: "codex", cwd: "/tmp/paseo-create-archived" }, + workspaceId: "ws-archived", + labels: {}, + provisionalTitle: null, + firstAgentContext: {}, + buildSessionConfig: async (config) => ({ sessionConfig: config }), + }, + ), + ).rejects.toThrow("Workspace not found: ws-archived"); + + expect(requireActiveWorkspaceForOwnership).toHaveBeenCalledWith("ws-archived"); + expect(createAgent).not.toHaveBeenCalled(); +}); + test("session create applies the resolved mode from the provider create config", async () => { const snapshot = { id: "agent-1", @@ -264,7 +739,11 @@ test("session create stamps the new worktree's workspaceId when a setup continua firstAgentContext: { attachments: [] }, buildSessionConfig: async (config) => ({ sessionConfig: config, - setupContinuation: { kind: "agent", startAfterAgentCreate: () => {} }, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: () => {}, + releaseWithoutStarting: () => {}, + }, createdWorkspaceId: "ws-new-worktree", }), }, @@ -380,6 +859,91 @@ test("mcp create exposes the created worktree before dispatching the initial pro } }); +test("mcp worktree create journals the exact path incarnation through durable agent publication", async () => { + const workdir = mkdtempSync(join(tmpdir(), "create-agent-mcp-journal-test-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const agentManager = createRealAgentManager(storage); + const agentId = "00000000-0000-4000-8000-000000000561"; + const worktreePath = join(workdir, "worktree"); + const workspaceCwd = join(worktreePath, "packages", "app"); + const worktreeIncarnationId = randomUUID(); + let observedPending: Awaited> = []; + const createAgent = agentManager.createAgent.bind(agentManager); + vi.spyOn(agentManager, "allocateAgentId").mockReturnValue(agentId); + vi.spyOn(agentManager, "createAgent").mockImplementation(async (...args) => { + observedPending = await storage.listPendingAgentCreations(); + return createAgent(...args); + }); + + try { + await createAgentCommand( + { + agentManager, + agentStorage: storage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + createPaseoWorktree: async (input: CreatePaseoWorktreeInput) => { + await input.onWorktreePathPlanned?.(worktreePath, { + worktreeIncarnationId, + metadataBaseRefName: "main", + }); + mkdirSync(workspaceCwd, { recursive: true }); + const directoryStat = statSync(worktreePath, { bigint: true }); + await input.onWorktreePathResolved?.(worktreePath, { + worktreeIncarnationId, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, + metadataBaseRefName: "main", + }); + return { + worktree: { worktreePath }, + intent: {}, + workspace: { workspaceId: "ws-mcp-journal", cwd: workspaceCwd }, + repoRoot: workdir, + created: true, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: () => {}, + releaseWithoutStarting: () => {}, + }, + } as unknown as CreatePaseoWorktreeWorkflowResult; + }, + }, + { + kind: "mcp", + provider: "codex", + cwd: workdir, + title: "journaled worktree", + background: true, + notifyOnFinish: false, + worktree: { worktreeName: "feature", baseBranch: "main" }, + }, + ); + + expect(observedPending).toEqual([ + expect.objectContaining({ + agentId, + cleanupTarget: { + kind: "worktree", + targetPath: worktreePath, + worktreeIncarnationId, + directoryIdentity: { + device: statSync(worktreePath, { bigint: true }).dev.toString(), + inode: statSync(worktreePath, { bigint: true }).ino.toString(), + }, + metadataBaseRefName: "main", + }, + }), + ]); + await expect(storage.listPendingAgentCreations()).resolves.toEqual([]); + await expect(storage.get(agentId)).resolves.toMatchObject({ id: agentId }); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("session create keeps the prompt title after the initial prompt settles", async () => { const workdir = mkdtempSync(join(tmpdir(), "create-agent-title-test-")); const storage = new AgentStorage(join(workdir, "agents"), logger); diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index b31a858f34..d18b0acd22 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -1,6 +1,7 @@ import type { Logger } from "pino"; import type { TerminalManager } from "../../../terminal/terminal-manager.js"; +import type { WorktreeCreationJournalCallbacks } from "../../../utils/worktree.js"; import type { CreatePaseoWorktreeInput } from "../../paseo-worktree-service.js"; import { expandUserPath, resolvePathFromBase } from "../../path-utils.js"; import { toWorktreeRequestError } from "../../worktree-errors.js"; @@ -13,7 +14,7 @@ import type { import type { AgentAttachment, FirstAgentContext, GitSetupOptions } from "../../messages.js"; import type { AgentManager, CreateAgentOptions, ManagedAgent } from "../agent-manager.js"; import type { AgentPromptInput, AgentRunOptions, AgentSessionConfig } from "../agent-sdk-types.js"; -import type { AgentStorage } from "../agent-storage.js"; +import type { AgentStorage, AutoArchiveObligation } from "../agent-storage.js"; import type { AgentOwner } from "../agent-owner.js"; import type { ProviderSnapshotManager } from "../provider-snapshot-manager.js"; import { setupFinishNotification, startCreatedAgentInitialPrompt } from "../agent-prompt.js"; @@ -26,6 +27,10 @@ import { emitLiveTimelineItemIfAgentKnown, } from "../timeline-append.js"; import { resolveCreateAgentIntent } from "./intent.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "../../workspace-lifecycle-coordinator.js"; export interface CreateAgentSessionWorktreeResult { sessionConfig: AgentSessionConfig; @@ -46,6 +51,8 @@ export interface CreateAgentCommandDependencies { createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn; // Mints a fresh directory workspace for a cwd and returns its id. ensureWorkspaceForCreate?: EnsureWorkspaceForCreate; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; + requireActiveWorkspaceForOwnership?: (workspaceId: string) => Promise; } export type EnsureWorkspaceForCreate = ( @@ -68,11 +75,15 @@ export interface CreateAgentFromSessionInput { env?: Record; provisionalTitle: string | null; firstAgentContext: FirstAgentContext; + autoArchiveObligation?: AutoArchiveObligation; + agentId?: string; + onCreated?: (created: { agentId: string; autoArchiveObligation?: AutoArchiveObligation }) => void; buildSessionConfig: ( config: AgentSessionConfig, gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, + worktreeCreationJournal?: WorktreeCreationJournalCallbacks, ) => Promise; } @@ -100,6 +111,7 @@ export interface CreateAgentFromMcpInput { agentId: string; createdWorktree: CreatePaseoWorktreeWorkflowResult | null; }) => void; + onPendingAgentCreation?: (agentId: string) => void; onWorktreeCreated?: (createdWorktree: CreatePaseoWorktreeWorkflowResult) => void; callerAgentId?: string; callerContext?: { @@ -174,16 +186,33 @@ export async function createAgentCommand( dependencies: CreateAgentCommandDependencies, input: CreateAgentCommandInput, ): Promise { - const resolved = - input.kind === "session" - ? await resolveSessionCreateAgent(dependencies, input) - : await resolveMcpCreateAgent(dependencies, input); + const pendingAgentId = await reservePendingAgentId(dependencies, input); + if (input.kind === "mcp" && pendingAgentId) { + input.onPendingAgentCreation?.(pendingAgentId); + } + const resolved = await resolveCreateAgent(dependencies, input, pendingAgentId); - const snapshot = await dependencies.agentManager.createAgent( - resolved.config, - undefined, - resolved.createOptions, - ); + let snapshot: ManagedAgent; + const lifecycleCoordinator = + dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const workspaceId = requireResolvedWorkspaceId(resolved.createOptions.workspaceId); + try { + snapshot = await lifecycleCoordinator.runWorkspaceOwnershipMutation( + workspaceId, + async () => { + await dependencies.requireActiveWorkspaceForOwnership?.(workspaceId); + }, + () => + dependencies.agentManager.createAgent( + resolved.config, + pendingAgentId, + resolved.createOptions, + ), + ); + } catch (error) { + resolved.setupContinuation?.releaseWithoutStarting(); + throw error; + } resolved.setupContinuation?.startAfterAgentCreate({ agentId: snapshot.id, @@ -192,9 +221,8 @@ export async function createAgentCommand( let liveSnapshot = snapshot; let initialPromptStarted = false; let initialPromptError: unknown | null = null; - if (input.kind === "mcp") { - input.onCreated?.({ agentId: snapshot.id, createdWorktree: resolved.createdWorktree ?? null }); - } + await completePendingAgentCreation(dependencies, pendingAgentId); + notifyAgentCreated(input, resolved, snapshot); if (resolved.prompt !== undefined) { const sendResult = await sendInitialPrompt(dependencies, resolved, snapshot); initialPromptStarted = sendResult.started; @@ -223,11 +251,92 @@ export async function createAgentCommand( }; } +async function reservePendingAgentId( + dependencies: CreateAgentCommandDependencies, + input: CreateAgentCommandInput, +): Promise { + if (input.kind === "session") return input.agentId; + if (!shouldCreateMcpWorktree(input.worktree)) return undefined; + + const pendingAgentId = dependencies.agentManager.allocateAgentId(); + await dependencies.agentStorage.beginPendingAgentCreation(pendingAgentId); + return pendingAgentId; +} + +async function resolveCreateAgent( + dependencies: CreateAgentCommandDependencies, + input: CreateAgentCommandInput, + pendingAgentId: string | undefined, +): Promise { + try { + return input.kind === "session" + ? await resolveSessionCreateAgent(dependencies, input) + : await resolveMcpCreateAgent(dependencies, input, pendingAgentId); + } catch (error) { + await removeSideEffectFreePendingCreation(dependencies.agentStorage, pendingAgentId); + throw error; + } +} + +async function completePendingAgentCreation( + dependencies: CreateAgentCommandDependencies, + pendingAgentId: string | undefined, +): Promise { + if (!pendingAgentId) return; + await dependencies.agentStorage.removePendingAgentCreation(pendingAgentId).catch((error) => { + dependencies.logger.warn( + { err: error, agentId: pendingAgentId }, + "Failed to remove completed agent creation journal", + ); + }); +} + +function notifyAgentCreated( + input: CreateAgentCommandInput, + resolved: ResolvedCreateAgent, + snapshot: ManagedAgent, +): void { + if (input.kind === "session") { + input.onCreated?.({ + agentId: snapshot.id, + autoArchiveObligation: resolved.createOptions.autoArchiveObligation, + }); + return; + } + input.onCreated?.({ agentId: snapshot.id, createdWorktree: resolved.createdWorktree ?? null }); +} + +function shouldCreateMcpWorktree(worktree: CreateAgentFromMcpInput["worktree"]): boolean { + return Boolean( + worktree && + (worktree.worktreeName || worktree.refName || worktree.action || worktree.githubPrNumber), + ); +} + +async function removeSideEffectFreePendingCreation( + agentStorage: AgentStorage, + pendingAgentId: string | undefined, +): Promise { + if (!pendingAgentId) return; + try { + const pending = (await agentStorage.listPendingAgentCreations()).find( + (record) => record.agentId === pendingAgentId, + ); + if (pending?.cleanupTarget.kind === "agent") { + await agentStorage.removePendingAgentCreation(pendingAgentId); + } + } catch { + // Preserve the original create failure. Startup recovery owns any journal + // whose side-effect-free cleanup could not be confirmed here. + } +} + async function resolveSessionCreateAgent( dependencies: CreateAgentCommandDependencies, input: CreateAgentFromSessionInput, ): Promise { const trimmedPrompt = input.initialPrompt?.trim(); + const pendingAgentId = input.agentId; const { sessionConfig: builtSessionConfig, setupContinuation, @@ -237,6 +346,9 @@ async function resolveSessionCreateAgent( input.git, input.worktreeName, input.firstAgentContext, + pendingAgentId + ? pendingWorktreeCreationJournal(dependencies.agentStorage, pendingAgentId) + : undefined, ); // Validate the requested mode against the provider's modes for the resolved // cwd. The app remembers mode preferences globally, so a saved mode can be @@ -251,57 +363,76 @@ async function resolveSessionCreateAgent( // this for the worktree path via cleanupCreatedWorktreeAfterFailedAgentCreate; // this is a pre-existing gap for directory-only workspace creates, not // introduced by this validation). - const resolvedCreateConfig = await dependencies.providerSnapshotManager.resolveCreateConfig({ - cwd: builtSessionConfig.cwd, - provider: builtSessionConfig.provider, - requestedMode: builtSessionConfig.modeId, - featureValues: builtSessionConfig.featureValues, - parent: null, - unattended: false, - }); - const sessionConfig: AgentSessionConfig = { - ...builtSessionConfig, - modeId: resolvedCreateConfig.modeId, - featureValues: resolvedCreateConfig.featureValues, - }; - const prompt = buildAgentPrompt(trimmedPrompt ?? "", input.images, input.attachments); - const hasPromptContent = Array.isArray(prompt) ? prompt.length > 0 : prompt.length > 0; - const clientMessageId = normalizeClientMessageId(input.clientMessageId); - const runOptions: AgentRunOptions | undefined = - input.outputSchema || clientMessageId - ? { - ...(input.outputSchema ? { outputSchema: input.outputSchema } : {}), - ...(clientMessageId ? { clientMessageId } : {}), - } - : undefined; - const workspaceId = setupContinuation ? createdWorkspaceId : input.workspaceId; - - return { - config: sessionConfig, - createOptions: { - labels: input.labels, - initialPrompt: trimmedPrompt, - env: input.env, - initialTitle: input.provisionalTitle, - // A legacy git/worktreeName worktree creates a fresh workspace, so the - // agent belongs to that workspace, not the source one. createdWorkspaceId - // is the freshly created worktree's workspace. - workspaceId: requireResolvedWorkspaceId(workspaceId), - }, - prompt: hasPromptContent ? prompt : undefined, - runOptions, - setupContinuation, - background: true, - promptFailure: "throw", - promptLogger: dependencies.logger.child({ - clientMessageId: resolveClientMessageId(input.clientMessageId), - }), - }; + try { + const resolvedCreateConfig = await dependencies.providerSnapshotManager.resolveCreateConfig({ + cwd: builtSessionConfig.cwd, + provider: builtSessionConfig.provider, + requestedMode: builtSessionConfig.modeId, + featureValues: builtSessionConfig.featureValues, + parent: null, + unattended: false, + }); + const sessionConfig: AgentSessionConfig = { + ...builtSessionConfig, + modeId: resolvedCreateConfig.modeId, + featureValues: resolvedCreateConfig.featureValues, + }; + const prompt = buildAgentPrompt(trimmedPrompt ?? "", input.images, input.attachments); + const hasPromptContent = Array.isArray(prompt) ? prompt.length > 0 : prompt.length > 0; + const clientMessageId = normalizeClientMessageId(input.clientMessageId); + const runOptions: AgentRunOptions | undefined = + input.outputSchema || clientMessageId + ? { + ...(input.outputSchema ? { outputSchema: input.outputSchema } : {}), + ...(clientMessageId ? { clientMessageId } : {}), + } + : undefined; + const workspaceId = setupContinuation ? createdWorkspaceId : input.workspaceId; + + return { + config: sessionConfig, + createOptions: { + labels: input.labels, + initialPrompt: trimmedPrompt, + env: input.env, + initialTitle: input.provisionalTitle, + // A legacy git/worktreeName worktree creates a fresh workspace, so the + // agent belongs to that workspace, not the source one. createdWorkspaceId + // is the freshly created worktree's workspace. + workspaceId: requireResolvedWorkspaceId(workspaceId), + autoArchiveObligation: + input.autoArchiveObligation && setupContinuation + ? { + phase: input.autoArchiveObligation.phase, + target: { + kind: "workspace", + workspaceId: requireResolvedWorkspaceId(createdWorkspaceId), + }, + } + : input.autoArchiveObligation, + }, + prompt: hasPromptContent ? prompt : undefined, + runOptions, + setupContinuation, + background: true, + // Agent publication is already durable before the first prompt is + // dispatched. Keep a prompt-start failure separate from creation truth so + // callers do not retry and create a duplicate agent/worktree. + promptFailure: "return-error", + promptLogger: dependencies.logger.child({ + clientMessageId: resolveClientMessageId(input.clientMessageId), + }), + }; + } catch (error) { + setupContinuation?.releaseWithoutStarting(); + throw error; + } } async function resolveMcpCreateAgent( dependencies: CreateAgentCommandDependencies, input: CreateAgentFromMcpInput, + pendingAgentId?: string, ): Promise { const resolvedProviderModel = resolveProviderModel(input.provider); const provider = resolvedProviderModel.provider; @@ -315,56 +446,64 @@ async function resolveMcpCreateAgent( cwd, worktree: input.worktree, initialPrompt: input.initialPrompt ?? "", + worktreeCreationJournal: pendingAgentId + ? pendingWorktreeCreationJournal(dependencies.agentStorage, pendingAgentId) + : undefined, }); - if (createdWorktree) input.onWorktreeCreated?.(createdWorktree); - - const intent = await resolveCreateAgentIntent({ - explicitWorkspaceId: setupContinuation ? createdWorkspaceId : input.workspaceId, - caller: parentAgent - ? { id: parentAgent.id, cwd: parentAgent.cwd, workspaceId: parentAgent.workspaceId } - : null, - labels: input.labels, - childAgentDefaultLabels: input.callerContext?.childAgentDefaultLabels, - legacyDetached: input.detached ?? false, - resolveWorkspace: async (workspaceId) => ({ workspaceId, cwd: resolvedCwd }), - createWorkspace: async () => ({ - workspaceId: requireResolvedWorkspaceId( - await ensureWorkspaceForMcpCreate(dependencies, resolvedCwd, input.initialPrompt ?? ""), - ), - cwd: resolvedCwd, - }), - }); - const resolvedCreateConfig = await resolveMcpProviderCreateConfig({ - dependencies, - input, - provider, - resolvedCwd, - parentAgent, - }); + try { + if (createdWorktree) input.onWorktreeCreated?.(createdWorktree); - const trimmedPrompt = input.initialPrompt?.trim() ?? ""; - return { - config: buildMcpSessionConfig({ + const intent = await resolveCreateAgentIntent({ + explicitWorkspaceId: setupContinuation ? createdWorkspaceId : input.workspaceId, + caller: parentAgent + ? { id: parentAgent.id, cwd: parentAgent.cwd, workspaceId: parentAgent.workspaceId } + : null, + labels: input.labels, + childAgentDefaultLabels: input.callerContext?.childAgentDefaultLabels, + legacyDetached: input.detached ?? false, + resolveWorkspace: async (workspaceId) => ({ workspaceId, cwd: resolvedCwd }), + createWorkspace: async () => ({ + workspaceId: requireResolvedWorkspaceId( + await ensureWorkspaceForMcpCreate(dependencies, resolvedCwd, input.initialPrompt ?? ""), + ), + cwd: resolvedCwd, + }), + }); + const resolvedCreateConfig = await resolveMcpProviderCreateConfig({ + dependencies, input, - resolvedProviderModel, provider, - resolvedCwd: intent.cwd, - trimmedPrompt, - resolvedMode: resolvedCreateConfig.modeId, - resolvedFeatures: resolvedCreateConfig.featureValues, - }), - createOptions: { - ...(Object.keys(intent.labels).length > 0 ? { labels: intent.labels } : {}), - workspaceId: intent.workspaceId, - owner: input.owner, - env: input.env, - }, - prompt: trimmedPrompt ? trimmedPrompt : undefined, - setupContinuation, - createdWorktree, - background: input.background, - promptFailure: input.promptFailure ?? "log", - }; + resolvedCwd, + parentAgent, + }); + + const trimmedPrompt = input.initialPrompt?.trim() ?? ""; + return { + config: buildMcpSessionConfig({ + input, + resolvedProviderModel, + provider, + resolvedCwd: intent.cwd, + trimmedPrompt, + resolvedMode: resolvedCreateConfig.modeId, + resolvedFeatures: resolvedCreateConfig.featureValues, + }), + createOptions: { + ...(Object.keys(intent.labels).length > 0 ? { labels: intent.labels } : {}), + workspaceId: intent.workspaceId, + owner: input.owner, + env: input.env, + }, + prompt: trimmedPrompt ? trimmedPrompt : undefined, + setupContinuation, + createdWorktree, + background: input.background, + promptFailure: input.promptFailure ?? "log", + }; + } catch (error) { + setupContinuation?.releaseWithoutStarting(); + throw error; + } } function resolveMcpInitialCwd( @@ -468,7 +607,11 @@ async function sendInitialPrompt( throw error; } if (resolved.promptFailure === "return-error") { - return { started: false, liveSnapshot: snapshot, error }; + return { + started: false, + liveSnapshot: dependencies.agentManager.getAgent(snapshot.id) ?? snapshot, + error, + }; } dependencies.logger.error({ err: error, agentId: snapshot.id }, "Failed to run initial prompt"); return { started: false, liveSnapshot: snapshot }; @@ -507,6 +650,7 @@ async function resolveMcpCwd(params: { cwd: string; initialPrompt: string; worktree: CreateAgentFromMcpInput["worktree"]; + worktreeCreationJournal?: WorktreeCreationJournalCallbacks; }): Promise<{ resolvedCwd: string; setupContinuation?: AgentWorktreeSetupContinuation; @@ -517,9 +661,7 @@ async function resolveMcpCwd(params: { if (!worktree) { return { resolvedCwd: params.cwd }; } - const shouldCreateWorktree = Boolean( - worktree.worktreeName || worktree.refName || worktree.action || worktree.githubPrNumber, - ); + const shouldCreateWorktree = shouldCreateMcpWorktree(worktree); if (!shouldCreateWorktree) { return { resolvedCwd: params.cwd }; } @@ -545,6 +687,8 @@ async function resolveMcpCwd(params: { runSetup: false, paseoHome: dependencies.paseoHome, worktreesRoot: dependencies.worktreesRoot, + onWorktreePathPlanned: params.worktreeCreationJournal?.onWorktreePathPlanned, + onWorktreePathResolved: params.worktreeCreationJournal?.onWorktreePathResolved, }, createPaseoWorktree: dependencies.createPaseoWorktree, resolveDefaultBranch: baseBranch ? async () => baseBranch : undefined, @@ -574,6 +718,18 @@ async function resolveMcpCwd(params: { }; } +function pendingWorktreeCreationJournal( + agentStorage: AgentStorage, + agentId: string, +): WorktreeCreationJournalCallbacks { + return { + onWorktreePathPlanned: (worktreePath, plan) => + agentStorage.planPendingAgentCreationWorktree(agentId, worktreePath, plan), + onWorktreePathResolved: (worktreePath, reservation) => + agentStorage.identifyPendingAgentCreationWorktree(agentId, worktreePath, reservation), + }; +} + interface CreateMcpWorktreeOptions { input: CreatePaseoWorktreeInput; createPaseoWorktree: CreatePaseoWorktreeWorkflowFn | undefined; diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index b29b0a39f5..86cee3993e 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -1,4 +1,5 @@ import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { describe, expect, it, vi } from "vitest"; @@ -56,6 +57,7 @@ import type { BrowserToolsBroker, BrowserToolsExecuteInput } from "../browser-to import type { BrowserToolsResponsePayload } from "../browser-tools/errors.js"; import { readPaseoWorktreeMetadata } from "../../utils/worktree-metadata.js"; import { createWorkspaceProvisioningService } from "../session/workspace-provisioning/workspace-provisioning-service.js"; +import { WorkspaceLifecycleCoordinator } from "../workspace-lifecycle-coordinator.js"; const REPO_CWD = resolvePath("/tmp/repo"); const TARGET_CWD = resolvePath("/tmp/target"); @@ -193,7 +195,9 @@ interface TestDeps { } function buildAgentManagerSpies() { + const getAgent = vi.fn(); return { + allocateAgentId: vi.fn(() => randomUUID()), createAgent: vi.fn(), waitForAgentEvent: vi.fn().mockResolvedValue({ status: "idle", @@ -209,7 +213,11 @@ function buildAgentManagerSpies() { updateAgentMetadata: vi.fn().mockResolvedValue(undefined), archiveAgent: vi.fn().mockResolvedValue({ archivedAt: new Date().toISOString() }), notifyAgentState: vi.fn(), - getAgent: vi.fn(), + getAgent, + getAgentInitializationState: vi.fn((agentId: string) => ({ + agent: getAgent(agentId), + closeInFlight: false, + })), listAgents: vi.fn().mockReturnValue([]), getTimeline: vi.fn().mockReturnValue([]), resumeAgentFromPersistence: vi.fn(), @@ -236,6 +244,16 @@ function buildAgentStorageSpies() { upsert: vi.fn().mockResolvedValue(undefined), applySnapshot: vi.fn(), list: vi.fn().mockResolvedValue([]), + beginPendingAgentCreation: vi.fn().mockImplementation(async (agentId: string) => ({ + agentId, + createdAt: new Date().toISOString(), + ownerKind: "agent" as const, + cleanupTarget: { kind: "agent" as const }, + })), + planPendingAgentCreationWorktree: vi.fn().mockResolvedValue(undefined), + identifyPendingAgentCreationWorktree: vi.fn().mockResolvedValue(undefined), + listPendingAgentCreations: vi.fn().mockResolvedValue([]), + removePendingAgentCreation: vi.fn().mockResolvedValue(undefined), remove: vi.fn(), }; } @@ -272,6 +290,89 @@ function createTerminalManagerStub(overrides: Partial = {}): Te } as unknown as TerminalManager; } +it("holds workspace ownership while an MCP terminal is being attached", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + let releaseTerminal: (() => void) | undefined; + let markTerminalStarted: (() => void) | undefined; + const terminalStarted = new Promise((resolve) => { + markTerminalStarted = resolve; + }); + const terminalGate = new Promise((resolve) => { + releaseTerminal = resolve; + }); + const createTerminal = vi.fn(async ({ cwd }: { cwd: string }) => { + markTerminalStarted?.(); + await terminalGate; + return { id: "terminal-owned", name: "Owned terminal", cwd }; + }); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + terminalManager: createTerminalManagerStub({ + createTerminal: createTerminal as unknown as TerminalManager["createTerminal"], + }), + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + ensureWorkspaceForCreate: async () => "ws-terminal-owned", + listActiveWorkspaces: async () => [ + { + workspaceId: "ws-terminal-owned", + cwd: "/tmp/terminal-owned", + kind: "local_checkout", + }, + ], + lifecycleCoordinator, + logger: createTestLogger(), + }); + + const terminalTask = registeredTool(server, "create_terminal").handler({ + cwd: "/tmp/terminal-owned", + name: "Owned terminal", + }); + await terminalStarted; + const archiveReservation = lifecycleCoordinator.reserveWorkspaceArchive(["ws-terminal-owned"]); + let ownershipReleased = false; + const waitTask = lifecycleCoordinator + .waitForWorkspaceOwnershipMutations(["ws-terminal-owned"]) + .then(() => { + ownershipReleased = true; + return undefined; + }); + await Promise.resolve(); + expect(ownershipReleased).toBe(false); + + releaseTerminal?.(); + await terminalTask; + await waitTask; + expect(ownershipReleased).toBe(true); + archiveReservation.release(); +}); + +it("rejects an MCP terminal attach after workspace archive", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const createTerminal = vi.fn(); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + terminalManager: createTerminalManagerStub({ + createTerminal: createTerminal as unknown as TerminalManager["createTerminal"], + }), + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + ensureWorkspaceForCreate: async () => "ws-terminal-archived", + listActiveWorkspaces: async () => [], + lifecycleCoordinator: new WorkspaceLifecycleCoordinator(), + logger: createTestLogger(), + }); + + await expect( + registeredTool(server, "create_terminal").handler({ + cwd: "/tmp/terminal-archived", + name: "Archived terminal", + }), + ).rejects.toThrow("Workspace not found: ws-terminal-archived"); + expect(createTerminal).not.toHaveBeenCalled(); +}); + type ProviderSnapshotManagerStub = ReturnType; interface ConfigureProviderEntry { @@ -758,6 +859,10 @@ function createPaseoWorktreeForMcpTest(options: { logger: createTestLogger(), generateWorkspaceName: options.generateWorkspaceName ?? (async () => null), }); + const worktreeCreationStorage = new AgentStorage( + join(options.paseoHome, "test-worktree-creation-journal"), + createTestLogger(), + ); return async (input, serviceOptions) => { options.setupContinuations?.push(serviceOptions?.setupContinuation?.kind); @@ -776,9 +881,30 @@ function createPaseoWorktreeForMcpTest(options: { warmWorkspaceGitData: async () => {}, autoNameWorkspaceBranchForFirstAgent: (autoNameInput) => workspaceAutoName.scheduleForWorktree(autoNameInput), - emitWorkspaceUpdateForWorkspaceId: async (workspaceId) => { - options.broadcasts.push(workspaceId); + worktreeCreationJournal: { + beginPendingAgentCreation: (creationId, journalOptions) => + worktreeCreationStorage.beginPendingAgentCreation(creationId, journalOptions), + planPendingAgentCreationWorktree: (creationId, worktreePath, plan) => + worktreeCreationStorage.planPendingAgentCreationWorktree( + creationId, + worktreePath, + plan, + ), + identifyPendingAgentCreationWorktree: (creationId, worktreePath, reservation) => + worktreeCreationStorage.identifyPendingAgentCreationWorktree( + creationId, + worktreePath, + reservation, + ), + removePendingAgentCreation: (creationId) => + worktreeCreationStorage.removePendingAgentCreation(creationId), + recoverPendingCreation: (creationId) => + worktreeCreationStorage.removePendingAgentCreation(creationId), }, + // The helper records the initial publication synchronously below. Keep + // background setup refreshes out of that assertion stream; auto-name + // refreshes use the WorkspaceAutoName dependency above. + emitWorkspaceUpdateForWorkspaceId: async () => {}, cacheWorkspaceSetupSnapshot: () => {}, emit: () => {}, sessionLogger: createTestLogger(), @@ -803,6 +929,7 @@ function createPaseoWorktreeForMcpTest(options: { startAfterAgentCreate: ({ agentId }) => { options.startedAgentSetupIds?.push(agentId); }, + releaseWithoutStarting: () => {}, }, }; } @@ -1783,7 +1910,7 @@ describe("create_agent MCP tool", () => { expect.objectContaining({ cwd: expect.stringContaining("agent-worktree"), }), - undefined, + expect.any(String), { workspaceId: createdWorkspaceIds[0] }, ); } finally { @@ -1791,6 +1918,76 @@ describe("create_agent MCP tool", () => { } }); + it("cleans and recovers a native MCP worktree journal when create_agent fails", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + const createdWorktree = { + worktree: { + branchName: "failed-agent-worktree", + worktreePath: "/tmp/worktrees/failed-agent-worktree", + }, + intent: { + kind: "branch-off" as const, + branchName: "failed-agent-worktree", + baseBranch: "main", + }, + workspace: createPersistedWorkspaceRecord({ + workspaceId: "ws-failed-agent-worktree", + projectId: REPO_CWD, + cwd: "/tmp/worktrees/failed-agent-worktree", + kind: "worktree", + displayName: "failed-agent-worktree", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }), + repoRoot: REPO_CWD, + created: true, + setupContinuation: { + kind: "agent" as const, + startAfterAgentCreate: vi.fn(), + releaseWithoutStarting: vi.fn(), + }, + }; + const cleanupCreatedWorktreeAfterFailedAgentCreate = vi.fn().mockResolvedValue(undefined); + const recoverPendingAgentCreation = vi.fn().mockResolvedValue(undefined); + spies.agentManager.createAgent.mockRejectedValue(new Error("provider creation failed")); + + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + providerSnapshotManager: createOpenCodeManager().manager, + createPaseoWorktree: vi.fn().mockResolvedValue(createdWorktree), + createAgentLifecycleDispatch: { + cleanupCreatedWorktreeAfterFailedAgentCreate, + recoverPendingAgentCreation, + }, + logger, + }); + + await expect( + registeredTool(server, "create_agent").handler({ + ...detachedWorktreeWorkspace(REPO_CWD, { + kind: "branch-off", + worktreeSlug: "failed-agent-worktree", + branchName: "failed-agent-worktree", + baseBranch: "main", + }), + title: "Failed worktree agent", + provider: "codex/gpt-5.4", + initialPrompt: "Fail after creating the worktree", + background: true, + }), + ).rejects.toThrow("provider creation failed"); + + const pendingAgentId = z + .string() + .parse(spies.agentStorage.beginPendingAgentCreation.mock.calls[0]?.[0]); + expect(cleanupCreatedWorktreeAfterFailedAgentCreate).toHaveBeenCalledWith({ + createdWorktree, + createdAgentId: null, + }); + expect(recoverPendingAgentCreation).toHaveBeenCalledWith(pendingAgentId); + }); + it("creates a create_agent branch-off worktree without invoking the legacy metadata branch rename", async () => { const { agentManager, agentStorage, spies } = createTestDeps(); const tempDir = await mkdtemp(join(tmpdir(), "paseo-mcp-agent-worktree-name-context-")); @@ -2132,7 +2329,7 @@ describe("create_agent MCP tool", () => { expect.objectContaining({ title: "Explicit Agent Title", }), - undefined, + expect.any(String), { workspaceId }, ); expect(workspace).toMatchObject({ @@ -2401,6 +2598,7 @@ describe("create_agent MCP tool", () => { startAfterAgentCreate: ({ agentId }: { agentId: string }) => { startedAgentSetupIds.push(agentId); }, + releaseWithoutStarting: () => {}, }, } : {}), @@ -2455,7 +2653,7 @@ describe("create_agent MCP tool", () => { expect(startedAgentSetupIds).toEqual(["agent-pr-worktree"]); expect(spies.agentManager.createAgent).toHaveBeenCalledWith( expect.objectContaining({ cwd: "/tmp/worktrees/pr-123" }), - undefined, + expect.any(String), { workspaceId: "ws-pr-123" }, ); await waitForUnexpectedWorkspaceNamingSideEffects(); @@ -2469,6 +2667,7 @@ describe("create_agent MCP tool", () => { const paseoHome = join(tempDir, ".paseo"); const broadcasts: string[] = []; const setupContinuations: Array<"workspace" | "agent" | undefined> = []; + const runLifecycleMutation = vi.fn(async (operation: () => Promise) => operation()); try { execFileSync("git", ["init", repoDir], { stdio: "pipe" }); @@ -2505,6 +2704,7 @@ describe("create_agent MCP tool", () => { WorkspaceGitService, "getSnapshot" | "listWorktrees" | "resolveRepoRoot" >, + runLifecycleMutation, logger, }); const tool = registeredTool(server, "create_workspace"); @@ -2521,6 +2721,7 @@ describe("create_agent MCP tool", () => { expect(response.structuredContent.workspaceId).toBe(broadcasts[0]); expect(workspaceGitService.getSnapshot).not.toHaveBeenCalled(); expect(setupContinuations).toEqual([undefined]); + expect(runLifecycleMutation).toHaveBeenCalledOnce(); expect(broadcasts).toHaveLength(1); expect(broadcasts[0]).toMatch(/^wks_[0-9a-f]{16}$/); } finally { @@ -3581,6 +3782,34 @@ describe("send_agent_prompt MCP tool", () => { ); }); + it("runs prompts inside lifecycle shutdown admission", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + const runLifecycleMutation = vi.fn(async (operation: () => Promise) => operation()); + spies.agentManager.getAgent.mockReturnValue({ + id: "child-agent", + cwd: existingCwd, + lifecycle: "running", + currentModeId: null, + availableModes: [], + config: { title: "Child" }, + } as ManagedAgent); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + providerSnapshotManager: createOpenCodeManager().manager, + runLifecycleMutation, + logger, + }); + + await invokeToolWithParsedInput(registeredTool(server, "send_agent_prompt"), { + agentId: "child-agent", + prompt: "Follow up", + background: true, + }); + + expect(runLifecycleMutation).toHaveBeenCalledOnce(); + }); + it("keeps top-level prompts blocking by default", async () => { const { agentManager, agentStorage, spies } = createTestDeps(); spies.agentManager.getAgent.mockReturnValue({ diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index e681eb521f..86f8eda452 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -21,6 +21,7 @@ import { import { curateAgentActivity } from "../activity-curator.js"; import { selectItemsByProjectedLimit } from "../timeline-projection.js"; import type { AgentStorage } from "../agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "../create-agent-lifecycle-dispatch.js"; import { ensureAgentLoaded } from "../agent-loading.js"; import { isStoredAgentProviderAvailable } from "../../persistence-hooks.js"; import { @@ -35,7 +36,10 @@ import type { FirstAgentContext } from "../../messages.js"; import { everyMsToFiveFieldCron } from "@getpaseo/protocol/schedule/cadence"; import { expandUserPath, isSameOrDescendantPath, resolvePathFromBase } from "../../path-utils.js"; import type { TerminalManager } from "../../../terminal/terminal-manager.js"; -import type { CreatePaseoWorktreeWorkflowFn } from "../../worktree-session.js"; +import type { + CreatePaseoWorktreeWorkflowFn, + CreatePaseoWorktreeWorkflowResult, +} from "../../worktree-session.js"; import type { ScheduleService } from "../../schedule/service.js"; import { ScheduleRunSchema, @@ -90,6 +94,10 @@ import type { PaseoToolExecutionContext, PaseoToolResult, } from "./types.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "../../workspace-lifecycle-coordinator.js"; export interface PaseoToolHostDependencies { agentManager: AgentManager; @@ -107,7 +115,7 @@ export interface PaseoToolHostDependencies { listActiveWorkspaces?: ArchiveDependencies["listActiveWorkspaces"]; archiveWorkspaceRecord?: ArchiveDependencies["archiveWorkspaceRecord"]; emitWorkspaceUpdatesForWorkspaceIds?: ArchiveDependencies["emitWorkspaceUpdatesForWorkspaceIds"]; - workspaceRegistry?: Pick; + workspaceRegistry?: Pick; projectRegistry?: Pick; createDirectoryWorkspace?: ( cwd: string, @@ -127,6 +135,12 @@ export interface PaseoToolHostDependencies { browserToolsBroker?: BrowserToolsBroker | null; paseoHome?: string; worktreesRoot?: string; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; + runLifecycleMutation?: (operation: () => Promise) => Promise; + createAgentLifecycleDispatch?: Pick< + CreateAgentLifecycleDispatch, + "cleanupCreatedWorktreeAfterFailedAgentCreate" | "recoverPendingAgentCreation" + >; /** * ID of the agent that is using this tool catalog. * Used for cwd/mode inheritance when agents spawn child agents. @@ -143,6 +157,48 @@ export interface PaseoToolHostDependencies { logger: Logger; } +const LIFECYCLE_MUTATION_TOOL_NAMES = new Set([ + "create_workspace", + "archive_workspace", + "create_agent", + "send_agent_prompt", + "cancel_agent", + "archive_agent", + "kill_agent", + "update_agent", + "rename_workspace", + "start_workspace_script", + "stop_workspace_script", + "create_terminal", + "kill_terminal", + "send_terminal_keys", + "create_schedule", + "create_heartbeat", + "delete_heartbeat", + "pause_schedule", + "resume_schedule", + "delete_schedule", + "update_schedule", + "run_schedule_once", + "set_agent_mode", + "respond_to_permission", +]); + +async function requireActiveWorkspaceForOwnership( + options: Pick, + workspaceId: string, +): Promise { + if (!options.listActiveWorkspaces && !options.workspaceRegistry) return; + const workspace = options.listActiveWorkspaces + ? (await options.listActiveWorkspaces()).find( + (candidate) => candidate.workspaceId === workspaceId, + ) + : await options.workspaceRegistry?.get(workspaceId); + if (!workspace || ("archivedAt" in workspace && workspace.archivedAt)) { + throw new Error(`Workspace not found: ${workspaceId}`); + } +} + function parseTimestamp(value: string | null | undefined): number { if (!value) { return 0; @@ -595,7 +651,10 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase if (!tool) { throw new Error(`Paseo tool not found: ${name}`); } - return tool.handler(await parseToolInput(tool, input), context); + const operation = async () => tool.handler(await parseToolInput(tool, input), context); + return options.runLifecycleMutation && LIFECYCLE_MUTATION_TOOL_NAMES.has(name) + ? options.runLifecycleMutation(operation) + : operation(); }, }); @@ -1357,6 +1416,7 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase workspaceId: z.string(), archivedAgentIds: z.array(z.string()), removedDirectory: z.boolean(), + cleanupPending: z.boolean(), }, }, async ({ workspaceId }) => { @@ -1364,7 +1424,10 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase throw new Error("Active workspace lister is required to archive workspaces"); } const workspace = await requireActiveWorkspaceForArchive( - { listActiveWorkspaces: options.listActiveWorkspaces }, + { + listActiveWorkspaces: options.listActiveWorkspaces, + workspaceRegistry: options.workspaceRegistry, + }, workspaceId, ); const result = await archiveByScope( @@ -1385,6 +1448,7 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase workspaceId, archivedAgentIds: result.archivedAgentIds, removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPendingWorkspaceIds.length > 0, }), }; }, @@ -1422,43 +1486,71 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase requestedBackground = resolvedArgs.parsedArgs.background; notifyOnFinish = resolvedArgs.parsedArgs.notifyOnFinish ?? false; } - const { - snapshot, - background: createdInBackground, - initialPromptStarted, - } = await createAgentCommand( - { - agentManager, - agentStorage, - logger: childLogger, - paseoHome: options.paseoHome, - worktreesRoot: options.worktreesRoot, - terminalManager, - providerSnapshotManager, - createPaseoWorktree: options.createPaseoWorktree, - ...(options.ensureWorkspaceForCreate - ? { ensureWorkspaceForCreate: options.ensureWorkspaceForCreate } - : {}), - }, - { - kind: "mcp", - provider: parsedArgs.provider, - title: parsedArgs.title, - initialPrompt: parsedArgs.initialPrompt, - cwd: resolvedArgs.cwd, - workspaceId: resolvedArgs.workspaceId, - thinking: parsedArgs.settings?.thinkingOptionId, - features: parsedArgs.settings?.features, - labels: parsedArgs.labels, - mode: parsedArgs.settings?.modeId, - background: requestedBackground, - notifyOnFinish, - detached: resolvedArgs.detached, - callerAgentId, - callerContext, - worktree, - }, - ); + let createdWorktreeForCleanup: CreatePaseoWorktreeWorkflowResult | null = null; + let createdAgentId: string | null = null; + let pendingCreationAgentId: string | undefined; + const createResult = await (async () => { + try { + return await createAgentCommand( + { + agentManager, + agentStorage, + logger: childLogger, + paseoHome: options.paseoHome, + worktreesRoot: options.worktreesRoot, + terminalManager, + providerSnapshotManager, + lifecycleCoordinator: + options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator, + requireActiveWorkspaceForOwnership: (workspaceId) => + requireActiveWorkspaceForOwnership(options, workspaceId), + createPaseoWorktree: options.createPaseoWorktree, + ...(options.ensureWorkspaceForCreate + ? { ensureWorkspaceForCreate: options.ensureWorkspaceForCreate } + : {}), + }, + { + kind: "mcp", + provider: parsedArgs.provider, + title: parsedArgs.title, + initialPrompt: parsedArgs.initialPrompt, + cwd: resolvedArgs.cwd, + workspaceId: resolvedArgs.workspaceId, + thinking: parsedArgs.settings?.thinkingOptionId, + features: parsedArgs.settings?.features, + labels: parsedArgs.labels, + mode: parsedArgs.settings?.modeId, + background: requestedBackground, + notifyOnFinish, + detached: resolvedArgs.detached, + callerAgentId, + callerContext, + worktree, + onPendingAgentCreation: (agentId) => { + pendingCreationAgentId = agentId; + }, + onWorktreeCreated: (createdWorktree) => { + createdWorktreeForCleanup = createdWorktree; + }, + onCreated: ({ agentId }) => { + createdAgentId = agentId; + }, + }, + ); + } catch (error) { + await options.createAgentLifecycleDispatch?.cleanupCreatedWorktreeAfterFailedAgentCreate({ + createdWorktree: createdWorktreeForCleanup, + createdAgentId, + }); + if (pendingCreationAgentId) { + await options.createAgentLifecycleDispatch?.recoverPendingAgentCreation( + pendingCreationAgentId, + ); + } + throw error; + } + })(); + const { snapshot, background: createdInBackground, initialPromptStarted } = createResult; try { if (!createdInBackground && initialPromptStarted) { @@ -2370,12 +2462,18 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase const resolvedCwd = resolveScopedCwd(cwd, { required: true }); const workspaceId = await resolveTerminalWorkspaceId(resolvedCwd); - - const terminal = await terminalManager.createTerminal({ - cwd: resolvedCwd, + const lifecycleCoordinator = + options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const terminal = await lifecycleCoordinator.runWorkspaceOwnershipMutation( workspaceId, - ...(name?.trim() ? { name: name.trim() } : {}), - }); + () => requireActiveWorkspaceForOwnership(options, workspaceId), + () => + terminalManager.createTerminal({ + cwd: resolvedCwd, + workspaceId, + ...(name?.trim() ? { name: name.trim() } : {}), + }), + ); return { content: [], @@ -3167,9 +3265,11 @@ function archiveWorktreeDependencies( findWorkspaceIdForCwd: options.findWorkspaceIdForCwd, listActiveWorkspaces: options.listActiveWorkspaces, archiveWorkspaceRecord: options.archiveWorkspaceRecord, + workspaceRegistry: options.workspaceRegistry, emitWorkspaceUpdatesForWorkspaceIds: options.emitWorkspaceUpdatesForWorkspaceIds, markWorkspaceArchiving: options.markWorkspaceArchiving, clearWorkspaceArchiving: options.clearWorkspaceArchiving, + lifecycleCoordinator: options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator, killTerminalsForWorkspace: (workspaceId: string) => killTerminalsForWorkspace( { diff --git a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts index 5e8cf6c617..ca7284a568 100644 --- a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts +++ b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts @@ -16,6 +16,8 @@ import type { WorkspaceGitRuntimeSnapshot } from "../workspace-git-service.js"; import { createWorktree, type WorktreeConfig } from "../../utils/worktree.js"; import type { ForgeService } from "../../../services/forge-service.js"; import type { StoredAgentRecord } from "../agent/agent-storage.js"; +import { CreateAgentLifecycleDispatch } from "../agent/create-agent-lifecycle-dispatch.js"; +import { createPersistedWorkspaceRecord } from "../workspace-registry.js"; const CWD = "/tmp/paseo/worktrees/repo/branch"; const PASEO_HOME = "/tmp/paseo"; @@ -104,8 +106,15 @@ function createHarness(overrides?: { agentManager: {} as AutoArchiveArchiveOptions["agentManager"], agentStorage: {} as AutoArchiveArchiveOptions["agentStorage"], terminalManager: {} as AutoArchiveArchiveOptions["terminalManager"], + workspaceRegistry: { + get: vi.fn(async () => null), + list: vi.fn(async () => []), + update: vi.fn(async () => null), + }, findWorkspaceIdForCwd: vi.fn(async () => "ws-auto-archive"), - listActiveWorkspaces: vi.fn(async () => []), + listActiveWorkspaces: vi.fn(async () => [ + { workspaceId: "ws-auto-archive", cwd: CWD, kind: "worktree" }, + ]), getAutoArchivedChangeRequestUrl: vi.fn( async () => overrides?.autoArchivedChangeRequestUrl ?? null, ), @@ -121,6 +130,7 @@ function createHarness(overrides?: { archivedAgentIds: [], archivedWorkspaceIds: [], removedDirectory: false, + cleanupPendingWorkspaceIds: [], }) satisfies ArchiveResult), ) as unknown as ArchiveIfSafeDependencies["archiveByScope"]; const resolveWorkspaceIdAtPath = vi.fn( @@ -263,6 +273,26 @@ function createRealOutcomeHarness(input: { archivedWorkspaceIds: Set; }) { const active = [...input.activeWorkspaces]; + const now = new Date().toISOString(); + const workspaceRecords = new Map( + active.map((workspace) => [ + workspace.workspaceId, + createPersistedWorkspaceRecord({ + workspaceId: workspace.workspaceId, + projectId: "project-auto-archive", + cwd: workspace.cwd, + kind: workspace.kind, + displayName: workspace.workspaceId, + ...(workspace.worktreeRoot ? { worktreeRoot: workspace.worktreeRoot } : {}), + ...(workspace.isPaseoOwnedWorktree !== undefined + ? { isPaseoOwnedWorktree: workspace.isPaseoOwnedWorktree } + : {}), + ...(workspace.mainRepoRoot ? { mainRepoRoot: workspace.mainRepoRoot } : {}), + createdAt: now, + updatedAt: now, + }), + ]), + ); const logger = pino({ level: "silent" }); vi.spyOn(logger, "info").mockImplementation(() => undefined); vi.spyOn(logger, "warn").mockImplementation(() => undefined); @@ -314,6 +344,17 @@ function createRealOutcomeHarness(input: { listDirectories: () => [], getTerminals: vi.fn().mockResolvedValue([]), } as unknown as AutoArchiveArchiveOptions["terminalManager"], + workspaceRegistry: { + get: async (workspaceId) => workspaceRecords.get(workspaceId) ?? null, + list: async () => Array.from(workspaceRecords.values()), + update: async (workspaceId, updater) => { + const existing = workspaceRecords.get(workspaceId); + if (!existing) return null; + const updated = updater(existing); + workspaceRecords.set(workspaceId, updated); + return updated; + }, + }, findWorkspaceIdForCwd: async (cwd: string) => { const match = active.find((workspace) => workspace.cwd === cwd); return match?.workspaceId ?? null; @@ -323,6 +364,13 @@ function createRealOutcomeHarness(input: { getAutoArchivedChangeRequestUrl: async () => null, archiveWorkspaceRecord: async (workspaceId: string) => { input.archivedWorkspaceIds.add(workspaceId); + const existing = workspaceRecords.get(workspaceId); + if (existing) { + workspaceRecords.set(workspaceId, { + ...existing, + archivedAt: new Date().toISOString(), + }); + } const index = active.findIndex((workspace) => workspace.workspaceId === workspaceId); if (index !== -1) { active.splice(index, 1); @@ -470,6 +518,57 @@ describe("archiveIfSafe", () => { expect(harness.inFlight.has(CWD)).toBe(false); }); + test("does not report success for an inactive resolved workspace without cleanup", async () => { + const harness = createHarness(); + const now = new Date().toISOString(); + const archived = createPersistedWorkspaceRecord({ + workspaceId: "ws-auto-archive", + projectId: "project-auto-archive", + cwd: CWD, + kind: "worktree", + displayName: "Archived", + createdAt: now, + updatedAt: now, + archivedAt: now, + }); + harness.options.listActiveWorkspaces = vi.fn(async () => []); + harness.options.workspaceRegistry.get = vi.fn(async () => archived); + + await runArchiveIfSafe(harness); + + expect(harness.deps.archiveByScope).not.toHaveBeenCalled(); + expect(harness.log.info).not.toHaveBeenCalledWith( + { cwd: CWD }, + "Auto-archived worktree after PR merge", + ); + expect(harness.log.warn).toHaveBeenCalledWith( + { cwd: CWD, workspaceId: "ws-auto-archive" }, + "Auto-archive resolved an inactive workspace without pending cleanup; skipping", + ); + }); + + test("does not report success while durable cleanup remains pending", async () => { + const harness = createHarness({ + archiveByScope: async () => ({ + archivedAgentIds: [], + archivedWorkspaceIds: ["ws-auto-archive"], + removedDirectory: false, + cleanupPendingWorkspaceIds: ["ws-auto-archive"], + }), + }); + + await runArchiveIfSafe(harness); + + expect(harness.log.info).not.toHaveBeenCalledWith( + { cwd: CWD }, + "Auto-archived worktree after PR merge", + ); + expect(harness.log.warn).toHaveBeenCalledWith( + { err: expect.any(Error), cwd: CWD }, + "Auto-archive after merge failed", + ); + }); + test("archives a clean Paseo-owned worktree after merge", async () => { const harness = createHarness(); @@ -488,6 +587,7 @@ describe("archiveIfSafe", () => { expect.objectContaining({ paseoHome: PASEO_HOME, workspaceGitService: harness.options.workspaceGitService, + workspaceRegistry: harness.options.workspaceRegistry, }), { scope: { kind: "workspace", workspaceId: "ws-auto-archive" }, @@ -511,6 +611,123 @@ describe("archiveIfSafe", () => { expect(harness.deps.archiveByScope).not.toHaveBeenCalled(); }); + test("keeps a restored merged workspace active while recovering one interrupted create archive", async () => { + const restoredWorkspaceId = "ws-restored-merged"; + const interruptedWorkspaceId = "ws-interrupted-create"; + const interruptedAgentId = "agent-interrupted-create"; + const changeRequestUrl = "https://github.com/acme/repo/pull/123"; + const now = new Date().toISOString(); + const workspaceRecords = new Map([ + [ + restoredWorkspaceId, + createPersistedWorkspaceRecord({ + workspaceId: restoredWorkspaceId, + projectId: "project-auto-archive", + cwd: CWD, + kind: "worktree", + displayName: "Restored merged workspace", + createdAt: now, + updatedAt: now, + autoArchivedChangeRequestUrl: changeRequestUrl, + }), + ], + [ + interruptedWorkspaceId, + createPersistedWorkspaceRecord({ + workspaceId: interruptedWorkspaceId, + projectId: "project-auto-archive", + cwd: "/tmp/paseo/worktrees/repo/interrupted-create", + kind: "worktree", + displayName: "Interrupted create", + createdAt: now, + updatedAt: now, + }), + ], + ]); + const workspaceRegistry = { + get: vi.fn(async (workspaceId: string) => workspaceRecords.get(workspaceId) ?? null), + list: vi.fn(async () => Array.from(workspaceRecords.values())), + update: vi.fn(async () => null), + }; + const mergeArchiveHarness = createHarness(); + mergeArchiveHarness.options.workspaceRegistry = workspaceRegistry; + mergeArchiveHarness.options.findWorkspaceIdForCwd = vi.fn(async () => restoredWorkspaceId); + mergeArchiveHarness.options.listActiveWorkspaces = vi.fn(async () => [ + { workspaceId: restoredWorkspaceId, cwd: CWD, kind: "worktree" }, + ]); + mergeArchiveHarness.options.getAutoArchivedChangeRequestUrl = vi.fn( + async (workspaceId) => + workspaceRecords.get(workspaceId)?.autoArchivedChangeRequestUrl ?? null, + ); + + await runArchiveIfSafe(mergeArchiveHarness); + + expect(mergeArchiveHarness.deps.archiveByScope).not.toHaveBeenCalled(); + expect(workspaceRecords.get(restoredWorkspaceId)?.archivedAt).toBeNull(); + + const agentRecords = new Map([ + [ + interruptedAgentId, + { + id: interruptedAgentId, + autoArchiveObligation: { + phase: "pending", + target: { kind: "workspace", workspaceId: interruptedWorkspaceId }, + }, + } as StoredAgentRecord, + ], + ]); + const archiveInterruptedWorkspace = vi.fn(async (workspaceId: string) => { + const workspace = workspaceRecords.get(workspaceId); + if (!workspace) throw new Error(`Unknown workspace: ${workspaceId}`); + workspaceRecords.set(workspaceId, { + ...workspace, + archivedAt: new Date().toISOString(), + }); + }); + const lifecycleDispatch = new CreateAgentLifecycleDispatch({ + paseoHome: PASEO_HOME, + agentManager: {}, + agentStorage: { + list: async () => Array.from(agentRecords.values()), + update: async ( + agentId: string, + mutation: (record: StoredAgentRecord) => StoredAgentRecord, + ) => { + const next = mutation(agentRecords.get(agentId)!); + agentRecords.set(agentId, next); + return next; + }, + }, + github: {}, + workspaceGitService: {}, + archiveAgentForClose: async () => undefined, + archiveWorkspaceForClose: archiveInterruptedWorkspace, + drainWorkspaceLifecycleOperations: async () => undefined, + findWorkspaceIdForCwd: async () => null, + listActiveWorkspaces: async () => [], + archiveWorkspaceRecord: async () => undefined, + workspaceRegistry, + emit: () => undefined, + emitAgentRemove: () => undefined, + emitWorkspaceUpdatesForWorkspaceIds: async () => undefined, + markWorkspaceArchiving: () => undefined, + clearWorkspaceArchiving: () => undefined, + killTerminalsForWorkspace: async () => undefined, + logger: mergeArchiveHarness.log, + } as unknown as ConstructorParameters[0]); + + await lifecycleDispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => + expect(agentRecords.get(interruptedAgentId)?.autoArchiveObligation).toBeUndefined(), + ); + await lifecycleDispatch.recoverPersistedAutoArchives(); + + expect(archiveInterruptedWorkspace).toHaveBeenCalledOnce(); + expect(workspaceRecords.get(interruptedWorkspaceId)?.archivedAt).not.toBeNull(); + expect(workspaceRecords.get(restoredWorkspaceId)?.archivedAt).toBeNull(); + }); + test("archives a different merged change request", async () => { const harness = createHarness({ autoArchivedChangeRequestUrl: "https://github.com/acme/repo/pull/122", diff --git a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts index 02c44bb9ba..9fa493fddc 100644 --- a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts +++ b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts @@ -15,8 +15,8 @@ import type { } from "../workspace-git-service.js"; import type { ForgeService } from "../../services/forge-service.js"; import type { TerminalManager } from "../../terminal/terminal-manager.js"; +import type { WorkspaceArchiveContext, WorkspaceRegistry } from "../workspace-registry.js"; import { isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; -import type { WorkspaceArchiveContext } from "../workspace-registry.js"; export interface AutoArchiveArchiveOptions { paseoHome: string; @@ -27,6 +27,7 @@ export interface AutoArchiveArchiveOptions { agentManager: AgentManager; agentStorage: AgentStorage; terminalManager: TerminalManager; + workspaceRegistry: Pick; findWorkspaceIdForCwd: (cwd: string) => Promise; listActiveWorkspaces: () => Promise; getAutoArchivedChangeRequestUrl: (workspaceId: string) => Promise; @@ -119,7 +120,21 @@ export async function archiveIfSafe(input: { return; } - await deps.archiveByScope( + const activeWorkspace = (await options.listActiveWorkspaces()).find( + (workspace) => workspace.workspaceId === workspaceId, + ); + if (!activeWorkspace) { + const persistedWorkspace = await options.workspaceRegistry.get(workspaceId); + if (!persistedWorkspace?.cleanupPending) { + log.warn( + { cwd, workspaceId }, + "Auto-archive resolved an inactive workspace without pending cleanup; skipping", + ); + return; + } + } + + const result = await deps.archiveByScope( { paseoHome: options.paseoHome, paseoWorktreesBaseRoot: options.paseoWorktreesBaseRoot, @@ -133,6 +148,7 @@ export async function archiveIfSafe(input: { options.archiveWorkspaceRecord(workspaceIdToArchive, { autoArchivedChangeRequestUrl: pullRequest.url, }), + workspaceRegistry: options.workspaceRegistry, emitWorkspaceUpdatesForWorkspaceIds: options.emitWorkspaceUpdatesForWorkspaceIds, markWorkspaceArchiving: options.markWorkspaceArchiving, clearWorkspaceArchiving: options.clearWorkspaceArchiving, @@ -151,6 +167,11 @@ export async function archiveIfSafe(input: { requestId: "auto-archive-on-merge", }, ); + if (result.cleanupPendingWorkspaceIds.length > 0) { + throw new Error( + `Auto-archive cleanup remains pending for: ${result.cleanupPendingWorkspaceIds.join(", ")}`, + ); + } log.info({ cwd }, "Auto-archived worktree after PR merge"); } catch (error) { log.warn({ err: error, cwd }, "Auto-archive after merge failed"); diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 3509453322..9489e3d5f7 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -1,10 +1,13 @@ import os from "node:os"; import http from "node:http"; import path from "node:path"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import pino from "pino"; import { afterEach, describe, expect, test, vi } from "vitest"; import { WebSocket } from "ws"; +import { experimental_createMCPClient } from "ai"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { createPaseoDaemon, parseListenString, type PaseoDaemonConfig } from "./bootstrap.js"; import { AgentManagerShuttingDownError } from "./agent/agent-manager.js"; @@ -15,6 +18,12 @@ import { createTestAgentClients } from "./test-utils/fake-agent-client.js"; import { DaemonClient } from "./test-utils/daemon-client.js"; import { isPlatform } from "../test-utils/platform.js"; import { findFreePort } from "./service-proxy.js"; +import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; +import { readPaseoWorktreeIncarnationId } from "../utils/worktree-metadata.js"; +import { getPaseoWorktreesRoot } from "../utils/worktree.js"; +import { createRealpathAwarePathMatcher } from "../utils/path.js"; +import { Session } from "./session.js"; +import { FileBackedProjectRegistry, FileBackedWorkspaceRegistry } from "./workspace-registry.js"; import type { HubEnrollment, HubEnrollmentResult, @@ -42,6 +51,21 @@ type WebSocketProbeResult = | { status: "connected" } | { status: "rejected"; statusCode: number | null }; +function deferred(): { promise: Promise; resolve: (value: T) => void } { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +async function createBootstrapAgentMcpClient(port: number) { + const transport = new StreamableHTTPClientTransport( + new URL(`http://127.0.0.1:${port}/mcp/agents`), + ); + return experimental_createMCPClient({ transport }); +} + describe("paseo daemon bootstrap", () => { afterEach(() => { vi.restoreAllMocks(); @@ -342,6 +366,873 @@ describe("paseo daemon bootstrap", () => { } }); + test("shutdown joins a worktree create accepted on an established socket before ingress freezes", async () => { + const clients = createTestAgentClients(); + const createStarted = deferred(); + const allowCreate = deferred(); + const codexClient = clients.codex!; + const createSession = codexClient.createSession.bind(codexClient); + let createdSessionCwd: string | null = null; + vi.spyOn(codexClient, "createSession").mockImplementation(async (config, launchContext) => { + createdSessionCwd = config.cwd; + createStarted.resolve(); + await allowCreate.promise; + return createSession(config, launchContext); + }); + + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: clients, + }); + const { repoDir, tempRoot } = await createCommittedGitRepo("accepted-create"); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + + try { + await client.connect(); + const createPromise = client.createAgent({ + provider: "codex", + cwd: repoDir, + worktree: { mode: "branch-off", newBranch: "accepted-before-shutdown", base: "main" }, + autoArchive: false, + }); + await createStarted.promise; + + const [pendingCreation] = await daemonHandle.daemon.agentStorage.listPendingAgentCreations(); + expect(pendingCreation?.cleanupTarget).toMatchObject({ + kind: "worktree", + targetPath: createdSessionCwd, + }); + if (pendingCreation?.cleanupTarget.kind !== "worktree" || !createdSessionCwd) { + throw new Error("Expected an exact pending worktree creation journal"); + } + const directoryStat = await stat(createdSessionCwd, { bigint: true }); + expect(pendingCreation.cleanupTarget.directoryIdentity).toEqual({ + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }); + expect(readPaseoWorktreeIncarnationId(createdSessionCwd)).toBe( + pendingCreation.cleanupTarget.worktreeIncarnationId, + ); + + const stopPromise = daemonHandle.daemon.stop(); + const earlyStop = await Promise.race([ + stopPromise.then(() => "stopped" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(earlyStop).toBe("pending"); + + allowCreate.resolve(); + const created = await createPromise; + expect(created.cwd).toBe(createdSessionCwd); + await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + await stopPromise; + } finally { + allowCreate.resolve(); + await client.close().catch(() => undefined); + await daemonHandle.daemon.stop().catch(() => undefined); + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + + test("an established socket cannot start a worktree create after ingress freezes", async () => { + const heldAgentClose = holdAgentClose(); + const clients = createTestAgentClients({ closeSession: heldAgentClose.closeSession }); + const createSession = vi.spyOn(clients.codex!, "createSession"); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: clients, + }); + const { repoDir, tempRoot } = await createCommittedGitRepo("rejected-create"); + const initialAgentCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-shutdown-agent-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: initialAgentCwd }, + undefined, + { workspaceId: undefined }, + ); + expect(createSession).toHaveBeenCalledTimes(1); + + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + await heldAgentClose.started; + + const lateCreate = client + .createAgent({ + provider: "codex", + cwd: repoDir, + worktree: { mode: "branch-off", newBranch: "rejected-after-shutdown", base: "main" }, + }) + .then( + () => "created" as const, + () => "rejected" as const, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(createSession).toHaveBeenCalledTimes(1); + expect(listGitWorktreePaths(repoDir)).toHaveLength(1); + await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + + heldAgentClose.finish(); + await stopPromise; + await client.close(); + await expect(lateCreate).resolves.toBe("rejected"); + } finally { + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.daemon.stop().catch(() => undefined); + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { + recursive: true, + force: true, + }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(initialAgentCwd, { recursive: true, force: true }), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + + test("an established socket cannot start a workspace mutation after shutdown admission closes", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ closeSession: heldAgentClose.closeSession }), + }); + const blockerCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-late-workspace-blocker-")); + const workspaceCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-late-workspace-target-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: blockerCwd }, + undefined, + { workspaceId: undefined }, + ); + const originalHandleMessage = Session.prototype.handleMessage; + const handledWorkspaceCreates: string[] = []; + vi.spyOn(Session.prototype, "handleMessage").mockImplementation( + async function (message, source) { + if (message.type === "workspace.create.request") { + handledWorkspaceCreates.push(message.source.path); + } + return originalHandleMessage.call(this, message, source); + }, + ); + + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + await heldAgentClose.started; + + const lateCreate = client + .createWorkspace({ source: { kind: "directory", path: workspaceCwd } }) + .then( + () => "created" as const, + () => "rejected" as const, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(handledWorkspaceCreates).toEqual([]); + + heldAgentClose.finish(); + await stopPromise; + await expect(lateCreate).resolves.toBe("rejected"); + } finally { + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(blockerCwd, { recursive: true, force: true }), + rm(workspaceCwd, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins workspace archive paused before lifecycle coordinator admission", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ closeSession: heldAgentClose.closeSession }), + }); + const blockerCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-archive-blocker-")); + const workspaceCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-archive-target-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + const registryStarted = deferred(); + const releaseRegistry = deferred(); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: blockerCwd }, + undefined, + { workspaceId: undefined }, + ); + const created = await client.createWorkspace({ + source: { kind: "directory", path: workspaceCwd }, + }); + if (!created.workspace) throw new Error(created.error ?? "Failed to create target workspace"); + + const originalList = FileBackedWorkspaceRegistry.prototype.list; + let pauseNextList = true; + vi.spyOn(FileBackedWorkspaceRegistry.prototype, "list").mockImplementation(async function () { + if (pauseNextList) { + pauseNextList = false; + registryStarted.resolve(); + await releaseRegistry.promise; + } + return originalList.call(this); + }); + + const archivePromise = client.archiveWorkspace(created.workspace.id); + await registryStarted.promise; + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + + const closureBeforeRelease = await Promise.race([ + heldAgentClose.started.then(() => "closing" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(closureBeforeRelease).toBe("pending"); + + releaseRegistry.resolve(); + await expect(archivePromise).resolves.toMatchObject({ + workspaceId: created.workspace.id, + error: null, + }); + await heldAgentClose.started; + heldAgentClose.finish(); + await stopPromise; + } finally { + releaseRegistry.resolve(); + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(blockerCwd, { recursive: true, force: true }), + rm(workspaceCwd, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins project removal paused before lifecycle coordinator admission", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ closeSession: heldAgentClose.closeSession }), + }); + const blockerCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-project-remove-blocker-")); + const workspaceCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-project-remove-target-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + const registryStarted = deferred(); + const releaseRegistry = deferred(); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: blockerCwd }, + undefined, + { workspaceId: undefined }, + ); + const created = await client.createWorkspace({ + source: { kind: "directory", path: workspaceCwd }, + }); + if (!created.workspace) throw new Error(created.error ?? "Failed to create target project"); + + const originalGet = FileBackedProjectRegistry.prototype.get; + let pauseTargetGet = true; + vi.spyOn(FileBackedProjectRegistry.prototype, "get").mockImplementation( + async function (projectId) { + if (pauseTargetGet && projectId === created.workspace?.projectId) { + pauseTargetGet = false; + registryStarted.resolve(); + await releaseRegistry.promise; + } + return originalGet.call(this, projectId); + }, + ); + + const removePromise = client.removeProject(created.workspace.projectId); + await registryStarted.promise; + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + + const closureBeforeRelease = await Promise.race([ + heldAgentClose.started.then(() => "closing" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(closureBeforeRelease).toBe("pending"); + + releaseRegistry.resolve(); + await expect(removePromise).resolves.toEqual({ + removedWorkspaceIds: [created.workspace.id], + }); + await heldAgentClose.started; + heldAgentClose.finish(); + await stopPromise; + } finally { + releaseRegistry.resolve(); + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(blockerCwd, { recursive: true, force: true }), + rm(workspaceCwd, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins workspace create paused before session mutation dispatch", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ closeSession: heldAgentClose.closeSession }), + }); + const blockerCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-workspace-create-blocker-")); + const workspaceCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-workspace-create-target-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + const dispatchStarted = deferred(); + const releaseDispatch = deferred(); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: blockerCwd }, + undefined, + { workspaceId: undefined }, + ); + + const originalHandleMessage = Session.prototype.handleMessage; + let pauseWorkspaceCreate = true; + vi.spyOn(Session.prototype, "handleMessage").mockImplementation( + async function (message, source) { + if (pauseWorkspaceCreate && message.type === "workspace.create.request") { + pauseWorkspaceCreate = false; + dispatchStarted.resolve(); + await releaseDispatch.promise; + } + return originalHandleMessage.call(this, message, source); + }, + ); + + const createPromise = client.createWorkspace({ + source: { kind: "directory", path: workspaceCwd }, + }); + await dispatchStarted.promise; + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + + const closureBeforeRelease = await Promise.race([ + heldAgentClose.started.then(() => "closing" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(closureBeforeRelease).toBe("pending"); + + releaseDispatch.resolve(); + const created = await createPromise; + expect(created.error).toBeNull(); + expect(created.workspace?.id).toBeTruthy(); + await heldAgentClose.started; + heldAgentClose.finish(); + await stopPromise; + } finally { + releaseDispatch.resolve(); + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(blockerCwd, { recursive: true, force: true }), + rm(workspaceCwd, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins a standalone MCP worktree create accepted before ingress freezes", async () => { + const daemonHandle = await createTestPaseoDaemon({ cleanup: false }); + const { repoDir, tempRoot } = await createCommittedGitRepo("mcp-accepted-create"); + const mcpClient = await createBootstrapAgentMcpClient(daemonHandle.port); + const releaseMutation = deferred(); + const mutationStarted = deferred(); + const worktreesRoot = await getPaseoWorktreesRoot(repoDir, daemonHandle.paseoHome); + const heldMutation = defaultWorkspaceLifecycleCoordinator.runWorktreeMutationExclusive( + worktreesRoot, + async () => { + mutationStarted.resolve(); + await releaseMutation.promise; + }, + ); + await mutationStarted.promise; + + try { + const createPromise = mcpClient.callTool({ + name: "create_workspace", + args: { + isolation: "worktree", + path: repoDir, + worktreeSlug: "mcp-accepted-before-shutdown", + branchName: "feature/mcp-accepted-before-shutdown", + baseBranch: "main", + }, + }); + await vi.waitFor(async () => { + const pending = await daemonHandle.daemon.agentStorage.listPendingAgentCreations(); + expect(pending).toContainEqual( + expect.objectContaining({ ownerKind: "standalone-worktree" }), + ); + }); + + const stopPromise = daemonHandle.daemon.stop(); + const earlyStop = await Promise.race([ + stopPromise.then(() => "stopped" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(earlyStop).toBe("pending"); + + releaseMutation.resolve(); + const result = await createPromise; + expect(result.structuredContent).toMatchObject({ + isolation: "worktree", + cwd: expect.stringContaining("mcp-accepted-before-shutdown"), + }); + await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + await stopPromise; + } finally { + releaseMutation.resolve(); + await heldMutation.catch(() => undefined); + await mcpClient.close().catch(() => undefined); + await daemonHandle.daemon.stop().catch(() => undefined); + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + + test("failed native MCP create can retry the same worktree slug and survive startup recovery", async () => { + const paseoHomeRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-mcp-create-retry-home-")); + const { repoDir, tempRoot } = await createCommittedGitRepo("mcp-create-retry"); + let firstDaemon: Awaited> | null = null; + let secondDaemon: Awaited> | null = null; + let firstMcpClient: Awaited> | null = null; + let secondDaemonClient: DaemonClient | null = null; + const staticDirs: string[] = []; + + try { + firstDaemon = await createTestPaseoDaemon({ paseoHomeRoot, cleanup: false }); + staticDirs.push(firstDaemon.staticDir); + firstMcpClient = await createBootstrapAgentMcpClient(firstDaemon.port); + const originalWorkspaceResult = await firstMcpClient.callTool({ + name: "create_workspace", + args: { + isolation: "worktree", + path: repoDir, + worktreeSlug: "same-slug-retry", + branchName: "same-slug-retry", + baseBranch: "main", + }, + }); + const originalWorkspace = originalWorkspaceResult.structuredContent as + | { cwd?: unknown; workspaceId?: unknown } + | undefined; + if ( + typeof originalWorkspace?.cwd !== "string" || + typeof originalWorkspace.workspaceId !== "string" + ) { + throw new Error("Expected original MCP worktree workspace identifiers"); + } + const worktreeCwd = originalWorkspace.cwd; + const originalWorkspaceId = originalWorkspace.workspaceId; + expect(listGitWorktreePaths(repoDir)).toContain(worktreeCwd); + + const failed = await firstMcpClient.callTool({ + name: "create_agent", + args: { + cwd: repoDir, + worktreeName: "same-slug-retry", + refName: "same-slug-retry", + title: "Failed same-slug create", + provider: "codex/gpt-5.4", + mode: "invalid-test-mode", + initialPrompt: "This create should fail after reusing its worktree", + background: true, + }, + }); + expect(failed.isError).toBe(true); + await expect(firstDaemon.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + expect(await activeWorkspaceIdsAtWorktreeRoot(firstDaemon.paseoHome, worktreeCwd)).toEqual([ + originalWorkspaceId, + ]); + expect(listGitWorktreePaths(repoDir)).toContain(worktreeCwd); + + const retried = await firstMcpClient.callTool({ + name: "create_agent", + args: { + cwd: repoDir, + worktreeName: "same-slug-retry", + refName: "same-slug-retry", + title: "Successful same-slug retry", + provider: "codex/gpt-5.4", + mode: "full-access", + initialPrompt: "Complete the retry", + background: true, + }, + }); + expect(retried.isError).not.toBe(true); + const structured = retried.structuredContent as + | { agentId?: unknown; cwd?: unknown; workspaceId?: unknown } + | undefined; + if ( + typeof structured?.agentId !== "string" || + typeof structured.cwd !== "string" || + typeof structured.workspaceId !== "string" + ) { + throw new Error("Expected successful MCP retry agent/workspace identifiers"); + } + const { agentId, cwd, workspaceId } = structured as { + agentId: string; + cwd: string; + workspaceId: string; + }; + expect(cwd).toBe(worktreeCwd); + expect(workspaceId).toBe(originalWorkspaceId); + await expect(firstDaemon.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + expect(await activeWorkspaceIdsAtWorktreeRoot(firstDaemon.paseoHome, worktreeCwd)).toEqual([ + originalWorkspaceId, + ]); + + await firstMcpClient.close(); + firstMcpClient = null; + await firstDaemon.daemon.stop(); + await firstDaemon.daemon.agentManager.flush(); + + secondDaemon = await createTestPaseoDaemon({ paseoHomeRoot, cleanup: false }); + staticDirs.push(secondDaemon.staticDir); + secondDaemonClient = new DaemonClient({ + url: `ws://127.0.0.1:${secondDaemon.port}/ws`, + appVersion: "0.1.82", + }); + await secondDaemonClient.connect(); + + const recoveredAgent = await secondDaemon.daemon.agentStorage.get(agentId); + expect(recoveredAgent).toMatchObject({ id: agentId, workspaceId }); + expect(recoveredAgent?.archivedAt).toBeFalsy(); + await expect(secondDaemon.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + await expect(stat(cwd)).resolves.toMatchObject({}); + expect(listGitWorktreePaths(repoDir)).toContain(cwd); + expect(await activeWorkspaceIdsAtWorktreeRoot(secondDaemon.paseoHome, worktreeCwd)).toEqual([ + workspaceId, + ]); + + await expect(secondDaemonClient.archiveWorkspace(workspaceId)).resolves.toMatchObject({ + workspaceId, + error: null, + }); + await expect(stat(worktreeCwd)).rejects.toMatchObject({ code: "ENOENT" }); + expect(listGitWorktreePaths(repoDir)).not.toContain(worktreeCwd); + expect(await activeWorkspaceIdsAtWorktreeRoot(secondDaemon.paseoHome, worktreeCwd)).toEqual( + [], + ); + } finally { + await secondDaemonClient?.close().catch(() => undefined); + await firstMcpClient?.close().catch(() => undefined); + await secondDaemon?.close().catch(() => undefined); + await firstDaemon?.close().catch(() => undefined); + await Promise.all([ + rm(paseoHomeRoot, { recursive: true, force: true }), + ...staticDirs.map((staticDir) => rm(staticDir, { recursive: true, force: true })), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + + test("MCP cannot create a worktree after the shutdown closure snapshot", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ + closeSession: heldAgentClose.closeSession, + }), + }); + const { repoDir, tempRoot } = await createCommittedGitRepo("mcp-rejected-create"); + const initialAgentCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-mcp-shutdown-agent-")); + const mcpClient = await createBootstrapAgentMcpClient(daemonHandle.port); + + try { + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: initialAgentCwd }, + undefined, + { workspaceId: undefined }, + ); + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + await heldAgentClose.started; + const closureWorktrees = listGitWorktreePaths(repoDir); + + const lateCreate = await mcpClient.callTool({ + name: "create_workspace", + args: { + isolation: "worktree", + path: repoDir, + worktreeSlug: "mcp-rejected-after-shutdown", + branchName: "feature/mcp-rejected-after-shutdown", + baseBranch: "main", + }, + }); + expect(lateCreate).toMatchObject({ + isError: true, + content: [expect.objectContaining({ text: "Lifecycle mutation ingress is closed" })], + }); + expect(listGitWorktreePaths(repoDir)).toEqual(closureWorktrees); + await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + + heldAgentClose.finish(); + await stopPromise; + } finally { + heldAgentClose.finish(); + await mcpClient.close().catch(() => undefined); + await daemonHandle.daemon.stop().catch(() => undefined); + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { + recursive: true, + force: true, + }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(initialAgentCwd, { recursive: true, force: true }), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins an admitted native MCP prompt and rejects prompts after ingress closes", async () => { + const heldAgentClose = holdAgentClose(); + const startedPrompts: unknown[] = []; + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ + closeSession: heldAgentClose.closeSession, + onStartTurn: (prompt) => startedPrompts.push(prompt), + }), + }); + const agentCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-mcp-prompt-shutdown-agent-")); + const mcpClient = await createBootstrapAgentMcpClient(daemonHandle.port); + const storageReadStarted = deferred(); + const releaseStorageRead = deferred(); + + try { + const agent = await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: agentCwd }, + undefined, + { workspaceId: undefined }, + ); + const originalGet = daemonHandle.daemon.agentStorage.get.bind( + daemonHandle.daemon.agentStorage, + ); + let pauseTargetRead = true; + vi.spyOn(daemonHandle.daemon.agentStorage, "get").mockImplementation(async (agentId) => { + if (pauseTargetRead && agentId === agent.id) { + pauseTargetRead = false; + storageReadStarted.resolve(); + await releaseStorageRead.promise; + } + return originalGet(agentId); + }); + + const admittedPrompt = mcpClient.callTool({ + name: "send_agent_prompt", + args: { + agentId: agent.id, + prompt: "Admitted before shutdown", + background: true, + }, + }); + await storageReadStarted.promise; + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + + const closureBeforeRelease = await Promise.race([ + heldAgentClose.started.then(() => "closing" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(closureBeforeRelease).toBe("pending"); + + releaseStorageRead.resolve(); + await expect(admittedPrompt).resolves.toMatchObject({ + structuredContent: { success: true }, + }); + expect(startedPrompts).toEqual(["Admitted before shutdown"]); + await heldAgentClose.started; + + const latePrompt = await mcpClient.callTool({ + name: "send_agent_prompt", + args: { + agentId: agent.id, + prompt: "Rejected after shutdown", + background: true, + }, + }); + expect(latePrompt).toMatchObject({ + isError: true, + content: [expect.objectContaining({ text: "Lifecycle mutation ingress is closed" })], + }); + expect(startedPrompts).toEqual(["Admitted before shutdown"]); + + heldAgentClose.finish(); + await stopPromise; + } finally { + releaseStorageRead.resolve(); + heldAgentClose.finish(); + await mcpClient.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(agentCwd, { recursive: true, force: true }), + ]); + } + }); + + test("continues teardown after a shutdown step fails", async () => { + const daemonHandle = await createTestPaseoDaemon(); + const failure = new Error("service proxy stop failed"); + vi.spyOn(daemonHandle.daemon.serviceProxy, "stopStandalone").mockRejectedValueOnce(failure); + + try { + await expect(daemonHandle.daemon.stop()).rejects.toThrow( + "One or more daemon shutdown steps failed", + ); + await expect(fetch(`http://127.0.0.1:${daemonHandle.port}/api/health`)).rejects.toThrow(); + } finally { + await daemonHandle.close(); + } + }); + + test("shutdown waits for a real workspace operation after its abortable wrapper rejects", async () => { + const daemonHandle = await createTestPaseoDaemon(); + const controller = new AbortController(); + let releaseOperation = () => {}; + const physicalOperation = new Promise((resolve) => { + releaseOperation = resolve; + }); + const wrappedOperation = defaultWorkspaceLifecycleCoordinator.runArchive( + `bootstrap-drain-${daemonHandle.port}`, + () => physicalOperation, + controller.signal, + ); + let stopSettled = false; + + try { + await Promise.resolve(); + controller.abort(); + await expect(wrappedOperation).rejects.toThrow("Workspace lifecycle operation canceled"); + + const stopping = daemonHandle.daemon.stop().then(() => { + stopSettled = true; + return undefined; + }); + await Promise.resolve(); + expect(stopSettled).toBe(false); + + releaseOperation(); + await stopping; + expect(stopSettled).toBe(true); + } finally { + releaseOperation(); + await daemonHandle.close(); + } + }); + + test("a blocked cleanup stop cannot prevent later teardown before the outer deadline", async () => { + let markCleanupStopStarted = () => {}; + const cleanupStopStarted = new Promise((resolve) => { + markCleanupStopStarted = resolve; + }); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + dependencies: { + workspaceCleanupRetryService: { + start: async () => undefined, + stop: () => { + markCleanupStopStarted(); + return new Promise(() => undefined); + }, + }, + }, + }); + const killAll = vi.spyOn(daemonHandle.daemon.terminalManager, "killAll"); + const outerDeadlineAt = Date.now() + 1_000; + + try { + const stopPromise = daemonHandle.daemon.stop({ deadlineAt: outerDeadlineAt }); + await cleanupStopStarted; + await expect(stopPromise).rejects.toMatchObject({ + errors: expect.arrayContaining([ + expect.objectContaining({ name: "DaemonShutdownDeadlineError" }), + ]), + }); + + expect(killAll).toHaveBeenCalledOnce(); + expect(Date.now()).toBeLessThan(outerDeadlineAt); + } finally { + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { + recursive: true, + force: true, + }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + ]); + } + }); + test("standalone listener exposes services only", async () => { const standalonePort = await findFreePort(); const upstream = http.createServer((_req, res) => { @@ -685,6 +1576,62 @@ function holdAgentClose(): HeldAgentClose { }; } +async function createCommittedGitRepo(slug: string): Promise<{ + repoDir: string; + tempRoot: string; +}> { + const tempRoot = await mkdtemp(path.join(os.tmpdir(), `paseo-shutdown-${slug}-`)); + const repoDir = path.join(tempRoot, "repo"); + execFileSync("git", ["init", "-b", "main", repoDir], { stdio: "pipe" }); + execFileSync("git", ["config", "user.email", "test@getpaseo.local"], { + cwd: repoDir, + stdio: "pipe", + }); + execFileSync("git", ["config", "user.name", "Paseo Test"], { + cwd: repoDir, + stdio: "pipe", + }); + await writeFile(path.join(repoDir, "README.md"), "shutdown lifecycle\n"); + execFileSync("git", ["add", "README.md"], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "initial"], { + cwd: repoDir, + stdio: "pipe", + }); + return { repoDir, tempRoot }; +} + +function listGitWorktreePaths(repoDir: string): string[] { + return execFileSync("git", ["worktree", "list", "--porcelain"], { + cwd: repoDir, + stdio: "pipe", + }) + .toString() + .split("\n") + .flatMap((line) => (line.startsWith("worktree ") ? [line.slice("worktree ".length)] : [])); +} + +async function activeWorkspaceIdsAtWorktreeRoot( + paseoHome: string, + worktreeRoot: string, +): Promise { + const records = JSON.parse( + await readFile(path.join(paseoHome, "projects", "workspaces.json"), "utf8"), + ) as Array<{ + workspaceId: string; + cwd: string; + worktreeRoot?: string | null; + archivedAt?: string | null; + }>; + const matchesWorktreeRoot = createRealpathAwarePathMatcher(worktreeRoot); + return records + .filter( + (workspace) => + !workspace.archivedAt && matchesWorktreeRoot(workspace.worktreeRoot ?? workspace.cwd), + ) + .map((workspace) => workspace.workspaceId) + .sort(); +} + async function beginDaemonShutdownWithAgentClosing(): Promise { const heldAgentClose = holdAgentClose(); const daemonHandle = await createTestPaseoDaemon({ diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 734f619326..509717b528 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -156,8 +156,12 @@ import { archiveByScope, archivePersistedWorkspaceRecord, killTerminalsForWorkspace, + requireArchiveCleanupComplete, + retryPendingWorkspaceCleanup, type ActiveWorkspaceRef, } from "./workspace-archive-service.js"; +import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service.js"; +import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { setupAutoArchiveOnMerge } from "./auto-archive-on-merge/index.js"; import { wrapSessionMessage, type SessionOutboundMessage } from "./messages.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; @@ -218,6 +222,8 @@ import { DaemonExecutions } from "./hub/daemon-executions.js"; const MAX_MCP_DEBUG_BATCH_ITEMS = 10; const REDACTED_LOG_VALUE = "[redacted]"; +const DEFAULT_DAEMON_SHUTDOWN_TIMEOUT_MS = 10_000; +const FORCE_EXIT_SHUTDOWN_RESERVE_MS = 250; const DOWNLOAD_OPEN_FLAGS = process.platform === "win32" ? constants.O_RDONLY : constants.O_RDONLY | constants.O_NOFOLLOW; @@ -444,21 +450,93 @@ export interface PaseoDaemon { scriptRuntimeStore: WorkspaceScriptRuntimeStore; browserToolsBroker: BrowserToolsBroker; start(): Promise; - stop(): Promise; + stop(options?: PaseoDaemonStopOptions): Promise; getListenTarget(): ListenTarget | null; } +export interface PaseoDaemonStopOptions { + deadlineAt?: number; +} + +export interface WorkspaceCleanupRetryLifecycle { + start(): Promise; + stop(): Promise; +} + export interface PaseoDaemonDependencies { hubRelationshipRemote?: HubRelationshipRemote; hubRelationshipClock?: HubRelationshipClock; hubRelationshipRetryPolicy?: HubRelationshipRetryPolicy; createHubDaemonId?: () => string; + workspaceCleanupRetryService?: WorkspaceCleanupRetryLifecycle; serverFeatureOverrides?: { daemonStatusRpc?: boolean; relayConfig?: boolean; }; } +class DaemonShutdownDeadlineError extends Error { + constructor(label: string) { + super(`Daemon shutdown deadline reached during: ${label}`); + this.name = "DaemonShutdownDeadlineError"; + } +} + +class LifecycleMutationIngressClosedError extends Error { + constructor() { + super("Lifecycle mutation ingress is closed"); + this.name = "LifecycleMutationIngressClosedError"; + } +} + +class LifecycleMutationIngress { + private accepting = true; + private readonly inFlight = new Set>(); + + run(operation: () => Promise): Promise { + if (!this.accepting) { + return Promise.reject(new LifecycleMutationIngressClosedError()); + } + const task = Promise.resolve().then(operation); + this.inFlight.add(task); + void task.then( + () => this.inFlight.delete(task), + () => this.inFlight.delete(task), + ); + return task; + } + + async closeAndDrain(): Promise { + this.accepting = false; + while (this.inFlight.size > 0) { + await Promise.allSettled(Array.from(this.inFlight)); + } + } +} + +function waitForShutdownStep( + task: Promise, + signal: AbortSignal, + label: string, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const finish = (callback: () => void) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + callback(); + }; + const onAbort = () => finish(() => reject(new DaemonShutdownDeadlineError(label))); + signal.addEventListener("abort", onAbort, { once: true }); + void task.then( + () => finish(resolve), + (error) => finish(() => reject(error)), + ); + if (signal.aborted) onAbort(); + }); +} + function createBootstrapManagedProcessRegistry( config: Pick, logger: Logger, @@ -988,6 +1066,7 @@ export async function createPaseoDaemon( agentManager, agentStorage, terminalManager, + workspaceRegistry, logger, findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, listActiveWorkspaces: listActiveWorkspacesExternal, @@ -1028,6 +1107,22 @@ export async function createPaseoDaemon( }, autoNameWorkspaceBranchForFirstAgent: (autoNameInput) => workspaceAutoName.scheduleForWorktree(autoNameInput), + worktreeCreationJournal: { + beginPendingAgentCreation: (creationId, options) => + agentStorage.beginPendingAgentCreation(creationId, options), + planPendingAgentCreationWorktree: (creationId, worktreePath, plan) => + agentStorage.planPendingAgentCreationWorktree(creationId, worktreePath, plan), + identifyPendingAgentCreationWorktree: (creationId, worktreePath, reservation) => + agentStorage.identifyPendingAgentCreationWorktree( + creationId, + worktreePath, + reservation, + ), + removePendingAgentCreation: (creationId) => + agentStorage.removePendingAgentCreation(creationId), + recoverPendingCreation: (creationId) => + hubAgentLifecycle.recoverPendingAgentCreation(creationId), + }, emitWorkspaceUpdateForWorkspaceId: async (workspaceId) => { await emitWorkspaceUpdatesExternal([workspaceId]); }, @@ -1058,30 +1153,91 @@ export async function createPaseoDaemon( providerSnapshotManager, createPaseoWorktree: createPaseoWorktreeForTools, ensureWorkspaceForCreate: ensureWorkspaceForCreateAndBroadcastExternal, + requireActiveWorkspaceForOwnership: async (workspaceId) => { + const workspace = await workspaceRegistry.get(workspaceId); + if (!workspace || workspace.archivedAt) { + throw new Error(`Workspace not found: ${workspaceId}`); + } + }, }; const createAgent = (input: Parameters[1]) => createAgentCommand(createAgentCommandDependencies, input); - const archiveWorkspaceByIdExternal = (workspaceId: string, requestId: string) => - archiveByScope( - { - paseoHome: config.paseoHome, - paseoWorktreesBaseRoot: config.worktreesRoot, - github, - workspaceGitService, - agentManager, - agentStorage, - findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, - listActiveWorkspaces: listActiveWorkspacesExternal, - archiveWorkspaceRecord: archiveWorkspaceRecordExternal, - emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, - markWorkspaceArchiving: markWorkspaceArchivingExternal, - clearWorkspaceArchiving: clearWorkspaceArchivingExternal, - killTerminalsForWorkspace: (workspaceIdToKill) => - killTerminalsForWorkspace({ terminalManager, sessionLogger: logger }, workspaceIdToKill), - sessionLogger: logger, - }, - { scope: { kind: "workspace", workspaceId }, requestId }, + const archiveWorkspaceByIdExternal = async ( + workspaceId: string, + requestId: string, + signal?: AbortSignal, + ) => + requireArchiveCleanupComplete( + await archiveByScope( + { + paseoHome: config.paseoHome, + paseoWorktreesBaseRoot: config.worktreesRoot, + github, + workspaceGitService, + agentManager, + agentStorage, + findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, + listActiveWorkspaces: listActiveWorkspacesExternal, + archiveWorkspaceRecord: archiveWorkspaceRecordExternal, + emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, + markWorkspaceArchiving: markWorkspaceArchivingExternal, + clearWorkspaceArchiving: clearWorkspaceArchivingExternal, + killTerminalsForWorkspace: (workspaceIdToKill) => + killTerminalsForWorkspace( + { terminalManager, sessionLogger: logger }, + workspaceIdToKill, + ), + workspaceRegistry, + sessionLogger: logger, + }, + { scope: { kind: "workspace", workspaceId }, requestId, signal }, + ), + "Workspace archive", + ); + const retryPendingWorktreeCleanupExternal: ConstructorParameters< + typeof WorkspaceCleanupRetryService + >[0]["retryWorktreeCleanup"] = async (target, signal) => { + requireArchiveCleanupComplete( + await retryPendingWorkspaceCleanup( + { + paseoHome: config.paseoHome, + paseoWorktreesBaseRoot: config.worktreesRoot, + github, + workspaceGitService, + agentManager, + agentStorage, + findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, + listActiveWorkspaces: listActiveWorkspacesExternal, + archiveWorkspaceRecord: archiveWorkspaceRecordExternal, + emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, + markWorkspaceArchiving: markWorkspaceArchivingExternal, + clearWorkspaceArchiving: clearWorkspaceArchivingExternal, + killTerminalsForWorkspace: (workspaceIdToKill) => + killTerminalsForWorkspace( + { terminalManager, sessionLogger: logger }, + workspaceIdToKill, + ), + workspaceRegistry, + sessionLogger: logger, + }, + { + directoryPath: target.directoryPath, + worktreeIncarnationId: target.worktreeIncarnationId, + requestId: `cleanup-retry:${randomUUID()}`, + signal, + }, + ), + "Workspace cleanup retry", ); + }; + const workspaceCleanupRetryService = + dependencies.workspaceCleanupRetryService ?? + new WorkspaceCleanupRetryService({ + workspaceRegistry, + retryWorktreeCleanup: retryPendingWorktreeCleanupExternal, + logger, + }); + await workspaceCleanupRetryService.start(); const hubAgentLifecycle = new CreateAgentLifecycleDispatch({ paseoHome: config.paseoHome, worktreesRoot: config.worktreesRoot, @@ -1089,12 +1245,15 @@ export async function createPaseoDaemon( agentStorage, github, workspaceGitService, - createPaseoWorktreeWorkflow: createPaseoWorktreeForTools, archiveAgentForClose: (agentId) => archiveAgentCommand({ agentManager, agentStorage, logger }, agentId), + archiveWorkspaceForClose: (workspaceId, signal) => + archiveWorkspaceByIdExternal(workspaceId, randomUUID(), signal), + drainWorkspaceLifecycleOperations: () => defaultWorkspaceLifecycleCoordinator.drain(), findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, listActiveWorkspaces: listActiveWorkspacesExternal, archiveWorkspaceRecord: archiveWorkspaceRecordExternal, + workspaceRegistry, emit: emitExternalSessionMessage, emitAgentRemove: async () => undefined, emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, @@ -1170,7 +1329,7 @@ export async function createPaseoDaemon( return result; }; const archiveScheduleWorkspaceExternal = async (workspaceId: string) => { - await archiveByScope( + const archiveResult = await archiveByScope( { paseoHome: config.paseoHome, paseoWorktreesBaseRoot: config.worktreesRoot, @@ -1192,6 +1351,7 @@ export async function createPaseoDaemon( }, workspaceIdToKill, ), + workspaceRegistry, sessionLogger: logger, }, { @@ -1199,6 +1359,7 @@ export async function createPaseoDaemon( requestId: "schedule-run-finish", }, ); + requireArchiveCleanupComplete(archiveResult, "Schedule workspace archive"); }; const scheduleService = new ScheduleService({ paseoHome: config.paseoHome, @@ -1221,6 +1382,8 @@ export async function createPaseoDaemon( logger.info({ elapsed: elapsed() }, "Schedule service initialized"); logger.info({ elapsed: elapsed() }, "Loading persisted agent registry"); const persistedRecords = await agentStorage.list(); + await hubAgentLifecycle.recoverPendingAgentCreations(); + await hubAgentLifecycle.recoverPersistedAutoArchives(); logger.info( { elapsed: elapsed() }, `Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`, @@ -1230,6 +1393,7 @@ export async function createPaseoDaemon( ); logger.info({ elapsed: elapsed() }, "Preparing voice and MCP runtime"); + const lifecycleMutationIngress = new LifecycleMutationIngress(); const createAgentToolHostDependencies = ( runtime: PaseoToolRuntimeContext, ): PaseoToolHostDependencies => ({ @@ -1282,6 +1446,8 @@ export async function createPaseoDaemon( browserToolsBroker, paseoHome: config.paseoHome, worktreesRoot: config.worktreesRoot, + runLifecycleMutation: (operation) => lifecycleMutationIngress.run(operation), + createAgentLifecycleDispatch: hubAgentLifecycle, callerAgentId: runtime.callerAgentId, enableVoiceTools: runtime.enableVoiceTools, voiceOnly: runtime.voiceOnly, @@ -1510,6 +1676,7 @@ export async function createPaseoDaemon( relayConfig: dependencies.serverFeatureOverrides?.relayConfig, }, workspaceAutoName, + hubAgentLifecycle, config.auth, speechService, terminalManager, @@ -1609,26 +1776,61 @@ export async function createPaseoDaemon( } }; - const stop = async () => { - await hubRelationships.stop(); - workspaceReconciliation.dispose(); - scriptHealthMonitor.stop(); - // Freeze both ingress and registration before taking the agent closure snapshot. - wsServer?.prepareForShutdown(); - agentManager.prepareForShutdown(); - await closeAllAgents(logger, agentManager); - await agentManager.flushForShutdown().catch(() => undefined); - detachAgentStoragePersistence(); - await agentStorage.flush().catch(() => undefined); - await providerSnapshotManager.shutdown(); - terminalManager.killAll(); - speechService.stop(); - await scheduleService.stop().catch(() => undefined); - await relayRuntime?.stop().catch(() => undefined); - if (wsServer) { - await wsServer.close(); - } - await serviceProxy.stopStandalone(); + const stop = async (options?: PaseoDaemonStopOptions) => { + const errors: unknown[] = []; + const outerDeadlineAt = options?.deadlineAt ?? Date.now() + DEFAULT_DAEMON_SHUTDOWN_TIMEOUT_MS; + const shutdownDeadlineAt = options?.deadlineAt + ? Math.max(Date.now(), outerDeadlineAt - FORCE_EXIT_SHUTDOWN_RESERVE_MS) + : outerDeadlineAt; + const shutdownController = new AbortController(); + const deadlineTimer = setTimeout( + () => shutdownController.abort(), + Math.max(0, shutdownDeadlineAt - Date.now()), + ); + deadlineTimer.unref(); + const attempt = async (label: string, action: () => void | Promise): Promise => { + const task = Promise.resolve().then(action); + try { + await waitForShutdownStep(task, shutdownController.signal, label); + } catch (error) { + errors.push(error); + logger.error({ err: error }, `Daemon shutdown step failed: ${label}`); + } + }; + + await attempt("hub relationships", () => hubRelationships.stop()); + await attempt("workspace cleanup retry service", () => workspaceCleanupRetryService.stop()); + await attempt("workspace reconciliation", () => workspaceReconciliation.dispose()); + await attempt("script health monitor", () => scriptHealthMonitor.stop()); + // Freeze every worktree/agent mutation producer and join operations accepted + // before the boundary before taking the agent closure snapshot. + await attempt("lifecycle mutation ingress", async () => { + await Promise.all([ + wsServer?.prepareForShutdown(), + lifecycleMutationIngress.closeAndDrain(), + scheduleService.stop(), + ]); + }); + await attempt("agent registration", () => agentManager.prepareForShutdown()); + await attempt("create-agent lifecycle", async () => { + const remainingMs = Math.max(0, shutdownDeadlineAt - Date.now()); + const lifecycleShutdown = await hubAgentLifecycle.shutdown({ timeoutMs: remainingMs }); + if (!lifecycleShutdown.completed) { + throw new Error( + `Create-agent lifecycle shutdown remains incomplete for agents: ${lifecycleShutdown.pendingAgentIds.join(", ")}`, + ); + } + }); + await attempt("agents", () => closeAllAgents(logger, agentManager)); + await attempt("agent persistence", () => agentManager.flushForShutdown()); + await attempt("agent storage detachment", () => detachAgentStoragePersistence()); + await attempt("agent storage", () => agentStorage.flush()); + await attempt("provider snapshots", () => providerSnapshotManager.shutdown()); + await attempt("terminals", () => terminalManager.killAll()); + await attempt("speech", () => speechService.stop()); + await attempt("relay", () => relayRuntime?.stop()); + await attempt("WebSocket server", () => wsServer?.close()); + await attempt("service proxy", () => serviceProxy.stopStandalone()); // Force-drop remaining sockets so httpServer.close() resolves promptly. // We've already closed wsServer (which sent ws-layer close frames) and // stopped every other service, so anything still attached is a TCP @@ -1636,13 +1838,23 @@ export async function createPaseoDaemon( // upgraded WS sockets in the closing handshake, or HTTP keep-alive // sockets in CLOSE_WAIT). closeIdleConnections() does not catch // upgraded sockets, so we use closeAllConnections() here. - httpServer.closeAllConnections(); - await new Promise((resolve) => { - httpServer.close(() => resolve()); - }); + await attempt("HTTP connections", () => httpServer.closeAllConnections()); + await attempt( + "HTTP server", + () => + new Promise((resolve) => { + httpServer.close(() => resolve()); + }), + ); // Clean up socket files - if (listenTarget.type === "socket" && existsSync(listenTarget.path)) { - unlinkSync(listenTarget.path); + await attempt("socket file", () => { + if (listenTarget.type === "socket" && existsSync(listenTarget.path)) { + unlinkSync(listenTarget.path); + } + }); + clearTimeout(deadlineTimer); + if (errors.length > 0) { + throw new AggregateError(errors, "One or more daemon shutdown steps failed"); } }; @@ -1662,13 +1874,20 @@ export async function createPaseoDaemon( async function closeAllAgents(logger: Logger, agentManager: AgentManager): Promise { const agents = agentManager.listAgents(); - await Promise.all( + const results = await Promise.allSettled( agents.map(async (agent) => { try { await agentManager.closeAgent(agent.id); } catch (err) { logger.error({ err, agentId: agent.id }, "Failed to close agent"); + throw err; } }), ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length > 0) { + throw new AggregateError(failures, "Failed to close one or more agents"); + } } diff --git a/packages/server/src/server/daemon-worker.ts b/packages/server/src/server/daemon-worker.ts index 4656523ef7..74eac3f9ba 100644 --- a/packages/server/src/server/daemon-worker.ts +++ b/packages/server/src/server/daemon-worker.ts @@ -33,6 +33,8 @@ interface BootstrapResult { config: ReturnType; } +const SHUTDOWN_TIMEOUT_MS = 10_000; + function isPidAlive(pid: number): boolean { try { process.kill(pid, 0); @@ -142,13 +144,14 @@ async function main() { ); shutdownPromise = (async () => { + const deadlineAt = Date.now() + SHUTDOWN_TIMEOUT_MS; const forceExit = setTimeout(() => { logger.warn( { signal, reason, ...getProcessDiagnostics() }, "Forcing shutdown - HTTP server didn't close in time", ); process.exit(1); - }, 10000); + }, SHUTDOWN_TIMEOUT_MS); try { if (!daemon) { @@ -156,7 +159,7 @@ async function main() { clearTimeout(forceExit); return 1; } - await daemon.stop(); + await daemon.stop({ deadlineAt }); clearTimeout(forceExit); logger.info("Server closed"); return options?.successExitCode ?? 0; diff --git a/packages/server/src/server/paseo-worktree-service.test.ts b/packages/server/src/server/paseo-worktree-service.test.ts index 5fec6bad05..f24a932487 100644 --- a/packages/server/src/server/paseo-worktree-service.test.ts +++ b/packages/server/src/server/paseo-worktree-service.test.ts @@ -460,7 +460,7 @@ test("an explicit project FK remains unchanged when its worktree comes from anot // POSIX-only: Windows git worktree paths need separate canonicalization coverage. test.skipIf(isPlatform("win32"))( - "reuses an existing worktree and still upserts the workspace", + "reuses an existing worktree with its canonical workspace identity", async () => { const { repoDir, tempDir } = createGitRepo(); cleanupPaths.push(tempDir); @@ -493,11 +493,20 @@ test.skipIf(isPlatform("win32"))( ); expect(second.created).toBe(false); - expect(second.worktree.worktreePath).toBe(first.worktree.worktreePath); - expect(events).toContain(`workspace:${second.workspace.workspaceId}`); - // Creation never dedupes by directory: the same worktree path yields a - // distinct workspace record on the second call. - expect(second.workspace.workspaceId).not.toBe(first.workspace.workspaceId); + expect( + createRealpathAwarePathMatcher(first.worktree.worktreePath)(second.worktree.worktreePath), + ).toBe(true); + expect(second.workspace.workspaceId).toBe(first.workspace.workspaceId); + expect(events).toEqual([]); + expect( + Array.from(deps.workspaces.values()).filter( + (workspace) => + !workspace.archivedAt && + createRealpathAwarePathMatcher(first.worktree.worktreePath)( + workspace.worktreeRoot ?? workspace.cwd, + ), + ), + ).toEqual([first.workspace]); }, ); diff --git a/packages/server/src/server/schedule/service.test.ts b/packages/server/src/server/schedule/service.test.ts index a7fad87beb..a0bb625d50 100644 --- a/packages/server/src/server/schedule/service.test.ts +++ b/packages/server/src/server/schedule/service.test.ts @@ -2055,6 +2055,57 @@ describe("ScheduleService", () => { expect(inspected.runs[0]?.status).toBe("succeeded"); }); + test("stop freezes new runs and joins a run accepted before shutdown", async () => { + let markRunStarted: (() => void) | null = null; + const runStarted = new Promise((resolve) => { + markRunStarted = resolve; + }); + let finishRun: (() => void) | null = null; + const runBlocked = new Promise((resolve) => { + finishRun = resolve; + }); + const service = createScheduleService({ + paseoHome: tempDir, + logger: createTestLogger(), + agentManager: new AgentManager({ logger: createTestLogger() }), + agentStorage, + providerSnapshotManager: NO_UNATTENDED_SCHEDULE_POLICY, + now: () => now, + runner: async () => { + markRunStarted?.(); + await runBlocked; + return { agentId: null, output: "finished before shutdown" }; + }, + }); + const created = await service.create({ + prompt: "join on shutdown", + cadence: { type: "every", everyMs: 60_000 }, + target: { + type: "new-agent", + config: { provider: "claude", cwd: tempDir }, + }, + }); + now = new Date("2026-01-01T00:01:00.000Z"); + const tickPromise = service.tick(); + await runStarted; + + const stopPromise = service.stop(); + await expect( + Promise.race([ + stopPromise.then(() => "stopped" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 20)), + ]), + ).resolves.toBe("pending"); + await expect(service.runOnce(created.id)).rejects.toThrow("shutting down"); + + finishRun?.(); + await Promise.all([tickPromise, stopPromise]); + expect((await service.inspect(created.id)).runs[0]).toMatchObject({ + status: "succeeded", + output: "finished before shutdown", + }); + }); + test("rejects archived target agents before loading them", async () => { const manager = new AgentManager({ logger: createTestLogger() }); const service = createScheduleService({ diff --git a/packages/server/src/server/schedule/service.ts b/packages/server/src/server/schedule/service.ts index 6a650ac92a..d5e71c2835 100644 --- a/packages/server/src/server/schedule/service.ts +++ b/packages/server/src/server/schedule/service.ts @@ -199,6 +199,7 @@ type ScheduleAgentManager = Pick< AgentManager, | "createAgent" | "getAgent" + | "getAgentInitializationState" | "getRegisteredProviderIds" | "hasInFlightRun" | "hydrateTimelineFromProvider" @@ -249,6 +250,9 @@ export class ScheduleService { runId: string, ) => Promise; private readonly runningScheduleIds = new Set(); + private readonly inFlightTicks = new Set>(); + private readonly inFlightRuns = new Set>(); + private acceptingRuns = true; private tickTimer: ReturnType | null = null; constructor(options: ScheduleServiceOptions) { @@ -265,6 +269,7 @@ export class ScheduleService { } async start(): Promise { + this.acceptingRuns = true; await this.recoverInterruptedRuns(); await this.sweepOrphanedSchedules(); if (this.tickTimer) { @@ -280,10 +285,14 @@ export class ScheduleService { } async stop(): Promise { + this.acceptingRuns = false; if (this.tickTimer) { clearInterval(this.tickTimer); this.tickTimer = null; } + while (this.inFlightTicks.size > 0 || this.inFlightRuns.size > 0) { + await Promise.allSettled([...this.inFlightTicks, ...this.inFlightRuns]); + } } async create(input: CreateScheduleInput): Promise { @@ -523,6 +532,9 @@ export class ScheduleService { } async runOnce(id: string): Promise { + if (!this.acceptingRuns) { + throw new Error("Schedule service is shutting down"); + } const schedule = await this.inspect(id); if (schedule.status === "completed") { throw new Error(`Schedule ${id} is already completed`); @@ -530,11 +542,22 @@ export class ScheduleService { if (this.runningScheduleIds.has(id)) { throw new Error(`Schedule ${id} is already running`); } - await this.runSchedule(schedule, this.now(), { manual: true }); + await this.runScheduleTracked(schedule, this.now(), { manual: true }); return this.inspect(id); } async tick(): Promise { + if (!this.acceptingRuns) return; + const task = this.tickInternal(); + this.inFlightTicks.add(task); + try { + await task; + } finally { + this.inFlightTicks.delete(task); + } + } + + private async tickInternal(): Promise { const now = this.now(); const schedules = await this.store.list(); for (const schedule of schedules) { @@ -551,7 +574,25 @@ export class ScheduleService { if (new Date(schedule.nextRunAt).getTime() > now.getTime()) { continue; } - await this.runSchedule(schedule, now); + if (!this.acceptingRuns) return; + await this.runScheduleTracked(schedule, now); + } + } + + private async runScheduleTracked( + schedule: StoredSchedule, + now: Date, + options?: { manual?: boolean }, + ): Promise { + if (!this.acceptingRuns) { + throw new Error("Schedule service is shutting down"); + } + const task = this.runSchedule(schedule, now, options); + this.inFlightRuns.add(task); + try { + await task; + } finally { + this.inFlightRuns.delete(task); } } diff --git a/packages/server/src/server/session.create-agent-worktree-autoarchive.e2e.test.ts b/packages/server/src/server/session.create-agent-worktree-autoarchive.e2e.test.ts index 8507a935ef..3e6ff5870c 100644 --- a/packages/server/src/server/session.create-agent-worktree-autoarchive.e2e.test.ts +++ b/packages/server/src/server/session.create-agent-worktree-autoarchive.e2e.test.ts @@ -82,7 +82,8 @@ async function expectWorktreePresentInList(repoDir: string, worktreePath: string .poll( async () => { const listed = await ctx.client.getPaseoWorktreeList({ cwd: repoDir }); - return listed.worktrees.map((worktree) => worktree.worktreePath).includes(worktreePath); + const matchesWorktree = createRealpathAwarePathMatcher(worktreePath); + return listed.worktrees.some((worktree) => matchesWorktree(worktree.worktreePath)); }, { timeout: 5000, interval: 100 }, ) @@ -97,6 +98,7 @@ async function expectWorktreeListEmpty(repoDir: string): Promise { async function createAgentInBranchOffWorktree(options?: { autoArchive?: boolean; branchName?: string; + deferInitialPrompt?: boolean; repoDir?: string; }): Promise<{ repoDir: string; agentId: string; worktreePath: string }> { const repoDir = options?.repoDir ?? createGitRepo(); @@ -112,7 +114,7 @@ async function createAgentInBranchOffWorktree(options?: { base: "main", }, ...(options?.autoArchive !== undefined ? { autoArchive: options.autoArchive } : {}), - initialPrompt: "Say done.", + ...(options?.deferInitialPrompt ? {} : { initialPrompt: "Say done." }), }); return { repoDir, agentId: created.id, worktreePath: created.cwd }; } @@ -259,6 +261,30 @@ test("failed nested worktree creation cleans up the created workspace and backin await ctx.client.archivePaseoWorktree({ worktreePath: source.worktreePath }); }, 30000); +test("failed auto-archive agent creation leaves a reused worktree active", async () => { + const source = await createAgentInBranchOffWorktree({ branchName: "reused-failure-source" }); + await ctx.client.waitForFinish(source.agentId, 10000); + + await expect( + ctx.client.createAgent({ + config: { provider: "unknown-provider", cwd: source.repoDir }, + worktree: { + mode: "branch-off", + newBranch: "reused-failure-source", + base: "main", + }, + autoArchive: true, + initialPrompt: "This agent cannot be created.", + }), + ).rejects.toThrow(); + + await expectWorktreePresentInList(source.repoDir, source.worktreePath); + expect(existsSync(source.worktreePath)).toBe(true); + await expectAgentPresentInActiveList(source.agentId); + + await ctx.client.archivePaseoWorktree({ worktreePath: source.worktreePath }); +}, 30000); + test("create_agent_request with autoArchive archives only the agent when no worktree was created", async () => { const repoDir = createGitRepo(); const created = await ctx.client.createAgent({ @@ -335,9 +361,12 @@ test("archiving a created worktree removes the directory on last reference", asy }); test("auto-archiving a created worktree keeps the directory when a sibling workspace references it", async () => { - const created = await createAgentInBranchOffWorktree({ autoArchive: true }); + const created = await createAgentInBranchOffWorktree({ + autoArchive: true, + deferInitialPrompt: true, + }); - // Create a sibling workspace that shares the same backing directory. + // Register the sibling before the first terminal event can trigger auto-archive. const sibling = await ctx.client.createWorkspace({ source: { kind: "directory", path: created.worktreePath }, title: "sibling", @@ -346,6 +375,7 @@ test("auto-archiving a created worktree keeps the directory when a sibling works throw new Error(sibling.error ?? "Failed to create sibling workspace"); } + await ctx.client.sendMessage(created.agentId, "Say done."); await ctx.client.waitForFinish(created.agentId, 10000); await expectAgentAbsentFromActiveList(created.agentId); diff --git a/packages/server/src/server/session.test.ts b/packages/server/src/server/session.test.ts index 5677750165..d7bcbf1209 100644 --- a/packages/server/src/server/session.test.ts +++ b/packages/server/src/server/session.test.ts @@ -41,6 +41,7 @@ import { asGitHubService, asWorkspaceGitService, asDaemonConfigStore, + createAgentLifecycleDispatchStub, createProviderSnapshotManagerStub, } from "./test-utils/session-stubs.js"; import { isPlatform } from "../test-utils/platform.js"; @@ -375,6 +376,7 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session { list: vi.fn().mockResolvedValue([]), ...options.agentStorage, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: { list: vi.fn().mockResolvedValue([]), get: vi.fn(), @@ -4965,9 +4967,11 @@ describe("agent config setters", () => { function liveAgentManager(overrides: { [K in keyof SessionOptions["agentManager"]]?: unknown }): { [K in keyof SessionOptions["agentManager"]]?: unknown; } { + const agent = { id: "agent-1" }; return { waitForAgentClose: vi.fn().mockResolvedValue(undefined), - getAgent: vi.fn(() => ({ id: "agent-1" })), + getAgent: vi.fn(() => agent), + getAgentInitializationState: vi.fn(() => ({ agent, closeInFlight: false })), ...overrides, }; } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 0e2e6e2dcc..16b67c141e 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -8,6 +8,7 @@ import { serializeAgentStreamEvent, type AgentSnapshotPayload, type AgentAttachment, + type CreateAgentWorktreeTarget, type FirstAgentContext, type SessionInboundMessage, type SessionOutboundMessage, @@ -26,6 +27,7 @@ import type { TerminalManager, TerminalWorkspaceContributionChangedEvent, } from "../terminal/terminal-manager.js"; +import type { WorktreeCreationJournalCallbacks } from "../utils/worktree.js"; import { TerminalSessionController } from "../terminal/terminal-session-controller.js"; import type { TerminalActivity } from "@getpaseo/protocol/terminal-activity"; import type { BinaryFrame } from "@getpaseo/protocol/binary-frames/index"; @@ -107,7 +109,7 @@ import { type AgentRunOptions, type AgentSessionConfig, } from "./agent/agent-sdk-types.js"; -import type { StoredAgentRecord } from "./agent/agent-storage.js"; +import type { AutoArchiveObligation, StoredAgentRecord } from "./agent/agent-storage.js"; import type { AgentStorage } from "./agent/agent-storage.js"; import { ImportSessionsRequestError, @@ -163,7 +165,7 @@ import { DownloadTokenStore } from "./file-download/token-store.js"; import { PushTokenStore } from "./push/token-store.js"; import { archivePersistedWorkspaceRecord, - archiveWorkspaceContents, + requireArchiveCleanupComplete, requireActiveWorkspaceForArchive, } from "./workspace-archive-service.js"; import type { ServiceProxySubsystem } from "./service-proxy.js"; @@ -229,8 +231,10 @@ import { handlePaseoWorktreeArchiveRequest as handleWorktreeArchiveRequest, handlePaseoWorktreeListRequest as handleWorktreeListRequest, handleWorkspaceSetupStatusRequest as handleWorkspaceSetupStatusRequestMessage, + cacheWorkspaceSetupSnapshot, } from "./worktree-session.js"; import { archiveByScope, type ActiveWorkspaceRef } from "./workspace-archive-service.js"; +import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { WorktreeRequestError, toWorktreeWireError } from "./worktree-errors.js"; import { parseGitRemoteLocation } from "@getpaseo/protocol/git-remote"; import { @@ -238,7 +242,7 @@ import { ProjectDirectoryRequestError, } from "./project-directory-service.js"; import { runGitCommand } from "../utils/run-git-command.js"; -import { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import { resolveWorktreeSourceCwd } from "./workspace-source.js"; // TODO: Remove once all app store clients are on >=0.1.45 and understand arbitrary provider strings. @@ -253,6 +257,12 @@ function errorToFriendlyMessage(error: unknown): string { return "Unknown error"; } +function shouldJournalAgentCreation(msg: CreateAgentRequestMessage): boolean { + return ( + msg.autoArchive === true || Boolean(msg.worktree || msg.worktreeName || msg.git?.createWorktree) + ); +} + function resolveSubscriptionId( subscribe: unknown, requestedSubscriptionId: string | undefined, @@ -345,6 +355,11 @@ interface ResolvedSessionCreateAgentIntent { createdDirectoryWorkspace: boolean; } +interface PendingAgentCreationReservation { + agentId?: string; + worktreeCreationJournal?: WorktreeCreationJournalCallbacks; +} + type FetchWorkspacesRequestMessage = Extract< SessionInboundMessage, { type: "fetch_workspaces_request" } @@ -417,6 +432,7 @@ export interface SessionOptions { worktreesRoot?: string; agentManager: AgentManager; agentStorage: AgentStorage; + createAgentLifecycleDispatch: CreateAgentLifecycleDispatch; projectRegistry: ProjectRegistry; workspaceRegistry: WorkspaceRegistry; filesystem?: SessionFileSystem; @@ -662,6 +678,7 @@ export class Session { worktreesRoot, agentManager, agentStorage, + createAgentLifecycleDispatch, projectRegistry, workspaceRegistry, filesystem, @@ -745,6 +762,7 @@ export class Session { projectRegistry: this.projectRegistry, workspaceGitService: this.workspaceGitService, logger: this.sessionLogger, + isDirectory: (targetPath) => this.filesystem.isDirectory(targetPath), }); this.workspaceRecovery = createWorkspaceRecoveryService({ paseoHome: this.paseoHome, @@ -890,6 +908,7 @@ export class Session { isPathWithinRoot: (rootPath, candidatePath) => this.isPathWithinRoot(rootPath, candidatePath), sessionLogger: this.sessionLogger, listTerminalWorkspaceRefs: () => this.listActiveWorkspaceRefs(), + lifecycleCoordinator: defaultWorkspaceLifecycleCoordinator, clientSupportsWrapReflow: () => this.clientCapabilities.has(CLIENT_CAPS.terminalReflowableSnapshot), getClientBufferedAmount: () => this.getTransportBufferedAmount(), @@ -905,30 +924,7 @@ export class Session { this.emitWorkspaceUpdateForWorkspaceId(workspaceId), logger: this.sessionLogger, }); - this.createAgentLifecycleDispatch = new CreateAgentLifecycleDispatch({ - paseoHome: this.paseoHome, - worktreesRoot: this.worktreesRoot, - agentManager: this.agentManager, - agentStorage: this.agentStorage, - github: this.github, - workspaceGitService: this.workspaceGitService, - createPaseoWorktreeWorkflow: (input, workflowOptions) => - this.createPaseoWorktreeWorkflow(input, workflowOptions), - archiveAgentForClose: (agentId) => this.archiveAgentForClose(agentId), - findWorkspaceIdForCwd: (cwd) => this.findWorkspaceIdForCwd(cwd), - listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), - archiveWorkspaceRecord: (workspaceId) => this.archiveWorkspaceRecord(workspaceId), - emit: (message) => this.emit(message), - emitAgentRemove: (agentId) => this.agentUpdates.removeAgent(agentId), - emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds) => - this.emitWorkspaceUpdatesForWorkspaceIds(workspaceIds), - markWorkspaceArchiving: (workspaceIds, archivingAt) => - this.markWorkspaceArchiving(workspaceIds, archivingAt), - clearWorkspaceArchiving: (workspaceIds) => this.clearWorkspaceArchiving(workspaceIds), - killTerminalsForWorkspace: (workspaceId) => - this.terminalController.killTerminalsForWorkspace(workspaceId), - logger: this.sessionLogger, - }); + this.createAgentLifecycleDispatch = createAgentLifecycleDispatch; this.providerSnapshotManager = providerSnapshotManager; this.serviceProxy = serviceProxy ?? null; this.scriptRuntimeStore = scriptRuntimeStore ?? null; @@ -2732,48 +2728,56 @@ export class Session { const projectWorkspaces = (await this.workspaceRegistry.list()).filter( (workspace) => workspace.projectId === resolvedProjectId, ); - const activeWorkspaceIds = projectWorkspaces - .filter((workspace) => !workspace.archivedAt) + const workspaceIdsToArchive = projectWorkspaces + .filter((workspace) => !workspace.archivedAt || workspace.cleanupPending) .map((workspace) => workspace.workspaceId); - if (activeWorkspaceIds.length > 0) { - this.markWorkspaceArchiving(activeWorkspaceIds, new Date().toISOString()); - await this.emitWorkspaceUpdatesForWorkspaceIds(activeWorkspaceIds); - } - const removedWorkspaceIds: string[] = []; - try { - for (const workspaceId of activeWorkspaceIds) { - await archiveWorkspaceContents( - { - agentManager: this.agentManager, - agentStorage: this.agentStorage, - killTerminalsForWorkspace: (id) => - this.terminalController.killTerminalsForWorkspace(id), - sessionLogger: this.sessionLogger, - }, - workspaceId, - ); - await this.archiveWorkspaceRecord(workspaceId); - removedWorkspaceIds.push(workspaceId); - } - - await this.projectRegistry.remove(resolvedProjectId); - await removeProjectCustomIcon({ - paseoHome: this.paseoHome, - projectId: resolvedProjectId, - }).catch((error) => { - this.sessionLogger.warn( - { err: error, projectId: resolvedProjectId }, - "Failed to clean up removed project icon", - ); - }); - } finally { - if (activeWorkspaceIds.length > 0) { - this.clearWorkspaceArchiving(activeWorkspaceIds); - } + for (const workspaceId of workspaceIdsToArchive) { + const archiveResult = await archiveByScope( + { + paseoHome: this.paseoHome, + paseoWorktreesBaseRoot: this.worktreesRoot, + github: this.github, + workspaceGitService: this.workspaceGitService, + agentManager: this.agentManager, + agentStorage: this.agentStorage, + findWorkspaceIdForCwd: (cwd) => this.findWorkspaceIdForCwd(cwd), + listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), + archiveWorkspaceRecord: (id) => this.archiveWorkspaceRecord(id), + workspaceRegistry: this.workspaceRegistry, + emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds) => + this.emitWorkspaceUpdatesForWorkspaceIds(workspaceIds), + markWorkspaceArchiving: (workspaceIds, archivingAt) => + this.markWorkspaceArchiving(workspaceIds, archivingAt), + clearWorkspaceArchiving: (workspaceIds) => this.clearWorkspaceArchiving(workspaceIds), + killTerminalsForWorkspace: (id) => + this.terminalController.killTerminalsForWorkspace(id), + sessionLogger: this.sessionLogger, + }, + { + scope: { kind: "workspace", workspaceId }, + requestId, + }, + ); + requireArchiveCleanupComplete(archiveResult, "Project workspace archive"); + removedWorkspaceIds.push(workspaceId); } + await this.projectRegistry.remove(resolvedProjectId); + await removeProjectCustomIcon({ + paseoHome: this.paseoHome, + projectId: resolvedProjectId, + }).catch((error) => { + this.sessionLogger.warn( + { + err: error, + projectId: resolvedProjectId, + }, + "Failed to clean up removed project icon", + ); + }); + const updateIds = removedWorkspaceIds.length > 0 ? removedWorkspaceIds @@ -3060,6 +3064,7 @@ export class Session { let createdWorktreeForCleanup: CreatePaseoWorktreeWorkflowResult | null = null; let createdAgentId: string | null = null; + let pendingCreationAgentId: string | undefined; try { const requestedCwd = resolve(config.cwd); const needsRequestedDirectory = @@ -3078,11 +3083,16 @@ export class Session { ...(attachments && attachments.length > 0 ? { attachments } : {}), }; const workspacePromptTitle = resolveFirstAgentPromptTitle(firstAgentContext); - const createdWorktree = await this.createAgentLifecycleDispatch.createWorktreeForRequest({ + const pendingCreation = await this.reservePendingAgentCreation( + shouldJournalAgentCreation(msg), + ); + pendingCreationAgentId = pendingCreation.agentId; + const createdWorktree = await this.createWorktreeForCreateAgentRequest({ cwd: config.cwd, target: worktree, firstAgentContext, hasLegacyGitOptions: Boolean(git), + worktreeCreationJournal: pendingCreation.worktreeCreationJournal, }); createdWorktreeForCleanup = createdWorktree; const resolvedIntent = await this.resolveSessionCreateAgentIntent({ @@ -3095,7 +3105,16 @@ export class Session { throw new Error(`Working directory does not exist or is not a directory: ${resolvedCwd}`); } - const { snapshot, liveSnapshot } = await createAgentCommand( + const autoArchiveObligation: AutoArchiveObligation | undefined = + autoArchive === true + ? { + phase: "armed", + target: createdWorktree + ? { kind: "workspace", workspaceId: createdWorktree.workspace.workspaceId } + : { kind: "agent" }, + } + : undefined; + const { snapshot, liveSnapshot, initialPromptError } = await createAgentCommand( { agentManager: this.agentManager, agentStorage: this.agentStorage, @@ -3103,6 +3122,13 @@ export class Session { paseoHome: this.paseoHome, worktreesRoot: this.worktreesRoot, providerSnapshotManager: this.providerSnapshotManager, + lifecycleCoordinator: defaultWorkspaceLifecycleCoordinator, + requireActiveWorkspaceForOwnership: async (workspaceId) => { + const workspace = await this.workspaceRegistry.get(workspaceId); + if (!workspace || workspace.archivedAt) { + throw new Error(`Workspace not found: ${workspaceId}`); + } + }, }, { kind: "session", @@ -3119,11 +3145,22 @@ export class Session { env, provisionalTitle, firstAgentContext, - buildSessionConfig: (sessionConfig, gitOptions, legacyWorktreeName, ctx) => - this.buildAgentSessionConfig(sessionConfig, gitOptions, legacyWorktreeName, ctx), + autoArchiveObligation, + agentId: pendingCreationAgentId, + onCreated: ({ agentId, autoArchiveObligation: persistedObligation }) => { + createdAgentId = agentId; + this.registerCreatedAgentAutoArchive(agentId, persistedObligation); + }, + buildSessionConfig: (sessionConfig, gitOptions, legacyWorktreeName, ctx, onPath) => + this.buildAgentSessionConfig( + sessionConfig, + gitOptions, + legacyWorktreeName, + ctx, + onPath, + ), }, ); - createdAgentId = snapshot.id; await this.agentUpdates.forwardLiveAgent(snapshot); if (resolvedIntent.createdDirectoryWorkspace && trimmedPrompt) { this.workspaceAutoName.scheduleForDirectory( @@ -3135,23 +3172,8 @@ export class Session { { currentSelection: this.getFocusedAgentSelectionForCwd(resolvedIntent.config.cwd) }, ); } - this.createAgentLifecycleDispatch.registerAutoArchiveIfRequested({ - autoArchive, - agentId: snapshot.id, - createdWorktree, - }); - if (requestId) { - const agentPayload = await this.buildAgentPayload(liveSnapshot); - this.emit({ - type: "status", - payload: { - status: "agent_created", - agentId: liveSnapshot.id, - requestId, - agent: agentPayload, - }, - }); - } + await this.emitAgentCreatedStatus(requestId, liveSnapshot); + this.reportInitialPromptStartFailure(snapshot.id, initialPromptError); this.sessionLogger.info( { agentId: snapshot.id, provider: snapshot.provider }, @@ -3162,6 +3184,7 @@ export class Session { createdWorktree: createdWorktreeForCleanup, createdAgentId, }); + await this.recoverPendingAgentCreation(pendingCreationAgentId); const wireError = toWorktreeWireError(error); this.sessionLogger.error({ err: error }, "Failed to create agent"); if (requestId) { @@ -3187,6 +3210,76 @@ export class Session { } } + private async emitAgentCreatedStatus( + requestId: string | undefined, + liveSnapshot: ManagedAgent, + ): Promise { + if (!requestId) return; + const agentPayload = await this.buildAgentPayload(liveSnapshot); + this.emit({ + type: "status", + payload: { + status: "agent_created", + agentId: liveSnapshot.id, + requestId, + agent: agentPayload, + }, + }); + } + + private reportInitialPromptStartFailure(agentId: string, error: unknown | null): void { + if (!error) return; + const promptError = error instanceof Error ? error.message : String(error); + this.sessionLogger.warn( + { err: error, agentId }, + "Agent created but initial prompt start was not confirmed", + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "error", + content: `Agent ${agentId} was created, but its initial prompt start was not confirmed: ${promptError}`, + }, + }); + } + + private async reservePendingAgentCreation( + requiresDurableCreation: boolean, + ): Promise { + if (!requiresDurableCreation) return {}; + + const agentId = this.agentManager.allocateAgentId(); + await this.agentStorage.beginPendingAgentCreation(agentId); + return { + agentId, + worktreeCreationJournal: { + onWorktreePathPlanned: (worktreePath, plan) => + this.agentStorage.planPendingAgentCreationWorktree(agentId, worktreePath, plan), + onWorktreePathResolved: (worktreePath, reservation) => + this.agentStorage.identifyPendingAgentCreationWorktree( + agentId, + worktreePath, + reservation, + ), + }, + }; + } + + private registerCreatedAgentAutoArchive( + agentId: string, + obligation: AutoArchiveObligation | undefined, + ): void { + if (!obligation) return; + this.createAgentLifecycleDispatch.registerAutoArchive({ agentId, obligation }); + } + + private async recoverPendingAgentCreation(agentId: string | undefined): Promise { + if (!agentId) return; + await this.createAgentLifecycleDispatch.recoverPendingAgentCreation(agentId); + } + private async resolveSessionCreateAgentIntent(input: { request: CreateAgentRequestMessage; createdWorktree: CreatePaseoWorktreeWorkflowResult | null; @@ -3537,6 +3630,7 @@ export class Session { gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, + worktreeCreationJournal?: WorktreeCreationJournalCallbacks, ): Promise<{ sessionConfig: AgentSessionConfig; setupContinuation?: CreatePaseoWorktreeWorkflowResult["setupContinuation"]; @@ -3577,6 +3671,7 @@ export class Session { gitOptions, legacyWorktreeName, firstAgentContext, + worktreeCreationJournal, ); } @@ -3874,6 +3969,7 @@ export class Session { findWorkspaceIdForCwd: (cwd) => this.findWorkspaceIdForCwd(cwd), listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), archiveWorkspaceRecord: (workspaceId) => this.archiveWorkspaceRecord(workspaceId), + workspaceRegistry: this.workspaceRegistry, emit: (message) => this.emit(message), emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds) => this.emitWorkspaceUpdatesForWorkspaceIds(workspaceIds), @@ -4630,6 +4726,7 @@ export class Session { archivedAt: archiveTimestamp, workspaceRegistry: this.workspaceRegistry, }); + this.workspaceSetupSnapshots.delete(workspaceId); if (!existingWorkspace) { this.workspaceGitObserver.removeForWorkspaceId(workspaceId); return; @@ -5768,10 +5865,26 @@ export class Session { this.workspaceAutoName.scheduleForWorktree(autoNameInput, { currentSelection: this.getFocusedAgentSelectionForCwd(autoNameInput.workspace.cwd), }), + worktreeCreationJournal: { + beginPendingAgentCreation: (creationId, journalOptions) => + this.agentStorage.beginPendingAgentCreation(creationId, journalOptions), + planPendingAgentCreationWorktree: (creationId, worktreePath, plan) => + this.agentStorage.planPendingAgentCreationWorktree(creationId, worktreePath, plan), + identifyPendingAgentCreationWorktree: (creationId, worktreePath, reservation) => + this.agentStorage.identifyPendingAgentCreationWorktree( + creationId, + worktreePath, + reservation, + ), + removePendingAgentCreation: (creationId) => + this.agentStorage.removePendingAgentCreation(creationId), + recoverPendingCreation: (creationId) => + this.createAgentLifecycleDispatch.recoverPendingAgentCreation(creationId), + }, emitWorkspaceUpdateForWorkspaceId: (workspaceId) => this.emitWorkspaceUpdateForWorkspaceId(workspaceId), cacheWorkspaceSetupSnapshot: (workspaceId, snapshot) => { - this.workspaceSetupSnapshots.set(workspaceId, snapshot); + cacheWorkspaceSetupSnapshot(this.workspaceSetupSnapshots, workspaceId, snapshot); }, emit: (message) => this.emit(message), sessionLogger: this.sessionLogger, @@ -5791,6 +5904,59 @@ export class Session { ); } + private async createWorktreeForCreateAgentRequest(input: { + cwd: string; + target: CreateAgentWorktreeTarget | undefined; + firstAgentContext: FirstAgentContext; + hasLegacyGitOptions: boolean; + worktreeCreationJournal?: WorktreeCreationJournalCallbacks; + }): Promise { + if (input.target && input.hasLegacyGitOptions) { + throw new Error("create_agent_request worktree cannot be combined with git options"); + } + if (!input.target) { + return null; + } + + const baseInput = { + cwd: input.cwd, + firstAgentContext: input.firstAgentContext, + runSetup: false, + paseoHome: this.paseoHome, + worktreesRoot: this.worktreesRoot, + onWorktreePathPlanned: input.worktreeCreationJournal?.onWorktreePathPlanned, + onWorktreePathResolved: input.worktreeCreationJournal?.onWorktreePathResolved, + } as const; + switch (input.target.mode) { + case "branch-off": { + const base = input.target.base; + return this.createPaseoWorktreeWorkflow( + { + ...baseInput, + worktreeSlug: input.target.newBranch, + action: "branch-off", + ...(base ? { refName: base } : {}), + }, + base ? { resolveDefaultBranch: async () => base } : undefined, + ); + } + case "checkout-branch": + return this.createPaseoWorktreeWorkflow({ + ...baseInput, + action: "checkout", + refName: input.target.branch, + }); + case "checkout-pr": + return this.createPaseoWorktreeWorkflow({ + ...baseInput, + action: "checkout", + githubPrNumber: input.target.prNumber, + }); + default: + throw new Error("Unsupported create_agent_request worktree target"); + } + } + private async handleWorkspaceSetupStatusRequest( request: Extract, ): Promise { @@ -5808,11 +5974,14 @@ export class Session { ): Promise { try { const existing = await requireActiveWorkspaceForArchive( - { listActiveWorkspaces: () => this.listActiveWorkspaceRefs() }, + { + listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), + workspaceRegistry: this.workspaceRegistry, + }, request.workspaceId, ); - await archiveByScope( + const archiveResult = await archiveByScope( { paseoHome: this.paseoHome, paseoWorktreesBaseRoot: this.worktreesRoot, @@ -5830,6 +5999,7 @@ export class Session { clearWorkspaceArchiving: (workspaceIds) => this.clearWorkspaceArchiving(workspaceIds), killTerminalsForWorkspace: (workspaceId) => this.terminalController.killTerminalsForWorkspace(workspaceId), + workspaceRegistry: this.workspaceRegistry, sessionLogger: this.sessionLogger, }, { @@ -5837,9 +6007,13 @@ export class Session { requestId: request.requestId, }, ); + requireArchiveCleanupComplete(archiveResult, "Workspace archive"); const archivedWorkspace = await this.workspaceRegistry.get(request.workspaceId); - const archivedAt = archivedWorkspace?.archivedAt ?? new Date().toISOString(); + if (!archivedWorkspace?.archivedAt) { + throw new Error(`Workspace archive did not persist: ${request.workspaceId}`); + } + const archivedAt = archivedWorkspace.archivedAt; this.emit({ type: "archive_workspace_response", payload: { diff --git a/packages/server/src/server/session.workspace-git-watch.test.ts b/packages/server/src/server/session.workspace-git-watch.test.ts index 00f3f780fd..07ce64efe5 100644 --- a/packages/server/src/server/session.workspace-git-watch.test.ts +++ b/packages/server/src/server/session.workspace-git-watch.test.ts @@ -5,7 +5,10 @@ import { createBranchChangeRouteHandler } from "./script-route-branch-handler.js import { createServiceProxySubsystem, type ServiceProxySubsystem } from "./service-proxy.js"; import { Session, type SessionOptions } from "./session.js"; import { asInternals, createStub } from "./test-utils/class-mocks.js"; -import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { + createAgentLifecycleDispatchStub, + createProviderSnapshotManagerStub, +} from "./test-utils/session-stubs.js"; import { createTestLogger } from "../test-utils/test-logger.js"; import { WorkspaceScriptRuntimeStore } from "./workspace-script-runtime-store.js"; import type { @@ -13,7 +16,11 @@ import type { WorkspaceGitRuntimeSnapshot, WorkspaceGitService, } from "./workspace-git-service.js"; -import type { SessionOutboundMessage, WorkspaceDescriptorPayload } from "./messages.js"; +import type { + SessionOutboundMessage, + WorkspaceDescriptorPayload, + WorkspaceSetupSnapshot, +} from "./messages.js"; import { createPersistedProjectRecord, createPersistedWorkspaceRecord, @@ -120,6 +127,7 @@ function createSessionForWorkspaceGitWatchTests(options?: { ) => void; serviceProxy?: ServiceProxySubsystem; scriptRuntimeStore?: WorkspaceScriptRuntimeStore; + workspaceSetupSnapshots?: Map; }): { session: Session; emitted: Array<{ type: string; payload: unknown }>; @@ -197,6 +205,7 @@ function createSessionForWorkspaceGitWatchTests(options?: { list: async () => [], get: async () => null, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: createStub({ subscribeToMutations: () => () => {}, initialize: async () => {}, @@ -257,6 +266,7 @@ function createSessionForWorkspaceGitWatchTests(options?: { terminalManager: null, serviceProxy: options?.serviceProxy, scriptRuntimeStore: options?.scriptRuntimeStore, + workspaceSetupSnapshots: options?.workspaceSetupSnapshots, onBranchChanged: options?.onBranchChanged, getDaemonTcpPort: () => 6767, }); @@ -535,6 +545,43 @@ describe("workspace git watch targets", () => { await session.cleanup(); }); + test("archiving a workspace clears its cached setup snapshot", async () => { + const setupSnapshots = new Map([ + [ + "ws-10", + { + status: "completed", + detail: { + type: "worktree_setup", + worktreePath: "/tmp/repo", + branchName: "main", + log: "done", + commands: [], + }, + error: null, + }, + ], + ]); + const { session, projects, workspaces } = createSessionForWorkspaceGitWatchTests({ + workspaceSetupSnapshots: setupSnapshots, + }); + seedGitWorkspace({ + projects, + workspaces, + projectId: "proj-1", + workspaceId: "ws-10", + cwd: "/tmp/repo", + name: "main", + }); + + await asInternals<{ archiveWorkspaceRecord: (workspaceId: string) => Promise }>( + session, + ).archiveWorkspaceRecord("ws-10"); + + expect(setupSnapshots.has("ws-10")).toBe(false); + await session.cleanup(); + }); + test("archiving a workspace releases its git watch subscription for the directory", async () => { const { session, projects, workspaces, subscriptions } = createSessionForWorkspaceGitWatchTests(); diff --git a/packages/server/src/server/session.workspace-resolution-invariants.test.ts b/packages/server/src/server/session.workspace-resolution-invariants.test.ts index fdc3a8aa42..0d2c474dae 100644 --- a/packages/server/src/server/session.workspace-resolution-invariants.test.ts +++ b/packages/server/src/server/session.workspace-resolution-invariants.test.ts @@ -12,7 +12,10 @@ import { Session, type SessionOptions } from "./session.js"; import type { SessionOutboundMessage } from "@getpaseo/protocol/messages"; import { createNoopWorkspaceGitService } from "./test-utils/workspace-git-service-stub.js"; import { asInternals, createStub } from "./test-utils/class-mocks.js"; -import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { + createAgentLifecycleDispatchStub, + createProviderSnapshotManagerStub, +} from "./test-utils/session-stubs.js"; import { createPersistedProjectRecord, createPersistedWorkspaceRecord, @@ -108,6 +111,7 @@ function createHarness(input: { list: async () => [], get: async () => null, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: createStub({ subscribeToMutations: () => () => {}, initialize: async () => {}, diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index de33dfe2a8..67c3037c08 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -60,6 +60,7 @@ import { asDaemonConfigStore, asTerminalManager, asSessionInternals, + createAgentLifecycleDispatchStub, createProviderSnapshotManagerStub, isSessionOutboundMessage, filterByType, @@ -74,6 +75,7 @@ import { type PersistedWorkspaceRecord, type WorkspaceMutation, } from "./workspace-registry.js"; +import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; const REPO_CWD = path.resolve("/tmp/repo"); const UNREGISTERED_CWD = path.resolve("/tmp/unregistered"); @@ -550,6 +552,7 @@ function createSessionForWorkspaceTests( terminalManager?: TerminalManager | null; agentManager?: { [K in keyof SessionOptions["agentManager"]]?: unknown }; agentStorage?: { [K in keyof SessionOptions["agentStorage"]]?: unknown }; + createAgentLifecycleDispatch?: SessionOptions["createAgentLifecycleDispatch"]; projectRegistry?: SessionOptions["projectRegistry"]; workspaceRegistry?: SessionOptions["workspaceRegistry"]; github?: ForgeService; @@ -670,6 +673,8 @@ function createSessionForWorkspaceTests( upsert: async () => {}, ...options.agentStorage, }), + createAgentLifecycleDispatch: + options.createAgentLifecycleDispatch ?? createAgentLifecycleDispatchStub(), projectRegistry: options.projectRegistry ?? { initialize: async () => {}, existsOnDisk: async () => true, @@ -874,6 +879,100 @@ test("client heartbeat clears attention for the focused terminal", async () => { }); }); +test("create_agent_request returns the published agent when first-prompt start times out", async () => { + const agentId = "00000000-0000-4000-8000-000000000550"; + const updatedAt = "2026-08-01T12:00:00.000Z"; + const snapshot = makeManagedAgent({ + id: agentId, + cwd: REPO_CWD, + workspaceId: "ws-repo-running", + lifecycle: "idle", + updatedAt, + }) as unknown as ManagedAgent; + const stored = makeStoredAgent({ id: agentId, cwd: REPO_CWD, updatedAt }); + stored.workspaceId = "ws-repo-running"; + stored.lastStatus = "idle"; + stored.autoArchiveObligation = { phase: "armed", target: { kind: "agent" } }; + const promptStartTimeout = new Error("initial prompt start timed out"); + const emitted: SessionOutboundMessage[] = []; + const beginPendingAgentCreation = vi.fn(async () => ({ + agentId, + createdAt: updatedAt, + cleanupTarget: { kind: "agent" as const }, + })); + const removePendingAgentCreation = vi.fn(async () => undefined); + const registerAutoArchive = vi.fn(() => ({ + settled: Promise.resolve("cancelled" as const), + cancel: async () => undefined, + })); + const lifecycleDispatch = createAgentLifecycleDispatchStub(); + lifecycleDispatch.registerAutoArchive = registerAutoArchive; + const streamAgent = vi.fn(() => (async function* noEvents() {})()); + + const session = createSessionForWorkspaceTests({ + onMessage: (message) => emitted.push(message), + createAgentLifecycleDispatch: lifecycleDispatch, + agentManager: { + allocateAgentId: () => agentId, + createAgent: vi.fn(async (_config, requestedAgentId, options) => { + expect(requestedAgentId).toBe(agentId); + expect(options.autoArchiveObligation).toEqual({ + phase: "armed", + target: { kind: "agent" }, + }); + return snapshot; + }), + getAgent: () => snapshot, + tryRunOutOfBand: () => false, + hasInFlightRun: () => false, + streamAgent, + waitForAgentRunStart: vi.fn(async () => { + throw promptStartTimeout; + }), + }, + agentStorage: { + beginPendingAgentCreation, + removePendingAgentCreation, + get: async (requestedAgentId: string) => (requestedAgentId === agentId ? stored : null), + }, + }); + + await session.handleMessage({ + type: "create_agent_request", + requestId: "req-create-prompt-timeout", + workspaceId: "ws-repo-running", + config: { provider: "codex", cwd: REPO_CWD }, + initialPrompt: "start the assigned work", + autoArchive: true, + attachments: [], + }); + + const statuses = filterByType(emitted, "status").map((message) => message.payload); + expect(statuses).toEqual([ + expect.objectContaining({ + status: "agent_created", + requestId: "req-create-prompt-timeout", + agentId, + agent: expect.objectContaining({ id: agentId }), + }), + ]); + expect(filterByType(emitted, "activity_log")).toContainEqual( + expect.objectContaining({ + payload: expect.objectContaining({ + type: "error", + content: expect.stringContaining("initial prompt start was not confirmed"), + }), + }), + ); + expect(beginPendingAgentCreation).toHaveBeenCalledWith(agentId); + expect(removePendingAgentCreation).toHaveBeenCalledWith(agentId); + expect(registerAutoArchive).toHaveBeenCalledWith({ + agentId, + obligation: { phase: "armed", target: { kind: "agent" } }, + }); + expect(streamAgent).toHaveBeenCalledWith(agentId, "start the assigned work", undefined); +}); + test("create_agent_request keeps requested child cwd when grouped under an existing parent workspace", async () => { const workdir = mkdtempSync(path.join(tmpdir(), "paseo-create-agent-cwd-")); try { @@ -952,6 +1051,7 @@ test("create_agent_request keeps requested child cwd when grouped under an exist paseoHome: path.join(workdir, "paseo-home"), agentManager, agentStorage, + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry, workspaceRegistry, chatService: asChatService(), @@ -1106,6 +1206,7 @@ test("create_agent_request launches from an exact subdirectory in a created work paseoHome: path.join(workdir, "paseo-home"), agentManager, agentStorage, + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry, workspaceRegistry, chatService: asChatService(), @@ -1245,6 +1346,7 @@ test("create_agent_request does not title an existing workspace from the agent p paseoHome: path.join(workdir, "paseo-home"), agentManager, agentStorage, + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry, workspaceRegistry, chatService: asChatService(), @@ -1539,6 +1641,7 @@ test("archive emits an authoritative agent_update upsert for subscribed clients" Object.assign(archivedRecord, record); }, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: (() => { const proj = createPersistedProjectRecord({ projectId: "proj-1", @@ -1901,6 +2004,7 @@ test("close_items_request archives agents and kills terminals in one batch", asy return archivedRecord; }, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: (() => { const proj = createPersistedProjectRecord({ projectId: "proj-close", @@ -2088,6 +2192,7 @@ test("close_items_request archives stored agents that are not currently loaded", }, upsert: upsertStoredRecord, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: (() => { const proj = createPersistedProjectRecord({ projectId: "proj-stored", @@ -2237,6 +2342,7 @@ test("close_items_request continues after an archive failure", async () => { return goodRecord; }, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: (() => { const proj = createPersistedProjectRecord({ projectId: "proj-err", @@ -3320,6 +3426,7 @@ test("workspace update stream keeps persisted workspace visible after agents sto list: async () => [], get: async () => null, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: { initialize: async () => {}, existsOnDisk: async () => true, @@ -3571,6 +3678,9 @@ test("project.remove.request archives active workspaces and removes the project workspaces.get(workspaceId) ?? null; session.workspaceRegistry.list = async () => Array.from(workspaces.values()); session.workspaceRegistry.archive = async (workspaceId: string, archivedAt: string) => { + expect(() => + defaultWorkspaceLifecycleCoordinator.reserveWorkspaceOwnershipMutation(workspaceId), + ).toThrow(`Workspace ${workspaceId} is being archived`); const existing = workspaces.get(workspaceId); if (!existing) return; workspaces.set(workspaceId, { ...existing, updatedAt: archivedAt, archivedAt }); @@ -5619,6 +5729,79 @@ test("archive_workspace_request hides non-destructive workspace records", async expect(response?.payload.error).toBeNull(); }); +test("archive_workspace_request never fabricates success when persistence fails", async () => { + const emitted: SessionOutboundMessage[] = []; + const session = createSessionForWorkspaceTests(); + const workspace = createPersistedWorkspaceRecord({ + workspaceId: "ws-archive-persistence-failure", + projectId: "proj-archive-persistence-failure", + cwd: REPO_CWD, + kind: "directory", + displayName: "repo", + createdAt: "2026-03-01T12:00:00.000Z", + updatedAt: "2026-03-01T12:00:00.000Z", + }); + + session.emit = (message) => { + if (isSessionOutboundMessage(message)) emitted.push(message); + }; + session.workspaceRegistry.get = async () => workspace; + session.workspaceRegistry.list = async () => [workspace]; + session.workspaceRegistry.archive = async () => { + throw new Error("workspace persistence failed"); + }; + + await session.handleMessage({ + type: "archive_workspace_request", + workspaceId: workspace.workspaceId, + requestId: "req-archive-persistence-failure", + }); + + expect(workspace.archivedAt).toBeNull(); + const response = emitted.find((message) => message.type === "archive_workspace_response") as + | { payload: Record } + | undefined; + expect(response?.payload).toMatchObject({ + archivedAt: null, + error: "Failed to archive one or more workspaces", + }); +}); + +test("archive_workspace_request requires a persisted archived timestamp", async () => { + const emitted: SessionOutboundMessage[] = []; + const session = createSessionForWorkspaceTests(); + const workspace = createPersistedWorkspaceRecord({ + workspaceId: "ws-archive-no-timestamp", + projectId: "proj-archive-no-timestamp", + cwd: REPO_CWD, + kind: "directory", + displayName: "repo", + createdAt: "2026-03-01T12:00:00.000Z", + updatedAt: "2026-03-01T12:00:00.000Z", + }); + + session.emit = (message) => { + if (isSessionOutboundMessage(message)) emitted.push(message); + }; + session.workspaceRegistry.get = async () => workspace; + session.workspaceRegistry.list = async () => [workspace]; + session.workspaceRegistry.archive = async () => {}; + + await session.handleMessage({ + type: "archive_workspace_request", + workspaceId: workspace.workspaceId, + requestId: "req-archive-no-timestamp", + }); + + const response = emitted.find((message) => message.type === "archive_workspace_response") as + | { payload: Record } + | undefined; + expect(response?.payload).toMatchObject({ + archivedAt: null, + error: `Workspace archive did not persist: ${workspace.workspaceId}`, + }); +}); + test("archive_workspace_request archives a worktree-kind workspace and removes the directory on last reference", async () => { const tempDir = mkdtempSync(path.join(tmpdir(), "session-worktree-kind-archive-")); const repoDir = path.join(tempDir, "repo"); @@ -5649,7 +5832,7 @@ test("archive_workspace_request archives a worktree-kind workspace and removes t const workspaceId = "ws-worktree-kind-archive"; const projectId = "proj-worktree-kind-archive"; - const workspace = createPersistedWorkspaceRecord({ + let workspace = createPersistedWorkspaceRecord({ workspaceId, projectId, cwd: worktree.worktreePath, @@ -5701,8 +5884,12 @@ test("archive_workspace_request archives a worktree-kind workspace and removes t }; session.workspaceRegistry.get = async () => workspace; session.workspaceRegistry.list = async () => [workspace]; + session.workspaceRegistry.update = async (_id, updater) => { + workspace = updater(workspace); + return workspace; + }; session.workspaceRegistry.archive = async (_id: string, archivedAt: string) => { - workspace.archivedAt = archivedAt; + workspace = { ...workspace, archivedAt, updatedAt: archivedAt }; }; session.projectRegistry.list = async () => [project]; @@ -8732,7 +8919,6 @@ test("workspace auto-name replaces the unchanged prompt title", async () => { }); test("workspace auto-name uses the backing root for a nested worktree", async () => { - vi.useFakeTimers(); const tempDir = realpathSync(mkdtempSync(path.join(tmpdir(), "workspace-auto-name-rejected-"))); const repoDir = path.join(tempDir, "repo"); mkdirSync(repoDir); @@ -8803,7 +8989,10 @@ test("workspace auto-name uses the backing root for a nested worktree", async () workspace, firstAgentContext: { prompt: "Fix checkout title" }, }); - await vi.runAllTimersAsync(); + // This integration path performs real Git discovery after the 0ms + // scheduling boundary. Keep it on the real clock so draining fake timers + // cannot advance the command's 30s timeout ahead of process closure. + await vi.waitFor(() => expect(emittedCwds).toEqual([workspaceCwd])); expect(generateCalls).toBe(1); expect(stored.get(workspace.workspaceId)).toMatchObject({ @@ -8825,7 +9014,6 @@ test("workspace auto-name uses the backing root for a nested worktree", async () expect(gitMutations).toEqual([]); expect(emittedCwds).toEqual([workspaceCwd]); } finally { - vi.useRealTimers(); rmSync(tempDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); diff --git a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts index a97e82f85d..b9e98d96fd 100644 --- a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts +++ b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts @@ -107,6 +107,7 @@ beforeEach(async () => { projectRegistry, workspaceGitService: gitService(), logger, + isDirectory: async () => true, }); }); @@ -144,6 +145,44 @@ test("re-opening an active workspace by exact path returns the same record witho expect(await workspaceRegistry.list()).toHaveLength(1); }); +test("concurrent managed worktree provisioning preserves one canonical active owner", async () => { + const repoRoot = path.join(tmpDir, "repo"); + const worktreeRoot = path.join(tmpDir, "worktrees", "same-slug"); + gitRoots.add(repoRoot); + gitRoots.add(worktreeRoot); + gitBranches.set(worktreeRoot, "same-slug"); + const project = await projectRegistry.getOrCreateActiveByRoot({ + rootPath: repoRoot, + kind: "git", + displayName: "repo", + timestamp: "2026-08-01T00:00:00.000Z", + }); + const input = { + sourceCwd: repoRoot, + projectId: project.projectId, + repoRoot, + cwd: worktreeRoot, + worktreeRoot, + branch: "same-slug", + baseBranch: "main", + title: "Canonical owner", + expectsInitialAgent: false, + }; + + const [created, reused] = await Promise.all([ + provisioning.createWorkspaceForWorktree(input), + provisioning.createWorkspaceForWorktree({ ...input, title: "Duplicate attempt" }), + ]); + + expect(reused.workspaceId).toBe(created.workspaceId); + expect(reused.title).toBe("Canonical owner"); + expect( + (await workspaceRegistry.list()).filter( + (workspace) => !workspace.archivedAt && workspace.worktreeRoot === worktreeRoot, + ), + ).toEqual([created]); +}); + test("re-opening Windows-equivalent workspace cwd spellings reuses the active and archived record", async () => { const cwd = path.join(tmpDir, "workspace"); const created = await provisioning.findOrCreateWorkspaceForDirectory(cwd); @@ -199,6 +238,7 @@ test("persists manual worktree ownership separately from its workspace kind", as mainRepoRoot, }), }), + isDirectory: async () => true, }); const workspace = await manualWorktreeProvisioning.findOrCreateWorkspaceForDirectory(cwd); @@ -261,6 +301,7 @@ test("reopening archived exact-root records restores the fresh Git project", asy mainRepoRoot: null, }), }), + isDirectory: async () => true, }); const reopened = await archivedProvisioning.ensureWorkspaceRecordUnarchived(workspace); @@ -300,6 +341,7 @@ test("uses one workspace snapshot when reopening an archived workspace", async ( workspaceRegistry: snapshotRegistry, projectRegistry, workspaceGitService: gitService(), + isDirectory: async () => true, }); const reopened = await snapshotProvisioning.findOrCreateWorkspaceForDirectory(repo); diff --git a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts index 90b0c7c128..37ffae1803 100644 --- a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts +++ b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts @@ -1,3 +1,4 @@ +import { stat } from "node:fs/promises"; import { basename, resolve } from "node:path"; import type { Logger } from "pino"; import { @@ -16,6 +17,10 @@ import type { WorkspaceGitService } from "../../workspace-git-service.js"; import type { CreatePaseoWorktreeWorkflowResult } from "../../worktree-session.js"; import { deriveProjectKey } from "../../project-key.js"; import { areEquivalentPaths, createRealpathAwarePathMatcher } from "../../../utils/path.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "../../workspace-lifecycle-coordinator.js"; export interface ResolveOrCreateWorkspaceIdInput { createdWorktree: CreatePaseoWorktreeWorkflowResult | null; @@ -90,8 +95,20 @@ export function createWorkspaceProvisioningService(deps: { projectRegistry: ProjectRegistry; workspaceGitService: Pick; logger: Logger; + lifecycleCoordinator?: Pick; + isDirectory?: (path: string) => Promise; }): WorkspaceProvisioningService { const { serverId, workspaceRegistry, projectRegistry, workspaceGitService, logger } = deps; + const lifecycleCoordinator = deps.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const isDirectory = + deps.isDirectory ?? + (async (targetPath: string) => { + try { + return (await stat(targetPath)).isDirectory(); + } catch { + return false; + } + }); async function runInImportWorkspace( input: ImportWorkspaceInput, @@ -189,6 +206,18 @@ export function createWorkspaceProvisioningService(deps: { context?: { expectsInitialAgent?: boolean }, ): Promise { const normalizedCwd = resolve(cwd); + return lifecycleCoordinator.runDirectoryExclusive(normalizedCwd, async () => { + await requireDirectoryAfterLifecycleWait(normalizedCwd); + return createWorkspaceForDirectoryUnlocked(normalizedCwd, title, projectId, context); + }); + } + + async function createWorkspaceForDirectoryUnlocked( + normalizedCwd: string, + title?: string | null, + projectId?: string, + context?: { expectsInitialAgent?: boolean }, + ): Promise { const checkout = await workspaceGitService.getCheckout(normalizedCwd); const project = projectId ? await refreshProjectKind(await requireActiveProject(projectId), normalizedCwd, checkout) @@ -207,6 +236,12 @@ export function createWorkspaceProvisioningService(deps: { return workspace; } + async function requireDirectoryAfterLifecycleWait(cwd: string): Promise { + if (!(await isDirectory(cwd))) { + throw new Error(`Working directory does not exist or is not a directory: ${cwd}`); + } + } + async function createWorkspaceForWorktree( input: CreateWorktreeWorkspaceInput, ): Promise { @@ -214,31 +249,46 @@ export function createWorkspaceProvisioningService(deps: { const repoRoot = resolve(input.repoRoot); const cwd = resolve(input.cwd); const worktreeRoot = resolve(input.worktreeRoot); - const project = await resolveSourceProjectForWorktree({ - sourceCwd, - projectId: input.projectId, - repoRoot, - }); - const timestamp = new Date().toISOString(); - const workspace = createPersistedWorkspaceRecord({ - workspaceId: generateWorkspaceId(), - projectId: project.projectId, - ...initialWorkspacePlacement({ - source: "created_worktree", - cwd, - worktreeRoot, - branch: input.branch, - baseBranch: input.baseBranch, - mainRepoRoot: repoRoot, - }), - title: input.title, - createdAt: timestamp, - updatedAt: timestamp, - }); - await workspaceRegistry.upsert(workspace, { - expectsInitialAgent: input.expectsInitialAgent, + return lifecycleCoordinator.runDirectoryExclusive(worktreeRoot, async () => { + const matchesWorktreeRoot = createRealpathAwarePathMatcher(worktreeRoot); + const activeOwner = (await workspaceRegistry.list()) + .filter( + (workspace) => + !workspace.archivedAt && matchesWorktreeRoot(workspace.worktreeRoot ?? workspace.cwd), + ) + .sort( + (left, right) => + Date.parse(left.createdAt) - Date.parse(right.createdAt) || + left.workspaceId.localeCompare(right.workspaceId), + )[0]; + if (activeOwner) return activeOwner; + + const project = await resolveSourceProjectForWorktree({ + sourceCwd, + projectId: input.projectId, + repoRoot, + }); + const timestamp = new Date().toISOString(); + const workspace = createPersistedWorkspaceRecord({ + workspaceId: generateWorkspaceId(), + projectId: project.projectId, + ...initialWorkspacePlacement({ + source: "created_worktree", + cwd, + worktreeRoot, + branch: input.branch, + baseBranch: input.baseBranch, + mainRepoRoot: repoRoot, + }), + title: input.title, + createdAt: timestamp, + updatedAt: timestamp, + }); + await workspaceRegistry.upsert(workspace, { + expectsInitialAgent: input.expectsInitialAgent, + }); + return workspace; }); - return workspace; } async function resolveSourceProjectForWorktree(input: { @@ -284,31 +334,36 @@ export function createWorkspaceProvisioningService(deps: { async function findOrCreateWorkspaceForDirectory(cwd: string): Promise { const normalizedCwd = resolve(cwd); - const workspaces = await workspaceRegistry.list(); - const active = workspaces - .filter( - (workspace) => !workspace.archivedAt && areEquivalentPaths(workspace.cwd, normalizedCwd), - ) - .sort( - (left, right) => - Date.parse(left.createdAt) - Date.parse(right.createdAt) || - left.workspaceId.localeCompare(right.workspaceId), - )[0]; - if (active) return refreshWorkspaceRecord(active); - const archived = workspaces - .filter( - (workspace) => workspace.archivedAt && areEquivalentPaths(workspace.cwd, normalizedCwd), - ) - .sort( - (left, right) => - Date.parse(left.createdAt) - Date.parse(right.createdAt) || - left.workspaceId.localeCompare(right.workspaceId), - )[0]; - if (archived) { - const project = await projectRegistry.get(archived.projectId); - if (project && !project.archivedAt) return ensureWorkspaceRecordUnarchived(archived); - } - return createWorkspaceForDirectory(normalizedCwd); + return lifecycleCoordinator.runDirectoryExclusive(normalizedCwd, async () => { + await requireDirectoryAfterLifecycleWait(normalizedCwd); + const workspaces = await workspaceRegistry.list(); + const active = workspaces + .filter( + (workspace) => !workspace.archivedAt && areEquivalentPaths(workspace.cwd, normalizedCwd), + ) + .sort( + (left, right) => + Date.parse(left.createdAt) - Date.parse(right.createdAt) || + left.workspaceId.localeCompare(right.workspaceId), + )[0]; + if (active) return refreshWorkspaceRecord(active); + const archived = workspaces + .filter( + (workspace) => workspace.archivedAt && areEquivalentPaths(workspace.cwd, normalizedCwd), + ) + .sort( + (left, right) => + Date.parse(left.createdAt) - Date.parse(right.createdAt) || + left.workspaceId.localeCompare(right.workspaceId), + )[0]; + if (archived) { + const project = await projectRegistry.get(archived.projectId); + if (project && !project.archivedAt) { + return ensureWorkspaceRecordUnarchivedUnlocked(archived); + } + } + return createWorkspaceForDirectoryUnlocked(normalizedCwd); + }); } async function resolveOrCreateWorkspaceIdForCreateAgent( @@ -341,6 +396,16 @@ export function createWorkspaceProvisioningService(deps: { async function ensureWorkspaceRecordUnarchived( workspace: PersistedWorkspaceRecord, + ): Promise { + const normalizedCwd = resolve(workspace.cwd); + return lifecycleCoordinator.runDirectoryExclusive(normalizedCwd, async () => { + await requireDirectoryAfterLifecycleWait(normalizedCwd); + return ensureWorkspaceRecordUnarchivedUnlocked(workspace); + }); + } + + async function ensureWorkspaceRecordUnarchivedUnlocked( + workspace: PersistedWorkspaceRecord, ): Promise { const project = await projectRegistry.get(workspace.projectId); if (!project) throw new Error(`Unknown project: ${workspace.projectId}`); diff --git a/packages/server/src/server/snapshot-mutation-ownership.test.ts b/packages/server/src/server/snapshot-mutation-ownership.test.ts index 3733643484..d3fde990f7 100644 --- a/packages/server/src/server/snapshot-mutation-ownership.test.ts +++ b/packages/server/src/server/snapshot-mutation-ownership.test.ts @@ -7,7 +7,10 @@ import { Session } from "./session.js"; import type { SessionOptions } from "./session.js"; import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js"; import { asInternals, createStub } from "./test-utils/class-mocks.js"; -import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { + createAgentLifecycleDispatchStub, + createProviderSnapshotManagerStub, +} from "./test-utils/session-stubs.js"; interface SessionInternals { archiveAgentForClose(agentId: string): Promise<{ archivedAt: string }>; @@ -118,6 +121,7 @@ describe("snapshot mutation ownership boundary", () => { applySnapshot: directStorageWrite, upsert: directStorageWrite, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: createStub({ subscribeToMutations: () => () => {}, initialize: async () => {}, diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index ed4708ec7b..64f8884410 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -6,6 +6,7 @@ import pino from "pino"; import { createPaseoDaemon, type PaseoDaemonConfig, + type PaseoDaemonDependencies, type PaseoOpenAIConfig, type PaseoSpeechConfig, } from "../bootstrap.js"; @@ -45,6 +46,7 @@ interface TestPaseoDaemonOptions { serviceProxy?: PaseoDaemonConfig["serviceProxy"]; webUi?: PaseoDaemonConfig["webUi"]; trustedProxies?: PaseoDaemonConfig["trustedProxies"]; + dependencies?: PaseoDaemonDependencies; } export interface TestPaseoDaemon { @@ -95,7 +97,9 @@ export async function createTestPaseoDaemon( const { config, paseoHomeRoot, paseoHome, staticDir } = await prepareTestDaemonConfig(options); const logger = options.logger ?? pino({ level: "silent" }); const daemon = await createPaseoDaemon(config, logger, { + ...options.dependencies, serverFeatureOverrides: { + ...options.dependencies?.serverFeatureOverrides, daemonStatusRpc: options.daemonStatusRpcCapability, relayConfig: options.relayConfigCapability, }, diff --git a/packages/server/src/server/test-utils/session-stubs.ts b/packages/server/src/server/test-utils/session-stubs.ts index f7ddbdd95d..124104510a 100644 --- a/packages/server/src/server/test-utils/session-stubs.ts +++ b/packages/server/src/server/test-utils/session-stubs.ts @@ -42,6 +42,21 @@ export function asAgentStorage(stub: { return createStub(stub); } +export function createAgentLifecycleDispatchStub(): SessionOptions["createAgentLifecycleDispatch"] { + return createStub({ + registerAutoArchive: () => ({ + settled: Promise.resolve("cancelled"), + cancel: async () => {}, + }), + registerAutoArchiveIfRequested: () => ({ + settled: Promise.resolve("cancelled"), + cancel: async () => {}, + }), + cleanupCreatedWorktreeAfterFailedAgentCreate: async () => {}, + recoverPendingAgentCreation: async () => {}, + }); +} + export function asDownloadTokenStore(): SessionOptions["downloadTokenStore"] { return createStub({}); } diff --git a/packages/server/src/server/websocket-server.browser-tools.test.ts b/packages/server/src/server/websocket-server.browser-tools.test.ts index 4ee596c420..b1ab6c8825 100644 --- a/packages/server/src/server/websocket-server.browser-tools.test.ts +++ b/packages/server/src/server/websocket-server.browser-tools.test.ts @@ -13,6 +13,7 @@ import { afterEach, describe, expect, it } from "vitest"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import { BrowserToolsBroker } from "./browser-tools/broker.js"; import type { CheckoutDiffManager } from "./checkout-diff-manager.js"; import type { FileBackedChatService } from "./chat/chat-service.js"; @@ -299,6 +300,7 @@ function createVoiceAssistantWebSocketServer(params: { null, { allowedOrigins: new Set(["*"]) }, createWorkspaceAutoNameStub(), + createStub({}), undefined, undefined, undefined, diff --git a/packages/server/src/server/websocket-server.notifications.test.ts b/packages/server/src/server/websocket-server.notifications.test.ts index f662a044a9..f7f982c9e1 100644 --- a/packages/server/src/server/websocket-server.notifications.test.ts +++ b/packages/server/src/server/websocket-server.notifications.test.ts @@ -3,6 +3,7 @@ import type { Server as HTTPServer } from "http"; import type pino from "pino"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { DaemonConfigStore } from "./daemon-config-store.js"; import type { FileBackedChatService } from "./chat/chat-service.js"; @@ -117,6 +118,7 @@ function createServer(agentManagerOverrides?: Record) { null, { allowedOrigins: new Set() }, createWorkspaceAutoNameStub(), + createStub({}), undefined, undefined, undefined, diff --git a/packages/server/src/server/websocket-server.relay-reconnect.test.ts b/packages/server/src/server/websocket-server.relay-reconnect.test.ts index 650e5150f4..0409e89894 100644 --- a/packages/server/src/server/websocket-server.relay-reconnect.test.ts +++ b/packages/server/src/server/websocket-server.relay-reconnect.test.ts @@ -3,6 +3,7 @@ import type { Server as HTTPServer } from "http"; import type pino from "pino"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { DaemonConfigStore } from "./daemon-config-store.js"; import type { FileBackedChatService } from "./chat/chat-service.js"; @@ -259,6 +260,7 @@ function createServer(options?: { null, { allowedOrigins: new Set() }, createWorkspaceAutoNameStub(), + createStub({}), undefined, speechReadiness ? { diff --git a/packages/server/src/server/websocket-server.terminal-notifications.test.ts b/packages/server/src/server/websocket-server.terminal-notifications.test.ts index 7d3cba294f..1443079f24 100644 --- a/packages/server/src/server/websocket-server.terminal-notifications.test.ts +++ b/packages/server/src/server/websocket-server.terminal-notifications.test.ts @@ -3,6 +3,7 @@ import type { Server as HTTPServer } from "http"; import type pino from "pino"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { DaemonConfigStore } from "./daemon-config-store.js"; import type { FileBackedChatService } from "./chat/chat-service.js"; @@ -148,6 +149,7 @@ function createServer(terminalManager: TerminalManager, workspaceRegistry?: Work null, { allowedOrigins: new Set() }, createWorkspaceAutoNameStub(), + createStub({}), undefined, undefined, terminalManager, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 5991101fdb..9cba086222 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -6,6 +6,7 @@ import { randomUUID } from "node:crypto"; import { monitorEventLoopDelay } from "node:perf_hooks"; import type { AgentManager, AgentMetricsSnapshot } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import type pino from "pino"; @@ -18,6 +19,7 @@ import type { CheckoutDiffManager, CheckoutDiffMetrics } from "./checkout-diff-m import type { DaemonConfigStore, MutableDaemonConfig } from "./daemon-config-store.js"; import { type ServerInfoStatusPayload, + type SessionInboundMessage, type SessionOutboundMessage, type WorkspaceSetupSnapshot, type WSHelloMessage, @@ -455,6 +457,51 @@ const WS_CLOSE_SERVER_SHUTDOWN = 1001; const WS_PROTOCOL_VERSION = 1; const WS_RUNTIME_METRICS_FLUSH_MS = 30_000; +const SHUTDOWN_TRACKED_SESSION_MUTATIONS = new Set([ + "agent.detach.request", + "agent.fork_context.request", + "agent.rewind.request", + "agent_permission_response", + "archive_agent_request", + "archive_workspace_request", + "cancel_agent_request", + "close_items_request", + "create_agent_request", + "create_paseo_worktree_request", + "create_terminal_request", + "delete_agent_request", + "hub.execution.agent.create.request", + "hub.execution.control.request", + "import_agent_request", + "kill_terminal_request", + "loop/run", + "loop/stop", + "open_project_request", + "paseo_worktree_archive_request", + "project.add.request", + "project.create_directory.request", + "project.github.clone.request", + "project.remove.request", + "refresh_agent_request", + "resume_agent_request", + "schedule/create", + "schedule/delete", + "schedule/pause", + "schedule/resume", + "schedule/run-once", + "schedule/update", + "send_agent_message_request", + "set_agent_feature_request", + "set_agent_mode_request", + "set_agent_model_request", + "set_agent_thinking_request", + "start_workspace_script_request", + "workspace.create.request", + "workspace.recovery.restore.request", + "workspace.script.start.request", + "workspace.script.stop.request", +]); + export class MissingDaemonVersionError extends Error { constructor() { super("VoiceAssistantWebSocketServer requires a non-empty daemonVersion."); @@ -499,6 +546,8 @@ export class VoiceAssistantWebSocketServer { private readonly wss: WebSocketServer; private readonly pendingConnections: Map = new Map(); private readonly sessions: Map = new Map(); + private readonly inFlightLifecycleMutations = new Set>(); + private acceptingLifecycleMutations = true; private readonly socketIdentities: Map = new Map(); private readonly externalSessionsByKey: Map = new Map(); private readonly serverId: string; @@ -506,6 +555,7 @@ export class VoiceAssistantWebSocketServer { private readonly daemonRuntimeConfig: DaemonRuntimeConfig | undefined; private readonly agentManager: AgentManager; private readonly agentStorage: AgentStorage; + private readonly createAgentLifecycleDispatch: CreateAgentLifecycleDispatch; private readonly projectRegistry: ProjectRegistry; private readonly workspaceRegistry: WorkspaceRegistry; private readonly chatService: FileBackedChatService; @@ -571,6 +621,7 @@ export class VoiceAssistantWebSocketServer { mcpBaseUrl: string | null, wsConfig: WebSocketServerConfig, workspaceAutoName: WorkspaceAutoName, + createAgentLifecycleDispatch: CreateAgentLifecycleDispatch, auth?: DaemonAuthConfig, speech?: SpeechService | null, terminalManager?: TerminalManager | null, @@ -617,6 +668,7 @@ export class VoiceAssistantWebSocketServer { this.hubRelationships = hubRelationships ?? null; this.agentManager = agentManager; this.agentStorage = agentStorage; + this.createAgentLifecycleDispatch = createAgentLifecycleDispatch; this.projectRegistry = projectRegistry ?? createNoopProjectRegistry(); this.workspaceRegistry = workspaceRegistry ?? createNoopWorkspaceRegistry(); const requiredServices = requireWebSocketServices({ @@ -947,12 +999,16 @@ export class VoiceAssistantWebSocketServer { connectionLogger.info("Hub session attached"); } - public prepareForShutdown(): void { + public async prepareForShutdown(): Promise { this.acceptingConnections = false; + this.acceptingLifecycleMutations = false; + while (this.inFlightLifecycleMutations.size > 0) { + await Promise.allSettled(Array.from(this.inFlightLifecycleMutations)); + } } public async close(): Promise { - this.prepareForShutdown(); + await this.prepareForShutdown(); this.unsubscribeSpeechReadiness?.(); this.unsubscribeSpeechReadiness = null; this.unsubscribeDaemonConfigChange?.(); @@ -1325,6 +1381,7 @@ export class VoiceAssistantWebSocketServer { worktreesRoot: this.worktreesRoot, agentManager: this.agentManager, agentStorage: this.agentStorage, + createAgentLifecycleDispatch: this.createAgentLifecycleDispatch, projectRegistry: this.projectRegistry, workspaceRegistry: this.workspaceRegistry, chatService: this.chatService, @@ -2059,9 +2116,24 @@ export class VoiceAssistantWebSocketServer { } if (message.type === "session") { - void this.dispatchSessionMessage(ws, activeConnection, message).catch((error: unknown) => { - this.handleRawMessageError({ ws, data, error, log: activeConnection.connectionLogger }); - }); + const trackedForShutdown = SHUTDOWN_TRACKED_SESSION_MUTATIONS.has(message.message.type); + if (trackedForShutdown && !this.acceptingLifecycleMutations) { + ws.close(WS_CLOSE_SERVER_SHUTDOWN, "Server shutting down"); + return; + } + const dispatch = this.dispatchSessionMessage(ws, activeConnection, message); + if (trackedForShutdown) { + this.inFlightLifecycleMutations.add(dispatch); + } + void dispatch + .catch((error: unknown) => { + this.handleRawMessageError({ ws, data, error, log: activeConnection.connectionLogger }); + }) + .finally(() => { + if (trackedForShutdown) { + this.inFlightLifecycleMutations.delete(dispatch); + } + }); } } catch (error) { this.handleRawMessageError({ ws, data, error, log }); diff --git a/packages/server/src/server/wire-compat.test.ts b/packages/server/src/server/wire-compat.test.ts index 85e402e61c..9119d7b359 100644 --- a/packages/server/src/server/wire-compat.test.ts +++ b/packages/server/src/server/wire-compat.test.ts @@ -11,7 +11,10 @@ import { type SessionOutboundMessage, } from "@getpaseo/protocol/messages"; import { Session, type SessionOptions } from "./session.js"; -import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { + createAgentLifecycleDispatchStub, + createProviderSnapshotManagerStub, +} from "./test-utils/session-stubs.js"; import type { AgentTimelineRow } from "./agent/agent-manager.js"; import { InMemoryAgentTimelineStore } from "./agent/agent-timeline-store.js"; import type { AgentTimelineFetchOptions } from "./agent/agent-timeline-store-types.js"; @@ -215,6 +218,7 @@ function createSessionForWireCompatTest(options?: { options?.rows ?? rows, ) as unknown as SessionOptions["agentManager"], agentStorage: new EmptyAgentStorage() as unknown as SessionOptions["agentStorage"], + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: new EmptyProjectRegistry() as unknown as SessionOptions["projectRegistry"], workspaceRegistry: new EmptyWorkspaceRegistry() as unknown as SessionOptions["workspaceRegistry"], diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 4adbadb601..2ac415ead0 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -1,5 +1,13 @@ import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import pino, { type Logger } from "pino"; @@ -8,6 +16,7 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import type { ForgeService } from "../services/forge-service.js"; import { createRealpathAwarePathMatcher } from "../utils/path.js"; import { createWorktree, type WorktreeConfig } from "../utils/worktree.js"; +import { readPaseoWorktreeIncarnationId } from "../utils/worktree-metadata.js"; import type { ManagedAgent } from "./agent/agent-manager.js"; import type { AgentStorage, StoredAgentRecord } from "./agent/agent-storage.js"; import type { WorkspaceGitService } from "./workspace-git-service.js"; @@ -16,8 +25,20 @@ import { type ActiveWorkspaceRef, type ArchiveDependencies, type ArchiveResult, + requireActiveWorkspaceForArchive, + requireArchiveCleanupComplete, + retryPendingWorkspaceCleanup, resolveWorkspaceIdAtPath, + WorkspaceArchiveTargetNotFoundError, } from "./workspace-archive-service.js"; +import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service.js"; +import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; +import { + createPersistedWorkspaceRecord, + FileBackedProjectRegistry, + FileBackedWorkspaceRegistry, +} from "./workspace-registry.js"; +import { createWorkspaceProvisioningService } from "./session/workspace-provisioning/workspace-provisioning-service.js"; const cleanupPaths: string[] = []; @@ -95,6 +116,19 @@ function createGitRepo(): { tempDir: string; repoDir: string } { }); return { tempDir, repoDir }; } +function removeManagedAgentById(agents: ManagedAgent[], agentId: string): void { + const index = agents.findIndex((agent) => agent.id === agentId); + if (index !== -1) { + agents.splice(index, 1); + } +} + +function findStoredAgentById( + records: StoredAgentRecord[], + agentId: string, +): StoredAgentRecord | undefined { + return records.find((record) => record.id === agentId); +} async function createPaseoOwnedWorktree( repoDir: string, @@ -187,6 +221,304 @@ function assertArchiveResult( } describe("archiveByScope", () => { + test("treats persisted cleanup as a retryable workspace archive target", async () => { + const persisted = createPersistedWorkspaceRecord({ + workspaceId: "ws-cleanup-retry-target", + projectId: "project-cleanup-retry-target", + cwd: "/tmp/cleanup-retry-target", + kind: "worktree", + displayName: "Cleanup retry target", + createdAt: "2026-07-31T00:00:00.000Z", + updatedAt: "2026-07-31T00:00:00.000Z", + archivedAt: "2026-07-31T00:01:00.000Z", + cleanupPending: { + directoryPath: "/tmp/cleanup-retry-target", + teardownCwd: "/tmp/cleanup-retry-target", + mainRepoRoot: "/tmp/repo", + paseoWorktreesRoot: "/tmp/worktrees", + worktreeIncarnationId: "incarnation-retry", + }, + }); + + await expect( + requireActiveWorkspaceForArchive( + { + listActiveWorkspaces: async () => [], + workspaceRegistry: { get: async () => persisted }, + }, + persisted.workspaceId, + ), + ).resolves.toBe(persisted); + }); + + test("waits for registered setup before archiving its workspace", async () => { + const { tempDir } = createGitRepo(); + const workspaceId = "ws-setup-race"; + const workspaceCwd = path.join(tempDir, "local-checkout"); + mkdirSync(workspaceCwd); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + let finishSetup: (() => void) | undefined; + const setupTask = new Promise((resolveSetup) => { + finishSetup = resolveSetup; + }); + lifecycleCoordinator.trackWorkspaceSetup(workspaceId, setupTask); + let markWaitStarted: (() => void) | undefined; + const waitStarted = new Promise((resolveWaitStarted) => { + markWaitStarted = resolveWaitStarted; + }); + const waitForWorkspaceSetups = + lifecycleCoordinator.waitForWorkspaceSetups.bind(lifecycleCoordinator); + vi.spyOn(lifecycleCoordinator, "waitForWorkspaceSetups").mockImplementation( + async (workspaceIds) => { + markWaitStarted?.(); + await waitForWorkspaceSetups(workspaceIds); + }, + ); + const deps = createArchiveDeps({ + paseoHome: path.join(tempDir, ".paseo"), + activeWorkspaces: [{ workspaceId, cwd: workspaceCwd, kind: "local_checkout" }], + }); + deps.lifecycleCoordinator = lifecycleCoordinator; + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + + const archiveTask = archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-setup-race", + }); + + await waitStarted; + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + + finishSetup?.(); + await expect(archiveTask).resolves.toMatchObject({ + archivedWorkspaceIds: [workspaceId], + }); + }); + + test("rejects truthfully when an explicit workspace vanishes during final refresh", async () => { + const { tempDir } = createGitRepo(); + const workspaceId = "ws-vanished-before-archive"; + const workspaceCwd = path.join(tempDir, "local-checkout"); + mkdirSync(workspaceCwd); + const deps = createArchiveDeps({ + paseoHome: path.join(tempDir, ".paseo"), + activeWorkspaces: [{ workspaceId, cwd: workspaceCwd, kind: "local_checkout" }], + }); + let activeReadCount = 0; + deps.listActiveWorkspaces = async () => { + activeReadCount += 1; + return activeReadCount === 1 + ? [{ workspaceId, cwd: workspaceCwd, kind: "local_checkout" }] + : []; + }; + deps.workspaceRegistry = { get: async () => null }; + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-vanished-before-archive", + }), + ).rejects.toBeInstanceOf(WorkspaceArchiveTargetNotFoundError); + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + }); + + test("waits for ownership on a sibling discovered during final target refresh", async () => { + const { tempDir } = createGitRepo(); + const workspaceCwd = path.join(tempDir, "shared-checkout"); + mkdirSync(workspaceCwd); + const workspaceA = "ws-refresh-closure-a"; + const workspaceB = "ws-refresh-closure-b"; + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const ownershipReservation = lifecycleCoordinator.reserveWorkspaceOwnershipMutation(workspaceB); + const deps = createArchiveDeps({ + paseoHome: path.join(tempDir, ".paseo"), + activeWorkspaces: [{ workspaceId: workspaceA, cwd: workspaceCwd, kind: "local_checkout" }], + }); + deps.lifecycleCoordinator = lifecycleCoordinator; + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + const listActiveWorkspaces = deps.listActiveWorkspaces; + let activeReadCount = 0; + deps.listActiveWorkspaces = async () => { + activeReadCount += 1; + if (activeReadCount === 2) { + deps.activeWorkspaces.push({ + workspaceId: workspaceB, + cwd: workspaceCwd, + kind: "local_checkout", + }); + } + return listActiveWorkspaces(); + }; + let markLateOwnershipWaitStarted: (() => void) | undefined; + const lateOwnershipWaitStarted = new Promise((resolveWait) => { + markLateOwnershipWaitStarted = resolveWait; + }); + const waitForWorkspaceOwnershipMutations = + lifecycleCoordinator.waitForWorkspaceOwnershipMutations.bind(lifecycleCoordinator); + vi.spyOn(lifecycleCoordinator, "waitForWorkspaceOwnershipMutations").mockImplementation( + async (workspaceIds) => { + if (Array.from(workspaceIds).includes(workspaceB)) { + markLateOwnershipWaitStarted?.(); + } + await waitForWorkspaceOwnershipMutations(workspaceIds); + }, + ); + + const archiveTask = archiveByScope(deps, { + scope: { kind: "worktree", targetPath: workspaceCwd }, + requestId: "req-refresh-closure", + }); + + await lateOwnershipWaitStarted; + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + ownershipReservation.release(); + + await expect(archiveTask).resolves.toMatchObject({ + archivedWorkspaceIds: expect.arrayContaining([workspaceA, workspaceB]), + }); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(2); + }); + + test("coalesces simultaneous archive requests for the same backing directory", async () => { + const { tempDir } = createGitRepo(); + const workspaceId = "ws-simultaneous"; + const workspaceCwd = path.join(tempDir, "local-checkout"); + mkdirSync(workspaceCwd); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const deps = createArchiveDeps({ + paseoHome: path.join(tempDir, ".paseo"), + activeWorkspaces: [{ workspaceId, cwd: workspaceCwd, kind: "local_checkout" }], + }); + deps.lifecycleCoordinator = lifecycleCoordinator; + const originalArchiveWorkspaceRecord = deps.archiveWorkspaceRecord; + let releaseArchive: (() => void) | undefined; + const archiveGate = new Promise((resolveArchive) => { + releaseArchive = resolveArchive; + }); + let markArchiveStarted: (() => void) | undefined; + const archiveStarted = new Promise((resolveStarted) => { + markArchiveStarted = resolveStarted; + }); + deps.archiveWorkspaceRecord = vi.fn(async (id: string) => { + markArchiveStarted?.(); + await archiveGate; + await originalArchiveWorkspaceRecord(id); + }); + + const first = archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-simultaneous-first", + }); + const second = archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-simultaneous-second", + }); + await archiveStarted; + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(1); + expect(() => lifecycleCoordinator.reserveWorkspaceOwnershipMutation(workspaceId)).toThrow( + `Workspace ${workspaceId} is being archived`, + ); + + releaseArchive?.(); + const [firstResult, secondResult] = await Promise.all([first, second]); + expect(secondResult).toEqual(firstResult); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(1); + const postArchiveReservation = + lifecycleCoordinator.reserveWorkspaceOwnershipMutation(workspaceId); + postArchiveReservation.release(); + }); + + test("archives sibling workspace records separately while serializing shared cleanup", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "concurrent-siblings"); + const workspaceA = "ws-concurrent-sibling-a"; + const workspaceB = "ws-concurrent-sibling-b"; + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { workspaceId: workspaceA, cwd: worktree.worktreePath, kind: "worktree" }, + { workspaceId: workspaceB, cwd: worktree.worktreePath, kind: "worktree" }, + ], + }); + deps.lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + + const [resultA, resultB] = await Promise.all([ + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId: workspaceA }, + requestId: "req-concurrent-sibling-a", + }), + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId: workspaceB }, + requestId: "req-concurrent-sibling-b", + }), + ]); + + expect(resultA.archivedWorkspaceIds).toEqual([workspaceA]); + expect(resultB.archivedWorkspaceIds).toEqual([workspaceB]); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(2); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledWith(workspaceA); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledWith(workspaceB); + expect(existsSync(worktree.worktreePath)).toBe(false); + }); + + test("holds final owner recheck and deletion behind sibling setup reservation", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "reserved-sibling"); + const workspaceId = "ws-reserved-target"; + const siblingWorkspaceId = "ws-reserved-sibling"; + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const reservation = lifecycleCoordinator.reserveWorkspaceSetup( + siblingWorkspaceId, + worktree.worktreePath, + ); + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId, + cwd: worktree.worktreePath, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + }, + ], + }); + deps.lifecycleCoordinator = lifecycleCoordinator; + const originalArchiveWorkspaceRecord = deps.archiveWorkspaceRecord; + let markRecordArchived: (() => void) | undefined; + const recordArchived = new Promise((resolveArchived) => { + markRecordArchived = resolveArchived; + }); + deps.archiveWorkspaceRecord = async (id: string) => { + await originalArchiveWorkspaceRecord(id); + deps.activeWorkspaces.push({ + workspaceId: siblingWorkspaceId, + cwd: path.join(worktree.worktreePath, "packages", "app"), + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + }); + markRecordArchived?.(); + }; + + const archiveTask = archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-reserved-sibling", + }); + await recordArchived; + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(existsSync(worktree.worktreePath)).toBe(true); + + reservation.release(); + const result = await archiveTask; + expect(result.removedDirectory).toBe(false); + expect(existsSync(worktree.worktreePath)).toBe(true); + }); + test("workspace scope archives the record and removes the directory on last reference", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -217,7 +549,7 @@ describe("archiveByScope", () => { expect(existsSync(worktree.worktreePath)).toBe(false); }); - test("workspace scope runs teardown while keeping a directory referenced by a sibling", async () => { + test("workspace scope skips teardown while a sibling still references the directory", async () => { const { tempDir, repoDir } = createGitRepo(); writeFileSync( path.join(repoDir, "paseo.json"), @@ -258,7 +590,7 @@ describe("archiveByScope", () => { removedDirectory: false, }); expect(existsSync(worktree.worktreePath)).toBe(true); - expect(readFileSync(path.join(repoDir, "shared-teardown.log"), "utf8")).toBe("ok"); + expect(existsSync(path.join(repoDir, "shared-teardown.log"))).toBe(false); }); test("workspace scope keeps a worktree for an active workspace in a subdirectory", async () => { @@ -303,6 +635,44 @@ describe("archiveByScope", () => { expect(existsSync(worktree.worktreePath)).toBe(true); }); + test("rechecks owners after archival before removing the backing directory", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "late-owner"); + const workspaceId = "ws-late-owner-target"; + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId, + cwd: worktree.worktreePath, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + }, + ], + }); + const originalArchiveWorkspaceRecord = deps.archiveWorkspaceRecord; + deps.archiveWorkspaceRecord = async (id: string) => { + await originalArchiveWorkspaceRecord(id); + deps.activeWorkspaces.push({ + workspaceId: "ws-late-owner-sibling", + cwd: path.join(worktree.worktreePath, "packages", "app"), + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + }); + }; + + const result = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-late-owner", + }); + + expect(result.removedDirectory).toBe(false); + expect(existsSync(worktree.worktreePath)).toBe(true); + }); + test("archiving a subdirectory workspace keeps its active worktree root", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -527,18 +897,620 @@ describe("archiveByScope", () => { return originalArchiveWorkspaceRecord(workspaceId); }; - const result = await archiveByScope(deps, { + await expect( + archiveByScope(deps, { + scope: { kind: "worktree", targetPath: worktree.worktreePath }, + requestId: "req-partial-failure", + }), + ).rejects.toThrow("Failed to archive one or more workspaces"); + + expect((await deps.listActiveWorkspaces()).map((workspace) => workspace.workspaceId)).toEqual([ + workspaceA, + ]); + expect(existsSync(worktree.worktreePath)).toBe(true); + }); + + test("retries persisted physical cleanup after the workspace record is archived", async () => { + const { tempDir, repoDir } = createGitRepo(); + writeFileSync( + path.join(repoDir, "paseo.json"), + JSON.stringify({ + worktree: { + teardown: [ + "node -e \"const fs=require('fs');const marker=process.env.PASEO_SOURCE_CHECKOUT_PATH+'/cleanup-retry.marker';if(!fs.existsSync(marker)){fs.writeFileSync(marker,'retry');process.exit(1)}\"", + ], + }, + }), + ); + execFileSync("git", ["add", "."], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "retry cleanup"], { + cwd: repoDir, + stdio: "pipe", + }); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "cleanup-retry"); + const workspaceId = "ws-cleanup-retry"; + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId, + projectId: "project-cleanup-retry", + cwd: worktree.worktreePath, + kind: "worktree", + displayName: "Cleanup retry", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId, + cwd: worktree.worktreePath, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + ], + }); + deps.workspaceRegistry = registry; + const archiveActiveRecord = deps.archiveWorkspaceRecord; + deps.archiveWorkspaceRecord = async (id: string) => { + await archiveActiveRecord(id); + await registry.archive(id, new Date().toISOString()); + }; + + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-cleanup-retry-first", + }), + ).rejects.toThrow("Worktree teardown command failed"); + expect((await registry.get(workspaceId))?.archivedAt).not.toBeNull(); + expect((await registry.get(workspaceId))?.cleanupPending).toMatchObject({ + directoryPath: worktree.worktreePath, + teardownCwd: worktree.worktreePath, + }); + expect(existsSync(worktree.worktreePath)).toBe(true); + + const retryResult = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-cleanup-retry-second", + }); + expect(retryResult.removedDirectory).toBe(true); + expect(existsSync(worktree.worktreePath)).toBe(false); + expect((await registry.get(workspaceId))?.cleanupPending).toBeNull(); + }); + + test("a replacement incarnation archives before an old cleanup retry without cross-blocking", async () => { + const { tempDir, repoDir } = createGitRepo(); + writeFileSync( + path.join(repoDir, "paseo.json"), + JSON.stringify({ worktree: { teardown: ['node -e "process.exit(1)"'] } }), + ); + execFileSync("git", ["add", "."], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "fail teardown"], { + cwd: repoDir, + stdio: "pipe", + }); + const paseoHome = path.join(tempDir, ".paseo"); + const slug = "reused-incarnation"; + const firstWorktree = await createPaseoOwnedWorktree(repoDir, paseoHome, slug); + const workspaceId = "ws-reused-incarnation-a"; + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId, + projectId: "project-reused-incarnation", + cwd: firstWorktree.worktreePath, + kind: "worktree", + displayName: "Old incarnation", + worktreeRoot: firstWorktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId, + cwd: firstWorktree.worktreePath, + kind: "worktree", + worktreeRoot: firstWorktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + ], + }); + deps.workspaceRegistry = registry; + const archiveActiveRecord = deps.archiveWorkspaceRecord; + deps.archiveWorkspaceRecord = async (id: string) => { + await archiveActiveRecord(id); + await registry.archive(id, new Date().toISOString()); + }; + + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-reused-incarnation-first", + }), + ).rejects.toThrow("Worktree teardown command failed"); + expect((await registry.get(workspaceId))?.cleanupPending).not.toBeNull(); + + execFileSync("git", ["worktree", "remove", firstWorktree.worktreePath, "--force"], { + cwd: repoDir, + stdio: "pipe", + }); + execFileSync("git", ["branch", "-D", slug], { cwd: repoDir, stdio: "pipe" }); + writeFileSync(path.join(repoDir, "paseo.json"), JSON.stringify({ worktree: { teardown: [] } })); + execFileSync("git", ["add", "paseo.json"], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "repair teardown"], { + cwd: repoDir, + stdio: "pipe", + }); + const replacementWorktree = await createPaseoOwnedWorktree(repoDir, paseoHome, slug); + const replacementWorkspaceId = "ws-reused-incarnation-b"; + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId: replacementWorkspaceId, + projectId: "project-reused-incarnation", + cwd: replacementWorktree.worktreePath, + kind: "worktree", + displayName: "Replacement incarnation", + worktreeRoot: replacementWorktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + deps.activeWorkspaces.push({ + workspaceId: replacementWorkspaceId, + cwd: replacementWorktree.worktreePath, + kind: "worktree", + worktreeRoot: replacementWorktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }); + + const replacementResult = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId: replacementWorkspaceId }, + requestId: "req-reused-incarnation-replacement", + }); + expect(replacementResult.removedDirectory).toBe(true); + expect(existsSync(replacementWorktree.worktreePath)).toBe(false); + expect((await registry.get(replacementWorkspaceId))?.cleanupPending).toBeNull(); + expect((await registry.get(workspaceId))?.cleanupPending).not.toBeNull(); + + const retryResult = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-reused-incarnation-retry", + }); + expect(retryResult.removedDirectory).toBe(false); + expect((await registry.get(workspaceId))?.cleanupPending).toBeNull(); + }); + + test("a partial worktree retry preserves every sibling teardown obligation", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "partial-sibling-retry"); + const nestedCwd = path.join(worktree.worktreePath, "packages", "app"); + mkdirSync(nestedCwd, { recursive: true }); + const workspaceA = "ws-partial-retry-a"; + const workspaceB = "ws-partial-retry-b"; + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + for (const [workspaceId, cwd] of [ + [workspaceA, worktree.worktreePath], + [workspaceB, nestedCwd], + ] as const) { + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId, + projectId: "project-partial-retry", + cwd, + kind: "worktree", + displayName: workspaceId, + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + } + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId: workspaceA, + cwd: worktree.worktreePath, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + { + workspaceId: workspaceB, + cwd: nestedCwd, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + ], + }); + deps.workspaceRegistry = registry; + const archiveActiveRecord = deps.archiveWorkspaceRecord; + let rejectWorkspaceBOnce = true; + deps.archiveWorkspaceRecord = async (workspaceId: string) => { + if (workspaceId === workspaceB && rejectWorkspaceBOnce) { + rejectWorkspaceBOnce = false; + throw new Error("registry archive failed once"); + } + await archiveActiveRecord(workspaceId); + await registry.archive(workspaceId, new Date().toISOString()); + }; + + await expect( + archiveByScope(deps, { + scope: { kind: "worktree", targetPath: worktree.worktreePath }, + requestId: "req-partial-retry-first", + }), + ).rejects.toThrow("Failed to archive one or more workspaces"); + expect((await registry.get(workspaceA))?.cleanupPending).not.toBeNull(); + expect((await registry.get(workspaceB))?.archivedAt).toBeNull(); + expect(existsSync(worktree.worktreePath)).toBe(true); + + const retry = await archiveByScope(deps, { scope: { kind: "worktree", targetPath: worktree.worktreePath }, - requestId: "req-partial-failure", + requestId: "req-partial-retry-second", + }); + + expect(retry.removedDirectory).toBe(true); + expect((await registry.get(workspaceA))?.cleanupPending).toBeNull(); + expect((await registry.get(workspaceB))?.cleanupPending).toBeNull(); + expect(existsSync(worktree.worktreePath)).toBe(false); + }); + + test("startup retry completes archived schedule cleanup without losing sibling teardown obligations", async () => { + const { tempDir, repoDir } = createGitRepo(); + const nestedRelative = path.join("packages", "app"); + const sourceNested = path.join(repoDir, nestedRelative); + mkdirSync(sourceNested, { recursive: true }); + writeFileSync( + path.join(repoDir, "paseo.json"), + JSON.stringify({ + worktree: { + teardown: [ + "node -e \"require('fs').writeFileSync(process.env.PASEO_SOURCE_CHECKOUT_PATH+'/retry-root.log','root')\"", + ], + }, + }), + ); + writeFileSync( + path.join(sourceNested, "paseo.json"), + JSON.stringify({ + worktree: { + teardown: [ + "node -e \"require('fs').writeFileSync(process.env.PASEO_SOURCE_CHECKOUT_PATH+'/retry-nested.log','nested')\"", + ], + }, + }), + ); + execFileSync("git", ["add", "."], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "retry teardown"], { + cwd: repoDir, + stdio: "pipe", + }); + + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "schedule-cleanup-retry"); + const nestedCwd = path.join(worktree.worktreePath, nestedRelative); + const incarnationId = readPaseoWorktreeIncarnationId(worktree.worktreePath); + expect(incarnationId).toBeTruthy(); + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + for (const [workspaceId, teardownCwd] of [ + ["ws-completed-schedule-root", worktree.worktreePath], + ["ws-completed-schedule-nested", nestedCwd], + ] as const) { + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId, + projectId: "project-completed-schedule-cleanup", + cwd: teardownCwd, + kind: "worktree", + displayName: workspaceId, + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + archivedAt: timestamp, + cleanupPending: { + directoryPath: worktree.worktreePath, + teardownCwd, + mainRepoRoot: repoDir, + paseoWorktreesRoot: null, + worktreeIncarnationId: incarnationId, + }, + }), + ); + } + + const deps = createArchiveDeps({ paseoHome, activeWorkspaces: [] }); + deps.workspaceRegistry = registry; + const service = new WorkspaceCleanupRetryService({ + workspaceRegistry: registry, + retryWorktreeCleanup: async (target, signal) => { + requireArchiveCleanupComplete( + await retryPendingWorkspaceCleanup(deps, { + directoryPath: target.directoryPath, + worktreeIncarnationId: target.worktreeIncarnationId, + requestId: "startup-schedule-cleanup-retry", + signal, + }), + "Startup schedule cleanup retry", + ); + }, + logger: createLogger(), + idlePollMs: 60_000, + }); + + await service.start(); + await vi.waitFor(async () => { + expect((await registry.get("ws-completed-schedule-root"))?.cleanupPending).toBeNull(); + expect((await registry.get("ws-completed-schedule-nested"))?.cleanupPending).toBeNull(); + }); + await service.stop(); + + expect(readFileSync(path.join(repoDir, "retry-root.log"), "utf8")).toBe("root"); + expect(readFileSync(path.join(repoDir, "retry-nested.log"), "utf8")).toBe("nested"); + expect(existsSync(worktree.worktreePath)).toBe(false); + expect((await registry.get("ws-completed-schedule-root"))?.cleanupPending).toBeNull(); + expect((await registry.get("ws-completed-schedule-nested"))?.cleanupPending).toBeNull(); + }); + + test("cleanup-only retry never archives a live replacement at a reused path", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const replacement = await createPaseoOwnedWorktree(repoDir, paseoHome, "reused-cleanup-path"); + const replacementIncarnation = readPaseoWorktreeIncarnationId(replacement.worktreePath); + expect(replacementIncarnation).toBeTruthy(); + const staleWorkspaceId = "ws-stale-cleanup-owner"; + const replacementWorkspaceId = "ws-live-replacement-owner"; + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId: staleWorkspaceId, + projectId: "project-reused-cleanup-path", + cwd: replacement.worktreePath, + kind: "worktree", + displayName: "Stale cleanup owner", + worktreeRoot: replacement.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + archivedAt: timestamp, + cleanupPending: { + directoryPath: replacement.worktreePath, + teardownCwd: replacement.worktreePath, + mainRepoRoot: repoDir, + paseoWorktreesRoot: null, + worktreeIncarnationId: "incarnation-before-replacement", + }, + }), + ); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId: replacementWorkspaceId, + projectId: "project-reused-cleanup-path", + cwd: replacement.worktreePath, + kind: "worktree", + displayName: "Live replacement owner", + worktreeRoot: replacement.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId: replacementWorkspaceId, + cwd: replacement.worktreePath, + kind: "worktree", + worktreeRoot: replacement.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + ], + }); + deps.workspaceRegistry = registry; + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + deps.killTerminalsForWorkspace = vi.fn(async () => undefined); + + const result = await retryPendingWorkspaceCleanup(deps, { + directoryPath: replacement.worktreePath, + worktreeIncarnationId: "incarnation-before-replacement", + requestId: "stale-cleanup-live-replacement", }); - expect(result.archivedWorkspaceIds).toEqual([workspaceB]); - expect(result.archivedWorkspaceIds).not.toContain(workspaceA); expect(result.removedDirectory).toBe(false); + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + expect(deps.killTerminalsForWorkspace).not.toHaveBeenCalled(); + expect(existsSync(replacement.worktreePath)).toBe(true); + expect((await registry.get(replacementWorkspaceId))?.archivedAt).toBeNull(); + expect((await registry.get(staleWorkspaceId))?.cleanupPending).toBeNull(); + }); + + test("directory workspace creation waits for cleanup and revalidates after final owner read", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "cleanup-create-race"); + const equivalentWorktreePath = path.join(tempDir, "cleanup-create-race-alias"); + symlinkSync(worktree.worktreePath, equivalentWorktreePath, "dir"); + const incarnationId = readPaseoWorktreeIncarnationId(worktree.worktreePath); + expect(incarnationId).toBeTruthy(); + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + const projectRegistry = new FileBackedProjectRegistry( + path.join(tempDir, "projects.json"), + createLogger(), + ); + await registry.initialize(); + await projectRegistry.initialize(); + const timestamp = new Date().toISOString(); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId: "ws-cleanup-create-race-stale", + projectId: "project-cleanup-create-race", + cwd: worktree.worktreePath, + kind: "worktree", + displayName: "Archived cleanup owner", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + archivedAt: timestamp, + cleanupPending: { + directoryPath: worktree.worktreePath, + teardownCwd: worktree.worktreePath, + mainRepoRoot: repoDir, + paseoWorktreesRoot: null, + worktreeIncarnationId: incarnationId, + }, + }), + ); + + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + let resolveFinalOwnerReadStarted: (() => void) | null = null; + const finalOwnerReadStarted = new Promise((resolvePromise) => { + resolveFinalOwnerReadStarted = resolvePromise; + }); + let allowFinalOwnerRead: (() => void) | null = null; + const finalOwnerReadAllowed = new Promise((resolvePromise) => { + allowFinalOwnerRead = resolvePromise; + }); + let ownerReads = 0; + const deps = createArchiveDeps({ paseoHome, activeWorkspaces: [] }); + deps.lifecycleCoordinator = lifecycleCoordinator; + deps.workspaceRegistry = registry; + deps.listActiveWorkspaces = vi.fn(async () => { + ownerReads += 1; + if (ownerReads === 2) { + resolveFinalOwnerReadStarted?.(); + await finalOwnerReadAllowed; + } + return []; + }); + const cleanup = retryPendingWorkspaceCleanup(deps, { + directoryPath: worktree.worktreePath, + worktreeIncarnationId: incarnationId!, + requestId: "cleanup-create-race", + }); + await finalOwnerReadStarted; + + const provisioning = createWorkspaceProvisioningService({ + workspaceRegistry: registry, + projectRegistry, + workspaceGitService: { + getCheckout: async (cwd) => ({ + cwd, + isGit: true, + currentBranch: "cleanup-create-race", + remoteUrl: null, + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }), + peekSnapshot: () => null, + }, + logger: createLogger(), + lifecycleCoordinator, + }); + let creationSettled = false; + const creation = provisioning + .createWorkspaceForDirectory(equivalentWorktreePath) + .finally(() => { + creationSettled = true; + }); + await Promise.resolve(); + expect(creationSettled).toBe(false); + + allowFinalOwnerRead?.(); + await expect(cleanup).resolves.toMatchObject({ removedDirectory: true }); + await expect(creation).rejects.toThrow("Working directory does not exist"); + expect(existsSync(worktree.worktreePath)).toBe(false); + expect((await registry.list()).filter((workspace) => workspace.archivedAt === null)).toEqual( + [], + ); + }); + + test("workspace teardown failure keeps the record active and prevents recursive deletion", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "strict-teardown"); + const workspaceId = "ws-strict-teardown"; + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [{ workspaceId, cwd: worktree.worktreePath, kind: "worktree" }], + }); + deps.killTerminalsForWorkspace = vi.fn(async () => { + throw new Error("terminal still owned"); + }); + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-strict-teardown", + }), + ).rejects.toThrow("Failed to archive one or more workspaces"); + + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); expect(existsSync(worktree.worktreePath)).toBe(true); }); - test("workspace scope with unknown workspace id is a clean no-op", async () => { + test("workspace scope rejects an unknown explicit workspace id", async () => { const { tempDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -551,15 +1523,12 @@ describe("archiveByScope", () => { return originalArchiveWorkspaceRecord(workspaceId); }); - const result = await archiveByScope(deps, { - scope: { kind: "workspace", workspaceId: "ws-does-not-exist" }, - requestId: "req-unknown-workspace", - }); - - assertArchiveResult(result, { - archivedWorkspaceIds: [], - removedDirectory: false, - }); + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId: "ws-does-not-exist" }, + requestId: "req-unknown-workspace", + }), + ).rejects.toBeInstanceOf(WorkspaceArchiveTargetNotFoundError); expect(deps.markWorkspaceArchiving).not.toHaveBeenCalled(); expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); expect(deps.emitWorkspaceUpdatesForWorkspaceIds).not.toHaveBeenCalled(); @@ -681,6 +1650,11 @@ describe("archiveByScope", () => { const liveAgentId = "agent-live"; const targetStoredAgentId = "agent-stored-target"; const otherStoredAgentId = "agent-stored-other"; + const liveAgents = [{ id: liveAgentId, workspaceId: targetWorkspaceId }] as ManagedAgent[]; + const storedRecords = [ + { id: targetStoredAgentId, workspaceId: targetWorkspaceId, archivedAt: null }, + { id: otherStoredAgentId, workspaceId: otherWorkspaceId, archivedAt: null }, + ] as StoredAgentRecord[]; const deps = createArchiveDeps({ paseoHome, @@ -689,22 +1663,21 @@ describe("archiveByScope", () => { ], }); deps.agentManager = { - listAgents: () => [{ id: liveAgentId, workspaceId: targetWorkspaceId }] as ManagedAgent[], + listAgents: () => liveAgents, archiveAgent: vi.fn(async (agentId: string) => { deps.archivedAgentIds.push(agentId); + removeManagedAgentById(liveAgents, agentId); return { archivedAt: new Date().toISOString() }; }), - archiveSnapshot: vi.fn(async (agentId: string, _archivedAt: string) => { + archiveSnapshot: vi.fn(async (agentId: string, archivedAt: string) => { deps.archivedSnapshotIds.push(agentId); + const record = findStoredAgentById(storedRecords, agentId); + if (record) record.archivedAt = archivedAt; return {}; }), }; deps.agentStorage = { - list: async () => - [ - { id: targetStoredAgentId, workspaceId: targetWorkspaceId, archivedAt: null }, - { id: otherStoredAgentId, workspaceId: otherWorkspaceId, archivedAt: null }, - ] as StoredAgentRecord[], + list: async () => storedRecords, } as Pick; const result = await archiveByScope(deps, { diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index cfdb59ffc1..ddc5abedbe 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -1,4 +1,5 @@ -import { resolve } from "node:path"; +import { existsSync } from "node:fs"; +import { dirname, resolve } from "node:path"; import type { Logger } from "pino"; @@ -8,17 +9,27 @@ import type { WorkspaceGitService } from "./workspace-git-service.js"; import type { ForgeService } from "../services/forge-service.js"; import { deletePaseoWorktree, + getPaseoWorktreesRoot, isPaseoOwnedWorktreeCwd, runWorktreeTeardownCommands, WorktreeTeardownError, } from "../utils/worktree.js"; +import { + ensurePaseoWorktreeIncarnationId, + readPaseoWorktreeIncarnationId, +} from "../utils/worktree-metadata.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import type { + PersistedWorkspaceCleanupPending, PersistedWorkspaceRecord, WorkspaceArchiveContext, WorkspaceRegistry, } from "./workspace-registry.js"; import { createRealpathAwarePathMatcher } from "../utils/path.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "./workspace-lifecycle-coordinator.js"; export type ActiveWorkspaceRef = Pick< PersistedWorkspaceRecord, @@ -47,6 +58,8 @@ export interface ArchiveDependencies { markWorkspaceArchiving: (workspaceIds: Iterable, archivingAt: string) => void; clearWorkspaceArchiving: (workspaceIds: Iterable) => void; killTerminalsForWorkspace: (workspaceId: string) => Promise; + workspaceRegistry?: Pick; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; sessionLogger?: Logger; } @@ -64,24 +77,64 @@ export interface ArchiveResult { archivedAgentIds: string[]; archivedWorkspaceIds: string[]; removedDirectory: boolean; + cleanupPendingWorkspaceIds: string[]; +} + +export class WorkspaceArchiveTargetNotFoundError extends Error { + constructor(workspaceId: string) { + super(`Workspace not found: ${workspaceId}`); + this.name = "WorkspaceArchiveTargetNotFoundError"; + } +} + +export class WorkspaceCleanupPendingError extends Error { + readonly workspaceIds: readonly string[]; + + constructor(operation: string, workspaceIds: readonly string[]) { + super(`${operation} left cleanup pending for: ${workspaceIds.join(", ")}`); + this.name = "WorkspaceCleanupPendingError"; + this.workspaceIds = [...workspaceIds]; + } +} + +export function requireArchiveCleanupComplete( + result: ArchiveResult, + operation: string, +): ArchiveResult { + if (result.cleanupPendingWorkspaceIds.length > 0) { + throw new WorkspaceCleanupPendingError(operation, result.cleanupPendingWorkspaceIds); + } + return result; } export interface ArchiveByScopeRequest { scope: ArchiveScope; requestId: string; + signal?: AbortSignal; +} + +export interface PendingWorkspaceCleanupRetryRequest { + directoryPath: string; + worktreeIncarnationId: string; + requestId: string; + signal?: AbortSignal; } export async function requireActiveWorkspaceForArchive( - dependencies: Pick, + dependencies: Pick, workspaceId: string, ): Promise { const workspace = (await dependencies.listActiveWorkspaces()).find( (candidate) => candidate.workspaceId === workspaceId, ); - if (!workspace) { - throw new Error(`Workspace not found: ${workspaceId}`); + if (workspace) { + return workspace; + } + const persistedWorkspace = await dependencies.workspaceRegistry?.get(workspaceId); + if (persistedWorkspace?.cleanupPending) { + return persistedWorkspace; } - return workspace; + throw new WorkspaceArchiveTargetNotFoundError(workspaceId); } interface BackingDirectory { @@ -118,7 +171,179 @@ export async function archiveByScope( dependencies: ArchiveDependencies, request: ArchiveByScopeRequest, ): Promise { - const target = await resolveArchiveTarget(dependencies, request.scope); + const lifecycleCoordinator = + dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const operationKey = + request.scope.kind === "workspace" + ? `workspace:${request.scope.workspaceId}` + : `worktree:${resolve(request.scope.targetPath)}`; + + return lifecycleCoordinator.runArchive( + operationKey, + async () => { + const initialTarget = await resolveArchiveTarget(dependencies, request.scope); + const archiveReservation = lifecycleCoordinator.reserveWorkspaceArchive( + initialTarget.workspaceIds, + ); + const archiveOperation = async () => { + const closureWorkspaceIds = new Set(initialTarget.workspaceIds); + let target = initialTarget; + while (true) { + await lifecycleCoordinator.waitForWorkspaceSetups(closureWorkspaceIds, request.signal); + await lifecycleCoordinator.waitForWorkspaceOwnershipMutations( + closureWorkspaceIds, + request.signal, + ); + + const refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); + archiveReservation.add(refreshedTarget.workspaceIds); + target = mergeArchiveTargets(target, refreshedTarget); + + const newlyDiscoveredWorkspaceIds = refreshedTarget.workspaceIds.filter( + (workspaceId) => !closureWorkspaceIds.has(workspaceId), + ); + if (newlyDiscoveredWorkspaceIds.length === 0) { + return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); + } + for (const workspaceId of newlyDiscoveredWorkspaceIds) { + closureWorkspaceIds.add(workspaceId); + } + } + }; + try { + const mutationRoot = + (await resolveWorktreeMutationRoot(dependencies, initialTarget.backing)) ?? + (request.scope.kind === "worktree" ? dirname(resolve(request.scope.targetPath)) : null); + return await (mutationRoot + ? lifecycleCoordinator.runWorktreeMutationExclusive( + mutationRoot, + archiveOperation, + request.signal, + ) + : archiveOperation()); + } finally { + archiveReservation.release(); + } + }, + request.signal, + ); +} + +// Retries only the physical teardown intent already persisted on archived +// workspace records. Unlike normal worktree-scope archive, this never archives +// active workspace owners that may now occupy a reused path. +export async function retryPendingWorkspaceCleanup( + dependencies: ArchiveDependencies, + request: PendingWorkspaceCleanupRetryRequest, +): Promise { + const workspaceRegistry = dependencies.workspaceRegistry; + if (!workspaceRegistry) { + return { + archivedAgentIds: [], + archivedWorkspaceIds: [], + removedDirectory: false, + cleanupPendingWorkspaceIds: [], + }; + } + const lifecycleCoordinator = + dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const directoryPath = resolve(request.directoryPath); + const operationKey = `cleanup:${directoryPath}:${request.worktreeIncarnationId}`; + + return lifecycleCoordinator.runArchive( + operationKey, + async () => { + const matchesDirectory = createRealpathAwarePathMatcher(directoryPath); + const pendingRecords = (await workspaceRegistry.list()).filter( + (workspace) => + workspace.archivedAt !== null && + workspace.cleanupPending !== null && + workspace.cleanupPending.worktreeIncarnationId === request.worktreeIncarnationId && + matchesDirectory(workspace.cleanupPending.directoryPath), + ); + const firstPending = pendingRecords[0]?.cleanupPending ?? null; + if (!firstPending) { + return { + archivedAgentIds: [], + archivedWorkspaceIds: [], + removedDirectory: false, + cleanupPendingWorkspaceIds: [], + }; + } + const target: ArchiveTarget = { + backing: { + path: firstPending.directoryPath, + isPaseoOwnedWorktree: true, + mainRepoRoot: firstPending.mainRepoRoot, + paseoWorktreesRoot: firstPending.paseoWorktreesRoot, + }, + teardownTargets: pendingRecords.map((workspace) => ({ + workspaceId: workspace.workspaceId, + cwd: workspace.cleanupPending!.teardownCwd, + })), + workspaceIds: [], + }; + const retry = async (): Promise => { + if (request.signal?.aborted) throw new Error("Workspace cleanup retry canceled"); + const removedDirectory = await maybeRemoveDirectory( + dependencies, + lifecycleCoordinator, + request, + target, + [], + request.signal, + ); + return { + archivedAgentIds: [], + archivedWorkspaceIds: [], + removedDirectory, + cleanupPendingWorkspaceIds: await getCleanupPendingWorkspaceIds(dependencies, target), + }; + }; + const mutationRoot = await resolveWorktreeMutationRoot(dependencies, target.backing); + return mutationRoot + ? lifecycleCoordinator.runWorktreeMutationExclusive(mutationRoot, retry, request.signal) + : retry(); + }, + request.signal, + ); +} + +async function resolveWorktreeMutationRoot( + dependencies: Pick, + backing: BackingDirectory | null, +): Promise { + if (!backing?.isPaseoOwnedWorktree) return null; + if (backing.paseoWorktreesRoot) return resolve(backing.paseoWorktreesRoot); + if (backing.mainRepoRoot) { + return resolve( + await getPaseoWorktreesRoot( + backing.mainRepoRoot, + dependencies.paseoHome, + dependencies.paseoWorktreesBaseRoot, + ), + ); + } + return dirname(backing.path); +} + +function mergeArchiveTargets(initial: ArchiveTarget, refreshed: ArchiveTarget): ArchiveTarget { + if (refreshed.backing !== null || refreshed.workspaceIds.length > 0) { + return refreshed; + } + return { + backing: initial.backing, + teardownTargets: initial.teardownTargets, + workspaceIds: [], + }; +} + +async function archiveResolvedTarget( + dependencies: ArchiveDependencies, + lifecycleCoordinator: WorkspaceLifecycleCoordinator, + request: ArchiveByScopeRequest, + target: ArchiveTarget, +): Promise { const targetWorkspaceIds = target.workspaceIds; if (targetWorkspaceIds.length > 0) { @@ -132,11 +357,17 @@ export async function archiveByScope( await dependencies.emitWorkspaceUpdatesForWorkspaceIds(targetWorkspaceIds); } - const { archivedAgents, archivedWorkspaceIds } = await archiveTargetRecords( + await persistTargetCleanupPending(dependencies, target); + + const { archivedAgents, archivedWorkspaceIds, failures } = await archiveTargetRecords( dependencies, targetWorkspaceIds, request.requestId, ); + await clearCleanupPendingForUnarchivedTargets(dependencies, target, archivedWorkspaceIds); + if (failures.length > 0) { + throw new AggregateError(failures, "Failed to archive one or more workspaces"); + } if (target.backing?.mainRepoRoot) { try { @@ -155,16 +386,20 @@ export async function archiveByScope( if (target.backing !== null) { removedDirectory = await maybeRemoveDirectory( dependencies, + lifecycleCoordinator, request, target, archivedWorkspaceIds, ); } + const cleanupPendingWorkspaceIds = await getCleanupPendingWorkspaceIds(dependencies, target); + return { archivedAgentIds: Array.from(archivedAgents), archivedWorkspaceIds, removedDirectory, + cleanupPendingWorkspaceIds, }; } finally { if (targetWorkspaceIds.length > 0) { @@ -184,11 +419,12 @@ async function resolveArchiveTarget( const workspaceId = scope.workspaceId; const record = activeWorkspaces.find((workspace) => workspace.workspaceId === workspaceId); if (!record) { - dependencies.sessionLogger?.warn( - { workspaceId }, - "Workspace not found for archive-by-scope; skipping", - ); - return { backing: null, teardownTargets: [], workspaceIds: [] }; + const persistedRecord = await dependencies.workspaceRegistry?.get(workspaceId); + if (persistedRecord?.cleanupPending) { + return archiveTargetFromPendingCleanup(workspaceId, persistedRecord.cleanupPending); + } + dependencies.sessionLogger?.warn({ workspaceId }, "Workspace not found for archive-by-scope"); + throw new WorkspaceArchiveTargetNotFoundError(workspaceId); } return { backing: await resolveWorkspaceBackingDirectory(record, dependencies), @@ -197,7 +433,15 @@ async function resolveArchiveTarget( }; } - const backing = await resolveBackingDirectory(scope.targetPath, dependencies); + return resolveWorktreeArchiveTarget(dependencies, scope.targetPath, activeWorkspaces); +} + +async function resolveWorktreeArchiveTarget( + dependencies: ArchiveDependencies, + targetPath: string, + activeWorkspaces: ActiveWorkspaceRef[], +): Promise { + const backing = await resolveBackingDirectory(targetPath, dependencies); const matchesBackingDirectory = createRealpathAwarePathMatcher(backing.path); const targetWorkspaces = ( await Promise.all( @@ -207,25 +451,65 @@ async function resolveArchiveTarget( }), ) ).filter((workspace): workspace is ActiveWorkspaceRef => workspace !== null); + const pendingRecords = (await dependencies.workspaceRegistry?.list())?.filter( + (workspace) => + workspace.cleanupPending !== null && + matchesBackingDirectory(workspace.cleanupPending.directoryPath), + ); + const pendingCleanup = pendingRecords?.[0]?.cleanupPending ?? null; const persistedMainRepoRoot = targetWorkspaces.find( (workspace) => workspace.mainRepoRoot, )?.mainRepoRoot; + let teardownTargets: ArchiveTarget["teardownTargets"]; + if (targetWorkspaces.length > 0 || (pendingRecords && pendingRecords.length > 0)) { + const activeWorkspaceIds = new Set(targetWorkspaces.map((workspace) => workspace.workspaceId)); + teardownTargets = targetWorkspaces.map((workspace) => ({ + workspaceId: workspace.workspaceId, + cwd: workspace.cwd, + })); + teardownTargets.push( + ...(pendingRecords ?? []).flatMap((workspace) => { + if (!workspace.cleanupPending || activeWorkspaceIds.has(workspace.workspaceId)) return []; + return [ + { + workspaceId: workspace.workspaceId, + cwd: workspace.cleanupPending.teardownCwd, + }, + ]; + }), + ); + } else { + teardownTargets = [{ workspaceId: null, cwd: targetPath }]; + } + return { backing: { - ...backing, - mainRepoRoot: persistedMainRepoRoot ?? backing.mainRepoRoot, + path: pendingCleanup?.directoryPath ?? backing.path, + isPaseoOwnedWorktree: pendingCleanup !== null || backing.isPaseoOwnedWorktree, + mainRepoRoot: persistedMainRepoRoot ?? pendingCleanup?.mainRepoRoot ?? backing.mainRepoRoot, + paseoWorktreesRoot: pendingCleanup?.paseoWorktreesRoot ?? backing.paseoWorktreesRoot, }, - teardownTargets: - targetWorkspaces.length > 0 - ? targetWorkspaces.map((workspace) => ({ - workspaceId: workspace.workspaceId, - cwd: workspace.cwd, - })) - : [{ workspaceId: null, cwd: scope.targetPath }], + teardownTargets, workspaceIds: targetWorkspaces.map((workspace) => workspace.workspaceId), }; } +function archiveTargetFromPendingCleanup( + workspaceId: string, + cleanupPending: PersistedWorkspaceCleanupPending, +): ArchiveTarget { + return { + backing: { + path: cleanupPending.directoryPath, + isPaseoOwnedWorktree: true, + mainRepoRoot: cleanupPending.mainRepoRoot, + paseoWorktreesRoot: cleanupPending.paseoWorktreesRoot, + }, + teardownTargets: [{ workspaceId, cwd: cleanupPending.teardownCwd }], + workspaceIds: [], + }; +} + async function resolveWorkspaceBackingDirectory( workspace: ActiveWorkspaceRef, dependencies: Pick, @@ -277,7 +561,11 @@ async function archiveTargetRecords( dependencies: ArchiveDependencies, targetWorkspaceIds: string[], requestId: string, -): Promise<{ archivedAgents: Set; archivedWorkspaceIds: string[] }> { +): Promise<{ + archivedAgents: Set; + archivedWorkspaceIds: string[]; + failures: unknown[]; +}> { const archivedAgents = new Set(); const archivedWorkspaceIds: string[] = []; @@ -298,34 +586,96 @@ async function archiveTargetRecords( } else { dependencies.sessionLogger?.warn( { err: result.reason, requestId }, - "archiveByScope workspace teardown failed; continuing", + "archiveByScope workspace teardown failed", ); } } - return { archivedAgents, archivedWorkspaceIds }; + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + return { archivedAgents, archivedWorkspaceIds, failures }; } async function maybeRemoveDirectory( dependencies: ArchiveDependencies, + lifecycleCoordinator: WorkspaceLifecycleCoordinator, request: Pick, target: ArchiveTarget, archivedWorkspaceIds: string[], + signal?: AbortSignal, ): Promise { const backing = target.backing; if (!backing?.isPaseoOwnedWorktree) { return false; } - const archivedWorkspaceIdSet = new Set(archivedWorkspaceIds); + return lifecycleCoordinator.runDirectoryExclusive( + backing.path, + async () => { + return maybeRemoveDirectoryExclusive( + dependencies, + request, + target, + archivedWorkspaceIds, + signal, + ); + }, + signal, + ); +} + +async function maybeRemoveDirectoryExclusive( + dependencies: ArchiveDependencies, + request: Pick, + target: ArchiveTarget, + archivedWorkspaceIds: string[], + signal?: AbortSignal, +): Promise { + const backing = target.backing; + if (!backing) return false; + + const pendingCleanupTargets = await listPendingCleanupTargets( + dependencies, + target, + archivedWorkspaceIds, + ); + if (pendingCleanupTargets.length === 0) { + return false; + } + + if (!existsSync(backing.path)) { + await clearPendingCleanup(dependencies, pendingCleanupTargets); + return false; + } + + const initialIncarnationState = await compareCleanupIncarnation( + dependencies, + backing, + pendingCleanupTargets, + ); + if (initialIncarnationState !== "match") { + if (initialIncarnationState === "mismatch") { + await clearPendingCleanup(dependencies, pendingCleanupTargets); + } + return false; + } + + const activeBeforeTeardown = await dependencies.listActiveWorkspaces(); + if ( + !(await isDirectoryUnreferenced( + activeBeforeTeardown, + backing.path, + new Set(archivedWorkspaceIds), + dependencies, + )) + ) { + await clearPendingCleanup(dependencies, pendingCleanupTargets); + return false; + } + const teardownCwds = uniqueFilesystemPaths( - target.teardownTargets - .filter( - (teardownTarget) => - teardownTarget.workspaceId === null || - archivedWorkspaceIdSet.has(teardownTarget.workspaceId), - ) - .map((teardownTarget) => teardownTarget.cwd), + pendingCleanupTargets.map((pendingCleanupTarget) => pendingCleanupTarget.teardownCwd), ); try { @@ -334,6 +684,7 @@ async function maybeRemoveDirectory( worktreePath: backing.path, teardownCwd, repoRootPath: backing.mainRepoRoot ?? undefined, + signal, }); } } catch (error) { @@ -342,11 +693,23 @@ async function maybeRemoveDirectory( { err: error, targetPath: backing.path, requestId: request.requestId }, "Worktree teardown failed during archive; workspace already archived", ); - return false; + throw error; } throw error; } + const finalIncarnationState = await compareCleanupIncarnation( + dependencies, + backing, + pendingCleanupTargets, + ); + if (finalIncarnationState !== "match") { + if (finalIncarnationState === "mismatch") { + await clearPendingCleanup(dependencies, pendingCleanupTargets); + } + return false; + } + const remainingActive = await dependencies.listActiveWorkspaces(); if ( !(await isDirectoryUnreferenced( @@ -356,6 +719,7 @@ async function maybeRemoveDirectory( dependencies, )) ) { + await clearPendingCleanup(dependencies, pendingCleanupTargets); return false; } @@ -367,8 +731,10 @@ async function maybeRemoveDirectory( worktreesRoot: backing.paseoWorktreesRoot ?? undefined, paseoHome: dependencies.paseoHome, worktreesBaseRoot: dependencies.paseoWorktreesBaseRoot, + signal, }); dependencies.github.invalidate({ cwd: backing.path }); + await clearPendingCleanup(dependencies, pendingCleanupTargets); return true; } catch (error) { if (error instanceof WorktreeTeardownError) { @@ -376,12 +742,188 @@ async function maybeRemoveDirectory( { err: error, targetPath: backing.path, requestId: request.requestId }, "Worktree disk removal failed during archive; workspace already archived", ); - return false; + throw error; } throw error; } } +async function compareCleanupIncarnation( + dependencies: Pick, + backing: BackingDirectory, + pendingCleanupTargets: PendingCleanupTarget[], +): Promise<"match" | "mismatch" | "unverifiable"> { + const expectedIncarnations = new Set( + pendingCleanupTargets + .map((target) => target.worktreeIncarnationId) + .filter((value): value is string => value !== null), + ); + if ( + expectedIncarnations.size !== 1 || + pendingCleanupTargets.some((target) => !target.worktreeIncarnationId) + ) { + dependencies.sessionLogger?.warn( + { targetPath: backing.path }, + "Refusing path-only or conflicting persisted worktree cleanup", + ); + return "unverifiable"; + } + let currentIncarnationId: string | null = null; + try { + currentIncarnationId = readPaseoWorktreeIncarnationId(backing.path); + } catch (error) { + dependencies.sessionLogger?.warn( + { err: error, targetPath: backing.path }, + "Could not verify persisted worktree cleanup incarnation", + ); + return "unverifiable"; + } + if (currentIncarnationId === null) return "unverifiable"; + return expectedIncarnations.has(currentIncarnationId) ? "match" : "mismatch"; +} + +interface PendingCleanupTarget extends PersistedWorkspaceCleanupPending { + workspaceId: string | null; +} + +async function persistTargetCleanupPending( + dependencies: ArchiveDependencies, + target: ArchiveTarget, +): Promise { + const backing = target.backing; + const workspaceRegistry = dependencies.workspaceRegistry; + if (!backing?.isPaseoOwnedWorktree || !workspaceRegistry) return; + // A target reconstructed from an archived record's cleanupPending state has + // no active workspace ids. Its incarnation is the authority for this retry: + // never rebind that stale cleanup intent to whatever now occupies the path. + const activeTargetWorkspaceIds = new Set(target.workspaceIds); + + const worktreeIncarnationId = existsSync(backing.path) + ? ensurePaseoWorktreeIncarnationId(backing.path) + : null; + + await Promise.all( + target.teardownTargets.flatMap((teardownTarget) => + teardownTarget.workspaceId && activeTargetWorkspaceIds.has(teardownTarget.workspaceId) + ? [ + workspaceRegistry.update(teardownTarget.workspaceId, (workspace) => ({ + ...workspace, + cleanupPending: { + directoryPath: backing.path, + teardownCwd: teardownTarget.cwd, + mainRepoRoot: backing.mainRepoRoot, + paseoWorktreesRoot: backing.paseoWorktreesRoot, + worktreeIncarnationId, + }, + })), + ] + : [], + ), + ); +} + +async function listPendingCleanupTargets( + dependencies: ArchiveDependencies, + target: ArchiveTarget, + archivedWorkspaceIds: string[], +): Promise { + const backingPath = target.backing?.path; + const workspaceRegistry = dependencies.workspaceRegistry; + if (backingPath && workspaceRegistry) { + const matchesBacking = createRealpathAwarePathMatcher(backingPath); + const targetWorkspaceIds = new Set( + target.teardownTargets.flatMap((teardownTarget) => + teardownTarget.workspaceId ? [teardownTarget.workspaceId] : [], + ), + ); + const records = await workspaceRegistry.list(); + return records.flatMap((workspace) => { + const cleanupPending = workspace.cleanupPending; + if ( + (targetWorkspaceIds.size > 0 && !targetWorkspaceIds.has(workspace.workspaceId)) || + !workspace.archivedAt || + !cleanupPending || + !matchesBacking(cleanupPending.directoryPath) + ) { + return []; + } + return [{ workspaceId: workspace.workspaceId, ...cleanupPending }]; + }); + } + + const archivedWorkspaceIdSet = new Set(archivedWorkspaceIds); + let fallbackIncarnationId: string | null = null; + if (target.backing && existsSync(target.backing.path)) { + try { + fallbackIncarnationId = readPaseoWorktreeIncarnationId(target.backing.path); + } catch { + fallbackIncarnationId = null; + } + } + return target.teardownTargets + .filter( + (teardownTarget) => + teardownTarget.workspaceId === null || + archivedWorkspaceIdSet.has(teardownTarget.workspaceId), + ) + .map((teardownTarget) => ({ + workspaceId: teardownTarget.workspaceId, + directoryPath: target.backing?.path ?? teardownTarget.cwd, + teardownCwd: teardownTarget.cwd, + mainRepoRoot: target.backing?.mainRepoRoot ?? null, + paseoWorktreesRoot: target.backing?.paseoWorktreesRoot ?? null, + worktreeIncarnationId: fallbackIncarnationId, + })); +} + +async function clearPendingCleanup( + dependencies: ArchiveDependencies, + cleanupTargets: PendingCleanupTarget[], +): Promise { + const workspaceRegistry = dependencies.workspaceRegistry; + if (!workspaceRegistry) return; + await Promise.all( + cleanupTargets.flatMap((cleanupTarget) => + cleanupTarget.workspaceId + ? [ + workspaceRegistry.update(cleanupTarget.workspaceId, (workspace) => ({ + ...workspace, + cleanupPending: null, + })), + ] + : [], + ), + ); +} + +async function clearCleanupPendingForUnarchivedTargets( + dependencies: ArchiveDependencies, + target: ArchiveTarget, + archivedWorkspaceIds: string[], +): Promise { + const archivedWorkspaceIdSet = new Set(archivedWorkspaceIds); + const targetWorkspaceIdSet = new Set(target.workspaceIds); + await clearPendingCleanup( + dependencies, + target.teardownTargets.flatMap((teardownTarget) => + teardownTarget.workspaceId && + targetWorkspaceIdSet.has(teardownTarget.workspaceId) && + !archivedWorkspaceIdSet.has(teardownTarget.workspaceId) + ? [ + { + workspaceId: teardownTarget.workspaceId, + directoryPath: target.backing?.path ?? teardownTarget.cwd, + teardownCwd: teardownTarget.cwd, + mainRepoRoot: target.backing?.mainRepoRoot ?? null, + paseoWorktreesRoot: target.backing?.paseoWorktreesRoot ?? null, + worktreeIncarnationId: null, + }, + ] + : [], + ), + ); +} + function uniqueFilesystemPaths(paths: string[]): string[] { const unique: string[] = []; for (const candidate of paths) { @@ -414,15 +956,7 @@ export async function archiveWorkspaceContents( archivedAgents.add(agent.id); } - let storedRecords: StoredAgentRecord[] = []; - try { - storedRecords = await dependencies.agentStorage.list(); - } catch (error) { - dependencies.sessionLogger?.warn( - { err: error, workspaceId }, - "Failed to list stored agents during workspace archive; continuing", - ); - } + const storedRecords: StoredAgentRecord[] = await dependencies.agentStorage.list(); const liveAgentIds = new Set(liveAgents.map((agent) => agent.id)); const matchingStoredRecords = storedRecords.filter( (record) => record.workspaceId === workspaceId, @@ -432,7 +966,7 @@ export async function archiveWorkspaceContents( } const archivedAt = new Date().toISOString(); - const archiveResults = await Promise.allSettled([ + await Promise.all([ ...liveAgents.map((agent) => dependencies.agentManager.archiveAgent(agent.id)), ...matchingStoredRecords .filter((record) => !liveAgentIds.has(record.id) && !record.archivedAt) @@ -440,13 +974,14 @@ export async function archiveWorkspaceContents( dependencies.killTerminalsForWorkspace(workspaceId), ]); - for (const result of archiveResults) { - if (result.status === "rejected") { - dependencies.sessionLogger?.warn( - { err: result.reason, workspaceId }, - "Workspace archive teardown step failed; continuing", - ); - } + const remainingLiveAgents = dependencies.agentManager + .listAgents() + .filter((agent) => agent.workspaceId === workspaceId); + const remainingStoredAgents = (await dependencies.agentStorage.list()).filter( + (record) => record.workspaceId === workspaceId && !record.archivedAt, + ); + if (remainingLiveAgents.length > 0 || remainingStoredAgents.length > 0) { + throw new Error(`Workspace ownership remains after archive: ${workspaceId}`); } return archivedAgents; @@ -480,20 +1015,18 @@ export async function killTerminalsForWorkspace( return; } + const listWorkspaceTerminals = async () => + ( + await Promise.all( + terminalManager + .listDirectories() + .map((terminalCwd) => terminalManager.getTerminals(terminalCwd, { workspaceId })), + ) + ) + .flat() + .filter((terminal) => terminal.workspaceId === workspaceId); const terminalIds: string[] = []; - const terminalLists = await Promise.all( - terminalManager.listDirectories().map(async (terminalCwd) => { - try { - return await terminalManager.getTerminals(terminalCwd, { workspaceId }); - } catch (error) { - dependencies.sessionLogger.warn( - { err: error, cwd: terminalCwd }, - "Failed to enumerate workspace terminals during archive", - ); - return []; - } - }), - ); + const terminalLists = [await listWorkspaceTerminals()]; for (const terminals of terminalLists) { for (const terminal of terminals) { if (terminal.workspaceId === workspaceId) { @@ -502,26 +1035,32 @@ export async function killTerminalsForWorkspace( } } - if (terminalIds.length === 0) { - return; - } - - await Promise.allSettled( + await Promise.all( terminalIds.map(async (terminalId) => { - try { - dependencies.detachTerminalStream?.(terminalId, { emitExit: true }); - await terminalManager.killTerminalAndWait(terminalId, { - gracefulTimeoutMs: 2000, - forceTimeoutMs: 1500, - }); - } catch (error) { - dependencies.sessionLogger.warn( - { err: error, terminalId }, - "Terminal kill escalation failed during archive; proceeding anyway", - ); - } + dependencies.detachTerminalStream?.(terminalId, { emitExit: true }); + await terminalManager.killTerminalAndWait(terminalId, { + gracefulTimeoutMs: 2000, + forceTimeoutMs: 1500, + }); }), ); + if ((await listWorkspaceTerminals()).length > 0) { + throw new Error(`Workspace terminals remain after archive: ${workspaceId}`); + } +} + +async function getCleanupPendingWorkspaceIds( + dependencies: Pick, + target: ArchiveTarget, +): Promise { + if (!dependencies.workspaceRegistry || !target.backing) return []; + const matchesBacking = createRealpathAwarePathMatcher(target.backing.path); + return (await dependencies.workspaceRegistry.list()) + .filter( + (workspace) => + workspace.cleanupPending !== null && matchesBacking(workspace.cleanupPending.directoryPath), + ) + .map((workspace) => workspace.workspaceId); } // Archiving the last workspace of a project leaves the project record active. diff --git a/packages/server/src/server/workspace-cleanup-retry-service.test.ts b/packages/server/src/server/workspace-cleanup-retry-service.test.ts new file mode 100644 index 0000000000..47250d29b5 --- /dev/null +++ b/packages/server/src/server/workspace-cleanup-retry-service.test.ts @@ -0,0 +1,195 @@ +import { afterEach, expect, test, vi } from "vitest"; + +import { createTestLogger } from "../test-utils/test-logger.js"; +import { + findWorkspaceCleanupRetryTargets, + WorkspaceCleanupRetryService, +} from "./workspace-cleanup-retry-service.js"; +import { createPersistedWorkspaceRecord } from "./workspace-registry.js"; + +afterEach(() => { + vi.useRealTimers(); +}); + +function pendingWorkspace(input: { + workspaceId: string; + directoryPath: string; + incarnationId: string | null; + archived?: boolean; +}) { + const timestamp = "2026-07-31T00:00:00.000Z"; + return createPersistedWorkspaceRecord({ + workspaceId: input.workspaceId, + projectId: "project-cleanup-retry", + cwd: input.directoryPath, + kind: "worktree", + displayName: input.workspaceId, + createdAt: timestamp, + updatedAt: timestamp, + archivedAt: input.archived === false ? null : timestamp, + cleanupPending: { + directoryPath: input.directoryPath, + teardownCwd: input.directoryPath, + mainRepoRoot: "/repo", + paseoWorktreesRoot: "/worktrees", + worktreeIncarnationId: input.incarnationId, + }, + }); +} + +test("groups archived cleanup by directory and requires one verified incarnation", () => { + const targets = findWorkspaceCleanupRetryTargets([ + pendingWorkspace({ + workspaceId: "ws-a", + directoryPath: "/worktrees/shared", + incarnationId: "inc-shared", + }), + pendingWorkspace({ + workspaceId: "ws-b", + directoryPath: "/worktrees/shared", + incarnationId: "inc-shared", + }), + pendingWorkspace({ + workspaceId: "ws-legacy", + directoryPath: "/worktrees/legacy", + incarnationId: null, + }), + pendingWorkspace({ + workspaceId: "ws-conflict-a", + directoryPath: "/worktrees/conflict", + incarnationId: "inc-a", + }), + pendingWorkspace({ + workspaceId: "ws-conflict-b", + directoryPath: "/worktrees/conflict", + incarnationId: "inc-b", + }), + pendingWorkspace({ + workspaceId: "ws-active", + directoryPath: "/worktrees/active", + incarnationId: "inc-active", + archived: false, + }), + ]); + + expect(targets).toEqual([ + { + directoryPath: "/worktrees/shared", + worktreeIncarnationId: "inc-shared", + workspaceIds: ["ws-a", "ws-b"], + }, + ]); +}); + +test("rotates bounded batches and preserves backoff after a failed target", async () => { + vi.useFakeTimers(); + const workspaces = [ + pendingWorkspace({ + workspaceId: "ws-first", + directoryPath: "/worktrees/first", + incarnationId: "inc-first", + }), + pendingWorkspace({ + workspaceId: "ws-second", + directoryPath: "/worktrees/second", + incarnationId: "inc-second", + }), + ]; + const attempts: string[] = []; + const retryWorktreeCleanup = vi.fn(async ({ directoryPath: targetPath }) => { + attempts.push(targetPath); + if (targetPath === "/worktrees/first") throw new Error("still busy"); + }); + const service = new WorkspaceCleanupRetryService({ + workspaceRegistry: { + list: async () => workspaces, + subscribeToMutations: () => () => undefined, + }, + retryWorktreeCleanup, + logger: createTestLogger(), + retryBaseMs: 10, + retryMaxMs: 10, + idlePollMs: 100, + maxTargetsPerCycle: 1, + }); + + await service.start(); + await vi.advanceTimersToNextTimerAsync(); + expect(attempts).toEqual(["/worktrees/first"]); + + await vi.advanceTimersByTimeAsync(9); + expect(attempts).toEqual(["/worktrees/first"]); + await vi.advanceTimersByTimeAsync(1); + expect(attempts).toContain("/worktrees/second"); + + await vi.advanceTimersToNextTimerAsync(); + expect(attempts.filter((path) => path === "/worktrees/first")).toHaveLength(2); + + await service.stop(); + await vi.advanceTimersByTimeAsync(1_000); + expect(retryWorktreeCleanup).toHaveBeenCalledTimes(attempts.length); +}); + +test("shutdown joins canceled cleanup through process closure and releases once", async () => { + vi.useFakeTimers(); + let abortCount = 0; + let closeProcess = () => {}; + const processClosed = new Promise((resolve) => { + closeProcess = resolve; + }); + const rejectAfterProcessClose = (reject: (error: Error) => void) => + processClosed.then(() => { + reject(new Error("cleanup canceled after process close")); + return undefined; + }); + const releaseSubscription = vi.fn(); + const retryWorktreeCleanup = vi.fn((_target, signal: AbortSignal) => { + return new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => { + abortCount += 1; + void rejectAfterProcessClose(reject); + }, + { once: true }, + ); + }); + }); + const service = new WorkspaceCleanupRetryService({ + workspaceRegistry: { + list: async () => [ + pendingWorkspace({ + workspaceId: "ws-hung", + directoryPath: "/worktrees/hung", + incarnationId: "inc-hung", + }), + ], + subscribeToMutations: () => releaseSubscription, + }, + retryWorktreeCleanup, + logger: createTestLogger(), + }); + + await expect(service.start()).resolves.toBeUndefined(); + await vi.advanceTimersToNextTimerAsync(); + expect(retryWorktreeCleanup).toHaveBeenCalledTimes(1); + + let stopSettled = false; + const stopping = service.stop().then(() => { + stopSettled = true; + return undefined; + }); + await Promise.resolve(); + + expect(abortCount).toBe(1); + expect(stopSettled).toBe(false); + expect(releaseSubscription).toHaveBeenCalledOnce(); + + closeProcess(); + await expect(stopping).resolves.toBeUndefined(); + expect(stopSettled).toBe(true); + + await expect(service.stop()).resolves.toBeUndefined(); + expect(abortCount).toBe(1); + expect(releaseSubscription).toHaveBeenCalledOnce(); +}); diff --git a/packages/server/src/server/workspace-cleanup-retry-service.ts b/packages/server/src/server/workspace-cleanup-retry-service.ts new file mode 100644 index 0000000000..b0b7fed56b --- /dev/null +++ b/packages/server/src/server/workspace-cleanup-retry-service.ts @@ -0,0 +1,209 @@ +import { resolve } from "node:path"; + +import type { Logger } from "pino"; + +import type { PersistedWorkspaceRecord, WorkspaceRegistry } from "./workspace-registry.js"; + +const DEFAULT_IDLE_POLL_MS = 30_000; +const DEFAULT_RETRY_BASE_MS = 1_000; +const DEFAULT_RETRY_MAX_MS = 60_000; +const DEFAULT_MAX_TARGETS_PER_CYCLE = 8; + +export interface WorkspaceCleanupRetryTarget { + directoryPath: string; + worktreeIncarnationId: string; + workspaceIds: string[]; +} + +export interface WorkspaceCleanupRetryServiceOptions { + workspaceRegistry: Pick; + retryWorktreeCleanup: (target: WorkspaceCleanupRetryTarget, signal: AbortSignal) => Promise; + logger: Logger; + idlePollMs?: number; + retryBaseMs?: number; + retryMaxMs?: number; + maxTargetsPerCycle?: number; +} + +export class WorkspaceCleanupRetryService { + private readonly logger: Logger; + private readonly idlePollMs: number; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly maxTargetsPerCycle: number; + private timer: ReturnType | null = null; + private activeCycle: Promise | null = null; + private unsubscribe: (() => void) | null = null; + private started = false; + private stopped = false; + private consecutiveFailedCycles = 0; + private wakeRequested = false; + private nextTargetDirectoryPath: string | null = null; + private activeAbortController: AbortController | null = null; + + constructor(private readonly options: WorkspaceCleanupRetryServiceOptions) { + this.logger = options.logger.child({ module: "workspace-cleanup-retry-service" }); + this.idlePollMs = options.idlePollMs ?? DEFAULT_IDLE_POLL_MS; + this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS; + this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS; + this.maxTargetsPerCycle = options.maxTargetsPerCycle ?? DEFAULT_MAX_TARGETS_PER_CYCLE; + } + + async start(): Promise { + if (this.started) return; + this.started = true; + this.stopped = false; + this.unsubscribe = + this.options.workspaceRegistry.subscribeToMutations?.((mutation) => { + if (mutation.workspace?.cleanupPending) { + this.requestWake(); + } + }) ?? null; + this.schedule(0); + } + + async stop(): Promise { + this.stopped = true; + this.started = false; + this.wakeRequested = false; + this.unsubscribe?.(); + this.unsubscribe = null; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + const activeCycle = this.activeCycle; + this.activeAbortController?.abort(); + this.activeAbortController = null; + await activeCycle; + } + + private requestWake(): void { + if (!this.started || this.stopped) return; + if (this.activeCycle) { + this.wakeRequested = true; + return; + } + this.schedule(0); + } + + private schedule(delayMs: number): void { + if (!this.started || this.stopped) return; + if (this.timer) clearTimeout(this.timer); + this.timer = setTimeout(() => { + this.timer = null; + void this.runCycle(); + }, delayMs); + (this.timer as unknown as { unref?: () => void }).unref?.(); + } + + private async runCycle(): Promise { + if (!this.started || this.stopped || this.activeCycle) return; + const task = this.executeCycle(); + this.activeCycle = task; + try { + await task; + } finally { + if (this.activeCycle === task) this.activeCycle = null; + if (this.started && !this.stopped) { + if (this.wakeRequested) { + this.wakeRequested = false; + this.schedule(0); + } else { + const delay = + this.consecutiveFailedCycles === 0 + ? this.idlePollMs + : Math.min( + this.retryMaxMs, + this.retryBaseMs * 2 ** (this.consecutiveFailedCycles - 1), + ); + this.schedule(delay); + } + } + } + } + + private async executeCycle(): Promise { + let targets: WorkspaceCleanupRetryTarget[]; + try { + targets = findWorkspaceCleanupRetryTargets(await this.options.workspaceRegistry.list()); + } catch (error) { + this.consecutiveFailedCycles += 1; + this.logger.warn({ err: error }, "Failed to list pending workspace cleanup"); + return; + } + + const rotatedTargets = rotateCleanupTargets(targets, this.nextTargetDirectoryPath); + const batch = rotatedTargets.slice(0, this.maxTargetsPerCycle); + const nextTarget = rotatedTargets[batch.length] ?? null; + this.nextTargetDirectoryPath = nextTarget?.directoryPath ?? null; + const abortController = new AbortController(); + this.activeAbortController = abortController; + + let failed = false; + try { + for (const target of batch) { + if (abortController.signal.aborted) break; + try { + await this.options.retryWorktreeCleanup(target, abortController.signal); + } catch (error) { + failed = true; + if (!abortController.signal.aborted) { + this.logger.warn( + { + err: error, + directoryPath: target.directoryPath, + workspaceIds: target.workspaceIds, + worktreeIncarnationId: target.worktreeIncarnationId, + }, + "Pending workspace cleanup retry failed", + ); + } + } + } + } finally { + if (this.activeAbortController === abortController) this.activeAbortController = null; + } + if (!failed && nextTarget) { + this.wakeRequested = true; + } + this.consecutiveFailedCycles = failed ? this.consecutiveFailedCycles + 1 : 0; + } +} + +function rotateCleanupTargets( + targets: readonly WorkspaceCleanupRetryTarget[], + nextDirectoryPath: string | null, +): WorkspaceCleanupRetryTarget[] { + if (!nextDirectoryPath) return [...targets]; + const index = targets.findIndex((target) => target.directoryPath === nextDirectoryPath); + if (index <= 0) return [...targets]; + return [...targets.slice(index), ...targets.slice(0, index)]; +} + +export function findWorkspaceCleanupRetryTargets( + workspaces: readonly PersistedWorkspaceRecord[], +): WorkspaceCleanupRetryTarget[] { + const byDirectory = new Map(); + for (const workspace of workspaces) { + if (!workspace.archivedAt || !workspace.cleanupPending) continue; + const directoryPath = resolve(workspace.cleanupPending.directoryPath); + const group = byDirectory.get(directoryPath) ?? []; + group.push(workspace); + byDirectory.set(directoryPath, group); + } + + const targets: WorkspaceCleanupRetryTarget[] = []; + for (const [directoryPath, group] of byDirectory) { + const incarnationIds = new Set( + group.map((workspace) => workspace.cleanupPending?.worktreeIncarnationId ?? null), + ); + if (incarnationIds.size !== 1 || incarnationIds.has(null)) continue; + targets.push({ + directoryPath, + worktreeIncarnationId: [...incarnationIds][0]!, + workspaceIds: group.map((workspace) => workspace.workspaceId).sort(), + }); + } + return targets.sort((left, right) => left.directoryPath.localeCompare(right.directoryPath)); +} diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.test.ts b/packages/server/src/server/workspace-lifecycle-coordinator.test.ts new file mode 100644 index 0000000000..94964dcea1 --- /dev/null +++ b/packages/server/src/server/workspace-lifecycle-coordinator.test.ts @@ -0,0 +1,104 @@ +import { expect, test, vi } from "vitest"; + +import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; + +test("cancels a workspace setup wait without waiting for the setup to settle", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + const reservation = coordinator.reserveWorkspaceSetup("ws-blocked", "/worktrees/blocked"); + const controller = new AbortController(); + + const waiting = coordinator.waitForWorkspaceSetups(["ws-blocked"], controller.signal); + controller.abort(); + + await expect(waiting).rejects.toThrow("Workspace lifecycle operation canceled"); + reservation.release(); +}); + +test("cancels a queued worktree mutation before it starts", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + let releaseFirst = () => {}; + const firstBlocked = new Promise((resolve) => { + releaseFirst = resolve; + }); + const first = coordinator.runWorktreeMutationExclusive("/worktrees", () => firstBlocked); + const secondOperation = vi.fn(async () => undefined); + const controller = new AbortController(); + const second = coordinator.runWorktreeMutationExclusive( + "/worktrees", + secondOperation, + controller.signal, + ); + + controller.abort(); + + await expect(second).rejects.toThrow("Workspace lifecycle operation canceled"); + expect(secondOperation).not.toHaveBeenCalled(); + releaseFirst(); + await first; + await Promise.resolve(); + expect(secondOperation).not.toHaveBeenCalled(); +}); + +test("drain waits for the real archive after its abortable wrapper rejects", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + const controller = new AbortController(); + let releaseArchive = () => {}; + const physicalArchive = new Promise((resolve) => { + releaseArchive = resolve; + }); + const archive = coordinator.runArchive( + "workspace:ws-drain", + () => physicalArchive, + controller.signal, + ); + await Promise.resolve(); + + controller.abort(); + await expect(archive).rejects.toThrow("Workspace lifecycle operation canceled"); + + let drained = false; + const drain = coordinator.drain().then(() => { + drained = true; + return undefined; + }); + await Promise.resolve(); + expect(drained).toBe(false); + + releaseArchive(); + await drain; + expect(drained).toBe(true); +}); + +test("drain waits for workspace setup reservations", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + const reservation = coordinator.reserveWorkspaceSetup("ws-setup", "/worktrees/setup"); + + let drained = false; + const drain = coordinator.drain().then(() => { + drained = true; + return undefined; + }); + await Promise.resolve(); + expect(drained).toBe(false); + + reservation.release(); + await drain; + expect(drained).toBe(true); +}); + +test("drain waits for workspace ownership mutation reservations", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + const reservation = coordinator.reserveWorkspaceOwnershipMutation("ws-ownership"); + + let drained = false; + const drain = coordinator.drain().then(() => { + drained = true; + return undefined; + }); + await Promise.resolve(); + expect(drained).toBe(false); + + reservation.release(); + await drain; + expect(drained).toBe(true); +}); diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts new file mode 100644 index 0000000000..b6a8561120 --- /dev/null +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -0,0 +1,296 @@ +import { normalizePathForIdentity } from "../utils/path.js"; + +function createCancellationError(message: string): Error { + const error = new Error(message); + error.name = "AbortError"; + return error; +} + +function waitForAbortable(task: Promise, signal: AbortSignal | undefined): Promise { + if (!signal) return task; + + return new Promise((resolve, reject) => { + let settled = false; + const finish = (callback: () => void) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + callback(); + }; + const onAbort = () => + finish(() => reject(createCancellationError("Workspace lifecycle operation canceled"))); + signal.addEventListener("abort", onAbort, { once: true }); + void task.then( + (value) => finish(() => resolve(value)), + (error) => finish(() => reject(error)), + ); + if (signal.aborted) onAbort(); + }); +} + +export class WorkspaceLifecycleCoordinator { + private readonly setupTasks = new Map>(); + private readonly setupTasksByDirectory = new Map>>(); + private readonly archiveOperations = new Map>(); + private readonly directoryOperations = new Map>(); + private readonly worktreeMutationOperations = new Map>(); + private readonly ownershipMutationTasks = new Map>>(); + private readonly archivingWorkspaceIds = new Map(); + + trackWorkspaceSetup(workspaceId: string, task: Promise, directoryPath?: string): void { + this.setupTasks.set(workspaceId, task); + const directoryKey = directoryPath ? normalizePathForIdentity(directoryPath) : null; + const directoryTasks = directoryKey + ? (this.setupTasksByDirectory.get(directoryKey) ?? new Set>()) + : null; + if (directoryKey && directoryTasks) { + directoryTasks.add(task); + this.setupTasksByDirectory.set(directoryKey, directoryTasks); + } + const clearTask = () => { + if (this.setupTasks.get(workspaceId) === task) { + this.setupTasks.delete(workspaceId); + } + if (directoryKey && directoryTasks) { + directoryTasks.delete(task); + if (directoryTasks.size === 0) { + this.setupTasksByDirectory.delete(directoryKey); + } + } + }; + void task.then(clearTask, clearTask); + } + + reserveWorkspaceSetup(workspaceId: string, directoryPath: string): WorkspaceSetupReservation { + let releaseReservation: (() => void) | null = null; + const reservationTask = new Promise((resolve) => { + releaseReservation = resolve; + }); + this.trackWorkspaceSetup(workspaceId, reservationTask, directoryPath); + + let released = false; + const release = () => { + if (released) return; + released = true; + releaseReservation?.(); + }; + return { + completeWith(task) { + void task.then(release, release); + }, + release, + }; + } + + async waitForWorkspaceSetups( + workspaceIds: Iterable, + signal?: AbortSignal, + ): Promise { + const tasks = Array.from(workspaceIds, (workspaceId) => + this.setupTasks.get(workspaceId), + ).filter((task): task is Promise => task !== undefined); + await waitForAbortable( + Promise.allSettled(tasks).then(() => undefined), + signal, + ); + } + + reserveWorkspaceOwnershipMutation(workspaceId: string): WorkspaceOwnershipMutationReservation { + if ((this.archivingWorkspaceIds.get(workspaceId) ?? 0) > 0) { + throw new Error(`Workspace ${workspaceId} is being archived`); + } + + let releaseReservation: (() => void) | null = null; + const reservationTask = new Promise((resolve) => { + releaseReservation = resolve; + }); + const tasks = this.ownershipMutationTasks.get(workspaceId) ?? new Set>(); + tasks.add(reservationTask); + this.ownershipMutationTasks.set(workspaceId, tasks); + + let released = false; + const release = () => { + if (released) return; + released = true; + releaseReservation?.(); + tasks.delete(reservationTask); + if (tasks.size === 0) { + this.ownershipMutationTasks.delete(workspaceId); + } + }; + return { release }; + } + + async runWorkspaceOwnershipMutation( + workspaceId: string, + requireActiveWorkspace: () => Promise, + operation: () => Promise, + ): Promise { + const reservation = this.reserveWorkspaceOwnershipMutation(workspaceId); + try { + // Validate after acquiring the reservation. An archive either already owns + // the workspace (and reservation fails), or must now wait for this check and + // mutation to finish. This also rejects attaches after archive completion. + await requireActiveWorkspace(); + return await operation(); + } finally { + reservation.release(); + } + } + + reserveWorkspaceArchive(workspaceIds: Iterable): WorkspaceArchiveReservation { + const ownedWorkspaceIds = new Set(); + const add = (additionalWorkspaceIds: Iterable) => { + for (const workspaceId of additionalWorkspaceIds) { + if (ownedWorkspaceIds.has(workspaceId)) continue; + this.archivingWorkspaceIds.set( + workspaceId, + (this.archivingWorkspaceIds.get(workspaceId) ?? 0) + 1, + ); + ownedWorkspaceIds.add(workspaceId); + } + }; + add(workspaceIds); + + let released = false; + return { + add, + release: () => { + if (released) return; + released = true; + for (const workspaceId of ownedWorkspaceIds) { + const remainingReservations = (this.archivingWorkspaceIds.get(workspaceId) ?? 1) - 1; + if (remainingReservations === 0) { + this.archivingWorkspaceIds.delete(workspaceId); + } else { + this.archivingWorkspaceIds.set(workspaceId, remainingReservations); + } + } + }, + }; + } + + async waitForWorkspaceOwnershipMutations( + workspaceIds: Iterable, + signal?: AbortSignal, + ): Promise { + const tasks = Array.from(workspaceIds).flatMap((workspaceId) => + Array.from(this.ownershipMutationTasks.get(workspaceId) ?? []), + ); + await waitForAbortable( + Promise.allSettled(tasks).then(() => undefined), + signal, + ); + } + + async drain(): Promise { + while (true) { + const tasks = new Set>([ + ...this.setupTasks.values(), + ...Array.from(this.setupTasksByDirectory.values()).flatMap((directoryTasks) => + Array.from(directoryTasks), + ), + ...this.archiveOperations.values(), + ...this.directoryOperations.values(), + ...this.worktreeMutationOperations.values(), + ...Array.from(this.ownershipMutationTasks.values()).flatMap((ownershipTasks) => + Array.from(ownershipTasks), + ), + ]); + if (tasks.size === 0) return; + await Promise.allSettled(tasks); + } + } + + runArchive(key: string, operation: () => Promise, signal?: AbortSignal): Promise { + const existing = this.archiveOperations.get(key); + if (existing) { + return waitForAbortable(existing as Promise, signal); + } + + const task = Promise.resolve().then(() => { + if (signal?.aborted) { + throw createCancellationError("Workspace archive canceled"); + } + return operation(); + }); + this.archiveOperations.set(key, task); + const clearTask = () => { + if (this.archiveOperations.get(key) === task) { + this.archiveOperations.delete(key); + } + }; + void task.then(clearTask, clearTask); + return waitForAbortable(task, signal); + } + + runDirectoryExclusive( + directoryPath: string, + operation: () => Promise, + signal?: AbortSignal, + ): Promise { + const directoryKey = normalizePathForIdentity(directoryPath); + const previous = this.directoryOperations.get(directoryKey) ?? Promise.resolve(); + const task = previous + .catch(() => undefined) + .then(async () => { + if (signal?.aborted) { + throw createCancellationError("Workspace directory operation canceled"); + } + while (true) { + const setupTasks = Array.from(this.setupTasksByDirectory.get(directoryKey) ?? []); + if (setupTasks.length === 0) break; + await waitForAbortable(Promise.allSettled(setupTasks), signal); + } + return operation(); + }); + this.directoryOperations.set(directoryKey, task); + const clearTask = () => { + if (this.directoryOperations.get(directoryKey) === task) { + this.directoryOperations.delete(directoryKey); + } + }; + void task.then(clearTask, clearTask); + return waitForAbortable(task, signal); + } + + runWorktreeMutationExclusive( + worktreesRoot: string, + operation: () => Promise, + signal?: AbortSignal, + ): Promise { + const previous = this.worktreeMutationOperations.get(worktreesRoot) ?? Promise.resolve(); + const task = previous + .catch(() => undefined) + .then(() => { + if (signal?.aborted) { + throw createCancellationError("Worktree mutation canceled"); + } + return operation(); + }); + this.worktreeMutationOperations.set(worktreesRoot, task); + const clearTask = () => { + if (this.worktreeMutationOperations.get(worktreesRoot) === task) { + this.worktreeMutationOperations.delete(worktreesRoot); + } + }; + void task.then(clearTask, clearTask); + return waitForAbortable(task, signal); + } +} + +export interface WorkspaceSetupReservation { + completeWith(task: Promise): void; + release(): void; +} + +export interface WorkspaceOwnershipMutationReservation { + release(): void; +} + +export interface WorkspaceArchiveReservation { + add(workspaceIds: Iterable): void; + release(): void; +} + +export const defaultWorkspaceLifecycleCoordinator = new WorkspaceLifecycleCoordinator(); diff --git a/packages/server/src/server/workspace-reconciliation-service.test.ts b/packages/server/src/server/workspace-reconciliation-service.test.ts index 9abe80e742..2000761b0a 100644 --- a/packages/server/src/server/workspace-reconciliation-service.test.ts +++ b/packages/server/src/server/workspace-reconciliation-service.test.ts @@ -627,6 +627,7 @@ describe("WorkspaceReconciliationService", () => { baseBranch: null, isPaseoOwnedWorktree: false, mainRepoRoot: null, + cleanupPending: null, createdAt: timestamp, updatedAt: expect.any(String), archivedAt: expect.any(String), diff --git a/packages/server/src/server/workspace-registry.ts b/packages/server/src/server/workspace-registry.ts index 5233e77e85..6d44fab432 100644 --- a/packages/server/src/server/workspace-registry.ts +++ b/packages/server/src/server/workspace-registry.ts @@ -40,6 +40,19 @@ const PersistedProjectRecordSchema = z.object({ archivedAt: z.string().nullable(), }); +const PersistedWorkspaceCleanupPendingSchema = z.object({ + directoryPath: z.string(), + teardownCwd: z.string(), + mainRepoRoot: z.string().nullable(), + paseoWorktreesRoot: z.string().nullable(), + // COMPAT(cleanupIncarnation): legacy path-only cleanup records fail closed. + worktreeIncarnationId: z + .string() + .nullable() + .optional() + .transform((value) => value ?? null), +}); + const PersistedWorkspaceRecordSchema = z.object({ workspaceId: z.string(), projectId: z.string(), @@ -77,6 +90,9 @@ const PersistedWorkspaceRecordSchema = z.object({ .transform((value) => value ?? null), isPaseoOwnedWorktree: z.boolean().default(false), mainRepoRoot: z.string().nullable().default(null), + cleanupPending: PersistedWorkspaceCleanupPendingSchema.nullable() + .optional() + .transform((value) => value ?? null), createdAt: z.string(), updatedAt: z.string(), archivedAt: z.string().nullable(), @@ -96,6 +112,9 @@ const PersistedWorkspaceRecordSchema = z.object({ export type PersistedProjectRecord = z.infer; export type PersistedWorkspaceRecord = z.infer; +export type PersistedWorkspaceCleanupPending = z.infer< + typeof PersistedWorkspaceCleanupPendingSchema +>; export interface WorkspaceMutation { kind: "upsert" | "archive" | "remove"; @@ -551,6 +570,7 @@ export function createPersistedWorkspaceRecord(input: { baseBranch?: string | null; isPaseoOwnedWorktree?: boolean; mainRepoRoot?: string | null; + cleanupPending?: PersistedWorkspaceCleanupPending | null; createdAt: string; updatedAt: string; archivedAt?: string | null; @@ -565,6 +585,7 @@ export function createPersistedWorkspaceRecord(input: { baseBranch: input.baseBranch ?? null, isPaseoOwnedWorktree: input.isPaseoOwnedWorktree ?? false, mainRepoRoot: input.mainRepoRoot ?? null, + cleanupPending: input.cleanupPending ?? null, archivedAt: input.archivedAt ?? null, autoArchivedChangeRequestUrl: input.autoArchivedChangeRequestUrl ?? null, pinnedAt: input.pinnedAt ?? null, diff --git a/packages/server/src/server/worktree-bootstrap.test.ts b/packages/server/src/server/worktree-bootstrap.test.ts index c878ef1387..eaff6c4f4d 100644 --- a/packages/server/src/server/worktree-bootstrap.test.ts +++ b/packages/server/src/server/worktree-bootstrap.test.ts @@ -5,7 +5,11 @@ import { join } from "path"; import { tmpdir } from "os"; import type { AgentTimelineItem } from "./agent/agent-sdk-types.js"; -import { runAsyncWorktreeBootstrap, spawnWorkspaceScript } from "./worktree-bootstrap.js"; +import { + createWorktreeSetupProgressCoalescer, + runAsyncWorktreeBootstrap, + spawnWorkspaceScript, +} from "./worktree-bootstrap.js"; import { ensureWorkspaceServicePortPlan } from "./workspace-service-port-registry.js"; import { ScriptRouteStore } from "./script-proxy.js"; import { createBranchChangeRouteHandler } from "./script-route-branch-handler.js"; @@ -17,6 +21,42 @@ import { import type { TerminalManager } from "../terminal/terminal-manager.js"; import type { TerminalSession } from "../terminal/terminal.js"; +describe("worktree setup progress coalescing", () => { + it("keeps only the latest pending update while an emission is in flight", async () => { + let releaseFirstEmission: (() => void) | null = null; + let markFirstEmissionStarted: (() => void) | null = null; + const firstEmissionGate = new Promise((resolve) => { + releaseFirstEmission = resolve; + }); + const firstEmissionStarted = new Promise((resolve) => { + markFirstEmissionStarted = resolve; + }); + const emittedVersions: number[] = []; + let version = 0; + const progress = createWorktreeSetupProgressCoalescer({ + emit: async () => { + emittedVersions.push(version); + if (emittedVersions.length === 1) { + markFirstEmissionStarted?.(); + await firstEmissionGate; + } + }, + }); + + progress.schedule(); + await firstEmissionStarted; + for (let nextVersion = 1; nextVersion <= 1_000; nextVersion += 1) { + version = nextVersion; + progress.schedule(); + } + + expect(emittedVersions).toEqual([0]); + releaseFirstEmission?.(); + await progress.flush(); + expect(emittedVersions).toEqual([0, 1_000]); + }); +}); + interface CreateAgentWorktreeTestOptions { cwd: string; branchName: string; @@ -142,7 +182,7 @@ describe("runAsyncWorktreeBootstrap", () => { } }); - it("truncates each command output to 64kb in the middle", async () => { + it("truncates each command output to 64kb with an exact omitted-byte count", async () => { const largeOutputCommand = "node -e \"process.stdout.write('prefix-'); process.stdout.write('x'.repeat(70000)); process.stdout.write('-suffix')\""; writeFileSync( @@ -194,12 +234,14 @@ describe("runAsyncWorktreeBootstrap", () => { expect(persistedSetupItem.detail.truncated).toBe(true); expect(persistedSetupItem.detail.log).toContain("prefix-"); expect(persistedSetupItem.detail.log).toContain("-suffix"); - expect(persistedSetupItem.detail.log).toContain("......"); + expect(persistedSetupItem.detail.log).toMatch(/\.\.\.<\d+ bytes omitted>\.\.\./); expect(persistedSetupItem.detail.commands[0]?.log).toContain("prefix-"); expect(persistedSetupItem.detail.commands[0]?.log).toContain("-suffix"); - expect(persistedSetupItem.detail.commands[0]?.log).toContain( - "......", - ); + const commandLog = persistedSetupItem.detail.commands[0]?.log ?? ""; + const marker = commandLog.match(/\n\.\.\.<(\d+) bytes omitted>\.\.\.\n/); + expect(marker).not.toBeNull(); + const retained = commandLog.replace(marker?.[0] ?? "", ""); + expect(Number(marker?.[1])).toBe(70_014 - Buffer.byteLength(retained)); }); it("waits for terminal output before sending bootstrap commands", async () => { diff --git a/packages/server/src/server/worktree-bootstrap.ts b/packages/server/src/server/worktree-bootstrap.ts index 75785057e4..643a0aa38b 100644 --- a/packages/server/src/server/worktree-bootstrap.ts +++ b/packages/server/src/server/worktree-bootstrap.ts @@ -31,6 +31,13 @@ import { refreshWorkspaceServicePort, } from "./workspace-service-port-registry.js"; import type { PaseoServicePortAllocation } from "@getpaseo/protocol/paseo-config-schema"; +import { + appendWorktreeSetupOutput, + createWorktreeSetupOutputAccumulator, + getWorktreeSetupCommandOutputLimit, + renderWorktreeSetupOutput, + type WorktreeSetupOutputAccumulator, +} from "../utils/worktree-setup-output.js"; export interface WorktreeBootstrapTerminalResult { name: string | null; @@ -54,120 +61,117 @@ export interface RunAsyncWorktreeBootstrapOptions { logger?: Logger; } -const MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES = 64 * 1024; -const WORKTREE_SETUP_TRUNCATION_MARKER = "\n......\n"; const WORKTREE_BOOTSTRAP_TERMINAL_READY_TIMEOUT_MS = 1_500; - -interface MiddleTruncationAccumulator { - totalBytes: number; - head: string; - tail: string; - truncated: boolean; -} - -export type WorktreeSetupOutputAccumulator = MiddleTruncationAccumulator; +const WORKTREE_SETUP_PROGRESS_EMIT_INTERVAL_MS = 100; export interface WorktreeSetupProgressAccumulator { resultsByIndex: Map; outputAccumulatorsByIndex: Map; } -function byteLength(text: string): number { - return Buffer.byteLength(text, "utf8"); +export interface WorktreeSetupProgressCoalescerOptions { + emit: () => unknown | Promise; + onError?: (error: unknown) => void; } -function sliceFirstBytes(text: string, maxBytes: number): string { - if (maxBytes <= 0 || text.length === 0) { - return ""; - } - const bytes = Buffer.from(text, "utf8"); - if (bytes.length <= maxBytes) { - return text; - } - return bytes.subarray(0, maxBytes).toString("utf8"); +export interface WorktreeSetupProgressCoalescer { + schedule: () => void; + flush: () => Promise; } -function sliceLastBytes(text: string, maxBytes: number): string { - if (maxBytes <= 0 || text.length === 0) { - return ""; - } - const bytes = Buffer.from(text, "utf8"); - if (bytes.length <= maxBytes) { - return text; - } - return bytes.subarray(bytes.length - maxBytes).toString("utf8"); -} - -export function createWorktreeSetupOutputAccumulator(): WorktreeSetupOutputAccumulator { - return { - totalBytes: 0, - head: "", - tail: "", - truncated: false, +export function createWorktreeSetupProgressCoalescer( + options: WorktreeSetupProgressCoalescerOptions, +): WorktreeSetupProgressCoalescer { + let pending = false; + let flushing = false; + let lastStartedAt: number | null = null; + let timer: ReturnType | null = null; + let inFlight: Promise | null = null; + + const handleError = (error: unknown) => { + try { + options.onError?.(error); + } catch { + // Progress reporting must not fail setup, including a failing error reporter. + } }; -} -function getHeadTailBudgets(maxBytes: number): { headBytes: number; tailBytes: number } { - const markerBytes = byteLength(WORKTREE_SETUP_TRUNCATION_MARKER); - const availableBytes = Math.max(0, maxBytes - markerBytes); - const headBytes = Math.floor(availableBytes / 2); - const tailBytes = availableBytes - headBytes; - return { headBytes, tailBytes }; -} + const startOrSchedule = () => { + if (!pending || inFlight || timer) { + return; + } -export function appendWorktreeSetupOutputAccumulator( - accumulator: WorktreeSetupOutputAccumulator, - chunk: string, -): void { - if (!chunk) { - return; - } - accumulator.totalBytes += byteLength(chunk); + const elapsedMs = lastStartedAt === null ? Infinity : Date.now() - lastStartedAt; + const delayMs = flushing + ? 0 + : Math.max(0, WORKTREE_SETUP_PROGRESS_EMIT_INTERVAL_MS - elapsedMs); + if (delayMs > 0) { + timer = setTimeout(() => { + timer = null; + startOrSchedule(); + }, delayMs); + return; + } - if (!accumulator.truncated) { - const combined = `${accumulator.head}${chunk}`; - if (byteLength(combined) <= MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES) { - accumulator.head = combined; + pending = false; + lastStartedAt = Date.now(); + let emitted: unknown | Promise; + try { + emitted = options.emit(); + } catch (error) { + handleError(error); + emitted = undefined; + } + const emission = Promise.resolve(emitted).catch(handleError); + let current: Promise; + current = emission.then(() => { + if (inFlight === current) { + inFlight = null; + startOrSchedule(); + } + return undefined; + }); + inFlight = current; + }; + + const waitForIdle = async (): Promise => { + startOrSchedule(); + const current = inFlight; + if (!current) { return; } - const { headBytes, tailBytes } = getHeadTailBudgets(MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES); - accumulator.head = sliceFirstBytes(combined, headBytes); - accumulator.tail = sliceLastBytes(combined, tailBytes); - accumulator.truncated = true; - return; - } + await current; + if (pending || inFlight) { + await waitForIdle(); + } + }; - const { tailBytes } = getHeadTailBudgets(MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES); - accumulator.tail = sliceLastBytes(`${accumulator.tail}${chunk}`, tailBytes); + return { + schedule: () => { + pending = true; + startOrSchedule(); + }, + flush: async () => { + flushing = true; + if (timer) { + clearTimeout(timer); + timer = null; + } + try { + await waitForIdle(); + } finally { + flushing = false; + } + }, + }; } function truncateTextInMiddle( text: string, maxBytes: number, ): { text: string; truncated: boolean } { - if (maxBytes <= 0 || !text) { - return { text: "", truncated: text.length > 0 }; - } - if (byteLength(text) <= maxBytes) { - return { text, truncated: false }; - } - const { headBytes, tailBytes } = getHeadTailBudgets(maxBytes); - return { - text: `${sliceFirstBytes(text, headBytes)}${WORKTREE_SETUP_TRUNCATION_MARKER}${sliceLastBytes(text, tailBytes)}`, - truncated: true, - }; -} - -function renderMiddleTruncationAccumulator(accumulator: MiddleTruncationAccumulator): { - text: string; - truncated: boolean; -} { - if (!accumulator.truncated) { - return { text: accumulator.head, truncated: false }; - } - return { - text: `${accumulator.head}${WORKTREE_SETUP_TRUNCATION_MARKER}${accumulator.tail}`, - truncated: true, - }; + const accumulator = createWorktreeSetupOutputAccumulator(maxBytes); + appendWorktreeSetupOutput(accumulator, text); + return renderWorktreeSetupOutput(accumulator); } function formatDurationMs(durationMs: number): string { @@ -231,10 +235,10 @@ function buildWorktreeSetupCommandLog(input: { const { index, result, outputAccumulatorsByIndex } = input; const accumulator = outputAccumulatorsByIndex?.get(index); const rendered = accumulator - ? renderMiddleTruncationAccumulator(accumulator) + ? renderWorktreeSetupOutput(accumulator) : truncateTextInMiddle( `${result.stdout ?? ""}${result.stderr ?? ""}`, - MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES, + getWorktreeSetupCommandOutputLimit(1), ); return { @@ -267,8 +271,8 @@ export function applyWorktreeSetupProgressEvent( if (event.type === "output") { const outputAccumulator = accumulator.outputAccumulatorsByIndex.get(event.index) ?? - createWorktreeSetupOutputAccumulator(); - appendWorktreeSetupOutputAccumulator(outputAccumulator, event.chunk); + createWorktreeSetupOutputAccumulator(getWorktreeSetupCommandOutputLimit(event.total)); + appendWorktreeSetupOutput(outputAccumulator, event.chunk); accumulator.outputAccumulatorsByIndex.set(event.index, outputAccumulator); accumulator.resultsByIndex.set(event.index, { ...baseResult, @@ -602,34 +606,27 @@ export async function runAsyncWorktreeBootstrap( const emitLiveTimelineItem = options.emitLiveTimelineItem; const progressAccumulator = createWorktreeSetupProgressAccumulator(); const workspaceCwd = options.workspaceCwd ?? options.worktree.worktreePath; - let liveEmitQueue = Promise.resolve(); - - const queueLiveRunningEmit = () => { - if (!emitLiveTimelineItem) { - return; - } - const runningResults = getWorktreeSetupProgressResults(progressAccumulator); - liveEmitQueue = liveEmitQueue.then(async () => { - try { - await emitLiveTimelineItem( - buildSetupTimelineItem({ - callId: setupCallId, - status: "running", - worktree: options.worktree, - results: runningResults, - outputAccumulatorsByIndex: progressAccumulator.outputAccumulatorsByIndex, - errorMessage: null, - }), - ); - } catch (error) { - options.logger?.warn( - { err: error, agentId: options.agentId }, - "Failed to emit live worktree setup timeline update", - ); - } - return; - }); - }; + const liveProgress = emitLiveTimelineItem + ? createWorktreeSetupProgressCoalescer({ + emit: () => + emitLiveTimelineItem( + buildSetupTimelineItem({ + callId: setupCallId, + status: "running", + worktree: options.worktree, + results: getWorktreeSetupProgressResults(progressAccumulator), + outputAccumulatorsByIndex: progressAccumulator.outputAccumulatorsByIndex, + errorMessage: null, + }), + ), + onError: (error) => { + options.logger?.warn( + { err: error, agentId: options.agentId }, + "Failed to emit live worktree setup timeline update", + ); + }, + }) + : null; try { runtimeEnv = await resolveWorktreeRuntimeEnv({ @@ -648,10 +645,10 @@ export async function runAsyncWorktreeBootstrap( runtimeEnv, onEvent: (event) => { applyWorktreeSetupProgressEvent(progressAccumulator, event); - queueLiveRunningEmit(); + liveProgress?.schedule(); }, }); - await liveEmitQueue; + await liveProgress?.flush(); const completed = await options.appendTimelineItem( buildSetupTimelineItem({ @@ -670,7 +667,7 @@ export async function runAsyncWorktreeBootstrap( if (error instanceof WorktreeSetupError) { setupResults = error.results; } - await liveEmitQueue; + await liveProgress?.flush(); const message = error instanceof Error ? error.message : String(error); await options.appendTimelineItem( buildSetupTimelineItem({ diff --git a/packages/server/src/server/worktree-core.posix.test.ts b/packages/server/src/server/worktree-core.posix.test.ts index 40bd983c7b..8a5c1867c0 100644 --- a/packages/server/src/server/worktree-core.posix.test.ts +++ b/packages/server/src/server/worktree-core.posix.test.ts @@ -8,6 +8,7 @@ import { readFileSync, realpathSync, rmSync, + statSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -24,7 +25,7 @@ import { getPaseoWorktreeMetadataPath, readPaseoWorktreeMetadata, } from "../utils/worktree-metadata.js"; -import { UnknownBranchError } from "../utils/worktree.js"; +import { computeWorktreePath, UnknownBranchError } from "../utils/worktree.js"; import { createWorktreeCore as createCoreWorktree } from "./worktree-core.js"; import { isPlatform } from "../test-utils/platform.js"; @@ -420,6 +421,48 @@ describe.skipIf(isPlatform("win32"))("worktree-core POSIX-only", () => { expect(existsSync(result.worktree.worktreePath)).toBe(true); }); + test("publishes the collision-resolved path before creating the worktree", async () => { + const { tempDir, repoDir, paseoHome } = createGitRepo(); + cleanupPaths.push(tempDir); + const occupiedPath = await computeWorktreePath(repoDir, "collision", paseoHome); + mkdirSync(occupiedPath, { recursive: true }); + const resolvedPath = `${occupiedPath}-1`; + let observedPath: string | null = null; + let observedIncarnationId: string | null = null; + + const result = await createCoreWorktree( + { + cwd: repoDir, + worktreeSlug: "collision", + paseoHome, + runSetup: false, + onWorktreePathPlanned: async (worktreePath, plan) => { + expect(worktreePath).toBe(resolvedPath); + expect(existsSync(worktreePath)).toBe(false); + observedIncarnationId = plan.worktreeIncarnationId; + }, + onWorktreePathResolved: async (worktreePath, reservation) => { + observedPath = worktreePath; + observedIncarnationId = reservation.worktreeIncarnationId; + expect(worktreePath).toBe(resolvedPath); + expect(existsSync(worktreePath)).toBe(true); + const directoryStat = statSync(worktreePath, { bigint: true }); + expect(reservation.directoryIdentity).toEqual({ + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }); + expect(reservation.metadataBaseRefName).toBe("main"); + }, + }, + createCoreDeps(), + ); + + expect(observedPath).toBe(resolvedPath); + expect(result.worktree.worktreePath).toBe(resolvedPath); + expect(existsSync(resolvedPath)).toBe(true); + expect(readPaseoWorktreeMetadata(resolvedPath)?.incarnationId).toBe(observedIncarnationId); + }); + test("creates branch-off worktrees from origin main without tracking origin main", async () => { const { tempDir, repoDir, paseoHome } = createGitRepoWithOriginMain(); cleanupPaths.push(tempDir); @@ -1443,19 +1486,32 @@ describe.skipIf(isPlatform("win32"))("worktree-core POSIX-only", () => { const { tempDir, repoDir, paseoHome } = createGitRepo(); cleanupPaths.push(tempDir); const deps = createCoreDeps(); + const resolvedPaths: string[] = []; const first = await createCoreWorktree( { cwd: repoDir, worktreeSlug: "reused-worktree", paseoHome, runSetup: false }, deps, ); const second = await createCoreWorktree( - { cwd: repoDir, worktreeSlug: "reused-worktree", paseoHome, runSetup: false }, + { + cwd: repoDir, + worktreeSlug: "reused-worktree", + paseoHome, + runSetup: false, + onWorktreePathPlanned: async (worktreePath) => { + resolvedPaths.push(worktreePath); + }, + onWorktreePathResolved: async (worktreePath) => { + resolvedPaths.push(worktreePath); + }, + }, deps, ); expect(first.created).toBe(true); expect(second.created).toBe(false); expect(second.worktree).toEqual(first.worktree); + expect(resolvedPaths).toEqual([]); }); // POSIX-only: Windows git worktree paths need separate canonicalization coverage. diff --git a/packages/server/src/server/worktree-core.ts b/packages/server/src/server/worktree-core.ts index 90d0788905..90ae1e53b7 100644 --- a/packages/server/src/server/worktree-core.ts +++ b/packages/server/src/server/worktree-core.ts @@ -6,6 +6,8 @@ import { resolveExistingWorktreeForSlug, slugify, validateBranchSlug, + type WorktreeCreationPlan, + type WorktreeCreationReservation, type WorktreeConfig, } from "../utils/worktree.js"; import { @@ -29,6 +31,11 @@ export interface CreateWorktreeCoreInput { paseoHome?: string; worktreesRoot?: string; runSetup?: boolean; + onWorktreePathPlanned?: (worktreePath: string, plan: WorktreeCreationPlan) => Promise; + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; } export interface CreateWorktreeCoreDeps { @@ -128,6 +135,8 @@ export async function createWorktreeCore( runSetup: input.runSetup ?? true, paseoHome: input.paseoHome, worktreesRoot: input.worktreesRoot, + onWorktreePathPlanned: input.onWorktreePathPlanned, + onWorktreePathResolved: input.onWorktreePathResolved, }), intent, repoRoot, diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index a5de5baedb..c710ea7fca 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -13,9 +13,14 @@ import path from "node:path"; import { afterEach, describe, expect, test, vi } from "vitest"; import pino, { type Logger } from "pino"; -import type { SessionOutboundMessage, WorkspaceDescriptorPayload } from "./messages.js"; +import type { + SessionOutboundMessage, + WorkspaceDescriptorPayload, + WorkspaceSetupSnapshot, +} from "./messages.js"; import { buildAgentSessionConfig, + cacheWorkspaceSetupSnapshot, createPaseoWorktreeWorkflow, handlePaseoWorktreeArchiveRequest, handlePaseoWorktreeListRequest, @@ -23,6 +28,7 @@ import { runWorktreeSetupInBackground, handleCreatePaseoWorktreeRequest, handleWorkspaceSetupStatusRequest, + MAX_WORKSPACE_SETUP_SNAPSHOTS, } from "./worktree-session.js"; import { createWorktree as createWorktreePrimitive, @@ -49,6 +55,7 @@ import { WorkspaceGitServiceImpl } from "./workspace-git-service.js"; import type { WorkspaceGitService } from "./workspace-git-service.js"; import { isPlatform } from "../test-utils/platform.js"; import { createWorkspaceProvisioningService } from "./session/workspace-provisioning/workspace-provisioning-service.js"; +import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; interface LegacyCreateWorktreeTestOptions { branchName: string; @@ -87,6 +94,21 @@ function createLogger(): Logger { return logger; } +function createWorktreeCreationJournalStub() { + return { + beginPendingAgentCreation: vi.fn().mockImplementation(async (creationId: string) => ({ + agentId: creationId, + createdAt: new Date().toISOString(), + ownerKind: "standalone-worktree" as const, + cleanupTarget: { kind: "agent" as const }, + })), + planPendingAgentCreationWorktree: vi.fn().mockResolvedValue(undefined), + identifyPendingAgentCreationWorktree: vi.fn().mockResolvedValue(undefined), + removePendingAgentCreation: vi.fn().mockResolvedValue(undefined), + recoverPendingCreation: vi.fn().mockResolvedValue(undefined), + }; +} + function createWorkflowForRequestTest(options: { paseoHome: string; createPaseoWorktree?: CreatePaseoWorktreeFn; @@ -108,6 +130,7 @@ function createWorkflowForRequestTest(options: { createPaseoWorktree, warmWorkspaceGitData: options.warmWorkspaceGitData ?? (async () => {}), autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: createWorktreeCreationJournalStub(), emitWorkspaceUpdateForWorkspaceId: async () => {}, cacheWorkspaceSetupSnapshot: () => {}, emit: () => {}, @@ -450,7 +473,121 @@ describe("resolveGitCreateBaseBranch", () => { }); }); -describe("create-agent worktree setup boundary", () => { +describe("create worktree setup boundary", () => { + test("journals standalone creation before mkdir and clears it after workspace publication", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const journal = createWorktreeCreationJournalStub(); + let plannedCreationId: string | null = null; + let plannedPath: string | null = null; + let plannedIncarnationId: string | null = null; + journal.planPendingAgentCreationWorktree.mockImplementation( + async (creationId, worktreePath, plan) => { + expect(existsSync(worktreePath)).toBe(false); + plannedCreationId = creationId; + plannedPath = worktreePath; + plannedIncarnationId = plan.worktreeIncarnationId; + }, + ); + journal.identifyPendingAgentCreationWorktree.mockImplementation( + async (creationId, worktreePath, reservation) => { + expect(creationId).toBe(plannedCreationId); + expect(worktreePath).toBe(plannedPath); + expect(reservation.worktreeIncarnationId).toBe(plannedIncarnationId); + expect(existsSync(worktreePath)).toBe(true); + }, + ); + + try { + const result = await createPaseoWorktreeWorkflow( + { + paseoHome, + createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), + warmWorkspaceGitData: async () => {}, + autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: journal, + emitWorkspaceUpdateForWorkspaceId: async () => {}, + cacheWorkspaceSetupSnapshot: () => {}, + emit: () => {}, + sessionLogger: createLogger(), + terminalManager: null, + archiveWorkspaceRecord: async () => {}, + serviceProxy: null, + scriptRuntimeStore: null, + getDaemonTcpPort: null, + getDaemonTcpHost: null, + onScriptsChanged: null, + }, + { + cwd: repoDir, + worktreeSlug: "standalone-journal-order", + runSetup: false, + paseoHome, + }, + ); + + expect(journal.beginPendingAgentCreation).toHaveBeenCalledWith(plannedCreationId, { + ownerKind: "standalone-worktree", + }); + expect(journal.identifyPendingAgentCreationWorktree).toHaveBeenCalledOnce(); + expect(journal.removePendingAgentCreation).toHaveBeenCalledWith(plannedCreationId); + expect(existsSync(result.worktree.worktreePath)).toBe(true); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + test("registers workspace setup before returning the created workspace", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const trackWorkspaceSetup = vi.spyOn(lifecycleCoordinator, "trackWorkspaceSetup"); + const runWorktreeMutationExclusive = vi.spyOn( + lifecycleCoordinator, + "runWorktreeMutationExclusive", + ); + + try { + const result = await createPaseoWorktreeWorkflow( + { + paseoHome, + createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), + warmWorkspaceGitData: async () => {}, + autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: createWorktreeCreationJournalStub(), + emitWorkspaceUpdateForWorkspaceId: async () => {}, + cacheWorkspaceSetupSnapshot: () => {}, + emit: () => {}, + sessionLogger: createLogger(), + terminalManager: null, + archiveWorkspaceRecord: async () => {}, + serviceProxy: null, + scriptRuntimeStore: null, + getDaemonTcpPort: null, + getDaemonTcpHost: null, + onScriptsChanged: null, + lifecycleCoordinator, + }, + { + cwd: repoDir, + worktreeSlug: "registered-before-return", + runSetup: false, + paseoHome, + }, + ); + + expect(trackWorkspaceSetup).toHaveBeenCalledWith( + result.workspace.workspaceId, + expect.any(Promise), + result.worktree.worktreePath, + ); + expect(runWorktreeMutationExclusive).toHaveBeenCalledOnce(); + await lifecycleCoordinator.waitForWorkspaceSetups([result.workspace.workspaceId]); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + test("agent setup continuation starts setup for the created agent timeline", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -465,6 +602,7 @@ describe("create-agent worktree setup boundary", () => { createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), warmWorkspaceGitData: async () => {}, autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: createWorktreeCreationJournalStub(), emitWorkspaceUpdateForWorkspaceId: async () => {}, cacheWorkspaceSetupSnapshot: () => {}, emit: (message) => workspaceSetupEvents.push(message), @@ -524,6 +662,65 @@ describe("create-agent worktree setup boundary", () => { rmSync(tempDir, { recursive: true, force: true }); } }); + + test("reserves agent setup before returning and releases an unused continuation", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const operation = vi.fn(); + + try { + const result = await createPaseoWorktreeWorkflow( + { + paseoHome, + createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), + warmWorkspaceGitData: async () => {}, + autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: createWorktreeCreationJournalStub(), + emitWorkspaceUpdateForWorkspaceId: async () => {}, + cacheWorkspaceSetupSnapshot: () => {}, + emit: () => {}, + sessionLogger: createLogger(), + terminalManager: null, + archiveWorkspaceRecord: async () => {}, + serviceProxy: null, + scriptRuntimeStore: null, + getDaemonTcpPort: null, + getDaemonTcpHost: null, + onScriptsChanged: null, + lifecycleCoordinator, + }, + { + cwd: repoDir, + worktreeSlug: "agent-setup-reservation", + runSetup: false, + paseoHome, + }, + { + setupContinuation: { + kind: "agent", + terminalManager: null, + appendTimelineItem: async () => true, + emitLiveTimelineItem: async () => true, + logger: createLogger(), + }, + }, + ); + + const directoryTask = lifecycleCoordinator.runDirectoryExclusive( + result.worktree.worktreePath, + async () => operation(), + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(operation).not.toHaveBeenCalled(); + + result.setupContinuation?.releaseWithoutStarting(); + await directoryTask; + expect(operation).toHaveBeenCalledOnce(); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); }); function createAgentStorageStub(): Pick { @@ -1274,6 +1471,29 @@ describe("runWorktreeSetupInBackground", () => { }, }); }); + + test("evicts the oldest setup snapshot when the shared cache reaches its bound", () => { + const snapshots = new Map(); + const snapshot: WorkspaceSetupSnapshot = { + status: "completed", + detail: { + type: "worktree_setup", + worktreePath: "/repo/worktree", + branchName: "feature", + log: "done", + commands: [], + }, + error: null, + }; + + for (let index = 0; index <= MAX_WORKSPACE_SETUP_SNAPSHOTS; index += 1) { + cacheWorkspaceSetupSnapshot(snapshots, `ws-${index}`, snapshot); + } + + expect(snapshots).toHaveLength(MAX_WORKSPACE_SETUP_SNAPSHOTS); + expect(snapshots.has("ws-0")).toBe(false); + expect(snapshots.has(`ws-${MAX_WORKSPACE_SETUP_SNAPSHOTS}`)).toBe(true); + }); }); describe("handleCreatePaseoWorktreeRequest", () => { @@ -1818,6 +2038,222 @@ describe("handlePaseoWorktreeArchiveRequest worktree scope", () => { } }); + test("reports archive teardown failure instead of emitting success", async () => { + const { tempDir } = createGitRepo(); + cleanupPaths.push(tempDir); + const workspaceId = "ws-archive-failure"; + const workspaceCwd = path.join(tempDir, "local-checkout"); + mkdirSync(workspaceCwd, { recursive: true }); + const emitted: SessionOutboundMessage[] = []; + + await handlePaseoWorktreeArchiveRequest( + { + paseoHome: path.join(tempDir, ".paseo"), + github: createGitHubServiceStub(), + workspaceGitService: { + getSnapshot: vi.fn(async () => null), + listWorktrees: vi.fn(async () => []), + }, + agentManager: { + listAgents: () => [], + archiveAgent: vi.fn(async () => ({ archivedAt: new Date().toISOString() })), + archiveSnapshot: vi.fn(async () => ({})), + }, + agentStorage: createAgentStorageStub(), + findWorkspaceIdForCwd: vi.fn(async () => workspaceId), + listActiveWorkspaces: vi.fn(async () => [ + { workspaceId, cwd: workspaceCwd, kind: "local_checkout" as const }, + ]), + archiveWorkspaceRecord: vi.fn(async () => { + throw new Error("registry archive failed"); + }), + emit: (message) => emitted.push(message), + emitWorkspaceUpdatesForWorkspaceIds: vi.fn(async () => {}), + markWorkspaceArchiving: vi.fn(), + clearWorkspaceArchiving: vi.fn(), + killTerminalsForWorkspace: vi.fn(async () => {}), + sessionLogger: createLogger(), + }, + { + type: "paseo_worktree_archive_request", + requestId: "req-archive-failure", + worktreePath: workspaceCwd, + repoRoot: workspaceCwd, + workspaceId, + scope: "workspace", + }, + ); + + expect( + emitted.find((message) => message.type === "paseo_worktree_archive_response"), + ).toMatchObject({ + payload: { + success: false, + error: { message: "Failed to archive one or more workspaces" }, + }, + }); + }); + + test("reports failure when an archive path no longer resolves to a workspace", async () => { + const { tempDir } = createGitRepo(); + cleanupPaths.push(tempDir); + const workspaceCwd = path.join(tempDir, "unresolved-workspace"); + mkdirSync(workspaceCwd, { recursive: true }); + const emitted: SessionOutboundMessage[] = []; + + await handlePaseoWorktreeArchiveRequest( + { + paseoHome: path.join(tempDir, ".paseo"), + github: createGitHubServiceStub(), + workspaceGitService: { + getSnapshot: vi.fn(async () => null), + listWorktrees: vi.fn(async () => []), + }, + agentManager: { + listAgents: () => [], + archiveAgent: vi.fn(async () => ({ archivedAt: new Date().toISOString() })), + archiveSnapshot: vi.fn(async () => ({})), + }, + agentStorage: createAgentStorageStub(), + findWorkspaceIdForCwd: vi.fn(async () => null), + listActiveWorkspaces: vi.fn(async () => []), + archiveWorkspaceRecord: vi.fn(async () => {}), + emit: (message) => emitted.push(message), + emitWorkspaceUpdatesForWorkspaceIds: vi.fn(async () => {}), + markWorkspaceArchiving: vi.fn(), + clearWorkspaceArchiving: vi.fn(), + killTerminalsForWorkspace: vi.fn(async () => {}), + sessionLogger: createLogger(), + }, + { + type: "paseo_worktree_archive_request", + requestId: "req-unresolved-workspace", + worktreePath: workspaceCwd, + repoRoot: workspaceCwd, + scope: "workspace", + }, + ); + + expect( + emitted.find((message) => message.type === "paseo_worktree_archive_response"), + ).toMatchObject({ + payload: { + success: false, + error: { + code: "UNKNOWN", + message: `Workspace not found for archive target: ${workspaceCwd}`, + }, + }, + }); + }); + + test("reports failure for an explicit unknown workspace id", async () => { + const { tempDir } = createGitRepo(); + cleanupPaths.push(tempDir); + const emitted: SessionOutboundMessage[] = []; + const archiveWorkspaceRecord = vi.fn(async () => {}); + + await handlePaseoWorktreeArchiveRequest( + { + paseoHome: path.join(tempDir, ".paseo"), + github: createGitHubServiceStub(), + workspaceGitService: { + getSnapshot: vi.fn(async () => null), + listWorktrees: vi.fn(async () => []), + }, + agentManager: { + listAgents: () => [], + archiveAgent: vi.fn(async () => ({ archivedAt: new Date().toISOString() })), + archiveSnapshot: vi.fn(async () => ({})), + }, + agentStorage: createAgentStorageStub(), + findWorkspaceIdForCwd: vi.fn(async () => null), + listActiveWorkspaces: vi.fn(async () => []), + archiveWorkspaceRecord, + emit: (message) => emitted.push(message), + emitWorkspaceUpdatesForWorkspaceIds: vi.fn(async () => {}), + markWorkspaceArchiving: vi.fn(), + clearWorkspaceArchiving: vi.fn(), + killTerminalsForWorkspace: vi.fn(async () => {}), + sessionLogger: createLogger(), + }, + { + type: "paseo_worktree_archive_request", + requestId: "req-explicit-unknown", + workspaceId: "ws-explicit-unknown", + scope: "workspace", + }, + ); + + expect(archiveWorkspaceRecord).not.toHaveBeenCalled(); + expect( + emitted.find((message) => message.type === "paseo_worktree_archive_response"), + ).toMatchObject({ + payload: { + success: false, + error: { + code: "UNKNOWN", + message: "Workspace not found: ws-explicit-unknown", + }, + }, + }); + }); + + test("preserves registry failures while validating an explicit workspace id", async () => { + const { tempDir } = createGitRepo(); + cleanupPaths.push(tempDir); + const emitted: SessionOutboundMessage[] = []; + const registryFailure = new Error("workspace registry read failed"); + + await handlePaseoWorktreeArchiveRequest( + { + paseoHome: path.join(tempDir, ".paseo"), + github: createGitHubServiceStub(), + workspaceGitService: { + getSnapshot: vi.fn(async () => null), + listWorktrees: vi.fn(async () => []), + }, + agentManager: { + listAgents: () => [], + archiveAgent: vi.fn(async () => ({ archivedAt: new Date().toISOString() })), + archiveSnapshot: vi.fn(async () => ({})), + }, + agentStorage: createAgentStorageStub(), + findWorkspaceIdForCwd: vi.fn(async () => null), + listActiveWorkspaces: vi.fn(async () => []), + workspaceRegistry: { + get: vi.fn(async () => { + throw registryFailure; + }), + list: vi.fn(async () => []), + update: vi.fn(async () => null), + }, + archiveWorkspaceRecord: vi.fn(async () => {}), + emit: (message) => emitted.push(message), + emitWorkspaceUpdatesForWorkspaceIds: vi.fn(async () => {}), + markWorkspaceArchiving: vi.fn(), + clearWorkspaceArchiving: vi.fn(), + killTerminalsForWorkspace: vi.fn(async () => {}), + sessionLogger: createLogger(), + }, + { + type: "paseo_worktree_archive_request", + requestId: "req-registry-failure", + workspaceId: "ws-registry-failure", + scope: "workspace", + }, + ); + + expect( + emitted.find((message) => message.type === "paseo_worktree_archive_response"), + ).toMatchObject({ + payload: { + success: false, + error: { message: "workspace registry read failed" }, + }, + }); + }); + test("archives every active workspace on the directory and removes it", async () => { const { tempDir, repoDir } = createGitRepo(); cleanupPaths.push(tempDir); @@ -2119,8 +2555,6 @@ describe("handlePaseoWorktreeArchiveRequest worktree scope", () => { await handlePaseoWorktreeArchiveRequest(deps, { type: "paseo_worktree_archive_request", requestId: "req-delete-flag-second", - worktreePath: sharedCwd, - repoRoot: repoDir, workspaceId: workspaceB, scope: "workspace", deleteWorktreeFromDisk: true, diff --git a/packages/server/src/server/worktree-session.ts b/packages/server/src/server/worktree-session.ts index 5573a945f8..6a9c6bbf7f 100644 --- a/packages/server/src/server/worktree-session.ts +++ b/packages/server/src/server/worktree-session.ts @@ -1,7 +1,9 @@ import type { Logger } from "pino"; -import { basename } from "node:path"; +import { randomUUID } from "node:crypto"; +import { basename, resolve } from "node:path"; import type { AgentSessionConfig } from "./agent/agent-sdk-types.js"; +import type { AgentStorage } from "./agent/agent-storage.js"; import { type GitSetupOptions, type FirstAgentContext, @@ -18,6 +20,7 @@ import { applyWorktreeSetupProgressEvent, buildWorktreeSetupDetail, createWorktreeSetupProgressAccumulator, + createWorktreeSetupProgressCoalescer, getWorktreeSetupProgressResults, } from "./worktree-bootstrap.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; @@ -27,10 +30,12 @@ import type { CheckoutExistingBranchResult } from "../utils/checkout-git.js"; import { expandTilde } from "../utils/path.js"; import { getWorktreeSetupCommands, + getPaseoWorktreesRoot, resolveWorktreeRuntimeEnv, runWorktreeSetupCommands, slugify, validateBranchSlug, + type WorktreeCreationJournalCallbacks, type WorktreeConfig, type WorktreeSetupCommandResult, WorktreeSetupError, @@ -47,6 +52,10 @@ import { createPaseoWorktreeCommand, listPaseoWorktreesCommand, } from "./worktree/commands.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "./workspace-lifecycle-coordinator.js"; const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._/-]+$/; @@ -109,6 +118,7 @@ interface CreatePaseoWorktreeInBackgroundDependencies { getDaemonTcpHost: (() => string | null) | null; serviceProxyPublicBaseUrl?: string | null; onScriptsChanged: ((workspaceId: string, workspaceDirectory: string) => void) | null; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; } interface CreatePaseoWorktreeWorkflowDependencies extends CreatePaseoWorktreeInBackgroundDependencies { @@ -123,6 +133,15 @@ interface CreatePaseoWorktreeWorkflowDependencies extends CreatePaseoWorktreeInB workspace: PersistedWorkspaceRecord; firstAgentContext: FirstAgentContext; }) => void; + worktreeCreationJournal: Pick< + AgentStorage, + | "beginPendingAgentCreation" + | "planPendingAgentCreationWorktree" + | "identifyPendingAgentCreationWorktree" + | "removePendingAgentCreation" + > & { + recoverPendingCreation: (creationId: string) => Promise; + }; } interface AgentWorktreeSetupContinuationInput { @@ -140,6 +159,7 @@ export type CreatePaseoWorktreeSetupContinuationInput = export interface AgentWorktreeSetupContinuation { kind: "agent"; startAfterAgentCreate: (input: { agentId: string }) => void; + releaseWithoutStarting: () => void; } export type CreatePaseoWorktreeWorkflowResult = CreatePaseoWorktreeResult & { @@ -159,6 +179,24 @@ interface HandleWorkspaceSetupStatusRequestDependencies { workspaceSetupSnapshots: ReadonlyMap; } +export const MAX_WORKSPACE_SETUP_SNAPSHOTS = 100; + +export function cacheWorkspaceSetupSnapshot( + snapshots: Map, + workspaceId: string, + snapshot: WorkspaceSetupSnapshot, +): void { + snapshots.delete(workspaceId); + snapshots.set(workspaceId, snapshot); + while (snapshots.size > MAX_WORKSPACE_SETUP_SNAPSHOTS) { + const oldestWorkspaceId = snapshots.keys().next().value; + if (oldestWorkspaceId === undefined) { + break; + } + snapshots.delete(oldestWorkspaceId); + } +} + interface HandleCreatePaseoWorktreeRequestDependencies { paseoHome?: string; worktreesRoot?: string; @@ -195,6 +233,7 @@ export async function buildAgentSessionConfig( gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, + worktreeCreationJournal?: WorktreeCreationJournalCallbacks, ): Promise<{ sessionConfig: AgentSessionConfig; setupContinuation?: AgentWorktreeSetupContinuation; @@ -232,6 +271,8 @@ export async function buildAgentSessionConfig( runSetup: false, paseoHome: dependencies.paseoHome, worktreesRoot: dependencies.worktreesRoot, + onWorktreePathPlanned: worktreeCreationJournal?.onWorktreePathPlanned, + onWorktreePathResolved: worktreeCreationJournal?.onWorktreePathResolved, }, { resolveDefaultBranch: normalized.baseBranch @@ -487,6 +528,8 @@ export async function handlePaseoWorktreeArchiveRequest( payload: { success: true, removedAgents: result.removedAgents, + removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPending, error: null, requestId, }, @@ -590,72 +633,158 @@ export async function createPaseoWorktreeWorkflow( setupContinuation?: CreatePaseoWorktreeSetupContinuationInput; }, ): Promise { - const createdWorktree = await dependencies.createPaseoWorktree( - { - ...input, - runSetup: false, - paseoHome: input.paseoHome ?? dependencies.paseoHome, - worktreesRoot: input.worktreesRoot ?? dependencies.worktreesRoot, - }, - options?.resolveDefaultBranch - ? { resolveDefaultBranch: options.resolveDefaultBranch } - : undefined, + const lifecycleCoordinator = + dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + if (Boolean(input.onWorktreePathPlanned) !== Boolean(input.onWorktreePathResolved)) { + throw new Error("Worktree creation journaling requires both planning and identity callbacks"); + } + const callerOwnsCreationJournal = Boolean( + input.onWorktreePathPlanned && input.onWorktreePathResolved, ); - const slug = basename(createdWorktree.worktree.worktreePath); - const workspace = createdWorktree.workspace; - const setupContinuation = options?.setupContinuation ?? { kind: "workspace" }; - - setTimeout(() => { - if (input.firstAgentContext) { - dependencies.autoNameWorkspaceBranchForFirstAgent({ - workspace, - firstAgentContext: input.firstAgentContext, - }); - } - void dependencies.warmWorkspaceGitData(workspace).catch((error) => { - dependencies.sessionLogger.warn( - { err: error, workspaceId: workspace.workspaceId }, - "Failed to warm workspace git data after creating worktree", - ); + const standaloneCreationId = callerOwnsCreationJournal ? null : randomUUID(); + if (standaloneCreationId) { + await dependencies.worktreeCreationJournal.beginPendingAgentCreation(standaloneCreationId, { + ownerKind: "standalone-worktree", }); - if (setupContinuation.kind === "workspace") { - void runWorktreeSetupInBackground(dependencies, { - requestCwd: input.cwd, - repoRoot: createdWorktree.repoRoot, - workspaceId: workspace.workspaceId, - worktree: createdWorktree.worktree, - shouldBootstrap: createdWorktree.created, - slug, - worktreePath: createdWorktree.worktree.worktreePath, - workspaceCwd: workspace.cwd, - }); - } - }, 0); + } + const journaledInput: CreatePaseoWorktreeInput = standaloneCreationId + ? { + ...input, + onWorktreePathPlanned: (worktreePath, plan) => + dependencies.worktreeCreationJournal.planPendingAgentCreationWorktree( + standaloneCreationId, + worktreePath, + plan, + ), + onWorktreePathResolved: (worktreePath, reservation) => + dependencies.worktreeCreationJournal.identifyPendingAgentCreationWorktree( + standaloneCreationId, + worktreePath, + reservation, + ), + } + : input; + const worktreesRoot = await getPaseoWorktreesRoot( + input.cwd, + input.paseoHome ?? dependencies.paseoHome, + input.worktreesRoot ?? dependencies.worktreesRoot, + ); - if (setupContinuation.kind === "agent") { - return { - ...createdWorktree, - setupContinuation: { - kind: "agent", - startAfterAgentCreate: ({ agentId }) => { - void runAsyncWorktreeBootstrap({ - agentId, - workspaceId: workspace.workspaceId, - worktree: createdWorktree.worktree, - workspaceCwd: workspace.cwd, - shouldBootstrap: createdWorktree.created, - terminalManager: setupContinuation.terminalManager, - appendTimelineItem: (item) => setupContinuation.appendTimelineItem({ agentId, item }), - emitLiveTimelineItem: (item) => - setupContinuation.emitLiveTimelineItem({ agentId, item }), - logger: setupContinuation.logger, + const creation = + lifecycleCoordinator.runWorktreeMutationExclusive( + resolve(worktreesRoot), + async (): Promise => { + const createdWorktree = await dependencies.createPaseoWorktree( + { + ...journaledInput, + runSetup: false, + paseoHome: input.paseoHome ?? dependencies.paseoHome, + worktreesRoot: input.worktreesRoot ?? dependencies.worktreesRoot, + }, + options?.resolveDefaultBranch + ? { resolveDefaultBranch: options.resolveDefaultBranch } + : undefined, + ); + const slug = basename(createdWorktree.worktree.worktreePath); + const workspace = createdWorktree.workspace; + const setupContinuation = options?.setupContinuation ?? { kind: "workspace" }; + + setTimeout(() => { + if (input.firstAgentContext) { + dependencies.autoNameWorkspaceBranchForFirstAgent({ + workspace, + firstAgentContext: input.firstAgentContext, + }); + } + void dependencies.warmWorkspaceGitData(workspace).catch((error) => { + dependencies.sessionLogger.warn( + { err: error, workspaceId: workspace.workspaceId }, + "Failed to warm workspace git data after creating worktree", + ); }); - }, - }, - }; - } + }, 0); + + if (setupContinuation.kind === "workspace") { + const setupTask = new Promise((resolveSetup) => { + setTimeout(() => { + void runWorktreeSetupInBackground(dependencies, { + requestCwd: input.cwd, + repoRoot: createdWorktree.repoRoot, + workspaceId: workspace.workspaceId, + worktree: createdWorktree.worktree, + shouldBootstrap: createdWorktree.created, + slug, + worktreePath: createdWorktree.worktree.worktreePath, + workspaceCwd: workspace.cwd, + }).then(resolveSetup, resolveSetup); + }, 0); + }); + lifecycleCoordinator.trackWorkspaceSetup( + workspace.workspaceId, + setupTask, + createdWorktree.worktree.worktreePath, + ); + } + + if (setupContinuation.kind === "agent") { + const reservation = lifecycleCoordinator.reserveWorkspaceSetup( + workspace.workspaceId, + createdWorktree.worktree.worktreePath, + ); + return { + ...createdWorktree, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: ({ agentId }: { agentId: string }) => { + const setupTask = runAsyncWorktreeBootstrap({ + agentId, + workspaceId: workspace.workspaceId, + worktree: createdWorktree.worktree, + workspaceCwd: workspace.cwd, + shouldBootstrap: createdWorktree.created, + terminalManager: setupContinuation.terminalManager, + appendTimelineItem: (item) => + setupContinuation.appendTimelineItem({ agentId, item }), + emitLiveTimelineItem: (item) => + setupContinuation.emitLiveTimelineItem({ agentId, item }), + logger: setupContinuation.logger, + }); + reservation.completeWith(setupTask); + void setupTask; + }, + releaseWithoutStarting: reservation.release, + }, + }; + } - return createdWorktree; + return createdWorktree; + }, + ); + if (!standaloneCreationId) return creation; + return creation.then( + async (result) => { + await dependencies.worktreeCreationJournal + .removePendingAgentCreation(standaloneCreationId) + .catch((error) => { + dependencies.sessionLogger.warn( + { err: error, creationId: standaloneCreationId }, + "Failed to clear a published standalone worktree creation journal", + ); + }); + return result; + }, + async (error) => { + await dependencies.worktreeCreationJournal + .recoverPendingCreation(standaloneCreationId) + .catch((recoveryError) => { + dependencies.sessionLogger.warn( + { err: recoveryError, creationId: standaloneCreationId }, + "Failed to recover a standalone worktree creation", + ); + }); + throw error; + }, + ); } export async function handleWorkspaceSetupStatusRequest( @@ -716,6 +845,15 @@ export async function runWorktreeSetupInBackground( }, }); }; + const runningProgress = createWorktreeSetupProgressCoalescer({ + emit: () => emitSetupProgress("running", null), + onError: (error) => { + dependencies.sessionLogger.warn( + { err: error, workspaceId }, + "Failed to emit worktree setup progress", + ); + }, + }); try { try { @@ -748,9 +886,10 @@ export async function runWorktreeSetupInBackground( runtimeEnv, onEvent: (event) => { applyWorktreeSetupProgressEvent(progressAccumulator, event); - emitSetupProgress("running", null); + runningProgress.schedule(); }, }); + await runningProgress.flush(); emitSetupProgress("completed", null); } } @@ -758,6 +897,7 @@ export async function runWorktreeSetupInBackground( if (error instanceof WorktreeSetupError) { setupResults = error.results; } + await runningProgress.flush(); const message = error instanceof Error ? error.message : String(error); emitSetupProgress("failed", message); diff --git a/packages/server/src/server/worktree/commands.ts b/packages/server/src/server/worktree/commands.ts index cedc5013c9..e8f93ed4ff 100644 --- a/packages/server/src/server/worktree/commands.ts +++ b/packages/server/src/server/worktree/commands.ts @@ -3,7 +3,9 @@ import { join } from "node:path"; import { getPaseoWorktreesRoot, isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; import { archiveByScope, + requireActiveWorkspaceForArchive, resolveWorkspaceIdAtPath, + WorkspaceArchiveTargetNotFoundError, type ArchiveDependencies, type ArchiveScope, } from "../workspace-archive-service.js"; @@ -110,10 +112,12 @@ export type ArchiveCommandResult = | { ok: true; removedAgents: string[]; + removedDirectory: boolean; + cleanupPending: boolean; } | { ok: false; - code: "NOT_ALLOWED"; + code: "NOT_ALLOWED" | "UNKNOWN"; message: string; removedAgents: []; }; @@ -122,8 +126,32 @@ export async function archiveCommand( dependencies: ArchiveCommandDependencies, input: ArchiveCommandInput, ): Promise { - const targetPath = await resolveArchiveTarget(dependencies, input); const scope = input.scope ?? "workspace"; + if (scope === "workspace" && input.workspaceId) { + try { + await requireActiveWorkspaceForArchive(dependencies, input.workspaceId); + } catch (error) { + if (!(error instanceof WorkspaceArchiveTargetNotFoundError)) throw error; + return { + ok: false, + code: "UNKNOWN", + message: `Workspace not found: ${input.workspaceId}`, + removedAgents: [], + }; + } + const result = await archiveByScope(dependencies, { + scope: { kind: "workspace", workspaceId: input.workspaceId }, + requestId: input.requestId, + }); + return { + ok: true, + removedAgents: result.archivedAgentIds, + removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPendingWorkspaceIds.length > 0, + }; + } + + const targetPath = await resolveArchiveTarget(dependencies, input); const ownership = await isPaseoOwnedWorktreeCwd(targetPath, { paseoHome: dependencies.paseoHome, worktreesRoot: dependencies.paseoWorktreesBaseRoot, @@ -147,6 +175,8 @@ export async function archiveCommand( return { ok: true, removedAgents: result.archivedAgentIds, + removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPendingWorkspaceIds.length > 0, }; } @@ -154,12 +184,11 @@ export async function archiveCommand( input.workspaceId ?? (await resolveWorkspaceIdAtPath(dependencies, targetPath)); if (!workspaceId) { - dependencies.sessionLogger?.warn( - { targetPath }, - "Could not resolve workspace for archive; skipping", - ); + dependencies.sessionLogger?.warn({ targetPath }, "Could not resolve workspace for archive"); return { - ok: true, + ok: false, + code: "UNKNOWN", + message: `Workspace not found for archive target: ${targetPath}`, removedAgents: [], }; } @@ -172,6 +201,8 @@ export async function archiveCommand( return { ok: true, removedAgents: result.archivedAgentIds, + removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPendingWorkspaceIds.length > 0, }; } diff --git a/packages/server/src/terminal/terminal-session-controller.test.ts b/packages/server/src/terminal/terminal-session-controller.test.ts index 81c3a15d6c..4846dbe84e 100644 --- a/packages/server/src/terminal/terminal-session-controller.test.ts +++ b/packages/server/src/terminal/terminal-session-controller.test.ts @@ -297,7 +297,7 @@ describe("terminal-session-controller legacy terminal creation", () => { hasBinaryChannel: () => true, isPathWithinRoot: isSameOrDescendantPath, sessionLogger: createLogger(), - listTerminalWorkspaceRefs: async () => [], + listTerminalWorkspaceRefs: async () => [{ workspaceId: "ws-1", cwd: "/work/repo" }], }); await controller.dispatch({ @@ -312,6 +312,37 @@ describe("terminal-session-controller legacy terminal creation", () => { expect.objectContaining({ cwd: "/work/repo", workspaceId: "ws-1", rows: 55, cols: 136 }), ); }); + + test("rejects a desktop terminal attach after its workspace was archived", async () => { + const outboundMessages: SessionOutboundMessage[] = []; + const createTerminal = vi.fn(); + const controller = new TerminalSessionController({ + terminalManager: { createTerminal } as unknown as TerminalManager, + emit: (message) => outboundMessages.push(message), + emitBinary: vi.fn(), + hasBinaryChannel: () => true, + isPathWithinRoot: isSameOrDescendantPath, + sessionLogger: createLogger(), + listTerminalWorkspaceRefs: async () => [], + }); + + await controller.dispatch({ + type: "create_terminal_request", + cwd: "/work/repo", + workspaceId: "ws-archived", + requestId: "req-archived", + }); + + expect(createTerminal).not.toHaveBeenCalled(); + expect(outboundMessages).toContainEqual({ + type: "create_terminal_response", + payload: { + terminal: null, + error: "Workspace not found: ws-archived", + requestId: "req-archived", + }, + }); + }); }); async function flushMicrotasks(): Promise { diff --git a/packages/server/src/terminal/terminal-session-controller.ts b/packages/server/src/terminal/terminal-session-controller.ts index dd7d053d62..9b175dff7a 100644 --- a/packages/server/src/terminal/terminal-session-controller.ts +++ b/packages/server/src/terminal/terminal-session-controller.ts @@ -14,6 +14,10 @@ import type { UnsubscribeTerminalsRequest, } from "../server/messages.js"; import { killTerminalsForWorkspace as killWorkspaceTerminals } from "../server/workspace-archive-service.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "../server/workspace-lifecycle-coordinator.js"; import { TerminalStreamOpcode, decodeTerminalResizePayload, @@ -69,6 +73,7 @@ export interface TerminalSessionControllerOptions { isPathWithinRoot: (rootPath: string, candidatePath: string) => boolean; sessionLogger: pino.Logger; listTerminalWorkspaceRefs?: () => Promise; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; listTerminalWorkspaceRoots?: () => Promise; // Whether the connected client can reflow restored snapshots. When true the // daemon attaches per-row soft-wrap flags to snapshots; otherwise it omits them @@ -128,6 +133,7 @@ export class TerminalSessionController { private readonly sessionLogger: pino.Logger; private readonly listTerminalWorkspaceRefs: () => Promise; private readonly listTerminalWorkspaceRoots: () => Promise; + private readonly lifecycleCoordinator: WorkspaceLifecycleCoordinator; private readonly clientSupportsWrapReflow: () => boolean; private readonly getClientBufferedAmount: () => number | null; @@ -153,6 +159,8 @@ export class TerminalSessionController { this.isPathWithinRoot = options.isPathWithinRoot; this.sessionLogger = options.sessionLogger; this.listTerminalWorkspaceRefs = options.listTerminalWorkspaceRefs ?? (async () => []); + this.lifecycleCoordinator = + options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; this.listTerminalWorkspaceRoots = options.listTerminalWorkspaceRoots ?? (async () => (await this.listTerminalWorkspaceRefs()).map((workspace) => workspace.cwd)); @@ -543,15 +551,27 @@ export class TerminalSessionController { return; } - const session = await this.terminalManager.createTerminal({ - cwd: msg.cwd, + const session = await this.lifecycleCoordinator.runWorkspaceOwnershipMutation( workspaceId, - name: msg.name, - command: msg.command, - args: msg.args, - rows: msg.size?.rows, - cols: msg.size?.cols, - }); + async () => { + const workspace = (await this.listTerminalWorkspaceRefs()).find( + (candidate) => candidate.workspaceId === workspaceId, + ); + if (!workspace) { + throw new Error(`Workspace not found: ${workspaceId}`); + } + }, + () => + this.terminalManager!.createTerminal({ + cwd: msg.cwd, + workspaceId, + name: msg.name, + command: msg.command, + args: msg.args, + rows: msg.size?.rows, + cols: msg.size?.cols, + }), + ); this.ensureExitSubscription(session); this.emit({ type: "create_terminal_response", diff --git a/packages/server/src/utils/run-git-command.test.ts b/packages/server/src/utils/run-git-command.test.ts index 371b0d3355..63476ae8ef 100644 --- a/packages/server/src/utils/run-git-command.test.ts +++ b/packages/server/src/utils/run-git-command.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; interface FakeSpawnBehavior { delayMs?: number; + deferCloseOnKill?: boolean; emitError?: Error; exitCode?: number | null; stderrData?: Buffer | string; @@ -69,6 +70,9 @@ class FakeChildProcess extends EventEmitter { this.killed = true; this.killSignals.push(signal); this.clearTimers(); + if (this.behavior.deferCloseOnKill) { + return true; + } this.schedule(() => { this.finishClose({ exitCode: null, @@ -78,6 +82,11 @@ class FakeChildProcess extends EventEmitter { return true; } + public closeKilledProcess(): void { + const signal = this.killSignals.at(-1) ?? null; + this.finishClose({ exitCode: null, signal }); + } + public dispose(): void { this.clearTimers(); this.closed = true; @@ -103,7 +112,10 @@ class FakeChildProcess extends EventEmitter { if (this.behavior.emitError) { this.schedule(() => { if (this.closed) return; - this.finishError(this.behavior.emitError); + this.emit("error", this.behavior.emitError); + this.schedule(() => { + this.finishClose({ exitCode: null, signal: null }); + }, 0); }, this.behavior.delayMs ?? 0); return; } @@ -131,15 +143,6 @@ class FakeChildProcess extends EventEmitter { this.emit("close", exitCode, signal); } - private finishError(error: Error): void { - if (this.closed) return; - - this.closed = true; - this.clearTimers(); - fakeSpawnController.activeCount -= 1; - this.emit("error", error); - } - private schedule(callback: () => void, delayMs: number): void { const timer = setTimeout(callback, delayMs); this.timers.push(timer); @@ -230,6 +233,74 @@ describe("runGitCommand", () => { }); }); + it("cancels active and queued git commands without starting the queued process", async () => { + const { runGitCommand } = await loadRunGitCommand(1); + const activeController = new AbortController(); + const queuedController = new AbortController(); + + enqueueSpawnBehaviors({ delayMs: 5_000 }, { delayMs: 0 }); + const active = runGitCommand(["status"], { + cwd: process.cwd(), + signal: activeController.signal, + }); + await vi.waitFor(() => expect(fakeSpawnController.processes).toHaveLength(1)); + const queued = runGitCommand(["rev-parse", "--show-toplevel"], { + cwd: process.cwd(), + signal: queuedController.signal, + }); + + queuedController.abort(); + await expect(queued).rejects.toThrow("Git command canceled: git rev-parse --show-toplevel"); + expect(fakeSpawnController.processes).toHaveLength(1); + + activeController.abort(); + await expect(active).rejects.toThrow("Git command canceled: git status"); + expect(fakeSpawnController.processes[0]?.killSignals).toEqual(["SIGKILL"]); + await vi.waitFor(() => expect(fakeSpawnController.activeCount).toBe(0)); + expect(fakeSpawnController.processes).toHaveLength(1); + + const preAbortedController = new AbortController(); + preAbortedController.abort(); + await expect( + runGitCommand(["status", "--short"], { + cwd: process.cwd(), + signal: preAbortedController.signal, + }), + ).rejects.toThrow("Git command canceled: git status --short"); + await Promise.resolve(); + expect(fakeSpawnController.processes).toHaveLength(1); + }); + + it("keeps an active cancellation pending until the killed process closes", async () => { + const { runGitCommand } = await loadRunGitCommand(1); + const controller = new AbortController(); + + enqueueSpawnBehaviors({ delayMs: 5_000, deferCloseOnKill: true }); + const command = runGitCommand(["worktree", "remove", "/tmp/example", "--force"], { + cwd: process.cwd(), + signal: controller.signal, + }); + const observedRejection = vi.fn(); + void command.catch(observedRejection); + await vi.waitFor(() => expect(fakeSpawnController.processes).toHaveLength(1)); + + controller.abort(); + await Promise.resolve(); + + expect(observedRejection).not.toHaveBeenCalled(); + expect(fakeSpawnController.activeCount).toBe(1); + expect(fakeSpawnController.processes[0]?.killSignals).toEqual(["SIGKILL"]); + + fakeSpawnController.processes[0]?.closeKilledProcess(); + + await expect(command).rejects.toMatchObject({ + name: "AbortError", + message: "Git command canceled: git worktree remove /tmp/example --force", + }); + expect(observedRejection).toHaveBeenCalledOnce(); + expect(fakeSpawnController.activeCount).toBe(0); + }); + it("resolves truncated stdout, caps output, and kills the child process", async () => { const { runGitCommand } = await loadRunGitCommand(1); @@ -252,17 +323,15 @@ describe("runGitCommand", () => { it("rejects process errors and frees the limiter for the next command", async () => { const { runGitCommand } = await loadRunGitCommand(1); + const spawnError = new Error("spawn exploded"); - enqueueSpawnBehaviors( - { emitError: new Error("spawn exploded") }, - { delayMs: 0, stdoutData: "ok" }, - ); + enqueueSpawnBehaviors({ emitError: spawnError }, { delayMs: 0, stdoutData: "ok" }); await expect( runGitCommand(["status"], { cwd: process.cwd(), }), - ).rejects.toThrow("spawn exploded"); + ).rejects.toBe(spawnError); await expect( runGitCommand(["status"], { diff --git a/packages/server/src/utils/run-git-command.ts b/packages/server/src/utils/run-git-command.ts index 254c980462..c66c587cd3 100644 --- a/packages/server/src/utils/run-git-command.ts +++ b/packages/server/src/utils/run-git-command.ts @@ -24,6 +24,7 @@ export interface GitCommandOptions { timeout?: number; maxOutputBytes?: number; acceptExitCodes?: number[]; + signal?: AbortSignal; } export interface GitCommandResult { @@ -131,216 +132,279 @@ function getEnvOverlayKeys(envOverlay: ProcessEnvRecord | undefined): string[] { return Object.keys(envOverlay ?? {}).sort(); } +function createGitCancellationError(args: readonly string[]): Error { + const error = new Error(`Git command canceled: ${formatGitCommand(args)}`); + error.name = "AbortError"; + return error; +} + +function waitForGitCommand( + task: Promise, + signal: AbortSignal | undefined, + args: readonly string[], + hasStarted: () => boolean, +): Promise { + if (!signal) return task; + + return new Promise((resolve, reject) => { + let settled = false; + const finish = (callback: () => void) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + callback(); + }; + const onAbort = () => { + // A queued command has no process to join, so it can be canceled + // immediately. Once the limiter has started the task, its process owns + // settlement and will preserve the cancellation error until `close`. + if (hasStarted()) return; + finish(() => reject(createGitCancellationError(args))); + }; + signal.addEventListener("abort", onAbort, { once: true }); + void task.then( + (value) => finish(() => resolve(value)), + (error) => finish(() => reject(error)), + ); + if (signal.aborted) onAbort(); + }); +} + export function runGitCommand( args: string[], options: GitCommandOptions, ): Promise { const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args)); - const promise = gitLimit( - () => - new Promise((resolve, reject) => { - gitRuntimeMetrics.start(runtimeMetric); - const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS; - const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; - const acceptExitCodes = options.acceptExitCodes ?? [0]; - const command = formatGitCommand(args); - const envOverlay = mergeEnvOverlays(options.env, options.envOverlay); - const startedAt = Date.now(); - const metricsState = beginGitCommandMetric(); - const logger = typeof options.logger?.trace === "function" ? options.logger : undefined; - const traceContext = logger - ? { - command: "git", - args, - cwd: options.cwd, - cwdExists: existsSync(options.cwd), - timeout, - maxOutputBytes, - acceptExitCodes, - envOverlayKeys: getEnvOverlayKeys(envOverlay), - } - : null; + let started = false; + const promise = gitLimit(() => { + started = true; + if (options.signal?.aborted) { + gitRuntimeMetrics.start(runtimeMetric); + gitRuntimeMetrics.finish(runtimeMetric, { success: false, timedOut: false }); + throw createGitCancellationError(args); + } + return new Promise((resolve, reject) => { + gitRuntimeMetrics.start(runtimeMetric); + const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS; + const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; + const acceptExitCodes = options.acceptExitCodes ?? [0]; + const command = formatGitCommand(args); + const envOverlay = mergeEnvOverlays(options.env, options.envOverlay); + const startedAt = Date.now(); + const metricsState = beginGitCommandMetric(); + const logger = typeof options.logger?.trace === "function" ? options.logger : undefined; + const traceContext = logger + ? { + command: "git", + args, + cwd: options.cwd, + cwdExists: existsSync(options.cwd), + timeout, + maxOutputBytes, + acceptExitCodes, + envOverlayKeys: getEnvOverlayKeys(envOverlay), + } + : null; + + if (logger && traceContext) { + logger.trace(traceContext, "Spawning git command"); + } + + // `core.quotepath=false` makes git emit raw UTF-8 paths instead of + // octal-escaping non-ASCII bytes (e.g. `测试文件.txt` vs `"\346\265\213..."`). + const child = spawnProcess("git", ["-c", "core.quotepath=false", ...args], { + cwd: options.cwd, + envOverlay, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + + let settled = false; + let metricFinished = false; + let pendingError: Error | null = null; + let pendingErrorTimedOut = false; + let requestedSignal: NodeJS.Signals | null = null; + let truncated = false; + let stdoutBytes = 0; + let stderrBytes = 0; + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + + const settle = (callback: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + callback(); + }; + + const finishMetricOnce = (metric: GitCommandMetric, timedOut = false) => { + if (metricFinished) return; + metricFinished = true; + finishGitCommandMetric(metricsState, metric); + gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut }); + }; + + const rememberError = (error: Error, timedOut = false) => { + if (pendingError) return; + pendingError = error; + pendingErrorTimedOut = timedOut; + }; + + const requestTermination = (error: Error, timedOut = false) => { + rememberError(error, timedOut); + if (requestedSignal) return; + requestedSignal = "SIGKILL"; + child.kill(requestedSignal); + }; + + const timer = setTimeout(() => { + const error = new Error(`Git command timed out after ${timeout}ms: ${command}`); + requestTermination(error, true); + }, timeout); + + const onAbort = () => { + requestTermination(createGitCancellationError(args)); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + if (options.signal?.aborted) onAbort(); + + child.stdout!.on("data", (chunk: Buffer | string) => { + if (settled || truncated) return; + + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const remainingBytes = maxOutputBytes - stdoutBytes; + + if (remainingBytes <= 0) { + truncated = true; + child.kill("SIGKILL"); + return; + } - if (logger && traceContext) { - logger.trace(traceContext, "Spawning git command"); + if (buffer.length > remainingBytes) { + stdoutChunks.push(buffer.subarray(0, remainingBytes)); + stdoutBytes += remainingBytes; + truncated = true; + child.kill("SIGKILL"); + return; } - // `core.quotepath=false` makes git emit raw UTF-8 paths instead of - // octal-escaping non-ASCII bytes (e.g. `测试文件.txt` vs `"\346\265\213..."`). - const child = spawnProcess("git", ["-c", "core.quotepath=false", ...args], { - cwd: options.cwd, - envOverlay, - shell: false, - stdio: ["ignore", "pipe", "pipe"], - }); + stdoutChunks.push(buffer); + stdoutBytes += buffer.length; + }); - let settled = false; - let metricFinished = false; - let truncated = false; - let stdoutBytes = 0; - let stderrBytes = 0; - const stdoutChunks: Buffer[] = []; - const stderrChunks: Buffer[] = []; - - const settle = (callback: () => void) => { - if (settled) return; - settled = true; - clearTimeout(timer); - callback(); - }; + child.stderr!.on("data", (chunk: Buffer | string) => { + if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return; + + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes; + + if (buffer.length > remainingBytes) { + stderrChunks.push(buffer.subarray(0, remainingBytes)); + stderrBytes += remainingBytes; + return; + } + + stderrChunks.push(buffer); + stderrBytes += buffer.length; + }); - const finishMetricOnce = (metric: GitCommandMetric, timedOut = false) => { - if (metricFinished) return; - metricFinished = true; - finishGitCommandMetric(metricsState, metric); - gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut }); + child.on("error", (error) => { + rememberError(error); + if (logger && traceContext) { + logger.trace( + { + ...traceContext, + err: error, + durationMs: Date.now() - startedAt, + }, + "Git command process error", + ); + } + }); + + child.on("close", (exitCode, signal) => { + const result: GitCommandResult = { + stdout: Buffer.concat(stdoutChunks).toString("utf8"), + stderr: Buffer.concat(stderrChunks).toString("utf8"), + truncated, + exitCode, + signal, }; + if (logger && traceContext) { + logger.trace( + { + ...traceContext, + durationMs: Date.now() - startedAt, + exitCode, + signal, + truncated, + stdoutBytes, + stderrBytes, + }, + "Git command closed", + ); + } - const timer = setTimeout(() => { - const error = new Error(`Git command timed out after ${timeout}ms: ${command}`); - child.kill("SIGKILL"); + if (pendingError) { + const error = pendingError; finishMetricOnce( { args, cwd: options.cwd, startedAtMs: startedAt, durationMs: Date.now() - startedAt, - exitCode: null, - signal: "SIGKILL", + exitCode, + signal: signal ?? requestedSignal, success: false, }, - true, + pendingErrorTimedOut, ); settle(() => reject(error)); - }, timeout); - - child.stdout!.on("data", (chunk: Buffer | string) => { - if (settled || truncated) return; - - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - const remainingBytes = maxOutputBytes - stdoutBytes; - - if (remainingBytes <= 0) { - truncated = true; - child.kill("SIGKILL"); - return; - } - - if (buffer.length > remainingBytes) { - stdoutChunks.push(buffer.subarray(0, remainingBytes)); - stdoutBytes += remainingBytes; - truncated = true; - child.kill("SIGKILL"); - return; - } - - stdoutChunks.push(buffer); - stdoutBytes += buffer.length; - }); - - child.stderr!.on("data", (chunk: Buffer | string) => { - if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return; - - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes; - - if (buffer.length > remainingBytes) { - stderrChunks.push(buffer.subarray(0, remainingBytes)); - stderrBytes += remainingBytes; - return; - } - - stderrChunks.push(buffer); - stderrBytes += buffer.length; - }); + return; + } - child.on("error", (error) => { + if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) { finishMetricOnce({ args, cwd: options.cwd, startedAtMs: startedAt, durationMs: Date.now() - startedAt, - exitCode: null, - signal: null, - success: false, - }); - if (logger && traceContext) { - logger.trace( - { - ...traceContext, - err: error, - durationMs: Date.now() - startedAt, - }, - "Git command process error", - ); - } - settle(() => reject(error)); - }); - - child.on("close", (exitCode, signal) => { - const result: GitCommandResult = { - stdout: Buffer.concat(stdoutChunks).toString("utf8"), - stderr: Buffer.concat(stderrChunks).toString("utf8"), - truncated, exitCode, signal, - }; - if (logger && traceContext) { - logger.trace( - { - ...traceContext, - durationMs: Date.now() - startedAt, - exitCode, - signal, - truncated, - stdoutBytes, - stderrBytes, - }, - "Git command closed", - ); - } + success: false, + }); + const stderrPreview = result.stderr.trim() || "(no stderr)"; + const truncationNote = result.truncated ? " (stdout truncated)" : ""; - if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) { - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, - exitCode, - signal, - success: false, - }); - const stderrPreview = result.stderr.trim() || "(no stderr)"; - const truncationNote = result.truncated ? " (stdout truncated)" : ""; - - settle(() => - reject( - new Error( - `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`, - ), + settle(() => + reject( + new Error( + `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`, ), - ); - return; - } + ), + ); + return; + } - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, - exitCode, - signal, - success: true, - }); - settle(() => resolve(result)); + finishMetricOnce({ + args, + cwd: options.cwd, + startedAtMs: startedAt, + durationMs: Date.now() - startedAt, + exitCode, + signal, + success: true, }); - }), - ); + settle(() => resolve(result)); + }); + }); + }); gitRuntimeMetrics.observeLimiter(gitLimit.activeCount, gitLimit.pendingCount); - return promise; + return waitForGitCommand(promise, options.signal, args, () => started); } -function formatGitCommand(args: string[]): string { +function formatGitCommand(args: readonly string[]): string { return ["git", ...args].join(" "); } diff --git a/packages/server/src/utils/worktree-metadata.ts b/packages/server/src/utils/worktree-metadata.ts index 7261ea3b2b..b335f39c1e 100644 --- a/packages/server/src/utils/worktree-metadata.ts +++ b/packages/server/src/utils/worktree-metadata.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "fs"; import { isAbsolute, join, resolve } from "path"; import { z } from "zod"; @@ -12,12 +13,14 @@ const ChangeRequestLookupTargetSchema = z.object({ const PaseoWorktreeMetadataV1Schema = z.object({ version: z.literal(1), baseRefName: z.string().min(1), + incarnationId: z.string().uuid().optional(), changeRequestLookupTarget: ChangeRequestLookupTargetSchema.optional(), }); const PaseoWorktreeMetadataV2Schema = z.object({ version: z.literal(2), baseRefName: z.string().min(1), + incarnationId: z.string().uuid().optional(), changeRequestLookupTarget: ChangeRequestLookupTargetSchema.optional(), firstAgentBranchAutoName: z .discriminatedUnion("status", [ @@ -155,6 +158,7 @@ export function writePaseoWorktreeMetadata( options: { baseRefName: string; changeRequestLookupTarget?: PaseoWorktreeChangeRequestHint; + incarnationId?: string; }, ): void { const baseRefName = normalizeBaseRefName(options.baseRefName); @@ -171,6 +175,7 @@ export function writePaseoWorktreeMetadata( const metadata: PaseoWorktreeMetadata = { version: 1, baseRefName, + ...(options.incarnationId ? { incarnationId: options.incarnationId } : {}), ...(options.changeRequestLookupTarget ? { changeRequestLookupTarget: options.changeRequestLookupTarget } : {}), @@ -194,6 +199,7 @@ export function writePaseoWorktreeRuntimeMetadata( const next: PaseoWorktreeMetadata = { version: 2, baseRefName: current.baseRefName, + ...(current.incarnationId ? { incarnationId: current.incarnationId } : {}), ...(current.changeRequestLookupTarget ? { changeRequestLookupTarget: current.changeRequestLookupTarget } : {}), @@ -224,6 +230,7 @@ export function writePaseoWorktreeFirstAgentBranchAutoNameMetadata( writePaseoWorktreeMetadataFile(worktreeRoot, { version: 2, baseRefName: current.baseRefName, + ...(current.incarnationId ? { incarnationId: current.incarnationId } : {}), ...(current.changeRequestLookupTarget ? { changeRequestLookupTarget: current.changeRequestLookupTarget } : {}), @@ -247,6 +254,7 @@ export function markPaseoWorktreeFirstAgentBranchAutoNameAttempted( const next: PaseoWorktreeMetadata = { version: 2, baseRefName: current.baseRefName, + ...(current.incarnationId ? { incarnationId: current.incarnationId } : {}), ...(current.changeRequestLookupTarget ? { changeRequestLookupTarget: current.changeRequestLookupTarget } : {}), @@ -270,6 +278,23 @@ export function readPaseoWorktreeMetadata(worktreeRoot: string): PaseoWorktreeMe return PaseoWorktreeMetadataSchema.parse(parsed); } +export function readPaseoWorktreeIncarnationId(worktreeRoot: string): string | null { + return readPaseoWorktreeMetadata(worktreeRoot)?.incarnationId ?? null; +} + +export function ensurePaseoWorktreeIncarnationId(worktreeRoot: string): string { + const current = readPaseoWorktreeMetadata(worktreeRoot); + if (!current) { + throw new Error("Cannot persist worktree incarnation: missing base metadata"); + } + if (current.incarnationId) { + return current.incarnationId; + } + const incarnationId = randomUUID(); + writePaseoWorktreeMetadataFile(worktreeRoot, { ...current, incarnationId }); + return incarnationId; +} + export function requirePaseoWorktreeBaseRefName(worktreeRoot: string): string { const metadataPath = getPaseoWorktreeMetadataPath(worktreeRoot); const metadata = readPaseoWorktreeMetadata(worktreeRoot); diff --git a/packages/server/src/utils/worktree-setup-output.test.ts b/packages/server/src/utils/worktree-setup-output.test.ts new file mode 100644 index 0000000000..5f2a2a74a2 --- /dev/null +++ b/packages/server/src/utils/worktree-setup-output.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; + +import { + appendWorktreeSetupOutput, + createWorktreeSetupOutputAccumulator, + renderWorktreeSetupOutput, +} from "./worktree-setup-output.js"; + +describe("worktree setup output", () => { + it("retains both ends when rendering buffered output under a smaller total budget", () => { + const accumulator = createWorktreeSetupOutputAccumulator(128); + appendWorktreeSetupOutput(accumulator, `prefix-${"x".repeat(80)}-suffix`); + + const rendered = renderWorktreeSetupOutput(accumulator, 64); + + expect(rendered.truncated).toBe(true); + expect(rendered.text).toContain("prefix-"); + expect(rendered.text).toContain("-suffix"); + expect(rendered.text).toContain(`...<${rendered.omittedBytes} bytes omitted>...`); + expect(Buffer.byteLength(rendered.text)).toBeLessThanOrEqual(64); + }); +}); diff --git a/packages/server/src/utils/worktree-setup-output.ts b/packages/server/src/utils/worktree-setup-output.ts new file mode 100644 index 0000000000..19e8425599 --- /dev/null +++ b/packages/server/src/utils/worktree-setup-output.ts @@ -0,0 +1,118 @@ +export const MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES = 64 * 1024; +export const MAX_WORKTREE_SETUP_TOTAL_OUTPUT_BYTES = 256 * 1024; + +export interface WorktreeSetupOutputAccumulator { + readonly maxBytes: number; + totalBytes: number; + head: Buffer; + tail: Buffer; + truncated: boolean; +} + +export interface RenderedWorktreeSetupOutput { + text: string; + truncated: boolean; + omittedBytes: number; +} + +export function getWorktreeSetupCommandOutputLimit(totalCommands: number): number { + const commandCount = Math.max(1, Math.floor(totalCommands)); + return Math.min( + MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES, + Math.floor(MAX_WORKTREE_SETUP_TOTAL_OUTPUT_BYTES / commandCount), + ); +} + +export function createWorktreeSetupOutputAccumulator( + maxBytes: number, +): WorktreeSetupOutputAccumulator { + return { + maxBytes: Math.max(0, Math.floor(maxBytes)), + totalBytes: 0, + head: Buffer.alloc(0), + tail: Buffer.alloc(0), + truncated: false, + }; +} + +export function appendWorktreeSetupOutput( + accumulator: WorktreeSetupOutputAccumulator, + chunk: string, +): void { + if (!chunk) { + return; + } + + const chunkBytes = Buffer.from(chunk, "utf8"); + accumulator.totalBytes += chunkBytes.length; + + if (accumulator.maxBytes === 0) { + accumulator.truncated = true; + return; + } + + const headCapacity = Math.floor(accumulator.maxBytes / 2); + const tailCapacity = accumulator.maxBytes - headCapacity; + if (!accumulator.truncated) { + const combined = Buffer.concat([accumulator.head, chunkBytes]); + if (combined.length <= accumulator.maxBytes) { + accumulator.head = combined; + return; + } + accumulator.head = combined.subarray(0, headCapacity); + accumulator.tail = combined.subarray(combined.length - tailCapacity); + accumulator.truncated = true; + return; + } + + const combinedTail = Buffer.concat([accumulator.tail, chunkBytes]); + accumulator.tail = combinedTail.subarray(Math.max(0, combinedTail.length - tailCapacity)); +} + +export function renderWorktreeSetupOutput( + accumulator: WorktreeSetupOutputAccumulator, + maxBytes: number = accumulator.maxBytes, +): RenderedWorktreeSetupOutput { + const outputLimit = Math.max(0, Math.min(accumulator.maxBytes, Math.floor(maxBytes))); + if (!accumulator.truncated && accumulator.totalBytes <= outputLimit) { + return { + text: accumulator.head.toString("utf8"), + truncated: false, + omittedBytes: 0, + }; + } + + let retainedBytes = Math.min(accumulator.totalBytes, outputLimit); + let omittedBytes = accumulator.totalBytes - retainedBytes; + for (;;) { + const markerBytes = Buffer.byteLength(buildOmissionMarker(omittedBytes), "utf8"); + const nextRetainedBytes = Math.min( + accumulator.totalBytes, + Math.max(0, outputLimit - markerBytes), + ); + const nextOmittedBytes = accumulator.totalBytes - nextRetainedBytes; + if (nextRetainedBytes === retainedBytes && nextOmittedBytes === omittedBytes) { + break; + } + retainedBytes = nextRetainedBytes; + omittedBytes = nextOmittedBytes; + } + + const headBytes = Math.floor(retainedBytes / 2); + const tailBytes = retainedBytes - headBytes; + const head = accumulator.head.subarray(0, headBytes); + const tailSource = accumulator.truncated ? accumulator.tail : accumulator.head; + const tail = tailSource.subarray(Math.max(0, tailSource.length - tailBytes)); + omittedBytes = accumulator.totalBytes - head.length - tail.length; + + return { + text: `${head.toString("utf8")}${buildOmissionMarker(omittedBytes)}${tail.toString("utf8")}`, + truncated: true, + omittedBytes, + }; +} + +function buildOmissionMarker(omittedBytes: number): string { + const unit = omittedBytes === 1 ? "byte" : "bytes"; + return `\n...<${omittedBytes} ${unit} omitted>...\n`; +} diff --git a/packages/server/src/utils/worktree.posix.test.ts b/packages/server/src/utils/worktree.posix.test.ts index 0cbbb9a040..37db6eaa49 100644 --- a/packages/server/src/utils/worktree.posix.test.ts +++ b/packages/server/src/utils/worktree.posix.test.ts @@ -21,6 +21,7 @@ import { type CreateWorktreeOptions, type WorktreeConfig, } from "./worktree"; +import { MAX_WORKTREE_SETUP_TOTAL_OUTPUT_BYTES } from "./worktree-setup-output.js"; import type { PaseoConfig } from "@getpaseo/protocol/paseo-config-schema"; import { getPaseoWorktreeMetadataPath } from "./worktree-metadata.js"; import { execFileSync } from "child_process"; @@ -813,6 +814,37 @@ describe.skipIf(isPlatform("win32"))("worktree POSIX-only", () => { expect(progressEvents.some((event) => event.type === "command_completed")).toBe(true); }); + it("bounds retained output across noisy setup commands and reports exact omitted bytes", async () => { + const outputBytesPerStream = Buffer.byteLength("prefix--suffix") + 70_000; + const noisyCommand = + "node -e \"process.stdout.write('prefix-' + 'x'.repeat(70000) + '-suffix'); process.stderr.write('prefix-' + 'y'.repeat(70000) + '-suffix')\""; + writeFileSync( + join(repoDir, "paseo.json"), + JSON.stringify({ worktree: { setup: Array.from({ length: 5 }, () => noisyCommand) } }), + ); + + const results = await runWorktreeSetupCommands({ + worktreePath: repoDir, + branchName: "main", + cleanupOnFailure: false, + }); + + expect(results).toHaveLength(5); + let retainedOutputBytes = 0; + for (const result of results) { + for (const output of [result.stdout, result.stderr]) { + const marker = output.match(/\n\.\.\.<(\d+) bytes omitted>\.\.\.\n/); + expect(marker).not.toBeNull(); + const retained = output.replace(marker?.[0] ?? "", ""); + expect(Number(marker?.[1])).toBe(outputBytesPerStream - Buffer.byteLength(retained)); + expect(retained).toContain("prefix-"); + expect(retained).toContain("-suffix"); + retainedOutputBytes += Buffer.byteLength(output); + } + } + expect(retainedOutputBytes).toBeLessThanOrEqual(MAX_WORKTREE_SETUP_TOTAL_OUTPUT_BYTES); + }, 15_000); + it("reuses persisted worktree runtime port across resolutions", async () => { const result = await createLegacyWorktreeForTest({ branchName: "main", @@ -1334,5 +1366,45 @@ describe.skipIf(isPlatform("win32"))("worktree POSIX-only", () => { expect(existsSync(created.worktreePath)).toBe(true); expect(existsSync(join(repoDir, "teardown-start.log"))).toBe(true); }); + + it("cancels a running teardown process tree before deleting the worktree", async () => { + writeFileSync( + join(repoDir, "paseo.json"), + JSON.stringify({ + worktree: { + teardown: ['node -e "setInterval(() => {}, 1000)"'], + }, + }), + ); + execFileSync("git", ["add", "paseo.json"], { cwd: repoDir }); + execFileSync( + "git", + ["-c", "commit.gpgsign=false", "commit", "-m", "add cancellable teardown"], + { cwd: repoDir }, + ); + const created = await createLegacyWorktreeForTest({ + branchName: "teardown-cancel-branch", + cwd: repoDir, + baseBranch: "main", + worktreeSlug: "teardown-cancel-test", + paseoHome, + }); + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(), 100); + + try { + await expect( + deletePaseoWorktree({ + cwd: repoDir, + worktreePath: created.worktreePath, + paseoHome, + signal: controller.signal, + }), + ).rejects.toThrow("Worktree teardown command failed"); + } finally { + clearTimeout(abortTimer); + } + expect(existsSync(created.worktreePath)).toBe(true); + }); }); }); diff --git a/packages/server/src/utils/worktree.ts b/packages/server/src/utils/worktree.ts index 7b2cf45609..7aaa0c1078 100644 --- a/packages/server/src/utils/worktree.ts +++ b/packages/server/src/utils/worktree.ts @@ -1,10 +1,17 @@ -import { execFile } from "child_process"; -import { promisify } from "util"; -import { existsSync, mkdirSync, realpathSync, rmSync, statSync } from "fs"; +import { + existsSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + statSync, + unlinkSync, + writeFileSync, +} from "fs"; import { copyFile, rm, stat } from "fs/promises"; import { join, basename, dirname, isAbsolute, resolve, sep } from "path"; import net from "node:net"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import stripAnsi from "strip-ansi"; import { buildStringCommandShellInvocation, @@ -32,15 +39,21 @@ import { } from "./worktree-metadata.js"; import { runGitCommand } from "./run-git-command.js"; import { spawnProcess } from "./spawn.js"; +import { terminateWithTreeKill } from "./tree-kill.js"; import { resolvePaseoHome } from "../server/paseo-home.js"; import { createExternalProcessEnv } from "../server/paseo-env.js"; import { parseGitRevParsePath, resolveGitRevParsePath } from "./git-rev-parse-path.js"; import { validateBranchSlug } from "@getpaseo/protocol/branch-slug"; import { expandTilde, getRealpathAwareRelativePath, isPathInsideRoot } from "./path.js"; +import { + appendWorktreeSetupOutput, + createWorktreeSetupOutputAccumulator, + getWorktreeSetupCommandOutputLimit, + renderWorktreeSetupOutput, +} from "./worktree-setup-output.js"; export { slugify, validateBranchSlug } from "@getpaseo/protocol/branch-slug"; -const execFileAsync = promisify(execFile); const READ_ONLY_GIT_ENV = { GIT_OPTIONAL_LOCKS: "0", } as const; @@ -204,6 +217,43 @@ export interface CreateWorktreeOptions { runSetup: boolean; paseoHome?: string; worktreesRoot?: string; + onWorktreePathPlanned?: (worktreePath: string, plan: WorktreeCreationPlan) => Promise; + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; +} + +export interface WorktreeCreationPlan { + worktreeIncarnationId: string; + metadataBaseRefName: string; +} + +export interface WorktreeCreationReservation { + worktreeIncarnationId: string; + directoryIdentity: { + device: string; + inode: string; + }; + metadataBaseRefName: string; +} + +export interface WorktreeCreationJournalCallbacks { + onWorktreePathPlanned: (worktreePath: string, plan: WorktreeCreationPlan) => Promise; + onWorktreePathResolved: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; +} + +const WORKTREE_CREATION_MARKER_FILENAME = ".paseo-worktree-creation"; + +export function readWorktreeCreationMarker(worktreePath: string): string | null { + try { + return readFileSync(join(worktreePath, WORKTREE_CREATION_MARKER_FILENAME), "utf8").trim(); + } catch { + return null; + } } interface ResolveExistingWorktreeForSlugOptions { @@ -411,50 +461,20 @@ export function processCarriageReturns(text: string): string { return output.join(""); } -async function execSetupCommand( - command: string, - options: { cwd: string; env: NodeJS.ProcessEnv }, -): Promise { - const startedAt = Date.now(); - const shellInvocation = buildStringCommandShellInvocation({ command }); - try { - const { stdout, stderr } = await execFileAsync(shellInvocation.shell, shellInvocation.args, { - cwd: options.cwd, - env: options.env, - }); - return { - command, - cwd: options.cwd, - stdout: stdout ?? "", - stderr: stderr ?? "", - exitCode: 0, - durationMs: Date.now() - startedAt, - }; - } catch (error) { - const execErr = error as { stdout?: string; stderr?: string; code?: unknown } | undefined; - return { - command, - cwd: options.cwd, - stdout: execErr?.stdout ?? "", - stderr: execErr?.stderr ?? (error instanceof Error ? error.message : String(error)), - exitCode: typeof execErr?.code === "number" ? execErr.code : null, - durationMs: Date.now() - startedAt, - }; - } -} - async function execSetupCommandStreamed(options: { command: string; cwd: string; env: NodeJS.ProcessEnv; index: number; total: number; + maxOutputBytes: number; onEvent?: (event: WorktreeSetupCommandProgressEvent) => void; + signal?: AbortSignal; }): Promise { return new Promise((resolvePromise) => { const startedAt = Date.now(); - const stdoutChunks: string[] = []; - const stderrChunks: string[] = []; + const stdoutAccumulator = createWorktreeSetupOutputAccumulator(options.maxOutputBytes); + const stderrAccumulator = createWorktreeSetupOutputAccumulator(options.maxOutputBytes); let settled = false; const emitOutput = (stream: "stdout" | "stderr", chunk: string) => { @@ -463,9 +483,9 @@ async function execSetupCommandStreamed(options: { return; } if (stream === "stdout") { - stdoutChunks.push(text); + appendWorktreeSetupOutput(stdoutAccumulator, text); } else { - stderrChunks.push(text); + appendWorktreeSetupOutput(stderrAccumulator, text); } options.onEvent?.({ type: "output", @@ -483,11 +503,20 @@ async function execSetupCommandStreamed(options: { return; } settled = true; + options.signal?.removeEventListener("abort", abortCommand); + const combinedBytes = stdoutAccumulator.totalBytes + stderrAccumulator.totalBytes; + const stdoutOutputBytes = + combinedBytes === 0 + ? 0 + : Math.floor((options.maxOutputBytes * stdoutAccumulator.totalBytes) / combinedBytes); + const stderrOutputBytes = options.maxOutputBytes - stdoutOutputBytes; + const stdout = renderWorktreeSetupOutput(stdoutAccumulator, stdoutOutputBytes).text; + const stderr = renderWorktreeSetupOutput(stderrAccumulator, stderrOutputBytes).text; const result: WorktreeSetupCommandResult = { command: options.command, cwd: options.cwd, - stdout: stdoutChunks.join(""), - stderr: stderrChunks.join(""), + stdout, + stderr, exitCode, durationMs: Date.now() - startedAt, }; @@ -520,6 +549,15 @@ async function execSetupCommandStreamed(options: { shell: false, stdio: ["ignore", "pipe", "pipe"], }); + const abortCommand = () => { + emitOutput("stderr", "Worktree lifecycle command canceled"); + void terminateWithTreeKill(child, { + gracefulTimeoutMs: 1_000, + forceTimeoutMs: 1_000, + }).finally(() => finish(null)); + }; + options.signal?.addEventListener("abort", abortCommand, { once: true }); + if (options.signal?.aborted) abortCommand(); child.stdout?.on("data", (chunk: Buffer | string) => { emitOutput("stdout", chunk.toString()); @@ -639,20 +677,17 @@ export async function runWorktreeSetupCommands(options: { const setupEnv = createStringCommandShellEnv(createExternalProcessEnv(process.env, runtimeEnv)); const results: WorktreeSetupCommandResult[] = []; + const maxOutputBytes = getWorktreeSetupCommandOutputLimit(setupCommands.length); for (const [index, cmd] of setupCommands.entries()) { - const result = options.onEvent - ? await execSetupCommandStreamed({ - command: cmd, - cwd: options.worktreePath, - env: setupEnv, - index: index + 1, - total: setupCommands.length, - onEvent: options.onEvent, - }) - : await execSetupCommand(cmd, { - cwd: options.worktreePath, - env: setupEnv, - }); + const result = await execSetupCommandStreamed({ + command: cmd, + cwd: options.worktreePath, + env: setupEnv, + index: index + 1, + total: setupCommands.length, + maxOutputBytes, + onEvent: options.onEvent, + }); results.push(result); if (result.exitCode !== 0) { @@ -732,6 +767,7 @@ export async function runWorktreeTeardownCommands(options: { teardownCwd?: string; branchName?: string; repoRootPath?: string; + signal?: AbortSignal; }): Promise { const teardownCwd = options.teardownCwd ?? options.worktreePath; if (getRealpathAwareRelativePath(options.worktreePath, teardownCwd) === null) { @@ -763,10 +799,16 @@ export async function runWorktreeTeardownCommands(options: { ); const results: WorktreeTeardownCommandResult[] = []; - for (const cmd of teardownCommands) { - const result = await execSetupCommand(cmd, { + const maxOutputBytes = getWorktreeSetupCommandOutputLimit(teardownCommands.length); + for (const [index, cmd] of teardownCommands.entries()) { + const result = await execSetupCommandStreamed({ + command: cmd, cwd: teardownCwd, env: teardownEnv, + index: index + 1, + total: teardownCommands.length, + maxOutputBytes, + signal: options.signal, }); results.push(result); @@ -909,6 +951,19 @@ function normalizePathForOwnership(input: string): string { } } +function normalizePlannedPathForOwnership(input: string): string { + const resolvedInput = resolve(input); + const missingSegments: string[] = []; + let existingAncestor = resolvedInput; + while (!existsSync(existingAncestor)) { + const parent = dirname(existingAncestor); + if (parent === existingAncestor) break; + missingSegments.unshift(basename(existingAncestor)); + existingAncestor = parent; + } + return join(normalizePathForOwnership(existingAncestor), ...missingSegments); +} + function resolveRepoRootFromGitCommonDir(commonDir: string): string { const normalizedCommonDir = normalizePathForOwnership(commonDir); return basename(normalizedCommonDir) === ".git" @@ -1084,6 +1139,36 @@ export interface DeletePaseoWorktreeOptions { worktreesRoot?: string; paseoHome?: string; worktreesBaseRoot?: string; + signal?: AbortSignal; +} + +function throwIfWorktreeDeletionCanceled(signal: AbortSignal | undefined): void { + if (!signal?.aborted) return; + const error = new Error("Worktree deletion canceled"); + error.name = "AbortError"; + throw error; +} + +async function waitForWorktreeDeletionRetry(delayMs: number, signal?: AbortSignal): Promise { + throwIfWorktreeDeletionCanceled(signal); + if (delayMs === 0) return; + + await new Promise((resolvePromise, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolvePromise(); + }, delayMs); + const onAbort = () => { + clearTimeout(timer); + reject( + Object.assign(new Error("Worktree deletion canceled"), { + name: "AbortError", + }), + ); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + }); } export async function deletePaseoWorktree({ @@ -1094,6 +1179,7 @@ export async function deletePaseoWorktree({ worktreesRoot, paseoHome, worktreesBaseRoot, + signal, }: DeletePaseoWorktreeOptions): Promise { if (!worktreePath && !worktreeSlug) { throw new Error("worktreePath or worktreeSlug is required"); @@ -1133,17 +1219,22 @@ export async function deletePaseoWorktree({ await runWorktreeTeardownCommands({ worktreePath: resolvedWorktree, teardownCwd, + signal, }); } } + throwIfWorktreeDeletionCanceled(signal); + if (cwd) { try { await runGitCommand(["worktree", "remove", resolvedWorktree, "--force"], { cwd, timeout: 120_000, + signal, }); } catch { + throwIfWorktreeDeletionCanceled(signal); // `git worktree remove` fails if the admin dir is already gone (e.g. a // prior archive attempt removed it before the working tree could be // fully cleaned up), or if the repo root has moved. Fall through to the @@ -1151,12 +1242,13 @@ export async function deletePaseoWorktree({ } } - await removeDirectoryWithRetries(resolvedWorktree); + await removeDirectoryWithRetries(resolvedWorktree, signal); if (cwd) { try { - await runGitCommand(["worktree", "prune"], { cwd, timeout: 30_000 }); + await runGitCommand(["worktree", "prune"], { cwd, timeout: 30_000, signal }); } catch { + throwIfWorktreeDeletionCanceled(signal); // not critical; git will prune lazily } } @@ -1195,7 +1287,8 @@ async function pathExists(path: string): Promise { } } -async function removeDirectoryWithRetries(path: string): Promise { +async function removeDirectoryWithRetries(path: string, signal?: AbortSignal): Promise { + throwIfWorktreeDeletionCanceled(signal); if (!(await pathExists(path))) { return; } @@ -1203,16 +1296,16 @@ async function removeDirectoryWithRetries(path: string): Promise { const delaysMs = [0, 100, 300, 700, 1500]; let lastError: unknown = null; for (const delay of delaysMs) { - if (delay > 0) { - await new Promise((resolvePromise) => setTimeout(resolvePromise, delay)); - } + await waitForWorktreeDeletionRetry(delay, signal); try { await rm(path, { recursive: true, force: true }); + throwIfWorktreeDeletionCanceled(signal); if (!(await pathExists(path))) { return; } lastError = new Error(`Directory still present after rm: ${path}`); } catch (error) { + throwIfWorktreeDeletionCanceled(signal); lastError = error; } } @@ -1234,17 +1327,74 @@ export const createWorktree = async ({ runSetup, paseoHome, worktreesRoot, + onWorktreePathPlanned, + onWorktreePathResolved, }: CreateWorktreeOptions): Promise => { + if (Boolean(onWorktreePathPlanned) !== Boolean(onWorktreePathResolved)) { + throw new Error("Worktree creation journaling requires both planning and identity callbacks"); + } const sourcePlan = await resolveWorktreeSourcePlan({ cwd, source, desiredSlug: worktreeSlug }); let worktreePath = join(await getPaseoWorktreesRoot(cwd, paseoHome, worktreesRoot), worktreeSlug); - mkdirSync(dirname(worktreePath), { recursive: true }); - // Also handle worktree path collision + // Also handle worktree path collision. Journaled creation first persists the + // exact candidate path and incarnation, then atomically claims the directory. let finalWorktreePath = worktreePath; let pathSuffix = 1; - while (existsSync(finalWorktreePath)) { - finalWorktreePath = `${worktreePath}-${pathSuffix}`; - pathSuffix++; + const worktreeIncarnationId = randomUUID(); + if (onWorktreePathPlanned && onWorktreePathResolved) { + while (true) { + const normalizedCandidatePath = normalizePlannedPathForOwnership(finalWorktreePath); + if (existsSync(normalizedCandidatePath)) { + finalWorktreePath = `${worktreePath}-${pathSuffix}`; + pathSuffix++; + continue; + } + await onWorktreePathPlanned(normalizedCandidatePath, { + worktreeIncarnationId, + metadataBaseRefName: sourcePlan.metadataBaseRefName, + }); + mkdirSync(dirname(normalizedCandidatePath), { recursive: true }); + try { + mkdirSync(normalizedCandidatePath); + finalWorktreePath = normalizedCandidatePath; + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + finalWorktreePath = `${worktreePath}-${pathSuffix}`; + pathSuffix++; + } + } + } else { + mkdirSync(dirname(finalWorktreePath), { recursive: true }); + while (existsSync(finalWorktreePath)) { + finalWorktreePath = `${worktreePath}-${pathSuffix}`; + pathSuffix++; + } + } + + const normalizedWorktreePath = normalizePathForOwnership(finalWorktreePath); + if (onWorktreePathPlanned && onWorktreePathResolved) { + const markerPath = join(normalizedWorktreePath, WORKTREE_CREATION_MARKER_FILENAME); + try { + writeFileSync(markerPath, `${worktreeIncarnationId}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + const directoryStat = statSync(normalizedWorktreePath, { bigint: true }); + await onWorktreePathResolved(normalizedWorktreePath, { + worktreeIncarnationId, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, + metadataBaseRefName: sourcePlan.metadataBaseRefName, + }); + unlinkSync(markerPath); + } catch (error) { + rmSync(normalizedWorktreePath, { recursive: true, force: true }); + throw error; + } } // Primitive owner for `git worktree add`; callers route through createWorktreeCore. @@ -1252,7 +1402,7 @@ export const createWorktree = async ({ cwd, timeout: 120_000, }); - worktreePath = normalizePathForOwnership(finalWorktreePath); + worktreePath = normalizedWorktreePath; if (sourcePlan.pushRemote) { await configureWorktreePushRemote({ @@ -1271,6 +1421,7 @@ export const createWorktree = async ({ writePaseoWorktreeMetadata(worktreePath, { baseRefName: sourcePlan.metadataBaseRefName, + incarnationId: worktreeIncarnationId, ...(sourcePlan.changeRequestLookupTarget ? { changeRequestLookupTarget: sourcePlan.changeRequestLookupTarget } : {}),