From aa305e190afef5990fa0b5ea24ad7f9842609a7b Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 08:19:29 +0100 Subject: [PATCH 01/43] fix: coordinate worktree setup and archive --- .../server/workspace-archive-service.test.ts | 126 ++++++++++++++++++ .../src/server/workspace-archive-service.ts | 42 +++++- .../server/workspace-lifecycle-coordinator.ts | 40 ++++++ .../src/server/worktree-session.test.ts | 47 ++++++- .../server/src/server/worktree-session.ts | 41 ++++-- 5 files changed, 281 insertions(+), 15 deletions(-) create mode 100644 packages/server/src/server/workspace-lifecycle-coordinator.ts diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 4adbadb601..80e41c8be2 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -18,6 +18,7 @@ import { type ArchiveResult, resolveWorkspaceIdAtPath, } from "./workspace-archive-service.js"; +import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; const cleanupPaths: string[] = []; @@ -187,6 +188,93 @@ function assertArchiveResult( } describe("archiveByScope", () => { + test("waits for registered setup before archiving its workspace", async () => { + const { tempDir } = createGitRepo(); + const workspaceId = "ws-setup-race"; + const workspaceCwd = path.join(tempDir, "local-checkout"); + mkdirSync(workspaceCwd); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + let finishSetup: (() => void) | undefined; + const setupTask = new Promise((resolveSetup) => { + finishSetup = resolveSetup; + }); + lifecycleCoordinator.trackWorkspaceSetup(workspaceId, setupTask); + let markWaitStarted: (() => void) | undefined; + const waitStarted = new Promise((resolveWaitStarted) => { + markWaitStarted = resolveWaitStarted; + }); + const waitForWorkspaceSetups = + lifecycleCoordinator.waitForWorkspaceSetups.bind(lifecycleCoordinator); + vi.spyOn(lifecycleCoordinator, "waitForWorkspaceSetups").mockImplementation( + async (workspaceIds) => { + markWaitStarted?.(); + await waitForWorkspaceSetups(workspaceIds); + }, + ); + const deps = createArchiveDeps({ + paseoHome: path.join(tempDir, ".paseo"), + activeWorkspaces: [{ workspaceId, cwd: workspaceCwd, kind: "local_checkout" }], + }); + deps.lifecycleCoordinator = lifecycleCoordinator; + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + + const archiveTask = archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-setup-race", + }); + + await waitStarted; + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + + finishSetup?.(); + await expect(archiveTask).resolves.toMatchObject({ + archivedWorkspaceIds: [workspaceId], + }); + }); + + test("coalesces simultaneous archive requests for the same backing directory", async () => { + const { tempDir } = createGitRepo(); + const workspaceId = "ws-simultaneous"; + const workspaceCwd = path.join(tempDir, "local-checkout"); + mkdirSync(workspaceCwd); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const deps = createArchiveDeps({ + paseoHome: path.join(tempDir, ".paseo"), + activeWorkspaces: [{ workspaceId, cwd: workspaceCwd, kind: "local_checkout" }], + }); + deps.lifecycleCoordinator = lifecycleCoordinator; + const originalArchiveWorkspaceRecord = deps.archiveWorkspaceRecord; + let releaseArchive: (() => void) | undefined; + const archiveGate = new Promise((resolveArchive) => { + releaseArchive = resolveArchive; + }); + let markArchiveStarted: (() => void) | undefined; + const archiveStarted = new Promise((resolveStarted) => { + markArchiveStarted = resolveStarted; + }); + deps.archiveWorkspaceRecord = vi.fn(async (id: string) => { + markArchiveStarted?.(); + await archiveGate; + await originalArchiveWorkspaceRecord(id); + }); + + const first = archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-simultaneous-first", + }); + const second = archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-simultaneous-second", + }); + await archiveStarted; + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(1); + + releaseArchive?.(); + const [firstResult, secondResult] = await Promise.all([first, second]); + expect(secondResult).toEqual(firstResult); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(1); + }); + test("workspace scope archives the record and removes the directory on last reference", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -303,6 +391,44 @@ describe("archiveByScope", () => { expect(existsSync(worktree.worktreePath)).toBe(true); }); + test("rechecks owners after archival before removing the backing directory", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "late-owner"); + const workspaceId = "ws-late-owner-target"; + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId, + cwd: worktree.worktreePath, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + }, + ], + }); + const originalArchiveWorkspaceRecord = deps.archiveWorkspaceRecord; + deps.archiveWorkspaceRecord = async (id: string) => { + await originalArchiveWorkspaceRecord(id); + deps.activeWorkspaces.push({ + workspaceId: "ws-late-owner-sibling", + cwd: path.join(worktree.worktreePath, "packages", "app"), + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + }); + }; + + const result = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-late-owner", + }); + + expect(result.removedDirectory).toBe(false); + expect(existsSync(worktree.worktreePath)).toBe(true); + }); + test("archiving a subdirectory workspace keeps its active worktree root", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 8b6045a447..ffcf22fd99 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -15,6 +15,10 @@ import { import type { TerminalManager } from "../terminal/terminal-manager.js"; import type { PersistedWorkspaceRecord, WorkspaceRegistry } from "./workspace-registry.js"; import { createRealpathAwarePathMatcher } from "../utils/path.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "./workspace-lifecycle-coordinator.js"; export type ActiveWorkspaceRef = Pick< PersistedWorkspaceRecord, @@ -43,6 +47,7 @@ export interface ArchiveDependencies { markWorkspaceArchiving: (workspaceIds: Iterable, archivingAt: string) => void; clearWorkspaceArchiving: (workspaceIds: Iterable) => void; killTerminalsForWorkspace: (workspaceId: string) => Promise; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; sessionLogger?: Logger; } @@ -114,7 +119,42 @@ export async function archiveByScope( dependencies: ArchiveDependencies, request: ArchiveByScopeRequest, ): Promise { - const target = await resolveArchiveTarget(dependencies, request.scope); + const initialTarget = await resolveArchiveTarget(dependencies, request.scope); + const lifecycleCoordinator = + dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + let operationKey: string; + if (initialTarget.backing) { + operationKey = `directory:${resolve(initialTarget.backing.path)}`; + } else if (request.scope.kind === "workspace") { + operationKey = `workspace:${request.scope.workspaceId}`; + } else { + operationKey = `directory:${resolve(request.scope.targetPath)}`; + } + + return lifecycleCoordinator.runArchive(operationKey, async () => { + await lifecycleCoordinator.waitForWorkspaceSetups(initialTarget.workspaceIds); + const refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); + const target = mergeArchiveTargets(initialTarget, refreshedTarget); + return archiveResolvedTarget(dependencies, request, target); + }); +} + +function mergeArchiveTargets(initial: ArchiveTarget, refreshed: ArchiveTarget): ArchiveTarget { + if (refreshed.backing !== null || refreshed.workspaceIds.length > 0) { + return refreshed; + } + return { + backing: initial.backing, + teardownTargets: initial.teardownTargets, + workspaceIds: [], + }; +} + +async function archiveResolvedTarget( + dependencies: ArchiveDependencies, + request: ArchiveByScopeRequest, + target: ArchiveTarget, +): Promise { const targetWorkspaceIds = target.workspaceIds; if (targetWorkspaceIds.length > 0) { diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts new file mode 100644 index 0000000000..727dcab4f7 --- /dev/null +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -0,0 +1,40 @@ +export class WorkspaceLifecycleCoordinator { + private readonly setupTasks = new Map>(); + private readonly archiveOperations = new Map>(); + + trackWorkspaceSetup(workspaceId: string, task: Promise): void { + this.setupTasks.set(workspaceId, task); + const clearTask = () => { + if (this.setupTasks.get(workspaceId) === task) { + this.setupTasks.delete(workspaceId); + } + }; + void task.then(clearTask, clearTask); + } + + async waitForWorkspaceSetups(workspaceIds: Iterable): Promise { + const tasks = Array.from(workspaceIds, (workspaceId) => + this.setupTasks.get(workspaceId), + ).filter((task): task is Promise => task !== undefined); + await Promise.allSettled(tasks); + } + + runArchive(key: string, operation: () => Promise): Promise { + const existing = this.archiveOperations.get(key); + if (existing) { + return existing as Promise; + } + + const task = Promise.resolve().then(operation); + this.archiveOperations.set(key, task); + const clearTask = () => { + if (this.archiveOperations.get(key) === task) { + this.archiveOperations.delete(key); + } + }; + void task.then(clearTask, clearTask); + return task; + } +} + +export const defaultWorkspaceLifecycleCoordinator = new WorkspaceLifecycleCoordinator(); diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index a5de5baedb..cee6e2ca8b 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -49,6 +49,7 @@ import { WorkspaceGitServiceImpl } from "./workspace-git-service.js"; import type { WorkspaceGitService } from "./workspace-git-service.js"; import { isPlatform } from "../test-utils/platform.js"; import { createWorkspaceProvisioningService } from "./session/workspace-provisioning/workspace-provisioning-service.js"; +import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; interface LegacyCreateWorktreeTestOptions { branchName: string; @@ -450,7 +451,51 @@ describe("resolveGitCreateBaseBranch", () => { }); }); -describe("create-agent worktree setup boundary", () => { +describe("create worktree setup boundary", () => { + test("registers workspace setup before returning the created workspace", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const trackWorkspaceSetup = vi.spyOn(lifecycleCoordinator, "trackWorkspaceSetup"); + + try { + const result = await createPaseoWorktreeWorkflow( + { + paseoHome, + createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), + warmWorkspaceGitData: async () => {}, + autoNameWorkspaceBranchForFirstAgent: () => {}, + emitWorkspaceUpdateForWorkspaceId: async () => {}, + cacheWorkspaceSetupSnapshot: () => {}, + emit: () => {}, + sessionLogger: createLogger(), + terminalManager: null, + archiveWorkspaceRecord: async () => {}, + serviceProxy: null, + scriptRuntimeStore: null, + getDaemonTcpPort: null, + getDaemonTcpHost: null, + onScriptsChanged: null, + lifecycleCoordinator, + }, + { + cwd: repoDir, + worktreeSlug: "registered-before-return", + runSetup: false, + paseoHome, + }, + ); + + expect(trackWorkspaceSetup).toHaveBeenCalledWith( + result.workspace.workspaceId, + expect.any(Promise), + ); + await lifecycleCoordinator.waitForWorkspaceSetups([result.workspace.workspaceId]); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + test("agent setup continuation starts setup for the created agent timeline", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); diff --git a/packages/server/src/server/worktree-session.ts b/packages/server/src/server/worktree-session.ts index 5573a945f8..9716ced730 100644 --- a/packages/server/src/server/worktree-session.ts +++ b/packages/server/src/server/worktree-session.ts @@ -47,6 +47,10 @@ import { createPaseoWorktreeCommand, listPaseoWorktreesCommand, } from "./worktree/commands.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "./workspace-lifecycle-coordinator.js"; const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._/-]+$/; @@ -109,6 +113,7 @@ interface CreatePaseoWorktreeInBackgroundDependencies { getDaemonTcpHost: (() => string | null) | null; serviceProxyPublicBaseUrl?: string | null; onScriptsChanged: ((workspaceId: string, workspaceDirectory: string) => void) | null; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; } interface CreatePaseoWorktreeWorkflowDependencies extends CreatePaseoWorktreeInBackgroundDependencies { @@ -604,6 +609,8 @@ export async function createPaseoWorktreeWorkflow( const slug = basename(createdWorktree.worktree.worktreePath); const workspace = createdWorktree.workspace; const setupContinuation = options?.setupContinuation ?? { kind: "workspace" }; + const lifecycleCoordinator = + dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; setTimeout(() => { if (input.firstAgentContext) { @@ -618,27 +625,33 @@ export async function createPaseoWorktreeWorkflow( "Failed to warm workspace git data after creating worktree", ); }); - if (setupContinuation.kind === "workspace") { - void runWorktreeSetupInBackground(dependencies, { - requestCwd: input.cwd, - repoRoot: createdWorktree.repoRoot, - workspaceId: workspace.workspaceId, - worktree: createdWorktree.worktree, - shouldBootstrap: createdWorktree.created, - slug, - worktreePath: createdWorktree.worktree.worktreePath, - workspaceCwd: workspace.cwd, - }); - } }, 0); + if (setupContinuation.kind === "workspace") { + const setupTask = new Promise((resolveSetup) => { + setTimeout(() => { + void runWorktreeSetupInBackground(dependencies, { + requestCwd: input.cwd, + repoRoot: createdWorktree.repoRoot, + workspaceId: workspace.workspaceId, + worktree: createdWorktree.worktree, + shouldBootstrap: createdWorktree.created, + slug, + worktreePath: createdWorktree.worktree.worktreePath, + workspaceCwd: workspace.cwd, + }).then(resolveSetup, resolveSetup); + }, 0); + }); + lifecycleCoordinator.trackWorkspaceSetup(workspace.workspaceId, setupTask); + } + if (setupContinuation.kind === "agent") { return { ...createdWorktree, setupContinuation: { kind: "agent", startAfterAgentCreate: ({ agentId }) => { - void runAsyncWorktreeBootstrap({ + const setupTask = runAsyncWorktreeBootstrap({ agentId, workspaceId: workspace.workspaceId, worktree: createdWorktree.worktree, @@ -650,6 +663,8 @@ export async function createPaseoWorktreeWorkflow( setupContinuation.emitLiveTimelineItem({ agentId, item }), logger: setupContinuation.logger, }); + lifecycleCoordinator.trackWorkspaceSetup(workspace.workspaceId, setupTask); + void setupTask; }, }, }; From 88d58b2bf32413a24e80598b0f488b491ca337ad Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 09:11:10 +0100 Subject: [PATCH 02/43] fix: close worktree lifecycle races --- .../agent/create-agent-lifecycle-dispatch.ts | 3 + .../server/agent/create-agent/create.test.ts | 95 ++++++- .../src/server/agent/create-agent/create.ts | 113 ++++---- .../src/server/agent/mcp-server.test.ts | 2 + .../src/server/agent/tools/paseo-tools.ts | 3 +- packages/server/src/server/bootstrap.ts | 3 + packages/server/src/server/session.ts | 3 + .../server/workspace-archive-service.test.ts | 175 ++++++++++++ .../src/server/workspace-archive-service.ts | 254 ++++++++++++++++-- .../server/workspace-lifecycle-coordinator.ts | 65 ++++- .../server/src/server/workspace-registry.ts | 15 ++ .../src/server/worktree-session.test.ts | 59 ++++ .../server/src/server/worktree-session.ts | 14 +- 13 files changed, 719 insertions(+), 85 deletions(-) diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index e4c29c958a..4e9937482f 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -9,6 +9,7 @@ import type { CreatePaseoWorktreeWorkflowResult, } from "../worktree-session.js"; import type { WorkspaceGitService } from "../workspace-git-service.js"; +import type { WorkspaceRegistry } from "../workspace-registry.js"; import type { CreateAgentWorktreeTarget, FirstAgentContext, @@ -29,6 +30,7 @@ interface CreateAgentLifecycleDispatchDependencies { findWorkspaceIdForCwd: (cwd: string) => Promise; listActiveWorkspaces: () => Promise; archiveWorkspaceRecord: (workspaceId: string) => Promise; + workspaceRegistry: Pick; emit: (message: SessionOutboundMessage) => void; emitAgentRemove: (agentId: string) => void; emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds: Iterable) => Promise; @@ -209,6 +211,7 @@ export class CreateAgentLifecycleDispatch { findWorkspaceIdForCwd: this.dependencies.findWorkspaceIdForCwd, listActiveWorkspaces: this.dependencies.listActiveWorkspaces, archiveWorkspaceRecord: this.dependencies.archiveWorkspaceRecord, + workspaceRegistry: this.dependencies.workspaceRegistry, emitWorkspaceUpdatesForWorkspaceIds: this.dependencies.emitWorkspaceUpdatesForWorkspaceIds, markWorkspaceArchiving: this.dependencies.markWorkspaceArchiving, clearWorkspaceArchiving: this.dependencies.clearWorkspaceArchiving, diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index 89cf7bab6c..870bf8249a 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -36,7 +36,11 @@ function fakeWorktreeCreator(args: { repoRoot: string; createdWorkspaceId: strin workspace: { workspaceId: args.createdWorkspaceId, cwd: workspaceCwd }, repoRoot: args.repoRoot, created: true, - setupContinuation: { kind: "agent" as const, startAfterAgentCreate: () => {} }, + setupContinuation: { + kind: "agent" as const, + startAfterAgentCreate: () => {}, + releaseWithoutStarting: () => {}, + }, }) as unknown as CreatePaseoWorktreeWorkflowResult; } @@ -122,6 +126,89 @@ test("session create validates the requested mode against the provider's modes", expect(createAgent).not.toHaveBeenCalled(); }); +test.each([ + ["validation failure", new Error("invalid provider mode")], + [ + "validation abort", + Object.assign(new Error("provider resolution aborted"), { name: "AbortError" }), + ], +])("session create releases setup reservation after %s", async (_caseName, failure) => { + const releaseWithoutStarting = vi.fn(); + const startAfterAgentCreate = vi.fn(); + const stub = createProviderSnapshotManagerStub(); + stub.resolveCreateConfig.mockRejectedValue(failure); + const dependencies: Parameters[0] = { + agentManager: { + createAgent: vi.fn(), + } as unknown as Parameters[0]["agentManager"], + agentStorage: {} as Parameters[0]["agentStorage"], + logger: createTestLogger(), + providerSnapshotManager: stub.manager, + }; + + await expect( + createAgentCommand(dependencies, { + kind: "session", + config: { provider: "opencode", cwd: "/tmp/paseo-create-test" }, + workspaceId: "ws-source", + labels: {}, + provisionalTitle: null, + firstAgentContext: { attachments: [] }, + buildSessionConfig: async (config) => ({ + sessionConfig: config, + setupContinuation: { + kind: "agent", + startAfterAgentCreate, + releaseWithoutStarting, + }, + createdWorkspaceId: "ws-created", + }), + }), + ).rejects.toThrow(failure.message); + + expect(releaseWithoutStarting).toHaveBeenCalledOnce(); + expect(startAfterAgentCreate).not.toHaveBeenCalled(); +}); + +test("session create releases setup reservation when agent creation fails", async () => { + const releaseWithoutStarting = vi.fn(); + const startAfterAgentCreate = vi.fn(); + const createFailure = new Error("agent creation failed"); + const dependencies: Parameters[0] = { + agentManager: { + createAgent: vi.fn(async () => { + throw createFailure; + }), + } as unknown as Parameters[0]["agentManager"], + agentStorage: {} as Parameters[0]["agentStorage"], + logger: createTestLogger(), + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + }; + + await expect( + createAgentCommand(dependencies, { + kind: "session", + config: { provider: "codex", cwd: "/tmp/paseo-create-test" }, + workspaceId: "ws-source", + labels: {}, + provisionalTitle: null, + firstAgentContext: { attachments: [] }, + buildSessionConfig: async (config) => ({ + sessionConfig: config, + setupContinuation: { + kind: "agent", + startAfterAgentCreate, + releaseWithoutStarting, + }, + createdWorkspaceId: "ws-created", + }), + }), + ).rejects.toThrow(createFailure.message); + + expect(releaseWithoutStarting).toHaveBeenCalledOnce(); + expect(startAfterAgentCreate).not.toHaveBeenCalled(); +}); + test("session create applies the resolved mode from the provider create config", async () => { const snapshot = { id: "agent-1", @@ -264,7 +351,11 @@ test("session create stamps the new worktree's workspaceId when a setup continua firstAgentContext: { attachments: [] }, buildSessionConfig: async (config) => ({ sessionConfig: config, - setupContinuation: { kind: "agent", startAfterAgentCreate: () => {} }, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: () => {}, + releaseWithoutStarting: () => {}, + }, createdWorkspaceId: "ws-new-worktree", }), }, diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index b31a858f34..94f42a50e6 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -179,11 +179,17 @@ export async function createAgentCommand( ? await resolveSessionCreateAgent(dependencies, input) : await resolveMcpCreateAgent(dependencies, input); - const snapshot = await dependencies.agentManager.createAgent( - resolved.config, - undefined, - resolved.createOptions, - ); + let snapshot: ManagedAgent; + try { + snapshot = await dependencies.agentManager.createAgent( + resolved.config, + undefined, + resolved.createOptions, + ); + } catch (error) { + resolved.setupContinuation?.releaseWithoutStarting(); + throw error; + } resolved.setupContinuation?.startAfterAgentCreate({ agentId: snapshot.id, @@ -251,52 +257,57 @@ async function resolveSessionCreateAgent( // this for the worktree path via cleanupCreatedWorktreeAfterFailedAgentCreate; // this is a pre-existing gap for directory-only workspace creates, not // introduced by this validation). - const resolvedCreateConfig = await dependencies.providerSnapshotManager.resolveCreateConfig({ - cwd: builtSessionConfig.cwd, - provider: builtSessionConfig.provider, - requestedMode: builtSessionConfig.modeId, - featureValues: builtSessionConfig.featureValues, - parent: null, - unattended: false, - }); - const sessionConfig: AgentSessionConfig = { - ...builtSessionConfig, - modeId: resolvedCreateConfig.modeId, - featureValues: resolvedCreateConfig.featureValues, - }; - const prompt = buildAgentPrompt(trimmedPrompt ?? "", input.images, input.attachments); - const hasPromptContent = Array.isArray(prompt) ? prompt.length > 0 : prompt.length > 0; - const clientMessageId = normalizeClientMessageId(input.clientMessageId); - const runOptions: AgentRunOptions | undefined = - input.outputSchema || clientMessageId - ? { - ...(input.outputSchema ? { outputSchema: input.outputSchema } : {}), - ...(clientMessageId ? { clientMessageId } : {}), - } - : undefined; - const workspaceId = setupContinuation ? createdWorkspaceId : input.workspaceId; - - return { - config: sessionConfig, - createOptions: { - labels: input.labels, - initialPrompt: trimmedPrompt, - env: input.env, - initialTitle: input.provisionalTitle, - // A legacy git/worktreeName worktree creates a fresh workspace, so the - // agent belongs to that workspace, not the source one. createdWorkspaceId - // is the freshly created worktree's workspace. - workspaceId: requireResolvedWorkspaceId(workspaceId), - }, - prompt: hasPromptContent ? prompt : undefined, - runOptions, - setupContinuation, - background: true, - promptFailure: "throw", - promptLogger: dependencies.logger.child({ - clientMessageId: resolveClientMessageId(input.clientMessageId), - }), - }; + try { + const resolvedCreateConfig = await dependencies.providerSnapshotManager.resolveCreateConfig({ + cwd: builtSessionConfig.cwd, + provider: builtSessionConfig.provider, + requestedMode: builtSessionConfig.modeId, + featureValues: builtSessionConfig.featureValues, + parent: null, + unattended: false, + }); + const sessionConfig: AgentSessionConfig = { + ...builtSessionConfig, + modeId: resolvedCreateConfig.modeId, + featureValues: resolvedCreateConfig.featureValues, + }; + const prompt = buildAgentPrompt(trimmedPrompt ?? "", input.images, input.attachments); + const hasPromptContent = Array.isArray(prompt) ? prompt.length > 0 : prompt.length > 0; + const clientMessageId = normalizeClientMessageId(input.clientMessageId); + const runOptions: AgentRunOptions | undefined = + input.outputSchema || clientMessageId + ? { + ...(input.outputSchema ? { outputSchema: input.outputSchema } : {}), + ...(clientMessageId ? { clientMessageId } : {}), + } + : undefined; + const workspaceId = setupContinuation ? createdWorkspaceId : input.workspaceId; + + return { + config: sessionConfig, + createOptions: { + labels: input.labels, + initialPrompt: trimmedPrompt, + env: input.env, + initialTitle: input.provisionalTitle, + // A legacy git/worktreeName worktree creates a fresh workspace, so the + // agent belongs to that workspace, not the source one. createdWorkspaceId + // is the freshly created worktree's workspace. + workspaceId: requireResolvedWorkspaceId(workspaceId), + }, + prompt: hasPromptContent ? prompt : undefined, + runOptions, + setupContinuation, + background: true, + promptFailure: "throw", + promptLogger: dependencies.logger.child({ + clientMessageId: resolveClientMessageId(input.clientMessageId), + }), + }; + } catch (error) { + setupContinuation?.releaseWithoutStarting(); + throw error; + } } async function resolveMcpCreateAgent( diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index b29b0a39f5..4e8dbb66da 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -803,6 +803,7 @@ function createPaseoWorktreeForMcpTest(options: { startAfterAgentCreate: ({ agentId }) => { options.startedAgentSetupIds?.push(agentId); }, + releaseWithoutStarting: () => {}, }, }; } @@ -2401,6 +2402,7 @@ describe("create_agent MCP tool", () => { startAfterAgentCreate: ({ agentId }: { agentId: string }) => { startedAgentSetupIds.push(agentId); }, + releaseWithoutStarting: () => {}, }, } : {}), diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index e681eb521f..6c822d43d9 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -107,7 +107,7 @@ export interface PaseoToolHostDependencies { listActiveWorkspaces?: ArchiveDependencies["listActiveWorkspaces"]; archiveWorkspaceRecord?: ArchiveDependencies["archiveWorkspaceRecord"]; emitWorkspaceUpdatesForWorkspaceIds?: ArchiveDependencies["emitWorkspaceUpdatesForWorkspaceIds"]; - workspaceRegistry?: Pick; + workspaceRegistry?: Pick; projectRegistry?: Pick; createDirectoryWorkspace?: ( cwd: string, @@ -3167,6 +3167,7 @@ function archiveWorktreeDependencies( findWorkspaceIdForCwd: options.findWorkspaceIdForCwd, listActiveWorkspaces: options.listActiveWorkspaces, archiveWorkspaceRecord: options.archiveWorkspaceRecord, + workspaceRegistry: options.workspaceRegistry, emitWorkspaceUpdatesForWorkspaceIds: options.emitWorkspaceUpdatesForWorkspaceIds, markWorkspaceArchiving: options.markWorkspaceArchiving, clearWorkspaceArchiving: options.clearWorkspaceArchiving, diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 09ff21faad..83afa33f1b 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -1062,6 +1062,7 @@ export async function createPaseoDaemon( clearWorkspaceArchiving: clearWorkspaceArchivingExternal, killTerminalsForWorkspace: (workspaceIdToKill) => killTerminalsForWorkspace({ terminalManager, sessionLogger: logger }, workspaceIdToKill), + workspaceRegistry, sessionLogger: logger, }, { scope: { kind: "workspace", workspaceId }, requestId }, @@ -1079,6 +1080,7 @@ export async function createPaseoDaemon( findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, listActiveWorkspaces: listActiveWorkspacesExternal, archiveWorkspaceRecord: archiveWorkspaceRecordExternal, + workspaceRegistry, emit: emitExternalSessionMessage, emitAgentRemove: () => undefined, emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, @@ -1176,6 +1178,7 @@ export async function createPaseoDaemon( }, workspaceIdToKill, ), + workspaceRegistry, sessionLogger: logger, }, { diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index a87aa42e6c..3bef5a6d5f 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -917,6 +917,7 @@ export class Session { findWorkspaceIdForCwd: (cwd) => this.findWorkspaceIdForCwd(cwd), listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), archiveWorkspaceRecord: (workspaceId) => this.archiveWorkspaceRecord(workspaceId), + workspaceRegistry: this.workspaceRegistry, emit: (message) => this.emit(message), emitAgentRemove: (agentId) => this.agentUpdates.removeAgent(agentId), emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds) => @@ -3810,6 +3811,7 @@ export class Session { findWorkspaceIdForCwd: (cwd) => this.findWorkspaceIdForCwd(cwd), listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), archiveWorkspaceRecord: (workspaceId) => this.archiveWorkspaceRecord(workspaceId), + workspaceRegistry: this.workspaceRegistry, emit: (message) => this.emit(message), emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds) => this.emitWorkspaceUpdatesForWorkspaceIds(workspaceIds), @@ -5756,6 +5758,7 @@ export class Session { clearWorkspaceArchiving: (workspaceIds) => this.clearWorkspaceArchiving(workspaceIds), killTerminalsForWorkspace: (workspaceId) => this.terminalController.killTerminalsForWorkspace(workspaceId), + workspaceRegistry: this.workspaceRegistry, sessionLogger: this.sessionLogger, }, { diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 80e41c8be2..1bc9d0eef8 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -19,6 +19,10 @@ import { resolveWorkspaceIdAtPath, } from "./workspace-archive-service.js"; import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; +import { + createPersistedWorkspaceRecord, + FileBackedWorkspaceRegistry, +} from "./workspace-registry.js"; const cleanupPaths: string[] = []; @@ -275,6 +279,96 @@ describe("archiveByScope", () => { expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(1); }); + test("archives sibling workspace records separately while serializing shared cleanup", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "concurrent-siblings"); + const workspaceA = "ws-concurrent-sibling-a"; + const workspaceB = "ws-concurrent-sibling-b"; + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { workspaceId: workspaceA, cwd: worktree.worktreePath, kind: "worktree" }, + { workspaceId: workspaceB, cwd: worktree.worktreePath, kind: "worktree" }, + ], + }); + deps.lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + + const [resultA, resultB] = await Promise.all([ + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId: workspaceA }, + requestId: "req-concurrent-sibling-a", + }), + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId: workspaceB }, + requestId: "req-concurrent-sibling-b", + }), + ]); + + expect(resultA.archivedWorkspaceIds).toEqual([workspaceA]); + expect(resultB.archivedWorkspaceIds).toEqual([workspaceB]); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(2); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledWith(workspaceA); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledWith(workspaceB); + expect(existsSync(worktree.worktreePath)).toBe(false); + }); + + test("holds final owner recheck and deletion behind sibling setup reservation", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "reserved-sibling"); + const workspaceId = "ws-reserved-target"; + const siblingWorkspaceId = "ws-reserved-sibling"; + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const reservation = lifecycleCoordinator.reserveWorkspaceSetup( + siblingWorkspaceId, + worktree.worktreePath, + ); + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId, + cwd: worktree.worktreePath, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + }, + ], + }); + deps.lifecycleCoordinator = lifecycleCoordinator; + const originalArchiveWorkspaceRecord = deps.archiveWorkspaceRecord; + let markRecordArchived: (() => void) | undefined; + const recordArchived = new Promise((resolveArchived) => { + markRecordArchived = resolveArchived; + }); + deps.archiveWorkspaceRecord = async (id: string) => { + await originalArchiveWorkspaceRecord(id); + deps.activeWorkspaces.push({ + workspaceId: siblingWorkspaceId, + cwd: path.join(worktree.worktreePath, "packages", "app"), + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + }); + markRecordArchived?.(); + }; + + const archiveTask = archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-reserved-sibling", + }); + await recordArchived; + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(existsSync(worktree.worktreePath)).toBe(true); + + reservation.release(); + const result = await archiveTask; + expect(result.removedDirectory).toBe(false); + expect(existsSync(worktree.worktreePath)).toBe(true); + }); + test("workspace scope archives the record and removes the directory on last reference", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -664,6 +758,87 @@ describe("archiveByScope", () => { expect(existsSync(worktree.worktreePath)).toBe(true); }); + test("retries persisted physical cleanup after the workspace record is archived", async () => { + const { tempDir, repoDir } = createGitRepo(); + writeFileSync( + path.join(repoDir, "paseo.json"), + JSON.stringify({ + worktree: { + teardown: [ + "node -e \"const fs=require('fs');const marker=process.env.PASEO_SOURCE_CHECKOUT_PATH+'/cleanup-retry.marker';if(!fs.existsSync(marker)){fs.writeFileSync(marker,'retry');process.exit(1)}\"", + ], + }, + }), + ); + execFileSync("git", ["add", "."], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "retry cleanup"], { + cwd: repoDir, + stdio: "pipe", + }); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "cleanup-retry"); + const workspaceId = "ws-cleanup-retry"; + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId, + projectId: "project-cleanup-retry", + cwd: worktree.worktreePath, + kind: "worktree", + displayName: "Cleanup retry", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId, + cwd: worktree.worktreePath, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + ], + }); + deps.workspaceRegistry = registry; + const archiveActiveRecord = deps.archiveWorkspaceRecord; + deps.archiveWorkspaceRecord = async (id: string) => { + await archiveActiveRecord(id); + await registry.archive(id, new Date().toISOString()); + }; + + const firstResult = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-cleanup-retry-first", + }); + expect(firstResult.removedDirectory).toBe(false); + expect((await registry.get(workspaceId))?.archivedAt).not.toBeNull(); + expect((await registry.get(workspaceId))?.cleanupPending).toMatchObject({ + directoryPath: worktree.worktreePath, + teardownCwd: worktree.worktreePath, + }); + expect(existsSync(worktree.worktreePath)).toBe(true); + + const retryResult = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-cleanup-retry-second", + }); + expect(retryResult.removedDirectory).toBe(true); + expect(existsSync(worktree.worktreePath)).toBe(false); + expect((await registry.get(workspaceId))?.cleanupPending).toBeNull(); + }); + test("workspace scope with unknown workspace id is a clean no-op", async () => { const { tempDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index ffcf22fd99..cda672ee60 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -1,3 +1,4 @@ +import { existsSync } from "node:fs"; import { resolve } from "node:path"; import type { Logger } from "pino"; @@ -13,7 +14,11 @@ import { WorktreeTeardownError, } from "../utils/worktree.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; -import type { PersistedWorkspaceRecord, WorkspaceRegistry } from "./workspace-registry.js"; +import type { + PersistedWorkspaceCleanupPending, + PersistedWorkspaceRecord, + WorkspaceRegistry, +} from "./workspace-registry.js"; import { createRealpathAwarePathMatcher } from "../utils/path.js"; import { defaultWorkspaceLifecycleCoordinator, @@ -47,6 +52,7 @@ export interface ArchiveDependencies { markWorkspaceArchiving: (workspaceIds: Iterable, archivingAt: string) => void; clearWorkspaceArchiving: (workspaceIds: Iterable) => void; killTerminalsForWorkspace: (workspaceId: string) => Promise; + workspaceRegistry?: Pick; lifecycleCoordinator?: WorkspaceLifecycleCoordinator; sessionLogger?: Logger; } @@ -122,20 +128,16 @@ export async function archiveByScope( const initialTarget = await resolveArchiveTarget(dependencies, request.scope); const lifecycleCoordinator = dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; - let operationKey: string; - if (initialTarget.backing) { - operationKey = `directory:${resolve(initialTarget.backing.path)}`; - } else if (request.scope.kind === "workspace") { - operationKey = `workspace:${request.scope.workspaceId}`; - } else { - operationKey = `directory:${resolve(request.scope.targetPath)}`; - } + const operationKey = + request.scope.kind === "workspace" + ? `workspace:${request.scope.workspaceId}` + : `worktree:${resolve(request.scope.targetPath)}`; return lifecycleCoordinator.runArchive(operationKey, async () => { await lifecycleCoordinator.waitForWorkspaceSetups(initialTarget.workspaceIds); const refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); const target = mergeArchiveTargets(initialTarget, refreshedTarget); - return archiveResolvedTarget(dependencies, request, target); + return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); }); } @@ -152,6 +154,7 @@ function mergeArchiveTargets(initial: ArchiveTarget, refreshed: ArchiveTarget): async function archiveResolvedTarget( dependencies: ArchiveDependencies, + lifecycleCoordinator: WorkspaceLifecycleCoordinator, request: ArchiveByScopeRequest, target: ArchiveTarget, ): Promise { @@ -168,11 +171,14 @@ async function archiveResolvedTarget( await dependencies.emitWorkspaceUpdatesForWorkspaceIds(targetWorkspaceIds); } + await persistTargetCleanupPending(dependencies, target); + const { archivedAgents, archivedWorkspaceIds } = await archiveTargetRecords( dependencies, targetWorkspaceIds, request.requestId, ); + await clearCleanupPendingForUnarchivedTargets(dependencies, target, archivedWorkspaceIds); if (target.backing?.mainRepoRoot) { try { @@ -191,6 +197,7 @@ async function archiveResolvedTarget( if (target.backing !== null) { removedDirectory = await maybeRemoveDirectory( dependencies, + lifecycleCoordinator, request, target, archivedWorkspaceIds, @@ -220,6 +227,10 @@ async function resolveArchiveTarget( const workspaceId = scope.workspaceId; const record = activeWorkspaces.find((workspace) => workspace.workspaceId === workspaceId); if (!record) { + const persistedRecord = await dependencies.workspaceRegistry?.get(workspaceId); + if (persistedRecord?.cleanupPending) { + return archiveTargetFromPendingCleanup(workspaceId, persistedRecord.cleanupPending); + } dependencies.sessionLogger?.warn( { workspaceId }, "Workspace not found for archive-by-scope; skipping", @@ -233,7 +244,15 @@ async function resolveArchiveTarget( }; } - const backing = await resolveBackingDirectory(scope.targetPath, dependencies); + return resolveWorktreeArchiveTarget(dependencies, scope.targetPath, activeWorkspaces); +} + +async function resolveWorktreeArchiveTarget( + dependencies: ArchiveDependencies, + targetPath: string, + activeWorkspaces: ActiveWorkspaceRef[], +): Promise { + const backing = await resolveBackingDirectory(targetPath, dependencies); const matchesBackingDirectory = createRealpathAwarePathMatcher(backing.path); const targetWorkspaces = ( await Promise.all( @@ -243,25 +262,63 @@ async function resolveArchiveTarget( }), ) ).filter((workspace): workspace is ActiveWorkspaceRef => workspace !== null); + const pendingRecords = (await dependencies.workspaceRegistry?.list())?.filter( + (workspace) => + workspace.cleanupPending !== null && + matchesBackingDirectory(workspace.cleanupPending.directoryPath), + ); + const pendingCleanup = pendingRecords?.[0]?.cleanupPending ?? null; const persistedMainRepoRoot = targetWorkspaces.find( (workspace) => workspace.mainRepoRoot, )?.mainRepoRoot; + let teardownTargets: ArchiveTarget["teardownTargets"]; + if (targetWorkspaces.length > 0) { + teardownTargets = targetWorkspaces.map((workspace) => ({ + workspaceId: workspace.workspaceId, + cwd: workspace.cwd, + })); + } else if (pendingRecords && pendingRecords.length > 0) { + teardownTargets = pendingRecords.flatMap((workspace) => { + if (!workspace.cleanupPending) return []; + return [ + { + workspaceId: workspace.workspaceId, + cwd: workspace.cleanupPending.teardownCwd, + }, + ]; + }); + } else { + teardownTargets = [{ workspaceId: null, cwd: targetPath }]; + } + return { backing: { - ...backing, - mainRepoRoot: persistedMainRepoRoot ?? backing.mainRepoRoot, + path: pendingCleanup?.directoryPath ?? backing.path, + isPaseoOwnedWorktree: pendingCleanup !== null || backing.isPaseoOwnedWorktree, + mainRepoRoot: persistedMainRepoRoot ?? pendingCleanup?.mainRepoRoot ?? backing.mainRepoRoot, + paseoWorktreesRoot: pendingCleanup?.paseoWorktreesRoot ?? backing.paseoWorktreesRoot, }, - teardownTargets: - targetWorkspaces.length > 0 - ? targetWorkspaces.map((workspace) => ({ - workspaceId: workspace.workspaceId, - cwd: workspace.cwd, - })) - : [{ workspaceId: null, cwd: scope.targetPath }], + teardownTargets, workspaceIds: targetWorkspaces.map((workspace) => workspace.workspaceId), }; } +function archiveTargetFromPendingCleanup( + workspaceId: string, + cleanupPending: PersistedWorkspaceCleanupPending, +): ArchiveTarget { + return { + backing: { + path: cleanupPending.directoryPath, + isPaseoOwnedWorktree: true, + mainRepoRoot: cleanupPending.mainRepoRoot, + paseoWorktreesRoot: cleanupPending.paseoWorktreesRoot, + }, + teardownTargets: [{ workspaceId, cwd: cleanupPending.teardownCwd }], + workspaceIds: [], + }; +} + async function resolveWorkspaceBackingDirectory( workspace: ActiveWorkspaceRef, dependencies: Pick, @@ -344,6 +401,7 @@ async function archiveTargetRecords( async function maybeRemoveDirectory( dependencies: ArchiveDependencies, + lifecycleCoordinator: WorkspaceLifecycleCoordinator, request: Pick, target: ArchiveTarget, archivedWorkspaceIds: string[], @@ -353,15 +411,36 @@ async function maybeRemoveDirectory( return false; } - const archivedWorkspaceIdSet = new Set(archivedWorkspaceIds); + return lifecycleCoordinator.runDirectoryExclusive(backing.path, async () => { + return maybeRemoveDirectoryExclusive(dependencies, request, target, archivedWorkspaceIds); + }); +} + +async function maybeRemoveDirectoryExclusive( + dependencies: ArchiveDependencies, + request: Pick, + target: ArchiveTarget, + archivedWorkspaceIds: string[], +): Promise { + const backing = target.backing; + if (!backing) return false; + + const pendingCleanupTargets = await listPendingCleanupTargets( + dependencies, + target, + archivedWorkspaceIds, + ); + if (pendingCleanupTargets.length === 0) { + return false; + } + + if (!existsSync(backing.path)) { + await clearPendingCleanup(dependencies, pendingCleanupTargets); + return false; + } + const teardownCwds = uniqueFilesystemPaths( - target.teardownTargets - .filter( - (teardownTarget) => - teardownTarget.workspaceId === null || - archivedWorkspaceIdSet.has(teardownTarget.workspaceId), - ) - .map((teardownTarget) => teardownTarget.cwd), + pendingCleanupTargets.map((pendingCleanupTarget) => pendingCleanupTarget.teardownCwd), ); try { @@ -392,6 +471,7 @@ async function maybeRemoveDirectory( dependencies, )) ) { + await clearPendingCleanup(dependencies, pendingCleanupTargets); return false; } @@ -405,6 +485,7 @@ async function maybeRemoveDirectory( worktreesBaseRoot: dependencies.paseoWorktreesBaseRoot, }); dependencies.github.invalidate({ cwd: backing.path }); + await clearPendingCleanup(dependencies, pendingCleanupTargets); return true; } catch (error) { if (error instanceof WorktreeTeardownError) { @@ -418,6 +499,123 @@ async function maybeRemoveDirectory( } } +interface PendingCleanupTarget extends PersistedWorkspaceCleanupPending { + workspaceId: string | null; +} + +async function persistTargetCleanupPending( + dependencies: ArchiveDependencies, + target: ArchiveTarget, +): Promise { + const backing = target.backing; + const workspaceRegistry = dependencies.workspaceRegistry; + if (!backing?.isPaseoOwnedWorktree || !workspaceRegistry) return; + + await Promise.all( + target.teardownTargets.flatMap((teardownTarget) => + teardownTarget.workspaceId + ? [ + workspaceRegistry.update(teardownTarget.workspaceId, (workspace) => ({ + ...workspace, + cleanupPending: { + directoryPath: backing.path, + teardownCwd: teardownTarget.cwd, + mainRepoRoot: backing.mainRepoRoot, + paseoWorktreesRoot: backing.paseoWorktreesRoot, + }, + })), + ] + : [], + ), + ); +} + +async function listPendingCleanupTargets( + dependencies: ArchiveDependencies, + target: ArchiveTarget, + archivedWorkspaceIds: string[], +): Promise { + const backingPath = target.backing?.path; + const workspaceRegistry = dependencies.workspaceRegistry; + if (backingPath && workspaceRegistry) { + const matchesBacking = createRealpathAwarePathMatcher(backingPath); + const records = await workspaceRegistry.list(); + return records.flatMap((workspace) => { + const cleanupPending = workspace.cleanupPending; + if ( + !workspace.archivedAt || + !cleanupPending || + !matchesBacking(cleanupPending.directoryPath) + ) { + return []; + } + return [{ workspaceId: workspace.workspaceId, ...cleanupPending }]; + }); + } + + const archivedWorkspaceIdSet = new Set(archivedWorkspaceIds); + return target.teardownTargets + .filter( + (teardownTarget) => + teardownTarget.workspaceId === null || + archivedWorkspaceIdSet.has(teardownTarget.workspaceId), + ) + .map((teardownTarget) => ({ + workspaceId: teardownTarget.workspaceId, + directoryPath: target.backing?.path ?? teardownTarget.cwd, + teardownCwd: teardownTarget.cwd, + mainRepoRoot: target.backing?.mainRepoRoot ?? null, + paseoWorktreesRoot: target.backing?.paseoWorktreesRoot ?? null, + })); +} + +async function clearPendingCleanup( + dependencies: ArchiveDependencies, + cleanupTargets: PendingCleanupTarget[], +): Promise { + const workspaceRegistry = dependencies.workspaceRegistry; + if (!workspaceRegistry) return; + await Promise.all( + cleanupTargets.flatMap((cleanupTarget) => + cleanupTarget.workspaceId + ? [ + workspaceRegistry.update(cleanupTarget.workspaceId, (workspace) => ({ + ...workspace, + cleanupPending: null, + })), + ] + : [], + ), + ); +} + +async function clearCleanupPendingForUnarchivedTargets( + dependencies: ArchiveDependencies, + target: ArchiveTarget, + archivedWorkspaceIds: string[], +): Promise { + const archivedWorkspaceIdSet = new Set(archivedWorkspaceIds); + const targetWorkspaceIdSet = new Set(target.workspaceIds); + await clearPendingCleanup( + dependencies, + target.teardownTargets.flatMap((teardownTarget) => + teardownTarget.workspaceId && + targetWorkspaceIdSet.has(teardownTarget.workspaceId) && + !archivedWorkspaceIdSet.has(teardownTarget.workspaceId) + ? [ + { + workspaceId: teardownTarget.workspaceId, + directoryPath: target.backing?.path ?? teardownTarget.cwd, + teardownCwd: teardownTarget.cwd, + mainRepoRoot: target.backing?.mainRepoRoot ?? null, + paseoWorktreesRoot: target.backing?.paseoWorktreesRoot ?? null, + }, + ] + : [], + ), + ); +} + function uniqueFilesystemPaths(paths: string[]): string[] { const unique: string[] = []; for (const candidate of paths) { diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts index 727dcab4f7..0632011726 100644 --- a/packages/server/src/server/workspace-lifecycle-coordinator.ts +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -1,17 +1,53 @@ export class WorkspaceLifecycleCoordinator { private readonly setupTasks = new Map>(); + private readonly setupTasksByDirectory = new Map>>(); private readonly archiveOperations = new Map>(); + private readonly directoryOperations = new Map>(); - trackWorkspaceSetup(workspaceId: string, task: Promise): void { + trackWorkspaceSetup(workspaceId: string, task: Promise, directoryPath?: string): void { this.setupTasks.set(workspaceId, task); + const directoryTasks = directoryPath + ? (this.setupTasksByDirectory.get(directoryPath) ?? new Set>()) + : null; + if (directoryPath && directoryTasks) { + directoryTasks.add(task); + this.setupTasksByDirectory.set(directoryPath, directoryTasks); + } const clearTask = () => { if (this.setupTasks.get(workspaceId) === task) { this.setupTasks.delete(workspaceId); } + if (directoryPath && directoryTasks) { + directoryTasks.delete(task); + if (directoryTasks.size === 0) { + this.setupTasksByDirectory.delete(directoryPath); + } + } }; void task.then(clearTask, clearTask); } + reserveWorkspaceSetup(workspaceId: string, directoryPath: string): WorkspaceSetupReservation { + let releaseReservation: (() => void) | null = null; + const reservationTask = new Promise((resolve) => { + releaseReservation = resolve; + }); + this.trackWorkspaceSetup(workspaceId, reservationTask, directoryPath); + + let released = false; + const release = () => { + if (released) return; + released = true; + releaseReservation?.(); + }; + return { + completeWith(task) { + void task.then(release, release); + }, + release, + }; + } + async waitForWorkspaceSetups(workspaceIds: Iterable): Promise { const tasks = Array.from(workspaceIds, (workspaceId) => this.setupTasks.get(workspaceId), @@ -35,6 +71,33 @@ export class WorkspaceLifecycleCoordinator { void task.then(clearTask, clearTask); return task; } + + runDirectoryExclusive(directoryPath: string, operation: () => Promise): Promise { + const previous = this.directoryOperations.get(directoryPath) ?? Promise.resolve(); + const task = previous + .catch(() => undefined) + .then(async () => { + while (true) { + const setupTasks = Array.from(this.setupTasksByDirectory.get(directoryPath) ?? []); + if (setupTasks.length === 0) break; + await Promise.allSettled(setupTasks); + } + return operation(); + }); + this.directoryOperations.set(directoryPath, task); + const clearTask = () => { + if (this.directoryOperations.get(directoryPath) === task) { + this.directoryOperations.delete(directoryPath); + } + }; + void task.then(clearTask, clearTask); + return task; + } +} + +export interface WorkspaceSetupReservation { + completeWith(task: Promise): void; + release(): void; } export const defaultWorkspaceLifecycleCoordinator = new WorkspaceLifecycleCoordinator(); diff --git a/packages/server/src/server/workspace-registry.ts b/packages/server/src/server/workspace-registry.ts index fe28048120..6e2d707606 100644 --- a/packages/server/src/server/workspace-registry.ts +++ b/packages/server/src/server/workspace-registry.ts @@ -34,6 +34,13 @@ const PersistedProjectRecordSchema = z.object({ archivedAt: z.string().nullable(), }); +const PersistedWorkspaceCleanupPendingSchema = z.object({ + directoryPath: z.string(), + teardownCwd: z.string(), + mainRepoRoot: z.string().nullable(), + paseoWorktreesRoot: z.string().nullable(), +}); + const PersistedWorkspaceRecordSchema = z.object({ workspaceId: z.string(), projectId: z.string(), @@ -71,6 +78,9 @@ const PersistedWorkspaceRecordSchema = z.object({ .transform((value) => value ?? null), isPaseoOwnedWorktree: z.boolean().default(false), mainRepoRoot: z.string().nullable().default(null), + cleanupPending: PersistedWorkspaceCleanupPendingSchema.nullable() + .optional() + .transform((value) => value ?? null), createdAt: z.string(), updatedAt: z.string(), archivedAt: z.string().nullable(), @@ -83,6 +93,9 @@ const PersistedWorkspaceRecordSchema = z.object({ export type PersistedProjectRecord = z.infer; export type PersistedWorkspaceRecord = z.infer; +export type PersistedWorkspaceCleanupPending = z.infer< + typeof PersistedWorkspaceCleanupPendingSchema +>; export interface WorkspaceMutation { kind: "upsert" | "archive" | "remove"; @@ -503,6 +516,7 @@ export function createPersistedWorkspaceRecord(input: { baseBranch?: string | null; isPaseoOwnedWorktree?: boolean; mainRepoRoot?: string | null; + cleanupPending?: PersistedWorkspaceCleanupPending | null; createdAt: string; updatedAt: string; archivedAt?: string | null; @@ -516,6 +530,7 @@ export function createPersistedWorkspaceRecord(input: { baseBranch: input.baseBranch ?? null, isPaseoOwnedWorktree: input.isPaseoOwnedWorktree ?? false, mainRepoRoot: input.mainRepoRoot ?? null, + cleanupPending: input.cleanupPending ?? null, archivedAt: input.archivedAt ?? null, pinnedAt: input.pinnedAt ?? null, }); diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index cee6e2ca8b..4aad98cbb0 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -489,6 +489,7 @@ describe("create worktree setup boundary", () => { expect(trackWorkspaceSetup).toHaveBeenCalledWith( result.workspace.workspaceId, expect.any(Promise), + result.worktree.worktreePath, ); await lifecycleCoordinator.waitForWorkspaceSetups([result.workspace.workspaceId]); } finally { @@ -569,6 +570,64 @@ describe("create worktree setup boundary", () => { rmSync(tempDir, { recursive: true, force: true }); } }); + + test("reserves agent setup before returning and releases an unused continuation", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const operation = vi.fn(); + + try { + const result = await createPaseoWorktreeWorkflow( + { + paseoHome, + createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), + warmWorkspaceGitData: async () => {}, + autoNameWorkspaceBranchForFirstAgent: () => {}, + emitWorkspaceUpdateForWorkspaceId: async () => {}, + cacheWorkspaceSetupSnapshot: () => {}, + emit: () => {}, + sessionLogger: createLogger(), + terminalManager: null, + archiveWorkspaceRecord: async () => {}, + serviceProxy: null, + scriptRuntimeStore: null, + getDaemonTcpPort: null, + getDaemonTcpHost: null, + onScriptsChanged: null, + lifecycleCoordinator, + }, + { + cwd: repoDir, + worktreeSlug: "agent-setup-reservation", + runSetup: false, + paseoHome, + }, + { + setupContinuation: { + kind: "agent", + terminalManager: null, + appendTimelineItem: async () => true, + emitLiveTimelineItem: async () => true, + logger: createLogger(), + }, + }, + ); + + const directoryTask = lifecycleCoordinator.runDirectoryExclusive( + result.worktree.worktreePath, + async () => operation(), + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(operation).not.toHaveBeenCalled(); + + result.setupContinuation?.releaseWithoutStarting(); + await directoryTask; + expect(operation).toHaveBeenCalledOnce(); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); }); function createAgentStorageStub(): Pick { diff --git a/packages/server/src/server/worktree-session.ts b/packages/server/src/server/worktree-session.ts index 9716ced730..4ffbb438d1 100644 --- a/packages/server/src/server/worktree-session.ts +++ b/packages/server/src/server/worktree-session.ts @@ -145,6 +145,7 @@ export type CreatePaseoWorktreeSetupContinuationInput = export interface AgentWorktreeSetupContinuation { kind: "agent"; startAfterAgentCreate: (input: { agentId: string }) => void; + releaseWithoutStarting: () => void; } export type CreatePaseoWorktreeWorkflowResult = CreatePaseoWorktreeResult & { @@ -642,10 +643,18 @@ export async function createPaseoWorktreeWorkflow( }).then(resolveSetup, resolveSetup); }, 0); }); - lifecycleCoordinator.trackWorkspaceSetup(workspace.workspaceId, setupTask); + lifecycleCoordinator.trackWorkspaceSetup( + workspace.workspaceId, + setupTask, + createdWorktree.worktree.worktreePath, + ); } if (setupContinuation.kind === "agent") { + const reservation = lifecycleCoordinator.reserveWorkspaceSetup( + workspace.workspaceId, + createdWorktree.worktree.worktreePath, + ); return { ...createdWorktree, setupContinuation: { @@ -663,9 +672,10 @@ export async function createPaseoWorktreeWorkflow( setupContinuation.emitLiveTimelineItem({ agentId, item }), logger: setupContinuation.logger, }); - lifecycleCoordinator.trackWorkspaceSetup(workspace.workspaceId, setupTask); + reservation.completeWith(setupTask); void setupTask; }, + releaseWithoutStarting: reservation.release, }, }; } From 3852880496b406aa8c1374b7081bcaf213cba149 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 09:32:54 +0100 Subject: [PATCH 03/43] fix: bound worktree setup updates --- packages/server/src/server/session.ts | 4 +- .../session.workspace-git-watch.test.ts | 45 +++- .../src/server/worktree-bootstrap.test.ts | 54 +++- .../server/src/server/worktree-bootstrap.ts | 249 +++++++++--------- .../src/server/worktree-session.test.ts | 31 ++- .../server/src/server/worktree-session.ts | 32 ++- .../src/utils/worktree-setup-output.test.ts | 22 ++ .../server/src/utils/worktree-setup-output.ts | 118 +++++++++ .../server/src/utils/worktree.posix.test.ts | 32 +++ packages/server/src/utils/worktree.ts | 94 +++---- 10 files changed, 489 insertions(+), 192 deletions(-) create mode 100644 packages/server/src/utils/worktree-setup-output.test.ts create mode 100644 packages/server/src/utils/worktree-setup-output.ts diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 3bef5a6d5f..562d5312c8 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -228,6 +228,7 @@ import { handlePaseoWorktreeArchiveRequest as handleWorktreeArchiveRequest, handlePaseoWorktreeListRequest as handleWorktreeListRequest, handleWorkspaceSetupStatusRequest as handleWorkspaceSetupStatusRequestMessage, + cacheWorkspaceSetupSnapshot, } from "./worktree-session.js"; import { archiveByScope, type ActiveWorkspaceRef } from "./workspace-archive-service.js"; import { WorktreeRequestError, toWorktreeWireError } from "./worktree-errors.js"; @@ -4558,6 +4559,7 @@ export class Session { archivedAt: archiveTimestamp, workspaceRegistry: this.workspaceRegistry, }); + this.workspaceSetupSnapshots.delete(workspaceId); if (!existingWorkspace) { this.workspaceGitObserver.removeForWorkspaceId(workspaceId); return; @@ -5699,7 +5701,7 @@ export class Session { emitWorkspaceUpdateForWorkspaceId: (workspaceId) => this.emitWorkspaceUpdateForWorkspaceId(workspaceId), cacheWorkspaceSetupSnapshot: (workspaceId, snapshot) => { - this.workspaceSetupSnapshots.set(workspaceId, snapshot); + cacheWorkspaceSetupSnapshot(this.workspaceSetupSnapshots, workspaceId, snapshot); }, emit: (message) => this.emit(message), sessionLogger: this.sessionLogger, diff --git a/packages/server/src/server/session.workspace-git-watch.test.ts b/packages/server/src/server/session.workspace-git-watch.test.ts index 00f3f780fd..9d119f3136 100644 --- a/packages/server/src/server/session.workspace-git-watch.test.ts +++ b/packages/server/src/server/session.workspace-git-watch.test.ts @@ -13,7 +13,11 @@ import type { WorkspaceGitRuntimeSnapshot, WorkspaceGitService, } from "./workspace-git-service.js"; -import type { SessionOutboundMessage, WorkspaceDescriptorPayload } from "./messages.js"; +import type { + SessionOutboundMessage, + WorkspaceDescriptorPayload, + WorkspaceSetupSnapshot, +} from "./messages.js"; import { createPersistedProjectRecord, createPersistedWorkspaceRecord, @@ -120,6 +124,7 @@ function createSessionForWorkspaceGitWatchTests(options?: { ) => void; serviceProxy?: ServiceProxySubsystem; scriptRuntimeStore?: WorkspaceScriptRuntimeStore; + workspaceSetupSnapshots?: Map; }): { session: Session; emitted: Array<{ type: string; payload: unknown }>; @@ -257,6 +262,7 @@ function createSessionForWorkspaceGitWatchTests(options?: { terminalManager: null, serviceProxy: options?.serviceProxy, scriptRuntimeStore: options?.scriptRuntimeStore, + workspaceSetupSnapshots: options?.workspaceSetupSnapshots, onBranchChanged: options?.onBranchChanged, getDaemonTcpPort: () => 6767, }); @@ -535,6 +541,43 @@ describe("workspace git watch targets", () => { await session.cleanup(); }); + test("archiving a workspace clears its cached setup snapshot", async () => { + const setupSnapshots = new Map([ + [ + "ws-10", + { + status: "completed", + detail: { + type: "worktree_setup", + worktreePath: "/tmp/repo", + branchName: "main", + log: "done", + commands: [], + }, + error: null, + }, + ], + ]); + const { session, projects, workspaces } = createSessionForWorkspaceGitWatchTests({ + workspaceSetupSnapshots: setupSnapshots, + }); + seedGitWorkspace({ + projects, + workspaces, + projectId: "proj-1", + workspaceId: "ws-10", + cwd: "/tmp/repo", + name: "main", + }); + + await asInternals<{ archiveWorkspaceRecord: (workspaceId: string) => Promise }>( + session, + ).archiveWorkspaceRecord("ws-10"); + + expect(setupSnapshots.has("ws-10")).toBe(false); + await session.cleanup(); + }); + test("archiving a workspace releases its git watch subscription for the directory", async () => { const { session, projects, workspaces, subscriptions } = createSessionForWorkspaceGitWatchTests(); diff --git a/packages/server/src/server/worktree-bootstrap.test.ts b/packages/server/src/server/worktree-bootstrap.test.ts index c878ef1387..eaff6c4f4d 100644 --- a/packages/server/src/server/worktree-bootstrap.test.ts +++ b/packages/server/src/server/worktree-bootstrap.test.ts @@ -5,7 +5,11 @@ import { join } from "path"; import { tmpdir } from "os"; import type { AgentTimelineItem } from "./agent/agent-sdk-types.js"; -import { runAsyncWorktreeBootstrap, spawnWorkspaceScript } from "./worktree-bootstrap.js"; +import { + createWorktreeSetupProgressCoalescer, + runAsyncWorktreeBootstrap, + spawnWorkspaceScript, +} from "./worktree-bootstrap.js"; import { ensureWorkspaceServicePortPlan } from "./workspace-service-port-registry.js"; import { ScriptRouteStore } from "./script-proxy.js"; import { createBranchChangeRouteHandler } from "./script-route-branch-handler.js"; @@ -17,6 +21,42 @@ import { import type { TerminalManager } from "../terminal/terminal-manager.js"; import type { TerminalSession } from "../terminal/terminal.js"; +describe("worktree setup progress coalescing", () => { + it("keeps only the latest pending update while an emission is in flight", async () => { + let releaseFirstEmission: (() => void) | null = null; + let markFirstEmissionStarted: (() => void) | null = null; + const firstEmissionGate = new Promise((resolve) => { + releaseFirstEmission = resolve; + }); + const firstEmissionStarted = new Promise((resolve) => { + markFirstEmissionStarted = resolve; + }); + const emittedVersions: number[] = []; + let version = 0; + const progress = createWorktreeSetupProgressCoalescer({ + emit: async () => { + emittedVersions.push(version); + if (emittedVersions.length === 1) { + markFirstEmissionStarted?.(); + await firstEmissionGate; + } + }, + }); + + progress.schedule(); + await firstEmissionStarted; + for (let nextVersion = 1; nextVersion <= 1_000; nextVersion += 1) { + version = nextVersion; + progress.schedule(); + } + + expect(emittedVersions).toEqual([0]); + releaseFirstEmission?.(); + await progress.flush(); + expect(emittedVersions).toEqual([0, 1_000]); + }); +}); + interface CreateAgentWorktreeTestOptions { cwd: string; branchName: string; @@ -142,7 +182,7 @@ describe("runAsyncWorktreeBootstrap", () => { } }); - it("truncates each command output to 64kb in the middle", async () => { + it("truncates each command output to 64kb with an exact omitted-byte count", async () => { const largeOutputCommand = "node -e \"process.stdout.write('prefix-'); process.stdout.write('x'.repeat(70000)); process.stdout.write('-suffix')\""; writeFileSync( @@ -194,12 +234,14 @@ describe("runAsyncWorktreeBootstrap", () => { expect(persistedSetupItem.detail.truncated).toBe(true); expect(persistedSetupItem.detail.log).toContain("prefix-"); expect(persistedSetupItem.detail.log).toContain("-suffix"); - expect(persistedSetupItem.detail.log).toContain("......"); + expect(persistedSetupItem.detail.log).toMatch(/\.\.\.<\d+ bytes omitted>\.\.\./); expect(persistedSetupItem.detail.commands[0]?.log).toContain("prefix-"); expect(persistedSetupItem.detail.commands[0]?.log).toContain("-suffix"); - expect(persistedSetupItem.detail.commands[0]?.log).toContain( - "......", - ); + const commandLog = persistedSetupItem.detail.commands[0]?.log ?? ""; + const marker = commandLog.match(/\n\.\.\.<(\d+) bytes omitted>\.\.\.\n/); + expect(marker).not.toBeNull(); + const retained = commandLog.replace(marker?.[0] ?? "", ""); + expect(Number(marker?.[1])).toBe(70_014 - Buffer.byteLength(retained)); }); it("waits for terminal output before sending bootstrap commands", async () => { diff --git a/packages/server/src/server/worktree-bootstrap.ts b/packages/server/src/server/worktree-bootstrap.ts index 75785057e4..643a0aa38b 100644 --- a/packages/server/src/server/worktree-bootstrap.ts +++ b/packages/server/src/server/worktree-bootstrap.ts @@ -31,6 +31,13 @@ import { refreshWorkspaceServicePort, } from "./workspace-service-port-registry.js"; import type { PaseoServicePortAllocation } from "@getpaseo/protocol/paseo-config-schema"; +import { + appendWorktreeSetupOutput, + createWorktreeSetupOutputAccumulator, + getWorktreeSetupCommandOutputLimit, + renderWorktreeSetupOutput, + type WorktreeSetupOutputAccumulator, +} from "../utils/worktree-setup-output.js"; export interface WorktreeBootstrapTerminalResult { name: string | null; @@ -54,120 +61,117 @@ export interface RunAsyncWorktreeBootstrapOptions { logger?: Logger; } -const MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES = 64 * 1024; -const WORKTREE_SETUP_TRUNCATION_MARKER = "\n......\n"; const WORKTREE_BOOTSTRAP_TERMINAL_READY_TIMEOUT_MS = 1_500; - -interface MiddleTruncationAccumulator { - totalBytes: number; - head: string; - tail: string; - truncated: boolean; -} - -export type WorktreeSetupOutputAccumulator = MiddleTruncationAccumulator; +const WORKTREE_SETUP_PROGRESS_EMIT_INTERVAL_MS = 100; export interface WorktreeSetupProgressAccumulator { resultsByIndex: Map; outputAccumulatorsByIndex: Map; } -function byteLength(text: string): number { - return Buffer.byteLength(text, "utf8"); +export interface WorktreeSetupProgressCoalescerOptions { + emit: () => unknown | Promise; + onError?: (error: unknown) => void; } -function sliceFirstBytes(text: string, maxBytes: number): string { - if (maxBytes <= 0 || text.length === 0) { - return ""; - } - const bytes = Buffer.from(text, "utf8"); - if (bytes.length <= maxBytes) { - return text; - } - return bytes.subarray(0, maxBytes).toString("utf8"); +export interface WorktreeSetupProgressCoalescer { + schedule: () => void; + flush: () => Promise; } -function sliceLastBytes(text: string, maxBytes: number): string { - if (maxBytes <= 0 || text.length === 0) { - return ""; - } - const bytes = Buffer.from(text, "utf8"); - if (bytes.length <= maxBytes) { - return text; - } - return bytes.subarray(bytes.length - maxBytes).toString("utf8"); -} - -export function createWorktreeSetupOutputAccumulator(): WorktreeSetupOutputAccumulator { - return { - totalBytes: 0, - head: "", - tail: "", - truncated: false, +export function createWorktreeSetupProgressCoalescer( + options: WorktreeSetupProgressCoalescerOptions, +): WorktreeSetupProgressCoalescer { + let pending = false; + let flushing = false; + let lastStartedAt: number | null = null; + let timer: ReturnType | null = null; + let inFlight: Promise | null = null; + + const handleError = (error: unknown) => { + try { + options.onError?.(error); + } catch { + // Progress reporting must not fail setup, including a failing error reporter. + } }; -} -function getHeadTailBudgets(maxBytes: number): { headBytes: number; tailBytes: number } { - const markerBytes = byteLength(WORKTREE_SETUP_TRUNCATION_MARKER); - const availableBytes = Math.max(0, maxBytes - markerBytes); - const headBytes = Math.floor(availableBytes / 2); - const tailBytes = availableBytes - headBytes; - return { headBytes, tailBytes }; -} + const startOrSchedule = () => { + if (!pending || inFlight || timer) { + return; + } -export function appendWorktreeSetupOutputAccumulator( - accumulator: WorktreeSetupOutputAccumulator, - chunk: string, -): void { - if (!chunk) { - return; - } - accumulator.totalBytes += byteLength(chunk); + const elapsedMs = lastStartedAt === null ? Infinity : Date.now() - lastStartedAt; + const delayMs = flushing + ? 0 + : Math.max(0, WORKTREE_SETUP_PROGRESS_EMIT_INTERVAL_MS - elapsedMs); + if (delayMs > 0) { + timer = setTimeout(() => { + timer = null; + startOrSchedule(); + }, delayMs); + return; + } - if (!accumulator.truncated) { - const combined = `${accumulator.head}${chunk}`; - if (byteLength(combined) <= MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES) { - accumulator.head = combined; + pending = false; + lastStartedAt = Date.now(); + let emitted: unknown | Promise; + try { + emitted = options.emit(); + } catch (error) { + handleError(error); + emitted = undefined; + } + const emission = Promise.resolve(emitted).catch(handleError); + let current: Promise; + current = emission.then(() => { + if (inFlight === current) { + inFlight = null; + startOrSchedule(); + } + return undefined; + }); + inFlight = current; + }; + + const waitForIdle = async (): Promise => { + startOrSchedule(); + const current = inFlight; + if (!current) { return; } - const { headBytes, tailBytes } = getHeadTailBudgets(MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES); - accumulator.head = sliceFirstBytes(combined, headBytes); - accumulator.tail = sliceLastBytes(combined, tailBytes); - accumulator.truncated = true; - return; - } + await current; + if (pending || inFlight) { + await waitForIdle(); + } + }; - const { tailBytes } = getHeadTailBudgets(MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES); - accumulator.tail = sliceLastBytes(`${accumulator.tail}${chunk}`, tailBytes); + return { + schedule: () => { + pending = true; + startOrSchedule(); + }, + flush: async () => { + flushing = true; + if (timer) { + clearTimeout(timer); + timer = null; + } + try { + await waitForIdle(); + } finally { + flushing = false; + } + }, + }; } function truncateTextInMiddle( text: string, maxBytes: number, ): { text: string; truncated: boolean } { - if (maxBytes <= 0 || !text) { - return { text: "", truncated: text.length > 0 }; - } - if (byteLength(text) <= maxBytes) { - return { text, truncated: false }; - } - const { headBytes, tailBytes } = getHeadTailBudgets(maxBytes); - return { - text: `${sliceFirstBytes(text, headBytes)}${WORKTREE_SETUP_TRUNCATION_MARKER}${sliceLastBytes(text, tailBytes)}`, - truncated: true, - }; -} - -function renderMiddleTruncationAccumulator(accumulator: MiddleTruncationAccumulator): { - text: string; - truncated: boolean; -} { - if (!accumulator.truncated) { - return { text: accumulator.head, truncated: false }; - } - return { - text: `${accumulator.head}${WORKTREE_SETUP_TRUNCATION_MARKER}${accumulator.tail}`, - truncated: true, - }; + const accumulator = createWorktreeSetupOutputAccumulator(maxBytes); + appendWorktreeSetupOutput(accumulator, text); + return renderWorktreeSetupOutput(accumulator); } function formatDurationMs(durationMs: number): string { @@ -231,10 +235,10 @@ function buildWorktreeSetupCommandLog(input: { const { index, result, outputAccumulatorsByIndex } = input; const accumulator = outputAccumulatorsByIndex?.get(index); const rendered = accumulator - ? renderMiddleTruncationAccumulator(accumulator) + ? renderWorktreeSetupOutput(accumulator) : truncateTextInMiddle( `${result.stdout ?? ""}${result.stderr ?? ""}`, - MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES, + getWorktreeSetupCommandOutputLimit(1), ); return { @@ -267,8 +271,8 @@ export function applyWorktreeSetupProgressEvent( if (event.type === "output") { const outputAccumulator = accumulator.outputAccumulatorsByIndex.get(event.index) ?? - createWorktreeSetupOutputAccumulator(); - appendWorktreeSetupOutputAccumulator(outputAccumulator, event.chunk); + createWorktreeSetupOutputAccumulator(getWorktreeSetupCommandOutputLimit(event.total)); + appendWorktreeSetupOutput(outputAccumulator, event.chunk); accumulator.outputAccumulatorsByIndex.set(event.index, outputAccumulator); accumulator.resultsByIndex.set(event.index, { ...baseResult, @@ -602,34 +606,27 @@ export async function runAsyncWorktreeBootstrap( const emitLiveTimelineItem = options.emitLiveTimelineItem; const progressAccumulator = createWorktreeSetupProgressAccumulator(); const workspaceCwd = options.workspaceCwd ?? options.worktree.worktreePath; - let liveEmitQueue = Promise.resolve(); - - const queueLiveRunningEmit = () => { - if (!emitLiveTimelineItem) { - return; - } - const runningResults = getWorktreeSetupProgressResults(progressAccumulator); - liveEmitQueue = liveEmitQueue.then(async () => { - try { - await emitLiveTimelineItem( - buildSetupTimelineItem({ - callId: setupCallId, - status: "running", - worktree: options.worktree, - results: runningResults, - outputAccumulatorsByIndex: progressAccumulator.outputAccumulatorsByIndex, - errorMessage: null, - }), - ); - } catch (error) { - options.logger?.warn( - { err: error, agentId: options.agentId }, - "Failed to emit live worktree setup timeline update", - ); - } - return; - }); - }; + const liveProgress = emitLiveTimelineItem + ? createWorktreeSetupProgressCoalescer({ + emit: () => + emitLiveTimelineItem( + buildSetupTimelineItem({ + callId: setupCallId, + status: "running", + worktree: options.worktree, + results: getWorktreeSetupProgressResults(progressAccumulator), + outputAccumulatorsByIndex: progressAccumulator.outputAccumulatorsByIndex, + errorMessage: null, + }), + ), + onError: (error) => { + options.logger?.warn( + { err: error, agentId: options.agentId }, + "Failed to emit live worktree setup timeline update", + ); + }, + }) + : null; try { runtimeEnv = await resolveWorktreeRuntimeEnv({ @@ -648,10 +645,10 @@ export async function runAsyncWorktreeBootstrap( runtimeEnv, onEvent: (event) => { applyWorktreeSetupProgressEvent(progressAccumulator, event); - queueLiveRunningEmit(); + liveProgress?.schedule(); }, }); - await liveEmitQueue; + await liveProgress?.flush(); const completed = await options.appendTimelineItem( buildSetupTimelineItem({ @@ -670,7 +667,7 @@ export async function runAsyncWorktreeBootstrap( if (error instanceof WorktreeSetupError) { setupResults = error.results; } - await liveEmitQueue; + await liveProgress?.flush(); const message = error instanceof Error ? error.message : String(error); await options.appendTimelineItem( buildSetupTimelineItem({ diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index 4aad98cbb0..0f467a0b69 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -13,9 +13,14 @@ import path from "node:path"; import { afterEach, describe, expect, test, vi } from "vitest"; import pino, { type Logger } from "pino"; -import type { SessionOutboundMessage, WorkspaceDescriptorPayload } from "./messages.js"; +import type { + SessionOutboundMessage, + WorkspaceDescriptorPayload, + WorkspaceSetupSnapshot, +} from "./messages.js"; import { buildAgentSessionConfig, + cacheWorkspaceSetupSnapshot, createPaseoWorktreeWorkflow, handlePaseoWorktreeArchiveRequest, handlePaseoWorktreeListRequest, @@ -23,6 +28,7 @@ import { runWorktreeSetupInBackground, handleCreatePaseoWorktreeRequest, handleWorkspaceSetupStatusRequest, + MAX_WORKSPACE_SETUP_SNAPSHOTS, } from "./worktree-session.js"; import { createWorktree as createWorktreePrimitive, @@ -1378,6 +1384,29 @@ describe("runWorktreeSetupInBackground", () => { }, }); }); + + test("evicts the oldest setup snapshot when the shared cache reaches its bound", () => { + const snapshots = new Map(); + const snapshot: WorkspaceSetupSnapshot = { + status: "completed", + detail: { + type: "worktree_setup", + worktreePath: "/repo/worktree", + branchName: "feature", + log: "done", + commands: [], + }, + error: null, + }; + + for (let index = 0; index <= MAX_WORKSPACE_SETUP_SNAPSHOTS; index += 1) { + cacheWorkspaceSetupSnapshot(snapshots, `ws-${index}`, snapshot); + } + + expect(snapshots).toHaveLength(MAX_WORKSPACE_SETUP_SNAPSHOTS); + expect(snapshots.has("ws-0")).toBe(false); + expect(snapshots.has(`ws-${MAX_WORKSPACE_SETUP_SNAPSHOTS}`)).toBe(true); + }); }); describe("handleCreatePaseoWorktreeRequest", () => { diff --git a/packages/server/src/server/worktree-session.ts b/packages/server/src/server/worktree-session.ts index 4ffbb438d1..5ac06704d3 100644 --- a/packages/server/src/server/worktree-session.ts +++ b/packages/server/src/server/worktree-session.ts @@ -18,6 +18,7 @@ import { applyWorktreeSetupProgressEvent, buildWorktreeSetupDetail, createWorktreeSetupProgressAccumulator, + createWorktreeSetupProgressCoalescer, getWorktreeSetupProgressResults, } from "./worktree-bootstrap.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; @@ -165,6 +166,24 @@ interface HandleWorkspaceSetupStatusRequestDependencies { workspaceSetupSnapshots: ReadonlyMap; } +export const MAX_WORKSPACE_SETUP_SNAPSHOTS = 100; + +export function cacheWorkspaceSetupSnapshot( + snapshots: Map, + workspaceId: string, + snapshot: WorkspaceSetupSnapshot, +): void { + snapshots.delete(workspaceId); + snapshots.set(workspaceId, snapshot); + while (snapshots.size > MAX_WORKSPACE_SETUP_SNAPSHOTS) { + const oldestWorkspaceId = snapshots.keys().next().value; + if (oldestWorkspaceId === undefined) { + break; + } + snapshots.delete(oldestWorkspaceId); + } +} + interface HandleCreatePaseoWorktreeRequestDependencies { paseoHome?: string; worktreesRoot?: string; @@ -741,6 +760,15 @@ export async function runWorktreeSetupInBackground( }, }); }; + const runningProgress = createWorktreeSetupProgressCoalescer({ + emit: () => emitSetupProgress("running", null), + onError: (error) => { + dependencies.sessionLogger.warn( + { err: error, workspaceId }, + "Failed to emit worktree setup progress", + ); + }, + }); try { try { @@ -773,9 +801,10 @@ export async function runWorktreeSetupInBackground( runtimeEnv, onEvent: (event) => { applyWorktreeSetupProgressEvent(progressAccumulator, event); - emitSetupProgress("running", null); + runningProgress.schedule(); }, }); + await runningProgress.flush(); emitSetupProgress("completed", null); } } @@ -783,6 +812,7 @@ export async function runWorktreeSetupInBackground( if (error instanceof WorktreeSetupError) { setupResults = error.results; } + await runningProgress.flush(); const message = error instanceof Error ? error.message : String(error); emitSetupProgress("failed", message); diff --git a/packages/server/src/utils/worktree-setup-output.test.ts b/packages/server/src/utils/worktree-setup-output.test.ts new file mode 100644 index 0000000000..5f2a2a74a2 --- /dev/null +++ b/packages/server/src/utils/worktree-setup-output.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; + +import { + appendWorktreeSetupOutput, + createWorktreeSetupOutputAccumulator, + renderWorktreeSetupOutput, +} from "./worktree-setup-output.js"; + +describe("worktree setup output", () => { + it("retains both ends when rendering buffered output under a smaller total budget", () => { + const accumulator = createWorktreeSetupOutputAccumulator(128); + appendWorktreeSetupOutput(accumulator, `prefix-${"x".repeat(80)}-suffix`); + + const rendered = renderWorktreeSetupOutput(accumulator, 64); + + expect(rendered.truncated).toBe(true); + expect(rendered.text).toContain("prefix-"); + expect(rendered.text).toContain("-suffix"); + expect(rendered.text).toContain(`...<${rendered.omittedBytes} bytes omitted>...`); + expect(Buffer.byteLength(rendered.text)).toBeLessThanOrEqual(64); + }); +}); diff --git a/packages/server/src/utils/worktree-setup-output.ts b/packages/server/src/utils/worktree-setup-output.ts new file mode 100644 index 0000000000..19e8425599 --- /dev/null +++ b/packages/server/src/utils/worktree-setup-output.ts @@ -0,0 +1,118 @@ +export const MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES = 64 * 1024; +export const MAX_WORKTREE_SETUP_TOTAL_OUTPUT_BYTES = 256 * 1024; + +export interface WorktreeSetupOutputAccumulator { + readonly maxBytes: number; + totalBytes: number; + head: Buffer; + tail: Buffer; + truncated: boolean; +} + +export interface RenderedWorktreeSetupOutput { + text: string; + truncated: boolean; + omittedBytes: number; +} + +export function getWorktreeSetupCommandOutputLimit(totalCommands: number): number { + const commandCount = Math.max(1, Math.floor(totalCommands)); + return Math.min( + MAX_WORKTREE_SETUP_COMMAND_OUTPUT_BYTES, + Math.floor(MAX_WORKTREE_SETUP_TOTAL_OUTPUT_BYTES / commandCount), + ); +} + +export function createWorktreeSetupOutputAccumulator( + maxBytes: number, +): WorktreeSetupOutputAccumulator { + return { + maxBytes: Math.max(0, Math.floor(maxBytes)), + totalBytes: 0, + head: Buffer.alloc(0), + tail: Buffer.alloc(0), + truncated: false, + }; +} + +export function appendWorktreeSetupOutput( + accumulator: WorktreeSetupOutputAccumulator, + chunk: string, +): void { + if (!chunk) { + return; + } + + const chunkBytes = Buffer.from(chunk, "utf8"); + accumulator.totalBytes += chunkBytes.length; + + if (accumulator.maxBytes === 0) { + accumulator.truncated = true; + return; + } + + const headCapacity = Math.floor(accumulator.maxBytes / 2); + const tailCapacity = accumulator.maxBytes - headCapacity; + if (!accumulator.truncated) { + const combined = Buffer.concat([accumulator.head, chunkBytes]); + if (combined.length <= accumulator.maxBytes) { + accumulator.head = combined; + return; + } + accumulator.head = combined.subarray(0, headCapacity); + accumulator.tail = combined.subarray(combined.length - tailCapacity); + accumulator.truncated = true; + return; + } + + const combinedTail = Buffer.concat([accumulator.tail, chunkBytes]); + accumulator.tail = combinedTail.subarray(Math.max(0, combinedTail.length - tailCapacity)); +} + +export function renderWorktreeSetupOutput( + accumulator: WorktreeSetupOutputAccumulator, + maxBytes: number = accumulator.maxBytes, +): RenderedWorktreeSetupOutput { + const outputLimit = Math.max(0, Math.min(accumulator.maxBytes, Math.floor(maxBytes))); + if (!accumulator.truncated && accumulator.totalBytes <= outputLimit) { + return { + text: accumulator.head.toString("utf8"), + truncated: false, + omittedBytes: 0, + }; + } + + let retainedBytes = Math.min(accumulator.totalBytes, outputLimit); + let omittedBytes = accumulator.totalBytes - retainedBytes; + for (;;) { + const markerBytes = Buffer.byteLength(buildOmissionMarker(omittedBytes), "utf8"); + const nextRetainedBytes = Math.min( + accumulator.totalBytes, + Math.max(0, outputLimit - markerBytes), + ); + const nextOmittedBytes = accumulator.totalBytes - nextRetainedBytes; + if (nextRetainedBytes === retainedBytes && nextOmittedBytes === omittedBytes) { + break; + } + retainedBytes = nextRetainedBytes; + omittedBytes = nextOmittedBytes; + } + + const headBytes = Math.floor(retainedBytes / 2); + const tailBytes = retainedBytes - headBytes; + const head = accumulator.head.subarray(0, headBytes); + const tailSource = accumulator.truncated ? accumulator.tail : accumulator.head; + const tail = tailSource.subarray(Math.max(0, tailSource.length - tailBytes)); + omittedBytes = accumulator.totalBytes - head.length - tail.length; + + return { + text: `${head.toString("utf8")}${buildOmissionMarker(omittedBytes)}${tail.toString("utf8")}`, + truncated: true, + omittedBytes, + }; +} + +function buildOmissionMarker(omittedBytes: number): string { + const unit = omittedBytes === 1 ? "byte" : "bytes"; + return `\n...<${omittedBytes} ${unit} omitted>...\n`; +} diff --git a/packages/server/src/utils/worktree.posix.test.ts b/packages/server/src/utils/worktree.posix.test.ts index d895df5395..fb29290572 100644 --- a/packages/server/src/utils/worktree.posix.test.ts +++ b/packages/server/src/utils/worktree.posix.test.ts @@ -21,6 +21,7 @@ import { type CreateWorktreeOptions, type WorktreeConfig, } from "./worktree"; +import { MAX_WORKTREE_SETUP_TOTAL_OUTPUT_BYTES } from "./worktree-setup-output.js"; import type { PaseoConfig } from "@getpaseo/protocol/paseo-config-schema"; import { getPaseoWorktreeMetadataPath } from "./worktree-metadata.js"; import { execFileSync } from "child_process"; @@ -799,6 +800,37 @@ describe.skipIf(isPlatform("win32"))("worktree POSIX-only", () => { expect(progressEvents.some((event) => event.type === "command_completed")).toBe(true); }); + it("bounds retained output across noisy setup commands and reports exact omitted bytes", async () => { + const outputBytesPerStream = Buffer.byteLength("prefix--suffix") + 70_000; + const noisyCommand = + "node -e \"process.stdout.write('prefix-' + 'x'.repeat(70000) + '-suffix'); process.stderr.write('prefix-' + 'y'.repeat(70000) + '-suffix')\""; + writeFileSync( + join(repoDir, "paseo.json"), + JSON.stringify({ worktree: { setup: Array.from({ length: 5 }, () => noisyCommand) } }), + ); + + const results = await runWorktreeSetupCommands({ + worktreePath: repoDir, + branchName: "main", + cleanupOnFailure: false, + }); + + expect(results).toHaveLength(5); + let retainedOutputBytes = 0; + for (const result of results) { + for (const output of [result.stdout, result.stderr]) { + const marker = output.match(/\n\.\.\.<(\d+) bytes omitted>\.\.\.\n/); + expect(marker).not.toBeNull(); + const retained = output.replace(marker?.[0] ?? "", ""); + expect(Number(marker?.[1])).toBe(outputBytesPerStream - Buffer.byteLength(retained)); + expect(retained).toContain("prefix-"); + expect(retained).toContain("-suffix"); + retainedOutputBytes += Buffer.byteLength(output); + } + } + expect(retainedOutputBytes).toBeLessThanOrEqual(MAX_WORKTREE_SETUP_TOTAL_OUTPUT_BYTES); + }, 15_000); + it("reuses persisted worktree runtime port across resolutions", async () => { const result = await createLegacyWorktreeForTest({ branchName: "main", diff --git a/packages/server/src/utils/worktree.ts b/packages/server/src/utils/worktree.ts index 4ab641806b..278ef9c368 100644 --- a/packages/server/src/utils/worktree.ts +++ b/packages/server/src/utils/worktree.ts @@ -1,5 +1,3 @@ -import { execFile } from "child_process"; -import { promisify } from "util"; import { existsSync, mkdirSync, realpathSync, rmSync, statSync } from "fs"; import { copyFile, rm, stat } from "fs/promises"; import { join, basename, dirname, isAbsolute, resolve, sep } from "path"; @@ -36,10 +34,15 @@ import { createExternalProcessEnv } from "../server/paseo-env.js"; import { parseGitRevParsePath, resolveGitRevParsePath } from "./git-rev-parse-path.js"; import { validateBranchSlug } from "@getpaseo/protocol/branch-slug"; import { expandTilde, getRealpathAwareRelativePath, isPathInsideRoot } from "./path.js"; +import { + appendWorktreeSetupOutput, + createWorktreeSetupOutputAccumulator, + getWorktreeSetupCommandOutputLimit, + renderWorktreeSetupOutput, +} from "./worktree-setup-output.js"; export { slugify, validateBranchSlug } from "@getpaseo/protocol/branch-slug"; -const execFileAsync = promisify(execFile); const READ_ONLY_GIT_ENV = { GIT_OPTIONAL_LOCKS: "0", } as const; @@ -410,50 +413,19 @@ export function processCarriageReturns(text: string): string { return output.join(""); } -async function execSetupCommand( - command: string, - options: { cwd: string; env: NodeJS.ProcessEnv }, -): Promise { - const startedAt = Date.now(); - const shellInvocation = buildStringCommandShellInvocation({ command }); - try { - const { stdout, stderr } = await execFileAsync(shellInvocation.shell, shellInvocation.args, { - cwd: options.cwd, - env: options.env, - }); - return { - command, - cwd: options.cwd, - stdout: stdout ?? "", - stderr: stderr ?? "", - exitCode: 0, - durationMs: Date.now() - startedAt, - }; - } catch (error) { - const execErr = error as { stdout?: string; stderr?: string; code?: unknown } | undefined; - return { - command, - cwd: options.cwd, - stdout: execErr?.stdout ?? "", - stderr: execErr?.stderr ?? (error instanceof Error ? error.message : String(error)), - exitCode: typeof execErr?.code === "number" ? execErr.code : null, - durationMs: Date.now() - startedAt, - }; - } -} - async function execSetupCommandStreamed(options: { command: string; cwd: string; env: NodeJS.ProcessEnv; index: number; total: number; + maxOutputBytes: number; onEvent?: (event: WorktreeSetupCommandProgressEvent) => void; }): Promise { return new Promise((resolvePromise) => { const startedAt = Date.now(); - const stdoutChunks: string[] = []; - const stderrChunks: string[] = []; + const stdoutAccumulator = createWorktreeSetupOutputAccumulator(options.maxOutputBytes); + const stderrAccumulator = createWorktreeSetupOutputAccumulator(options.maxOutputBytes); let settled = false; const emitOutput = (stream: "stdout" | "stderr", chunk: string) => { @@ -462,9 +434,9 @@ async function execSetupCommandStreamed(options: { return; } if (stream === "stdout") { - stdoutChunks.push(text); + appendWorktreeSetupOutput(stdoutAccumulator, text); } else { - stderrChunks.push(text); + appendWorktreeSetupOutput(stderrAccumulator, text); } options.onEvent?.({ type: "output", @@ -482,11 +454,19 @@ async function execSetupCommandStreamed(options: { return; } settled = true; + const combinedBytes = stdoutAccumulator.totalBytes + stderrAccumulator.totalBytes; + const stdoutOutputBytes = + combinedBytes === 0 + ? 0 + : Math.floor((options.maxOutputBytes * stdoutAccumulator.totalBytes) / combinedBytes); + const stderrOutputBytes = options.maxOutputBytes - stdoutOutputBytes; + const stdout = renderWorktreeSetupOutput(stdoutAccumulator, stdoutOutputBytes).text; + const stderr = renderWorktreeSetupOutput(stderrAccumulator, stderrOutputBytes).text; const result: WorktreeSetupCommandResult = { command: options.command, cwd: options.cwd, - stdout: stdoutChunks.join(""), - stderr: stderrChunks.join(""), + stdout, + stderr, exitCode, durationMs: Date.now() - startedAt, }; @@ -638,20 +618,17 @@ export async function runWorktreeSetupCommands(options: { const setupEnv = createStringCommandShellEnv(createExternalProcessEnv(process.env, runtimeEnv)); const results: WorktreeSetupCommandResult[] = []; + const maxOutputBytes = getWorktreeSetupCommandOutputLimit(setupCommands.length); for (const [index, cmd] of setupCommands.entries()) { - const result = options.onEvent - ? await execSetupCommandStreamed({ - command: cmd, - cwd: options.worktreePath, - env: setupEnv, - index: index + 1, - total: setupCommands.length, - onEvent: options.onEvent, - }) - : await execSetupCommand(cmd, { - cwd: options.worktreePath, - env: setupEnv, - }); + const result = await execSetupCommandStreamed({ + command: cmd, + cwd: options.worktreePath, + env: setupEnv, + index: index + 1, + total: setupCommands.length, + maxOutputBytes, + onEvent: options.onEvent, + }); results.push(result); if (result.exitCode !== 0) { @@ -762,10 +739,15 @@ export async function runWorktreeTeardownCommands(options: { ); const results: WorktreeTeardownCommandResult[] = []; - for (const cmd of teardownCommands) { - const result = await execSetupCommand(cmd, { + const maxOutputBytes = getWorktreeSetupCommandOutputLimit(teardownCommands.length); + for (const [index, cmd] of teardownCommands.entries()) { + const result = await execSetupCommandStreamed({ + command: cmd, cwd: teardownCwd, env: teardownEnv, + index: index + 1, + total: teardownCommands.length, + maxOutputBytes, }); results.push(result); From d8d104fe32c1ea0ebd237c7a06f6f86058db8950 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 00:09:38 +0100 Subject: [PATCH 04/43] fix: make worktree cleanup incarnation-safe --- packages/protocol/src/messages.ts | 2 + .../src/server/agent/tools/paseo-tools.ts | 2 + .../server/workspace-archive-service.test.ts | 123 +++++++++- .../src/server/workspace-archive-service.ts | 228 ++++++++++++++---- .../server/workspace-lifecycle-coordinator.ts | 14 ++ .../server/src/server/workspace-registry.ts | 6 + .../src/server/worktree-session.test.ts | 7 +- .../server/src/server/worktree-session.ts | 165 +++++++------ .../server/src/server/worktree/commands.ts | 23 +- .../server/src/utils/worktree-metadata.ts | 25 ++ packages/server/src/utils/worktree.ts | 3 +- 11 files changed, 457 insertions(+), 141 deletions(-) diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index af44e09f90..cbfad65b97 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -4664,6 +4664,8 @@ export const PaseoWorktreeArchiveResponseSchema = z.object({ payload: z.object({ success: z.boolean(), removedAgents: z.array(z.string()).optional(), + removedDirectory: z.boolean().optional(), + cleanupPending: z.boolean().optional(), error: CheckoutErrorSchema.nullable(), requestId: z.string(), }), diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index 6c822d43d9..f6a6ba80f5 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -1357,6 +1357,7 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase workspaceId: z.string(), archivedAgentIds: z.array(z.string()), removedDirectory: z.boolean(), + cleanupPending: z.boolean(), }, }, async ({ workspaceId }) => { @@ -1385,6 +1386,7 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase workspaceId, archivedAgentIds: result.archivedAgentIds, removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPendingWorkspaceIds.length > 0, }), }; }, diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 1bc9d0eef8..5bf418babd 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -839,6 +839,108 @@ describe("archiveByScope", () => { expect((await registry.get(workspaceId))?.cleanupPending).toBeNull(); }); + test("a persisted cleanup retry cannot remove a newer worktree incarnation at the same path", async () => { + const { tempDir, repoDir } = createGitRepo(); + writeFileSync( + path.join(repoDir, "paseo.json"), + JSON.stringify({ worktree: { teardown: ["node -e \"process.exit(1)\""] } }), + ); + execFileSync("git", ["add", "."], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "fail teardown"], { + cwd: repoDir, + stdio: "pipe", + }); + const paseoHome = path.join(tempDir, ".paseo"); + const slug = "reused-incarnation"; + const firstWorktree = await createPaseoOwnedWorktree(repoDir, paseoHome, slug); + const workspaceId = "ws-reused-incarnation-a"; + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId, + projectId: "project-reused-incarnation", + cwd: firstWorktree.worktreePath, + kind: "worktree", + displayName: "Old incarnation", + worktreeRoot: firstWorktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId, + cwd: firstWorktree.worktreePath, + kind: "worktree", + worktreeRoot: firstWorktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + ], + }); + deps.workspaceRegistry = registry; + const archiveActiveRecord = deps.archiveWorkspaceRecord; + deps.archiveWorkspaceRecord = async (id: string) => { + await archiveActiveRecord(id); + await registry.archive(id, new Date().toISOString()); + }; + + const firstResult = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-reused-incarnation-first", + }); + expect(firstResult.cleanupPendingWorkspaceIds).toEqual([workspaceId]); + + execFileSync("git", ["worktree", "remove", firstWorktree.worktreePath, "--force"], { + cwd: repoDir, + stdio: "pipe", + }); + execFileSync("git", ["branch", "-D", slug], { cwd: repoDir, stdio: "pipe" }); + const replacementWorktree = await createPaseoOwnedWorktree(repoDir, paseoHome, slug); + + const retryResult = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-reused-incarnation-retry", + }); + expect(retryResult.removedDirectory).toBe(false); + expect(existsSync(replacementWorktree.worktreePath)).toBe(true); + expect((await registry.get(workspaceId))?.cleanupPending).toBeNull(); + }); + + test("workspace teardown failure keeps the record active and prevents recursive deletion", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "strict-teardown"); + const workspaceId = "ws-strict-teardown"; + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [{ workspaceId, cwd: worktree.worktreePath, kind: "worktree" }], + }); + deps.killTerminalsForWorkspace = vi.fn(async () => { + throw new Error("terminal still owned"); + }); + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + + const result = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-strict-teardown", + }); + + expect(result.archivedWorkspaceIds).toEqual([]); + expect(result.removedDirectory).toBe(false); + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + expect(existsSync(worktree.worktreePath)).toBe(true); + }); + test("workspace scope with unknown workspace id is a clean no-op", async () => { const { tempDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -982,6 +1084,11 @@ describe("archiveByScope", () => { const liveAgentId = "agent-live"; const targetStoredAgentId = "agent-stored-target"; const otherStoredAgentId = "agent-stored-other"; + const liveAgents = [{ id: liveAgentId, workspaceId: targetWorkspaceId }] as ManagedAgent[]; + const storedRecords = [ + { id: targetStoredAgentId, workspaceId: targetWorkspaceId, archivedAt: null }, + { id: otherStoredAgentId, workspaceId: otherWorkspaceId, archivedAt: null }, + ] as StoredAgentRecord[]; const deps = createArchiveDeps({ paseoHome, @@ -990,22 +1097,24 @@ describe("archiveByScope", () => { ], }); deps.agentManager = { - listAgents: () => [{ id: liveAgentId, workspaceId: targetWorkspaceId }] as ManagedAgent[], + listAgents: () => liveAgents, archiveAgent: vi.fn(async (agentId: string) => { deps.archivedAgentIds.push(agentId); + liveAgents.splice( + liveAgents.findIndex((agent) => agent.id === agentId), + 1, + ); return { archivedAt: new Date().toISOString() }; }), - archiveSnapshot: vi.fn(async (agentId: string, _archivedAt: string) => { + archiveSnapshot: vi.fn(async (agentId: string, archivedAt: string) => { deps.archivedSnapshotIds.push(agentId); + const record = storedRecords.find((candidate) => candidate.id === agentId); + if (record) record.archivedAt = archivedAt; return {}; }), }; deps.agentStorage = { - list: async () => - [ - { id: targetStoredAgentId, workspaceId: targetWorkspaceId, archivedAt: null }, - { id: otherStoredAgentId, workspaceId: otherWorkspaceId, archivedAt: null }, - ] as StoredAgentRecord[], + list: async () => storedRecords, } as Pick; const result = await archiveByScope(deps, { diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index cda672ee60..6aad67fdfc 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -1,5 +1,5 @@ import { existsSync } from "node:fs"; -import { resolve } from "node:path"; +import { dirname, resolve } from "node:path"; import type { Logger } from "pino"; @@ -9,10 +9,15 @@ import type { WorkspaceGitService } from "./workspace-git-service.js"; import type { ForgeService } from "../services/forge-service.js"; import { deletePaseoWorktree, + getPaseoWorktreesRoot, isPaseoOwnedWorktreeCwd, runWorktreeTeardownCommands, WorktreeTeardownError, } from "../utils/worktree.js"; +import { + ensurePaseoWorktreeIncarnationId, + readPaseoWorktreeIncarnationId, +} from "../utils/worktree-metadata.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import type { PersistedWorkspaceCleanupPending, @@ -71,6 +76,7 @@ export interface ArchiveResult { archivedAgentIds: string[]; archivedWorkspaceIds: string[]; removedDirectory: boolean; + cleanupPendingWorkspaceIds: string[]; } export interface ArchiveByScopeRequest { @@ -125,7 +131,6 @@ export async function archiveByScope( dependencies: ArchiveDependencies, request: ArchiveByScopeRequest, ): Promise { - const initialTarget = await resolveArchiveTarget(dependencies, request.scope); const lifecycleCoordinator = dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; const operationKey = @@ -134,13 +139,42 @@ export async function archiveByScope( : `worktree:${resolve(request.scope.targetPath)}`; return lifecycleCoordinator.runArchive(operationKey, async () => { - await lifecycleCoordinator.waitForWorkspaceSetups(initialTarget.workspaceIds); - const refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); - const target = mergeArchiveTargets(initialTarget, refreshedTarget); - return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); + const initialTarget = await resolveArchiveTarget(dependencies, request.scope); + const archiveOperation = async () => { + await lifecycleCoordinator.waitForWorkspaceSetups(initialTarget.workspaceIds); + let refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); + await lifecycleCoordinator.waitForWorkspaceSetups(refreshedTarget.workspaceIds); + refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); + const target = mergeArchiveTargets(initialTarget, refreshedTarget); + return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); + }; + const mutationRoot = + (await resolveWorktreeMutationRoot(dependencies, initialTarget.backing)) ?? + (request.scope.kind === "worktree" ? dirname(resolve(request.scope.targetPath)) : null); + return mutationRoot + ? lifecycleCoordinator.runWorktreeMutationExclusive(mutationRoot, archiveOperation) + : archiveOperation(); }); } +async function resolveWorktreeMutationRoot( + dependencies: Pick, + backing: BackingDirectory | null, +): Promise { + if (!backing?.isPaseoOwnedWorktree) return null; + if (backing.paseoWorktreesRoot) return resolve(backing.paseoWorktreesRoot); + if (backing.mainRepoRoot) { + return resolve( + await getPaseoWorktreesRoot( + backing.mainRepoRoot, + dependencies.paseoHome, + dependencies.paseoWorktreesBaseRoot, + ), + ); + } + return dirname(backing.path); +} + function mergeArchiveTargets(initial: ArchiveTarget, refreshed: ArchiveTarget): ArchiveTarget { if (refreshed.backing !== null || refreshed.workspaceIds.length > 0) { return refreshed; @@ -204,10 +238,16 @@ async function archiveResolvedTarget( ); } + const cleanupPendingWorkspaceIds = await getCleanupPendingWorkspaceIds( + dependencies, + target, + ); + return { archivedAgentIds: Array.from(archivedAgents), archivedWorkspaceIds, removedDirectory, + cleanupPendingWorkspaceIds, }; } finally { if (targetWorkspaceIds.length > 0) { @@ -439,6 +479,31 @@ async function maybeRemoveDirectoryExclusive( return false; } + const initialIncarnationState = await compareCleanupIncarnation( + dependencies, + backing, + pendingCleanupTargets, + ); + if (initialIncarnationState !== "match") { + if (initialIncarnationState === "mismatch") { + await clearPendingCleanup(dependencies, pendingCleanupTargets); + } + return false; + } + + const activeBeforeTeardown = await dependencies.listActiveWorkspaces(); + if ( + !(await isDirectoryUnreferenced( + activeBeforeTeardown, + backing.path, + new Set(archivedWorkspaceIds), + dependencies, + )) + ) { + await clearPendingCleanup(dependencies, pendingCleanupTargets); + return false; + } + const teardownCwds = uniqueFilesystemPaths( pendingCleanupTargets.map((pendingCleanupTarget) => pendingCleanupTarget.teardownCwd), ); @@ -462,6 +527,18 @@ async function maybeRemoveDirectoryExclusive( throw error; } + const finalIncarnationState = await compareCleanupIncarnation( + dependencies, + backing, + pendingCleanupTargets, + ); + if (finalIncarnationState !== "match") { + if (finalIncarnationState === "mismatch") { + await clearPendingCleanup(dependencies, pendingCleanupTargets); + } + return false; + } + const remainingActive = await dependencies.listActiveWorkspaces(); if ( !(await isDirectoryUnreferenced( @@ -499,6 +576,40 @@ async function maybeRemoveDirectoryExclusive( } } +async function compareCleanupIncarnation( + dependencies: Pick, + backing: BackingDirectory, + pendingCleanupTargets: PendingCleanupTarget[], +): Promise<"match" | "mismatch" | "unverifiable"> { + const expectedIncarnations = new Set( + pendingCleanupTargets + .map((target) => target.worktreeIncarnationId) + .filter((value): value is string => value !== null), + ); + if ( + expectedIncarnations.size !== 1 || + pendingCleanupTargets.some((target) => !target.worktreeIncarnationId) + ) { + dependencies.sessionLogger?.warn( + { targetPath: backing.path }, + "Refusing path-only or conflicting persisted worktree cleanup", + ); + return "unverifiable"; + } + let currentIncarnationId: string | null = null; + try { + currentIncarnationId = readPaseoWorktreeIncarnationId(backing.path); + } catch (error) { + dependencies.sessionLogger?.warn( + { err: error, targetPath: backing.path }, + "Could not verify persisted worktree cleanup incarnation", + ); + return "unverifiable"; + } + if (currentIncarnationId === null) return "unverifiable"; + return expectedIncarnations.has(currentIncarnationId) ? "match" : "mismatch"; +} + interface PendingCleanupTarget extends PersistedWorkspaceCleanupPending { workspaceId: string | null; } @@ -511,6 +622,10 @@ async function persistTargetCleanupPending( const workspaceRegistry = dependencies.workspaceRegistry; if (!backing?.isPaseoOwnedWorktree || !workspaceRegistry) return; + const worktreeIncarnationId = existsSync(backing.path) + ? ensurePaseoWorktreeIncarnationId(backing.path) + : null; + await Promise.all( target.teardownTargets.flatMap((teardownTarget) => teardownTarget.workspaceId @@ -522,6 +637,7 @@ async function persistTargetCleanupPending( teardownCwd: teardownTarget.cwd, mainRepoRoot: backing.mainRepoRoot, paseoWorktreesRoot: backing.paseoWorktreesRoot, + worktreeIncarnationId, }, })), ] @@ -554,6 +670,14 @@ async function listPendingCleanupTargets( } const archivedWorkspaceIdSet = new Set(archivedWorkspaceIds); + let fallbackIncarnationId: string | null = null; + if (target.backing && existsSync(target.backing.path)) { + try { + fallbackIncarnationId = readPaseoWorktreeIncarnationId(target.backing.path); + } catch { + fallbackIncarnationId = null; + } + } return target.teardownTargets .filter( (teardownTarget) => @@ -566,6 +690,7 @@ async function listPendingCleanupTargets( teardownCwd: teardownTarget.cwd, mainRepoRoot: target.backing?.mainRepoRoot ?? null, paseoWorktreesRoot: target.backing?.paseoWorktreesRoot ?? null, + worktreeIncarnationId: fallbackIncarnationId, })); } @@ -609,6 +734,7 @@ async function clearCleanupPendingForUnarchivedTargets( teardownCwd: teardownTarget.cwd, mainRepoRoot: target.backing?.mainRepoRoot ?? null, paseoWorktreesRoot: target.backing?.paseoWorktreesRoot ?? null, + worktreeIncarnationId: null, }, ] : [], @@ -648,15 +774,7 @@ export async function archiveWorkspaceContents( archivedAgents.add(agent.id); } - let storedRecords: StoredAgentRecord[] = []; - try { - storedRecords = await dependencies.agentStorage.list(); - } catch (error) { - dependencies.sessionLogger?.warn( - { err: error, workspaceId }, - "Failed to list stored agents during workspace archive; continuing", - ); - } + const storedRecords: StoredAgentRecord[] = await dependencies.agentStorage.list(); const liveAgentIds = new Set(liveAgents.map((agent) => agent.id)); const matchingStoredRecords = storedRecords.filter( (record) => record.workspaceId === workspaceId, @@ -666,7 +784,7 @@ export async function archiveWorkspaceContents( } const archivedAt = new Date().toISOString(); - const archiveResults = await Promise.allSettled([ + await Promise.all([ ...liveAgents.map((agent) => dependencies.agentManager.archiveAgent(agent.id)), ...matchingStoredRecords .filter((record) => !liveAgentIds.has(record.id) && !record.archivedAt) @@ -674,13 +792,14 @@ export async function archiveWorkspaceContents( dependencies.killTerminalsForWorkspace(workspaceId), ]); - for (const result of archiveResults) { - if (result.status === "rejected") { - dependencies.sessionLogger?.warn( - { err: result.reason, workspaceId }, - "Workspace archive teardown step failed; continuing", - ); - } + const remainingLiveAgents = dependencies.agentManager + .listAgents() + .filter((agent) => agent.workspaceId === workspaceId); + const remainingStoredAgents = (await dependencies.agentStorage.list()).filter( + (record) => record.workspaceId === workspaceId && !record.archivedAt, + ); + if (remainingLiveAgents.length > 0 || remainingStoredAgents.length > 0) { + throw new Error(`Workspace ownership remains after archive: ${workspaceId}`); } return archivedAgents; @@ -714,20 +833,16 @@ export async function killTerminalsForWorkspace( return; } + const listWorkspaceTerminals = async () => + ( + await Promise.all( + terminalManager + .listDirectories() + .map((terminalCwd) => terminalManager.getTerminals(terminalCwd, { workspaceId })), + ) + ).flat().filter((terminal) => terminal.workspaceId === workspaceId); const terminalIds: string[] = []; - const terminalLists = await Promise.all( - terminalManager.listDirectories().map(async (terminalCwd) => { - try { - return await terminalManager.getTerminals(terminalCwd, { workspaceId }); - } catch (error) { - dependencies.sessionLogger.warn( - { err: error, cwd: terminalCwd }, - "Failed to enumerate workspace terminals during archive", - ); - return []; - } - }), - ); + const terminalLists = [await listWorkspaceTerminals()]; for (const terminals of terminalLists) { for (const terminal of terminals) { if (terminal.workspaceId === workspaceId) { @@ -736,26 +851,33 @@ export async function killTerminalsForWorkspace( } } - if (terminalIds.length === 0) { - return; - } - - await Promise.allSettled( + await Promise.all( terminalIds.map(async (terminalId) => { - try { - dependencies.detachTerminalStream?.(terminalId, { emitExit: true }); - await terminalManager.killTerminalAndWait(terminalId, { - gracefulTimeoutMs: 2000, - forceTimeoutMs: 1500, - }); - } catch (error) { - dependencies.sessionLogger.warn( - { err: error, terminalId }, - "Terminal kill escalation failed during archive; proceeding anyway", - ); - } + dependencies.detachTerminalStream?.(terminalId, { emitExit: true }); + await terminalManager.killTerminalAndWait(terminalId, { + gracefulTimeoutMs: 2000, + forceTimeoutMs: 1500, + }); }), ); + if ((await listWorkspaceTerminals()).length > 0) { + throw new Error(`Workspace terminals remain after archive: ${workspaceId}`); + } +} + +async function getCleanupPendingWorkspaceIds( + dependencies: Pick, + target: ArchiveTarget, +): Promise { + if (!dependencies.workspaceRegistry || !target.backing) return []; + const matchesBacking = createRealpathAwarePathMatcher(target.backing.path); + return (await dependencies.workspaceRegistry.list()) + .filter( + (workspace) => + workspace.cleanupPending !== null && + matchesBacking(workspace.cleanupPending.directoryPath), + ) + .map((workspace) => workspace.workspaceId); } // Archiving the last workspace of a project leaves the project record active. diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts index 0632011726..58ff2897a1 100644 --- a/packages/server/src/server/workspace-lifecycle-coordinator.ts +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -3,6 +3,7 @@ export class WorkspaceLifecycleCoordinator { private readonly setupTasksByDirectory = new Map>>(); private readonly archiveOperations = new Map>(); private readonly directoryOperations = new Map>(); + private readonly worktreeMutationOperations = new Map>(); trackWorkspaceSetup(workspaceId: string, task: Promise, directoryPath?: string): void { this.setupTasks.set(workspaceId, task); @@ -93,6 +94,19 @@ export class WorkspaceLifecycleCoordinator { void task.then(clearTask, clearTask); return task; } + + runWorktreeMutationExclusive(worktreesRoot: string, operation: () => Promise): Promise { + const previous = this.worktreeMutationOperations.get(worktreesRoot) ?? Promise.resolve(); + const task = previous.catch(() => undefined).then(operation); + this.worktreeMutationOperations.set(worktreesRoot, task); + const clearTask = () => { + if (this.worktreeMutationOperations.get(worktreesRoot) === task) { + this.worktreeMutationOperations.delete(worktreesRoot); + } + }; + void task.then(clearTask, clearTask); + return task; + } } export interface WorkspaceSetupReservation { diff --git a/packages/server/src/server/workspace-registry.ts b/packages/server/src/server/workspace-registry.ts index 6e2d707606..fd2ed92c51 100644 --- a/packages/server/src/server/workspace-registry.ts +++ b/packages/server/src/server/workspace-registry.ts @@ -39,6 +39,12 @@ const PersistedWorkspaceCleanupPendingSchema = z.object({ teardownCwd: z.string(), mainRepoRoot: z.string().nullable(), paseoWorktreesRoot: z.string().nullable(), + // COMPAT(cleanupIncarnation): legacy path-only cleanup records fail closed. + worktreeIncarnationId: z + .string() + .nullable() + .optional() + .transform((value) => value ?? null), }); const PersistedWorkspaceRecordSchema = z.object({ diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index 0f467a0b69..7c4bf294e0 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -463,6 +463,10 @@ describe("create worktree setup boundary", () => { const paseoHome = path.join(tempDir, ".paseo"); const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); const trackWorkspaceSetup = vi.spyOn(lifecycleCoordinator, "trackWorkspaceSetup"); + const runWorktreeMutationExclusive = vi.spyOn( + lifecycleCoordinator, + "runWorktreeMutationExclusive", + ); try { const result = await createPaseoWorktreeWorkflow( @@ -497,6 +501,7 @@ describe("create worktree setup boundary", () => { expect.any(Promise), result.worktree.worktreePath, ); + expect(runWorktreeMutationExclusive).toHaveBeenCalledOnce(); await lifecycleCoordinator.waitForWorkspaceSetups([result.workspace.workspaceId]); } finally { rmSync(tempDir, { recursive: true, force: true }); @@ -2252,8 +2257,6 @@ describe("handlePaseoWorktreeArchiveRequest worktree scope", () => { await handlePaseoWorktreeArchiveRequest(deps, { type: "paseo_worktree_archive_request", requestId: "req-delete-flag-second", - worktreePath: sharedCwd, - repoRoot: repoDir, workspaceId: workspaceB, scope: "workspace", deleteWorktreeFromDisk: true, diff --git a/packages/server/src/server/worktree-session.ts b/packages/server/src/server/worktree-session.ts index 5ac06704d3..f968929d7b 100644 --- a/packages/server/src/server/worktree-session.ts +++ b/packages/server/src/server/worktree-session.ts @@ -1,5 +1,5 @@ import type { Logger } from "pino"; -import { basename } from "node:path"; +import { basename, resolve } from "node:path"; import type { AgentSessionConfig } from "./agent/agent-sdk-types.js"; import { @@ -28,6 +28,7 @@ import type { CheckoutExistingBranchResult } from "../utils/checkout-git.js"; import { expandTilde } from "../utils/path.js"; import { getWorktreeSetupCommands, + getPaseoWorktreesRoot, resolveWorktreeRuntimeEnv, runWorktreeSetupCommands, slugify, @@ -512,6 +513,8 @@ export async function handlePaseoWorktreeArchiveRequest( payload: { success: true, removedAgents: result.removedAgents, + removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPending, error: null, requestId, }, @@ -615,91 +618,99 @@ export async function createPaseoWorktreeWorkflow( setupContinuation?: CreatePaseoWorktreeSetupContinuationInput; }, ): Promise { - const createdWorktree = await dependencies.createPaseoWorktree( - { - ...input, - runSetup: false, - paseoHome: input.paseoHome ?? dependencies.paseoHome, - worktreesRoot: input.worktreesRoot ?? dependencies.worktreesRoot, - }, - options?.resolveDefaultBranch - ? { resolveDefaultBranch: options.resolveDefaultBranch } - : undefined, - ); - const slug = basename(createdWorktree.worktree.worktreePath); - const workspace = createdWorktree.workspace; - const setupContinuation = options?.setupContinuation ?? { kind: "workspace" }; const lifecycleCoordinator = dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const worktreesRoot = await getPaseoWorktreesRoot( + input.cwd, + input.paseoHome ?? dependencies.paseoHome, + input.worktreesRoot ?? dependencies.worktreesRoot, + ); - setTimeout(() => { - if (input.firstAgentContext) { - dependencies.autoNameWorkspaceBranchForFirstAgent({ - workspace, - firstAgentContext: input.firstAgentContext, - }); - } - void dependencies.warmWorkspaceGitData(workspace).catch((error) => { - dependencies.sessionLogger.warn( - { err: error, workspaceId: workspace.workspaceId }, - "Failed to warm workspace git data after creating worktree", - ); - }); - }, 0); - - if (setupContinuation.kind === "workspace") { - const setupTask = new Promise((resolveSetup) => { - setTimeout(() => { - void runWorktreeSetupInBackground(dependencies, { - requestCwd: input.cwd, - repoRoot: createdWorktree.repoRoot, - workspaceId: workspace.workspaceId, - worktree: createdWorktree.worktree, - shouldBootstrap: createdWorktree.created, - slug, - worktreePath: createdWorktree.worktree.worktreePath, - workspaceCwd: workspace.cwd, - }).then(resolveSetup, resolveSetup); - }, 0); - }); - lifecycleCoordinator.trackWorkspaceSetup( - workspace.workspaceId, - setupTask, - createdWorktree.worktree.worktreePath, - ); - } - - if (setupContinuation.kind === "agent") { - const reservation = lifecycleCoordinator.reserveWorkspaceSetup( - workspace.workspaceId, - createdWorktree.worktree.worktreePath, + return lifecycleCoordinator.runWorktreeMutationExclusive(resolve(worktreesRoot), async () => { + const createdWorktree = await dependencies.createPaseoWorktree( + { + ...input, + runSetup: false, + paseoHome: input.paseoHome ?? dependencies.paseoHome, + worktreesRoot: input.worktreesRoot ?? dependencies.worktreesRoot, + }, + options?.resolveDefaultBranch + ? { resolveDefaultBranch: options.resolveDefaultBranch } + : undefined, ); - return { - ...createdWorktree, - setupContinuation: { - kind: "agent", - startAfterAgentCreate: ({ agentId }) => { - const setupTask = runAsyncWorktreeBootstrap({ - agentId, + const slug = basename(createdWorktree.worktree.worktreePath); + const workspace = createdWorktree.workspace; + const setupContinuation = options?.setupContinuation ?? { kind: "workspace" }; + + setTimeout(() => { + if (input.firstAgentContext) { + dependencies.autoNameWorkspaceBranchForFirstAgent({ + workspace, + firstAgentContext: input.firstAgentContext, + }); + } + void dependencies.warmWorkspaceGitData(workspace).catch((error) => { + dependencies.sessionLogger.warn( + { err: error, workspaceId: workspace.workspaceId }, + "Failed to warm workspace git data after creating worktree", + ); + }); + }, 0); + + if (setupContinuation.kind === "workspace") { + const setupTask = new Promise((resolveSetup) => { + setTimeout(() => { + void runWorktreeSetupInBackground(dependencies, { + requestCwd: input.cwd, + repoRoot: createdWorktree.repoRoot, workspaceId: workspace.workspaceId, worktree: createdWorktree.worktree, - workspaceCwd: workspace.cwd, shouldBootstrap: createdWorktree.created, - terminalManager: setupContinuation.terminalManager, - appendTimelineItem: (item) => setupContinuation.appendTimelineItem({ agentId, item }), - emitLiveTimelineItem: (item) => - setupContinuation.emitLiveTimelineItem({ agentId, item }), - logger: setupContinuation.logger, - }); - reservation.completeWith(setupTask); - void setupTask; + slug, + worktreePath: createdWorktree.worktree.worktreePath, + workspaceCwd: workspace.cwd, + }).then(resolveSetup, resolveSetup); + }, 0); + }); + lifecycleCoordinator.trackWorkspaceSetup( + workspace.workspaceId, + setupTask, + createdWorktree.worktree.worktreePath, + ); + } + + if (setupContinuation.kind === "agent") { + const reservation = lifecycleCoordinator.reserveWorkspaceSetup( + workspace.workspaceId, + createdWorktree.worktree.worktreePath, + ); + return { + ...createdWorktree, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: ({ agentId }) => { + const setupTask = runAsyncWorktreeBootstrap({ + agentId, + workspaceId: workspace.workspaceId, + worktree: createdWorktree.worktree, + workspaceCwd: workspace.cwd, + shouldBootstrap: createdWorktree.created, + terminalManager: setupContinuation.terminalManager, + appendTimelineItem: (item) => setupContinuation.appendTimelineItem({ agentId, item }), + emitLiveTimelineItem: (item) => + setupContinuation.emitLiveTimelineItem({ agentId, item }), + logger: setupContinuation.logger, + }); + reservation.completeWith(setupTask); + void setupTask; + }, + releaseWithoutStarting: reservation.release, }, - releaseWithoutStarting: reservation.release, - }, - }; - } + }; + } - return createdWorktree; + return createdWorktree; + }); } export async function handleWorkspaceSetupStatusRequest( diff --git a/packages/server/src/server/worktree/commands.ts b/packages/server/src/server/worktree/commands.ts index cedc5013c9..edd6b55966 100644 --- a/packages/server/src/server/worktree/commands.ts +++ b/packages/server/src/server/worktree/commands.ts @@ -110,6 +110,8 @@ export type ArchiveCommandResult = | { ok: true; removedAgents: string[]; + removedDirectory: boolean; + cleanupPending: boolean; } | { ok: false; @@ -122,8 +124,21 @@ export async function archiveCommand( dependencies: ArchiveCommandDependencies, input: ArchiveCommandInput, ): Promise { - const targetPath = await resolveArchiveTarget(dependencies, input); const scope = input.scope ?? "workspace"; + if (scope === "workspace" && input.workspaceId) { + const result = await archiveByScope(dependencies, { + scope: { kind: "workspace", workspaceId: input.workspaceId }, + requestId: input.requestId, + }); + return { + ok: true, + removedAgents: result.archivedAgentIds, + removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPendingWorkspaceIds.length > 0, + }; + } + + const targetPath = await resolveArchiveTarget(dependencies, input); const ownership = await isPaseoOwnedWorktreeCwd(targetPath, { paseoHome: dependencies.paseoHome, worktreesRoot: dependencies.paseoWorktreesBaseRoot, @@ -147,6 +162,8 @@ export async function archiveCommand( return { ok: true, removedAgents: result.archivedAgentIds, + removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPendingWorkspaceIds.length > 0, }; } @@ -161,6 +178,8 @@ export async function archiveCommand( return { ok: true, removedAgents: [], + removedDirectory: false, + cleanupPending: false, }; } @@ -172,6 +191,8 @@ export async function archiveCommand( return { ok: true, removedAgents: result.archivedAgentIds, + removedDirectory: result.removedDirectory, + cleanupPending: result.cleanupPendingWorkspaceIds.length > 0, }; } diff --git a/packages/server/src/utils/worktree-metadata.ts b/packages/server/src/utils/worktree-metadata.ts index 4300c4370e..732635e15a 100644 --- a/packages/server/src/utils/worktree-metadata.ts +++ b/packages/server/src/utils/worktree-metadata.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "fs"; import { isAbsolute, join, resolve } from "path"; import { z } from "zod"; @@ -11,12 +12,14 @@ const ChangeRequestLookupTargetSchema = z.object({ const PaseoWorktreeMetadataV1Schema = z.object({ version: z.literal(1), baseRefName: z.string().min(1), + incarnationId: z.string().uuid().optional(), changeRequestLookupTarget: ChangeRequestLookupTargetSchema.optional(), }); const PaseoWorktreeMetadataV2Schema = z.object({ version: z.literal(2), baseRefName: z.string().min(1), + incarnationId: z.string().uuid().optional(), changeRequestLookupTarget: ChangeRequestLookupTargetSchema.optional(), firstAgentBranchAutoName: z .discriminatedUnion("status", [ @@ -91,6 +94,7 @@ export function writePaseoWorktreeMetadata( options: { baseRefName: string; changeRequestLookupTarget?: PaseoWorktreeChangeRequestLookupTarget; + incarnationId?: string; }, ): void { const baseRefName = normalizeBaseRefName(options.baseRefName); @@ -107,6 +111,7 @@ export function writePaseoWorktreeMetadata( const metadata: PaseoWorktreeMetadata = { version: 1, baseRefName, + ...(options.incarnationId ? { incarnationId: options.incarnationId } : {}), ...(options.changeRequestLookupTarget ? { changeRequestLookupTarget: options.changeRequestLookupTarget } : {}), @@ -130,6 +135,7 @@ export function writePaseoWorktreeRuntimeMetadata( const next: PaseoWorktreeMetadata = { version: 2, baseRefName: current.baseRefName, + ...(current.incarnationId ? { incarnationId: current.incarnationId } : {}), ...(current.changeRequestLookupTarget ? { changeRequestLookupTarget: current.changeRequestLookupTarget } : {}), @@ -160,6 +166,7 @@ export function writePaseoWorktreeFirstAgentBranchAutoNameMetadata( writePaseoWorktreeMetadataFile(worktreeRoot, { version: 2, baseRefName: current.baseRefName, + ...(current.incarnationId ? { incarnationId: current.incarnationId } : {}), ...(current.changeRequestLookupTarget ? { changeRequestLookupTarget: current.changeRequestLookupTarget } : {}), @@ -183,6 +190,7 @@ export function markPaseoWorktreeFirstAgentBranchAutoNameAttempted( const next: PaseoWorktreeMetadata = { version: 2, baseRefName: current.baseRefName, + ...(current.incarnationId ? { incarnationId: current.incarnationId } : {}), ...(current.changeRequestLookupTarget ? { changeRequestLookupTarget: current.changeRequestLookupTarget } : {}), @@ -206,6 +214,23 @@ export function readPaseoWorktreeMetadata(worktreeRoot: string): PaseoWorktreeMe return PaseoWorktreeMetadataSchema.parse(parsed); } +export function readPaseoWorktreeIncarnationId(worktreeRoot: string): string | null { + return readPaseoWorktreeMetadata(worktreeRoot)?.incarnationId ?? null; +} + +export function ensurePaseoWorktreeIncarnationId(worktreeRoot: string): string { + const current = readPaseoWorktreeMetadata(worktreeRoot); + if (!current) { + throw new Error("Cannot persist worktree incarnation: missing base metadata"); + } + if (current.incarnationId) { + return current.incarnationId; + } + const incarnationId = randomUUID(); + writePaseoWorktreeMetadataFile(worktreeRoot, { ...current, incarnationId }); + return incarnationId; +} + export function requirePaseoWorktreeBaseRefName(worktreeRoot: string): string { const metadataPath = getPaseoWorktreeMetadataPath(worktreeRoot); const metadata = readPaseoWorktreeMetadata(worktreeRoot); diff --git a/packages/server/src/utils/worktree.ts b/packages/server/src/utils/worktree.ts index 278ef9c368..67c3c0a3de 100644 --- a/packages/server/src/utils/worktree.ts +++ b/packages/server/src/utils/worktree.ts @@ -2,7 +2,7 @@ import { existsSync, mkdirSync, realpathSync, rmSync, statSync } from "fs"; import { copyFile, rm, stat } from "fs/promises"; import { join, basename, dirname, isAbsolute, resolve, sep } from "path"; import net from "node:net"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import stripAnsi from "strip-ansi"; import { buildStringCommandShellInvocation, @@ -1252,6 +1252,7 @@ export const createWorktree = async ({ writePaseoWorktreeMetadata(worktreePath, { baseRefName: sourcePlan.metadataBaseRefName, + incarnationId: randomUUID(), ...(sourcePlan.changeRequestLookupTarget ? { changeRequestLookupTarget: sourcePlan.changeRequestLookupTarget } : {}), From 23492e25d15334b151764d7c5c697416c0b6a091 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 00:18:54 +0100 Subject: [PATCH 05/43] fix: preserve cleanup incarnation on retry --- packages/server/src/server/session.workspaces.test.ts | 8 ++++++-- .../server/src/server/workspace-archive-service.test.ts | 4 ++-- packages/server/src/server/workspace-archive-service.ts | 6 +++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index d4ad5d1c88..a1194a8297 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -5526,7 +5526,7 @@ test("archive_workspace_request archives a worktree-kind workspace and removes t const workspaceId = "ws-worktree-kind-archive"; const projectId = "proj-worktree-kind-archive"; - const workspace = createPersistedWorkspaceRecord({ + let workspace = createPersistedWorkspaceRecord({ workspaceId, projectId, cwd: worktree.worktreePath, @@ -5578,8 +5578,12 @@ test("archive_workspace_request archives a worktree-kind workspace and removes t }; session.workspaceRegistry.get = async () => workspace; session.workspaceRegistry.list = async () => [workspace]; + session.workspaceRegistry.update = async (_id, updater) => { + workspace = updater(workspace); + return workspace; + }; session.workspaceRegistry.archive = async (_id: string, archivedAt: string) => { - workspace.archivedAt = archivedAt; + workspace = { ...workspace, archivedAt, updatedAt: archivedAt }; }; session.projectRegistry.list = async () => [project]; diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 5bf418babd..068dce3a8a 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -399,7 +399,7 @@ describe("archiveByScope", () => { expect(existsSync(worktree.worktreePath)).toBe(false); }); - test("workspace scope runs teardown while keeping a directory referenced by a sibling", async () => { + test("workspace scope skips teardown while a sibling still references the directory", async () => { const { tempDir, repoDir } = createGitRepo(); writeFileSync( path.join(repoDir, "paseo.json"), @@ -440,7 +440,7 @@ describe("archiveByScope", () => { removedDirectory: false, }); expect(existsSync(worktree.worktreePath)).toBe(true); - expect(readFileSync(path.join(repoDir, "shared-teardown.log"), "utf8")).toBe("ok"); + expect(existsSync(path.join(repoDir, "shared-teardown.log"))).toBe(false); }); test("workspace scope keeps a worktree for an active workspace in a subdirectory", async () => { diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 6aad67fdfc..0aeac199e7 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -621,6 +621,10 @@ async function persistTargetCleanupPending( const backing = target.backing; const workspaceRegistry = dependencies.workspaceRegistry; if (!backing?.isPaseoOwnedWorktree || !workspaceRegistry) return; + // A target reconstructed from an archived record's cleanupPending state has + // no active workspace ids. Its incarnation is the authority for this retry: + // never rebind that stale cleanup intent to whatever now occupies the path. + const activeTargetWorkspaceIds = new Set(target.workspaceIds); const worktreeIncarnationId = existsSync(backing.path) ? ensurePaseoWorktreeIncarnationId(backing.path) @@ -628,7 +632,7 @@ async function persistTargetCleanupPending( await Promise.all( target.teardownTargets.flatMap((teardownTarget) => - teardownTarget.workspaceId + teardownTarget.workspaceId && activeTargetWorkspaceIds.has(teardownTarget.workspaceId) ? [ workspaceRegistry.update(teardownTarget.workspaceId, (workspace) => ({ ...workspace, From 8f96a87ed75f8bc3dc3dd8dd38a211223bb69168 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 00:23:58 +0100 Subject: [PATCH 06/43] test: close worktree lifecycle gates --- .../server/workspace-archive-service.test.ts | 22 ++++++++++++++----- .../src/server/workspace-archive-service.ts | 12 +++++----- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 068dce3a8a..14177c0605 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -100,6 +100,19 @@ function createGitRepo(): { tempDir: string; repoDir: string } { }); return { tempDir, repoDir }; } +function removeManagedAgentById(agents: ManagedAgent[], agentId: string): void { + const index = agents.findIndex((agent) => agent.id === agentId); + if (index !== -1) { + agents.splice(index, 1); + } +} + +function findStoredAgentById( + records: StoredAgentRecord[], + agentId: string, +): StoredAgentRecord | undefined { + return records.find((record) => record.id === agentId); +} async function createPaseoOwnedWorktree( repoDir: string, @@ -843,7 +856,7 @@ describe("archiveByScope", () => { const { tempDir, repoDir } = createGitRepo(); writeFileSync( path.join(repoDir, "paseo.json"), - JSON.stringify({ worktree: { teardown: ["node -e \"process.exit(1)\""] } }), + JSON.stringify({ worktree: { teardown: ['node -e "process.exit(1)"'] } }), ); execFileSync("git", ["add", "."], { cwd: repoDir, stdio: "pipe" }); execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "fail teardown"], { @@ -1100,15 +1113,12 @@ describe("archiveByScope", () => { listAgents: () => liveAgents, archiveAgent: vi.fn(async (agentId: string) => { deps.archivedAgentIds.push(agentId); - liveAgents.splice( - liveAgents.findIndex((agent) => agent.id === agentId), - 1, - ); + removeManagedAgentById(liveAgents, agentId); return { archivedAt: new Date().toISOString() }; }), archiveSnapshot: vi.fn(async (agentId: string, archivedAt: string) => { deps.archivedSnapshotIds.push(agentId); - const record = storedRecords.find((candidate) => candidate.id === agentId); + const record = findStoredAgentById(storedRecords, agentId); if (record) record.archivedAt = archivedAt; return {}; }), diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 0aeac199e7..020db793fa 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -238,10 +238,7 @@ async function archiveResolvedTarget( ); } - const cleanupPendingWorkspaceIds = await getCleanupPendingWorkspaceIds( - dependencies, - target, - ); + const cleanupPendingWorkspaceIds = await getCleanupPendingWorkspaceIds(dependencies, target); return { archivedAgentIds: Array.from(archivedAgents), @@ -844,7 +841,9 @@ export async function killTerminalsForWorkspace( .listDirectories() .map((terminalCwd) => terminalManager.getTerminals(terminalCwd, { workspaceId })), ) - ).flat().filter((terminal) => terminal.workspaceId === workspaceId); + ) + .flat() + .filter((terminal) => terminal.workspaceId === workspaceId); const terminalIds: string[] = []; const terminalLists = [await listWorkspaceTerminals()]; for (const terminals of terminalLists) { @@ -878,8 +877,7 @@ async function getCleanupPendingWorkspaceIds( return (await dependencies.workspaceRegistry.list()) .filter( (workspace) => - workspace.cleanupPending !== null && - matchesBacking(workspace.cleanupPending.directoryPath), + workspace.cleanupPending !== null && matchesBacking(workspace.cleanupPending.directoryPath), ) .map((workspace) => workspace.workspaceId); } From 1e7a87d7a54fe75a186fe751c7ab83b43c3ca5b8 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 20:26:59 -0400 Subject: [PATCH 07/43] fix: make workspace lifecycle failures durable --- .../create-agent-lifecycle-dispatch.test.ts | 78 ++++++++++- .../agent/create-agent-lifecycle-dispatch.ts | 66 ++++++--- .../server/agent/create-agent/create.test.ts | 124 +++++++++++++++++ .../src/server/agent/create-agent/create.ts | 117 +++++++++------- .../src/server/agent/mcp-server.test.ts | 52 +++++++ .../src/server/agent/tools/paseo-tools.ts | 32 ++++- packages/server/src/server/session.ts | 12 +- .../src/server/session.workspaces.test.ts | 73 ++++++++++ .../server/workspace-archive-service.test.ts | 129 ++++++++++++++---- .../src/server/workspace-archive-service.ts | 63 ++++++--- .../server/workspace-lifecycle-coordinator.ts | 76 +++++++++++ .../src/server/worktree-session.test.ts | 56 ++++++++ 12 files changed, 762 insertions(+), 116 deletions(-) diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index bdb9030435..e3af01335c 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -1,7 +1,11 @@ -import { expect, test } from "vitest"; +import { expect, test, vi } from "vitest"; import type { AgentManagerEvent, AgentSubscriber } from "./agent-manager.js"; -import { registerAgentAutoArchive } from "./create-agent-lifecycle-dispatch.js"; +import { createTestLogger } from "../../test-utils/test-logger.js"; +import { + CreateAgentLifecycleDispatch, + registerAgentAutoArchive, +} from "./create-agent-lifecycle-dispatch.js"; class AgentLifecycleEvents { private readonly listeners = new Set(); @@ -45,3 +49,73 @@ test("auto-archive self-releases once and later cancellation waits harmlessly", expect(archiveCount).toBe(1); expect(agents.listenerCount()).toBe(0); }); + +test("auto-archive retries a later terminal event after a transient failure", async () => { + const agentId = "agent-auto-archive-retry"; + const agents = new AgentLifecycleEvents(); + const archive = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("transient archive failure")) + .mockResolvedValueOnce(undefined); + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive, + }); + + agents.completeTurn(agentId); + await Promise.resolve(); + await Promise.resolve(); + expect(archive).toHaveBeenCalledTimes(1); + expect(agents.listenerCount()).toBe(1); + + agents.completeTurn(agentId); + await registration.cancel(); + expect(archive).toHaveBeenCalledTimes(2); + expect(agents.listenerCount()).toBe(0); +}); + +test("auto-archive cancellation observes an in-flight archive rejection", async () => { + const agentId = "agent-auto-archive-cancel-failure"; + const agents = new AgentLifecycleEvents(); + const failure = new Error("archive failed during cancellation"); + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive: async () => { + throw failure; + }, + }); + + agents.completeTurn(agentId); + await expect(registration.cancel()).rejects.toBe(failure); + expect(agents.listenerCount()).toBe(0); +}); + +test("lifecycle dispatch does not permanently suppress an agent after archive failure", async () => { + const agentId = "agent-dispatch-auto-archive-retry"; + const agents = new AgentLifecycleEvents(); + const archiveAgentForClose = vi + .fn<(agentId: string) => Promise>() + .mockRejectedValueOnce(new Error("transient close failure")) + .mockResolvedValueOnce(undefined); + const dependencies = { + agentManager: agents, + archiveAgentForClose, + logger: createTestLogger(), + } as unknown as ConstructorParameters[0]; + const dispatch = new CreateAgentLifecycleDispatch(dependencies); + const registration = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + + agents.completeTurn(agentId); + await new Promise((resolve) => setTimeout(resolve, 0)); + agents.completeTurn(agentId); + await registration.cancel(); + + expect(archiveAgentForClose).toHaveBeenCalledTimes(2); + expect(agents.listenerCount()).toBe(0); +}); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 4e9937482f..20b1039b6a 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -55,7 +55,7 @@ type AutoArchiveTarget = | { kind: "created-worktree"; result: CreatePaseoWorktreeWorkflowResult }; export class CreateAgentLifecycleDispatch { - private readonly autoArchiveAgentIds = new Set(); + private readonly autoArchiveTasks = new Map>(); constructor(private readonly dependencies: CreateAgentLifecycleDispatchDependencies) {} @@ -166,24 +166,33 @@ export class CreateAgentLifecycleDispatch { } private async autoArchiveAgentOnce(agentId: string, target: AutoArchiveTarget): Promise { - if (this.autoArchiveAgentIds.has(agentId)) { - return; - } - this.autoArchiveAgentIds.add(agentId); + const existingTask = this.autoArchiveTasks.get(agentId); + if (existingTask) return existingTask; + const archiveTask = this.runAutoArchiveAgent(agentId, target); + this.autoArchiveTasks.set(agentId, archiveTask); try { - if (target.kind === "created-worktree") { - await this.archiveAutoCreatedWorktree({ - agentId, - createdWorktree: target.result, - }); - return; - } - - await this.dependencies.archiveAgentForClose(agentId); + await archiveTask; } catch (error) { this.dependencies.logger.warn({ err: error, agentId }, "Failed to auto-archive agent"); + throw error; + } finally { + if (this.autoArchiveTasks.get(agentId) === archiveTask) { + this.autoArchiveTasks.delete(agentId); + } + } + } + + private async runAutoArchiveAgent(agentId: string, target: AutoArchiveTarget): Promise { + if (target.kind === "created-worktree") { + await this.archiveAutoCreatedWorktree({ + agentId, + createdWorktree: target.result, + }); + return; } + + await this.dependencies.archiveAgentForClose(agentId); } private async archiveAutoCreatedWorktree(options: { @@ -237,6 +246,7 @@ export function registerAgentAutoArchive(input: { }): LifecycleRegistration { let unsubscribe: (() => void) | null = null; let archiveTask: Promise | null = null; + let lastArchiveFailure: unknown | null = null; const release = () => { if (!unsubscribe) return; const subscribed = unsubscribe; @@ -246,7 +256,13 @@ export function registerAgentAutoArchive(input: { const registration: LifecycleRegistration = { async cancel() { release(); - await archiveTask; + const pendingArchive = archiveTask; + if (pendingArchive) { + await pendingArchive; + } + if (lastArchiveFailure) { + throw lastArchiveFailure; + } }, }; unsubscribe = input.agentManager.subscribe( @@ -259,8 +275,24 @@ export function registerAgentAutoArchive(input: { ) { return; } - release(); - archiveTask = input.archive(); + if (archiveTask) return; + const task = input.archive(); + archiveTask = task; + void task.then( + () => { + if (archiveTask !== task) return; + archiveTask = null; + lastArchiveFailure = null; + release(); + return undefined; + }, + (error: unknown) => { + if (archiveTask !== task) return; + archiveTask = null; + lastArchiveFailure = error; + return undefined; + }, + ); }, { agentId: input.agentId, replayState: false }, ); diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index 870bf8249a..ea515d4a3f 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -11,6 +11,7 @@ import { AgentStorage } from "../agent-storage.js"; import type { CreatePaseoWorktreeWorkflowResult } from "../../worktree-session.js"; import { createAgentCommand } from "./create.js"; import type { ManagedAgent } from "../agent-manager.js"; +import { WorkspaceLifecycleCoordinator } from "../../workspace-lifecycle-coordinator.js"; const logger = createTestLogger(); @@ -209,6 +210,129 @@ test("session create releases setup reservation when agent creation fails", asyn expect(startAfterAgentCreate).not.toHaveBeenCalled(); }); +test.each(["worktree callback", "provider resolution"] as const)( + "MCP create releases a reserved setup continuation after %s fails", + async (failurePoint) => { + const releaseWithoutStarting = vi.fn(); + const startAfterAgentCreate = vi.fn(); + const createAgent = vi.fn(); + const providerSnapshotManager = createProviderSnapshotManagerStub(); + if (failurePoint === "provider resolution") { + providerSnapshotManager.resolveCreateConfig.mockRejectedValue( + new Error("provider resolution failed"), + ); + } + const createdWorktree = { + worktree: { worktreePath: "/tmp/paseo-mcp-create/worktree" }, + intent: {}, + workspace: { + workspaceId: "ws-mcp-created", + cwd: "/tmp/paseo-mcp-create/worktree", + }, + repoRoot: "/tmp/paseo-mcp-create", + created: true, + setupContinuation: { + kind: "agent" as const, + startAfterAgentCreate, + releaseWithoutStarting, + }, + } as unknown as CreatePaseoWorktreeWorkflowResult; + + await expect( + createAgentCommand( + { + agentManager: { createAgent } as unknown as AgentManager, + agentStorage: {} as AgentStorage, + logger, + providerSnapshotManager: providerSnapshotManager.manager, + createPaseoWorktree: vi.fn(async () => createdWorktree), + }, + { + kind: "mcp", + provider: "opencode/test-model", + title: "MCP worktree child", + initialPrompt: "do the work", + background: true, + notifyOnFinish: false, + worktree: { worktreeName: "worktree", baseBranch: "main" }, + ...(failurePoint === "worktree callback" + ? { + onWorktreeCreated: () => { + throw new Error("worktree callback failed"); + }, + } + : {}), + }, + ), + ).rejects.toThrow(failurePoint === "worktree callback" ? "callback" : "provider resolution"); + + expect(releaseWithoutStarting).toHaveBeenCalledOnce(); + expect(startAfterAgentCreate).not.toHaveBeenCalled(); + expect(createAgent).not.toHaveBeenCalled(); + }, +); + +test("agent creation holds workspace ownership until the agent is attached", async () => { + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + let releaseCreate: (() => void) | undefined; + let markCreateStarted: (() => void) | undefined; + const createStarted = new Promise((resolve) => { + markCreateStarted = resolve; + }); + const createGate = new Promise((resolve) => { + releaseCreate = resolve; + }); + const snapshot = { + id: "agent-owned", + provider: "codex", + cwd: "/tmp/paseo-create-owned", + workspaceId: "ws-owned", + runtimeInfo: null, + } as ManagedAgent; + const createTask = createAgentCommand( + { + agentManager: { + createAgent: vi.fn(async () => { + markCreateStarted?.(); + await createGate; + return snapshot; + }), + } as unknown as AgentManager, + agentStorage: {} as AgentStorage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + lifecycleCoordinator, + }, + { + kind: "session", + config: { provider: "codex", cwd: snapshot.cwd }, + workspaceId: "ws-owned", + labels: {}, + provisionalTitle: null, + firstAgentContext: {}, + buildSessionConfig: async (config) => ({ sessionConfig: config }), + }, + ); + + await createStarted; + const archiveReservation = lifecycleCoordinator.reserveWorkspaceArchive(["ws-owned"]); + let ownershipReleased = false; + const waitTask = lifecycleCoordinator + .waitForWorkspaceOwnershipMutations(["ws-owned"]) + .then(() => { + ownershipReleased = true; + return undefined; + }); + await Promise.resolve(); + expect(ownershipReleased).toBe(false); + + releaseCreate?.(); + await createTask; + await waitTask; + expect(ownershipReleased).toBe(true); + archiveReservation.release(); +}); + test("session create applies the resolved mode from the provider create config", async () => { const snapshot = { id: "agent-1", diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index 94f42a50e6..13c4e927fc 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -26,6 +26,10 @@ import { emitLiveTimelineItemIfAgentKnown, } from "../timeline-append.js"; import { resolveCreateAgentIntent } from "./intent.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "../../workspace-lifecycle-coordinator.js"; export interface CreateAgentSessionWorktreeResult { sessionConfig: AgentSessionConfig; @@ -46,6 +50,7 @@ export interface CreateAgentCommandDependencies { createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn; // Mints a fresh directory workspace for a cwd and returns its id. ensureWorkspaceForCreate?: EnsureWorkspaceForCreate; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; } export type EnsureWorkspaceForCreate = ( @@ -180,6 +185,17 @@ export async function createAgentCommand( : await resolveMcpCreateAgent(dependencies, input); let snapshot: ManagedAgent; + const lifecycleCoordinator = + dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + let ownershipReservation; + try { + ownershipReservation = lifecycleCoordinator.reserveWorkspaceOwnershipMutation( + requireResolvedWorkspaceId(resolved.createOptions.workspaceId), + ); + } catch (error) { + resolved.setupContinuation?.releaseWithoutStarting(); + throw error; + } try { snapshot = await dependencies.agentManager.createAgent( resolved.config, @@ -189,6 +205,8 @@ export async function createAgentCommand( } catch (error) { resolved.setupContinuation?.releaseWithoutStarting(); throw error; + } finally { + ownershipReservation.release(); } resolved.setupContinuation?.startAfterAgentCreate({ @@ -327,55 +345,60 @@ async function resolveMcpCreateAgent( worktree: input.worktree, initialPrompt: input.initialPrompt ?? "", }); - if (createdWorktree) input.onWorktreeCreated?.(createdWorktree); - - const intent = await resolveCreateAgentIntent({ - explicitWorkspaceId: setupContinuation ? createdWorkspaceId : input.workspaceId, - caller: parentAgent - ? { id: parentAgent.id, cwd: parentAgent.cwd, workspaceId: parentAgent.workspaceId } - : null, - labels: input.labels, - childAgentDefaultLabels: input.callerContext?.childAgentDefaultLabels, - legacyDetached: input.detached ?? false, - resolveWorkspace: async (workspaceId) => ({ workspaceId, cwd: resolvedCwd }), - createWorkspace: async () => ({ - workspaceId: requireResolvedWorkspaceId( - await ensureWorkspaceForMcpCreate(dependencies, resolvedCwd, input.initialPrompt ?? ""), - ), - cwd: resolvedCwd, - }), - }); - const resolvedCreateConfig = await resolveMcpProviderCreateConfig({ - dependencies, - input, - provider, - resolvedCwd, - parentAgent, - }); - - const trimmedPrompt = input.initialPrompt?.trim() ?? ""; - return { - config: buildMcpSessionConfig({ + try { + if (createdWorktree) input.onWorktreeCreated?.(createdWorktree); + + const intent = await resolveCreateAgentIntent({ + explicitWorkspaceId: setupContinuation ? createdWorkspaceId : input.workspaceId, + caller: parentAgent + ? { id: parentAgent.id, cwd: parentAgent.cwd, workspaceId: parentAgent.workspaceId } + : null, + labels: input.labels, + childAgentDefaultLabels: input.callerContext?.childAgentDefaultLabels, + legacyDetached: input.detached ?? false, + resolveWorkspace: async (workspaceId) => ({ workspaceId, cwd: resolvedCwd }), + createWorkspace: async () => ({ + workspaceId: requireResolvedWorkspaceId( + await ensureWorkspaceForMcpCreate(dependencies, resolvedCwd, input.initialPrompt ?? ""), + ), + cwd: resolvedCwd, + }), + }); + const resolvedCreateConfig = await resolveMcpProviderCreateConfig({ + dependencies, input, - resolvedProviderModel, provider, - resolvedCwd: intent.cwd, - trimmedPrompt, - resolvedMode: resolvedCreateConfig.modeId, - resolvedFeatures: resolvedCreateConfig.featureValues, - }), - createOptions: { - ...(Object.keys(intent.labels).length > 0 ? { labels: intent.labels } : {}), - workspaceId: intent.workspaceId, - owner: input.owner, - env: input.env, - }, - prompt: trimmedPrompt ? trimmedPrompt : undefined, - setupContinuation, - createdWorktree, - background: input.background, - promptFailure: input.promptFailure ?? "log", - }; + resolvedCwd, + parentAgent, + }); + + const trimmedPrompt = input.initialPrompt?.trim() ?? ""; + return { + config: buildMcpSessionConfig({ + input, + resolvedProviderModel, + provider, + resolvedCwd: intent.cwd, + trimmedPrompt, + resolvedMode: resolvedCreateConfig.modeId, + resolvedFeatures: resolvedCreateConfig.featureValues, + }), + createOptions: { + ...(Object.keys(intent.labels).length > 0 ? { labels: intent.labels } : {}), + workspaceId: intent.workspaceId, + owner: input.owner, + env: input.env, + }, + prompt: trimmedPrompt ? trimmedPrompt : undefined, + setupContinuation, + createdWorktree, + background: input.background, + promptFailure: input.promptFailure ?? "log", + }; + } catch (error) { + setupContinuation?.releaseWithoutStarting(); + throw error; + } } function resolveMcpInitialCwd( diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 4e8dbb66da..8ff3d05045 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -56,6 +56,7 @@ import type { BrowserToolsBroker, BrowserToolsExecuteInput } from "../browser-to import type { BrowserToolsResponsePayload } from "../browser-tools/errors.js"; import { readPaseoWorktreeMetadata } from "../../utils/worktree-metadata.js"; import { createWorkspaceProvisioningService } from "../session/workspace-provisioning/workspace-provisioning-service.js"; +import { WorkspaceLifecycleCoordinator } from "../workspace-lifecycle-coordinator.js"; const REPO_CWD = resolvePath("/tmp/repo"); const TARGET_CWD = resolvePath("/tmp/target"); @@ -272,6 +273,57 @@ function createTerminalManagerStub(overrides: Partial = {}): Te } as unknown as TerminalManager; } +it("holds workspace ownership while an MCP terminal is being attached", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + let releaseTerminal: (() => void) | undefined; + let markTerminalStarted: (() => void) | undefined; + const terminalStarted = new Promise((resolve) => { + markTerminalStarted = resolve; + }); + const terminalGate = new Promise((resolve) => { + releaseTerminal = resolve; + }); + const createTerminal = vi.fn(async ({ cwd }: { cwd: string }) => { + markTerminalStarted?.(); + await terminalGate; + return { id: "terminal-owned", name: "Owned terminal", cwd }; + }); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + terminalManager: createTerminalManagerStub({ + createTerminal: createTerminal as unknown as TerminalManager["createTerminal"], + }), + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + ensureWorkspaceForCreate: async () => "ws-terminal-owned", + lifecycleCoordinator, + logger: createTestLogger(), + }); + + const terminalTask = registeredTool(server, "create_terminal").handler({ + cwd: "/tmp/terminal-owned", + name: "Owned terminal", + }); + await terminalStarted; + const archiveReservation = lifecycleCoordinator.reserveWorkspaceArchive(["ws-terminal-owned"]); + let ownershipReleased = false; + const waitTask = lifecycleCoordinator + .waitForWorkspaceOwnershipMutations(["ws-terminal-owned"]) + .then(() => { + ownershipReleased = true; + return undefined; + }); + await Promise.resolve(); + expect(ownershipReleased).toBe(false); + + releaseTerminal?.(); + await terminalTask; + await waitTask; + expect(ownershipReleased).toBe(true); + archiveReservation.release(); +}); + type ProviderSnapshotManagerStub = ReturnType; interface ConfigureProviderEntry { diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index f6a6ba80f5..5cd45fa3b4 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -90,6 +90,10 @@ import type { PaseoToolExecutionContext, PaseoToolResult, } from "./types.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "../../workspace-lifecycle-coordinator.js"; export interface PaseoToolHostDependencies { agentManager: AgentManager; @@ -127,6 +131,7 @@ export interface PaseoToolHostDependencies { browserToolsBroker?: BrowserToolsBroker | null; paseoHome?: string; worktreesRoot?: string; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; /** * ID of the agent that is using this tool catalog. * Used for cwd/mode inheritance when agents spawn child agents. @@ -1365,7 +1370,10 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase throw new Error("Active workspace lister is required to archive workspaces"); } const workspace = await requireActiveWorkspaceForArchive( - { listActiveWorkspaces: options.listActiveWorkspaces }, + { + listActiveWorkspaces: options.listActiveWorkspaces, + workspaceRegistry: options.workspaceRegistry, + }, workspaceId, ); const result = await archiveByScope( @@ -1437,6 +1445,8 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase worktreesRoot: options.worktreesRoot, terminalManager, providerSnapshotManager, + lifecycleCoordinator: + options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator, createPaseoWorktree: options.createPaseoWorktree, ...(options.ensureWorkspaceForCreate ? { ensureWorkspaceForCreate: options.ensureWorkspaceForCreate } @@ -2372,12 +2382,19 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase const resolvedCwd = resolveScopedCwd(cwd, { required: true }); const workspaceId = await resolveTerminalWorkspaceId(resolvedCwd); - - const terminal = await terminalManager.createTerminal({ - cwd: resolvedCwd, - workspaceId, - ...(name?.trim() ? { name: name.trim() } : {}), - }); + const ownershipReservation = ( + options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator + ).reserveWorkspaceOwnershipMutation(workspaceId); + let terminal; + try { + terminal = await terminalManager.createTerminal({ + cwd: resolvedCwd, + workspaceId, + ...(name?.trim() ? { name: name.trim() } : {}), + }); + } finally { + ownershipReservation.release(); + } return { content: [], @@ -3173,6 +3190,7 @@ function archiveWorktreeDependencies( emitWorkspaceUpdatesForWorkspaceIds: options.emitWorkspaceUpdatesForWorkspaceIds, markWorkspaceArchiving: options.markWorkspaceArchiving, clearWorkspaceArchiving: options.clearWorkspaceArchiving, + lifecycleCoordinator: options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator, killTerminalsForWorkspace: (workspaceId: string) => killTerminalsForWorkspace( { diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 562d5312c8..e222d20570 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -231,6 +231,7 @@ import { cacheWorkspaceSetupSnapshot, } from "./worktree-session.js"; import { archiveByScope, type ActiveWorkspaceRef } from "./workspace-archive-service.js"; +import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { WorktreeRequestError, toWorktreeWireError } from "./worktree-errors.js"; import { parseGitRemoteLocation } from "@getpaseo/protocol/git-remote"; import { @@ -3041,6 +3042,7 @@ export class Session { paseoHome: this.paseoHome, worktreesRoot: this.worktreesRoot, providerSnapshotManager: this.providerSnapshotManager, + lifecycleCoordinator: defaultWorkspaceLifecycleCoordinator, }, { kind: "session", @@ -5738,7 +5740,10 @@ export class Session { ): Promise { try { const existing = await requireActiveWorkspaceForArchive( - { listActiveWorkspaces: () => this.listActiveWorkspaceRefs() }, + { + listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), + workspaceRegistry: this.workspaceRegistry, + }, request.workspaceId, ); @@ -5770,7 +5775,10 @@ export class Session { ); const archivedWorkspace = await this.workspaceRegistry.get(request.workspaceId); - const archivedAt = archivedWorkspace?.archivedAt ?? new Date().toISOString(); + if (!archivedWorkspace?.archivedAt) { + throw new Error(`Workspace archive did not persist: ${request.workspaceId}`); + } + const archivedAt = archivedWorkspace.archivedAt; this.emit({ type: "archive_workspace_response", payload: { diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index a1194a8297..1f5fe70ba0 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -5496,6 +5496,79 @@ test("archive_workspace_request hides non-destructive workspace records", async expect(response?.payload.error).toBeNull(); }); +test("archive_workspace_request never fabricates success when persistence fails", async () => { + const emitted: SessionOutboundMessage[] = []; + const session = createSessionForWorkspaceTests(); + const workspace = createPersistedWorkspaceRecord({ + workspaceId: "ws-archive-persistence-failure", + projectId: "proj-archive-persistence-failure", + cwd: REPO_CWD, + kind: "directory", + displayName: "repo", + createdAt: "2026-03-01T12:00:00.000Z", + updatedAt: "2026-03-01T12:00:00.000Z", + }); + + session.emit = (message) => { + if (isSessionOutboundMessage(message)) emitted.push(message); + }; + session.workspaceRegistry.get = async () => workspace; + session.workspaceRegistry.list = async () => [workspace]; + session.workspaceRegistry.archive = async () => { + throw new Error("workspace persistence failed"); + }; + + await session.handleMessage({ + type: "archive_workspace_request", + workspaceId: workspace.workspaceId, + requestId: "req-archive-persistence-failure", + }); + + expect(workspace.archivedAt).toBeNull(); + const response = emitted.find((message) => message.type === "archive_workspace_response") as + | { payload: Record } + | undefined; + expect(response?.payload).toMatchObject({ + archivedAt: null, + error: "Failed to archive one or more workspaces", + }); +}); + +test("archive_workspace_request requires a persisted archived timestamp", async () => { + const emitted: SessionOutboundMessage[] = []; + const session = createSessionForWorkspaceTests(); + const workspace = createPersistedWorkspaceRecord({ + workspaceId: "ws-archive-no-timestamp", + projectId: "proj-archive-no-timestamp", + cwd: REPO_CWD, + kind: "directory", + displayName: "repo", + createdAt: "2026-03-01T12:00:00.000Z", + updatedAt: "2026-03-01T12:00:00.000Z", + }); + + session.emit = (message) => { + if (isSessionOutboundMessage(message)) emitted.push(message); + }; + session.workspaceRegistry.get = async () => workspace; + session.workspaceRegistry.list = async () => [workspace]; + session.workspaceRegistry.archive = async () => {}; + + await session.handleMessage({ + type: "archive_workspace_request", + workspaceId: workspace.workspaceId, + requestId: "req-archive-no-timestamp", + }); + + const response = emitted.find((message) => message.type === "archive_workspace_response") as + | { payload: Record } + | undefined; + expect(response?.payload).toMatchObject({ + archivedAt: null, + error: `Workspace archive did not persist: ${workspace.workspaceId}`, + }); +}); + test("archive_workspace_request archives a worktree-kind workspace and removes the directory on last reference", async () => { const tempDir = mkdtempSync(path.join(tmpdir(), "session-worktree-kind-archive-")); const repoDir = path.join(tempDir, "repo"); diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 14177c0605..4019fe9bb4 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -16,6 +16,7 @@ import { type ActiveWorkspaceRef, type ArchiveDependencies, type ArchiveResult, + requireActiveWorkspaceForArchive, resolveWorkspaceIdAtPath, } from "./workspace-archive-service.js"; import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; @@ -205,6 +206,36 @@ function assertArchiveResult( } describe("archiveByScope", () => { + test("treats persisted cleanup as a retryable workspace archive target", async () => { + const persisted = createPersistedWorkspaceRecord({ + workspaceId: "ws-cleanup-retry-target", + projectId: "project-cleanup-retry-target", + cwd: "/tmp/cleanup-retry-target", + kind: "worktree", + displayName: "Cleanup retry target", + createdAt: "2026-07-31T00:00:00.000Z", + updatedAt: "2026-07-31T00:00:00.000Z", + archivedAt: "2026-07-31T00:01:00.000Z", + cleanupPending: { + directoryPath: "/tmp/cleanup-retry-target", + teardownCwd: "/tmp/cleanup-retry-target", + mainRepoRoot: "/tmp/repo", + paseoWorktreesRoot: "/tmp/worktrees", + worktreeIncarnationId: "incarnation-retry", + }, + }); + + await expect( + requireActiveWorkspaceForArchive( + { + listActiveWorkspaces: async () => [], + workspaceRegistry: { get: async () => persisted }, + }, + persisted.workspaceId, + ), + ).resolves.toBe(persisted); + }); + test("waits for registered setup before archiving its workspace", async () => { const { tempDir } = createGitRepo(); const workspaceId = "ws-setup-race"; @@ -285,11 +316,17 @@ describe("archiveByScope", () => { }); await archiveStarted; expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(1); + expect(() => lifecycleCoordinator.reserveWorkspaceOwnershipMutation(workspaceId)).toThrow( + `Workspace ${workspaceId} is being archived`, + ); releaseArchive?.(); const [firstResult, secondResult] = await Promise.all([first, second]); expect(secondResult).toEqual(firstResult); expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(1); + const postArchiveReservation = + lifecycleCoordinator.reserveWorkspaceOwnershipMutation(workspaceId); + postArchiveReservation.release(); }); test("archives sibling workspace records separately while serializing shared cleanup", async () => { @@ -760,14 +797,16 @@ describe("archiveByScope", () => { return originalArchiveWorkspaceRecord(workspaceId); }; - const result = await archiveByScope(deps, { - scope: { kind: "worktree", targetPath: worktree.worktreePath }, - requestId: "req-partial-failure", - }); + await expect( + archiveByScope(deps, { + scope: { kind: "worktree", targetPath: worktree.worktreePath }, + requestId: "req-partial-failure", + }), + ).rejects.toThrow("Failed to archive one or more workspaces"); - expect(result.archivedWorkspaceIds).toEqual([workspaceB]); - expect(result.archivedWorkspaceIds).not.toContain(workspaceA); - expect(result.removedDirectory).toBe(false); + expect((await deps.listActiveWorkspaces()).map((workspace) => workspace.workspaceId)).toEqual([ + workspaceA, + ]); expect(existsSync(worktree.worktreePath)).toBe(true); }); @@ -831,11 +870,12 @@ describe("archiveByScope", () => { await registry.archive(id, new Date().toISOString()); }; - const firstResult = await archiveByScope(deps, { - scope: { kind: "workspace", workspaceId }, - requestId: "req-cleanup-retry-first", - }); - expect(firstResult.removedDirectory).toBe(false); + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-cleanup-retry-first", + }), + ).rejects.toThrow("Worktree teardown command failed"); expect((await registry.get(workspaceId))?.archivedAt).not.toBeNull(); expect((await registry.get(workspaceId))?.cleanupPending).toMatchObject({ directoryPath: worktree.worktreePath, @@ -852,7 +892,7 @@ describe("archiveByScope", () => { expect((await registry.get(workspaceId))?.cleanupPending).toBeNull(); }); - test("a persisted cleanup retry cannot remove a newer worktree incarnation at the same path", async () => { + test("a replacement incarnation archives before an old cleanup retry without cross-blocking", async () => { const { tempDir, repoDir } = createGitRepo(); writeFileSync( path.join(repoDir, "paseo.json"), @@ -907,25 +947,64 @@ describe("archiveByScope", () => { await registry.archive(id, new Date().toISOString()); }; - const firstResult = await archiveByScope(deps, { - scope: { kind: "workspace", workspaceId }, - requestId: "req-reused-incarnation-first", - }); - expect(firstResult.cleanupPendingWorkspaceIds).toEqual([workspaceId]); + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-reused-incarnation-first", + }), + ).rejects.toThrow("Worktree teardown command failed"); + expect((await registry.get(workspaceId))?.cleanupPending).not.toBeNull(); execFileSync("git", ["worktree", "remove", firstWorktree.worktreePath, "--force"], { cwd: repoDir, stdio: "pipe", }); execFileSync("git", ["branch", "-D", slug], { cwd: repoDir, stdio: "pipe" }); + writeFileSync(path.join(repoDir, "paseo.json"), JSON.stringify({ worktree: { teardown: [] } })); + execFileSync("git", ["add", "paseo.json"], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "repair teardown"], { + cwd: repoDir, + stdio: "pipe", + }); const replacementWorktree = await createPaseoOwnedWorktree(repoDir, paseoHome, slug); + const replacementWorkspaceId = "ws-reused-incarnation-b"; + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId: replacementWorkspaceId, + projectId: "project-reused-incarnation", + cwd: replacementWorktree.worktreePath, + kind: "worktree", + displayName: "Replacement incarnation", + worktreeRoot: replacementWorktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + deps.activeWorkspaces.push({ + workspaceId: replacementWorkspaceId, + cwd: replacementWorktree.worktreePath, + kind: "worktree", + worktreeRoot: replacementWorktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }); + + const replacementResult = await archiveByScope(deps, { + scope: { kind: "workspace", workspaceId: replacementWorkspaceId }, + requestId: "req-reused-incarnation-replacement", + }); + expect(replacementResult.removedDirectory).toBe(true); + expect(existsSync(replacementWorktree.worktreePath)).toBe(false); + expect((await registry.get(replacementWorkspaceId))?.cleanupPending).toBeNull(); + expect((await registry.get(workspaceId))?.cleanupPending).not.toBeNull(); const retryResult = await archiveByScope(deps, { scope: { kind: "workspace", workspaceId }, requestId: "req-reused-incarnation-retry", }); expect(retryResult.removedDirectory).toBe(false); - expect(existsSync(replacementWorktree.worktreePath)).toBe(true); expect((await registry.get(workspaceId))?.cleanupPending).toBeNull(); }); @@ -943,13 +1022,13 @@ describe("archiveByScope", () => { }); deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); - const result = await archiveByScope(deps, { - scope: { kind: "workspace", workspaceId }, - requestId: "req-strict-teardown", - }); + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-strict-teardown", + }), + ).rejects.toThrow("Failed to archive one or more workspaces"); - expect(result.archivedWorkspaceIds).toEqual([]); - expect(result.removedDirectory).toBe(false); expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); expect(existsSync(worktree.worktreePath)).toBe(true); }); diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 020db793fa..176f473025 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -85,16 +85,20 @@ export interface ArchiveByScopeRequest { } export async function requireActiveWorkspaceForArchive( - dependencies: Pick, + dependencies: Pick, workspaceId: string, ): Promise { const workspace = (await dependencies.listActiveWorkspaces()).find( (candidate) => candidate.workspaceId === workspaceId, ); - if (!workspace) { - throw new Error(`Workspace not found: ${workspaceId}`); + if (workspace) { + return workspace; } - return workspace; + const persistedWorkspace = await dependencies.workspaceRegistry?.get(workspaceId); + if (persistedWorkspace?.cleanupPending) { + return persistedWorkspace; + } + throw new Error(`Workspace not found: ${workspaceId}`); } interface BackingDirectory { @@ -140,20 +144,31 @@ export async function archiveByScope( return lifecycleCoordinator.runArchive(operationKey, async () => { const initialTarget = await resolveArchiveTarget(dependencies, request.scope); + const archiveReservation = lifecycleCoordinator.reserveWorkspaceArchive( + initialTarget.workspaceIds, + ); const archiveOperation = async () => { await lifecycleCoordinator.waitForWorkspaceSetups(initialTarget.workspaceIds); + await lifecycleCoordinator.waitForWorkspaceOwnershipMutations(initialTarget.workspaceIds); let refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); + archiveReservation.add(refreshedTarget.workspaceIds); await lifecycleCoordinator.waitForWorkspaceSetups(refreshedTarget.workspaceIds); + await lifecycleCoordinator.waitForWorkspaceOwnershipMutations(refreshedTarget.workspaceIds); refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); + archiveReservation.add(refreshedTarget.workspaceIds); const target = mergeArchiveTargets(initialTarget, refreshedTarget); return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); }; - const mutationRoot = - (await resolveWorktreeMutationRoot(dependencies, initialTarget.backing)) ?? - (request.scope.kind === "worktree" ? dirname(resolve(request.scope.targetPath)) : null); - return mutationRoot - ? lifecycleCoordinator.runWorktreeMutationExclusive(mutationRoot, archiveOperation) - : archiveOperation(); + try { + const mutationRoot = + (await resolveWorktreeMutationRoot(dependencies, initialTarget.backing)) ?? + (request.scope.kind === "worktree" ? dirname(resolve(request.scope.targetPath)) : null); + return await (mutationRoot + ? lifecycleCoordinator.runWorktreeMutationExclusive(mutationRoot, archiveOperation) + : archiveOperation()); + } finally { + archiveReservation.release(); + } }); } @@ -207,12 +222,15 @@ async function archiveResolvedTarget( await persistTargetCleanupPending(dependencies, target); - const { archivedAgents, archivedWorkspaceIds } = await archiveTargetRecords( + const { archivedAgents, archivedWorkspaceIds, failures } = await archiveTargetRecords( dependencies, targetWorkspaceIds, request.requestId, ); await clearCleanupPendingForUnarchivedTargets(dependencies, target, archivedWorkspaceIds); + if (failures.length > 0) { + throw new AggregateError(failures, "Failed to archive one or more workspaces"); + } if (target.backing?.mainRepoRoot) { try { @@ -407,7 +425,11 @@ async function archiveTargetRecords( dependencies: ArchiveDependencies, targetWorkspaceIds: string[], requestId: string, -): Promise<{ archivedAgents: Set; archivedWorkspaceIds: string[] }> { +): Promise<{ + archivedAgents: Set; + archivedWorkspaceIds: string[]; + failures: unknown[]; +}> { const archivedAgents = new Set(); const archivedWorkspaceIds: string[] = []; @@ -428,12 +450,15 @@ async function archiveTargetRecords( } else { dependencies.sessionLogger?.warn( { err: result.reason, requestId }, - "archiveByScope workspace teardown failed; continuing", + "archiveByScope workspace teardown failed", ); } } - return { archivedAgents, archivedWorkspaceIds }; + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + return { archivedAgents, archivedWorkspaceIds, failures }; } async function maybeRemoveDirectory( @@ -519,7 +544,7 @@ async function maybeRemoveDirectoryExclusive( { err: error, targetPath: backing.path, requestId: request.requestId }, "Worktree teardown failed during archive; workspace already archived", ); - return false; + throw error; } throw error; } @@ -567,7 +592,7 @@ async function maybeRemoveDirectoryExclusive( { err: error, targetPath: backing.path, requestId: request.requestId }, "Worktree disk removal failed during archive; workspace already archived", ); - return false; + throw error; } throw error; } @@ -656,10 +681,16 @@ async function listPendingCleanupTargets( const workspaceRegistry = dependencies.workspaceRegistry; if (backingPath && workspaceRegistry) { const matchesBacking = createRealpathAwarePathMatcher(backingPath); + const targetWorkspaceIds = new Set( + target.teardownTargets.flatMap((teardownTarget) => + teardownTarget.workspaceId ? [teardownTarget.workspaceId] : [], + ), + ); const records = await workspaceRegistry.list(); return records.flatMap((workspace) => { const cleanupPending = workspace.cleanupPending; if ( + (targetWorkspaceIds.size > 0 && !targetWorkspaceIds.has(workspace.workspaceId)) || !workspace.archivedAt || !cleanupPending || !matchesBacking(cleanupPending.directoryPath) diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts index 58ff2897a1..fe6f6a0733 100644 --- a/packages/server/src/server/workspace-lifecycle-coordinator.ts +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -4,6 +4,8 @@ export class WorkspaceLifecycleCoordinator { private readonly archiveOperations = new Map>(); private readonly directoryOperations = new Map>(); private readonly worktreeMutationOperations = new Map>(); + private readonly ownershipMutationTasks = new Map>>(); + private readonly archivingWorkspaceIds = new Map(); trackWorkspaceSetup(workspaceId: string, task: Promise, directoryPath?: string): void { this.setupTasks.set(workspaceId, task); @@ -56,6 +58,71 @@ export class WorkspaceLifecycleCoordinator { await Promise.allSettled(tasks); } + reserveWorkspaceOwnershipMutation(workspaceId: string): WorkspaceOwnershipMutationReservation { + if ((this.archivingWorkspaceIds.get(workspaceId) ?? 0) > 0) { + throw new Error(`Workspace ${workspaceId} is being archived`); + } + + let releaseReservation: (() => void) | null = null; + const reservationTask = new Promise((resolve) => { + releaseReservation = resolve; + }); + const tasks = this.ownershipMutationTasks.get(workspaceId) ?? new Set>(); + tasks.add(reservationTask); + this.ownershipMutationTasks.set(workspaceId, tasks); + + let released = false; + const release = () => { + if (released) return; + released = true; + releaseReservation?.(); + tasks.delete(reservationTask); + if (tasks.size === 0) { + this.ownershipMutationTasks.delete(workspaceId); + } + }; + return { release }; + } + + reserveWorkspaceArchive(workspaceIds: Iterable): WorkspaceArchiveReservation { + const ownedWorkspaceIds = new Set(); + const add = (additionalWorkspaceIds: Iterable) => { + for (const workspaceId of additionalWorkspaceIds) { + if (ownedWorkspaceIds.has(workspaceId)) continue; + this.archivingWorkspaceIds.set( + workspaceId, + (this.archivingWorkspaceIds.get(workspaceId) ?? 0) + 1, + ); + ownedWorkspaceIds.add(workspaceId); + } + }; + add(workspaceIds); + + let released = false; + return { + add, + release: () => { + if (released) return; + released = true; + for (const workspaceId of ownedWorkspaceIds) { + const remainingReservations = (this.archivingWorkspaceIds.get(workspaceId) ?? 1) - 1; + if (remainingReservations === 0) { + this.archivingWorkspaceIds.delete(workspaceId); + } else { + this.archivingWorkspaceIds.set(workspaceId, remainingReservations); + } + } + }, + }; + } + + async waitForWorkspaceOwnershipMutations(workspaceIds: Iterable): Promise { + const tasks = Array.from(workspaceIds).flatMap((workspaceId) => + Array.from(this.ownershipMutationTasks.get(workspaceId) ?? []), + ); + await Promise.allSettled(tasks); + } + runArchive(key: string, operation: () => Promise): Promise { const existing = this.archiveOperations.get(key); if (existing) { @@ -114,4 +181,13 @@ export interface WorkspaceSetupReservation { release(): void; } +export interface WorkspaceOwnershipMutationReservation { + release(): void; +} + +export interface WorkspaceArchiveReservation { + add(workspaceIds: Iterable): void; + release(): void; +} + export const defaultWorkspaceLifecycleCoordinator = new WorkspaceLifecycleCoordinator(); diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index 7c4bf294e0..ea62651ab5 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -1956,6 +1956,62 @@ describe("handlePaseoWorktreeArchiveRequest worktree scope", () => { } }); + test("reports archive teardown failure instead of emitting success", async () => { + const { tempDir } = createGitRepo(); + cleanupPaths.push(tempDir); + const workspaceId = "ws-archive-failure"; + const workspaceCwd = path.join(tempDir, "local-checkout"); + mkdirSync(workspaceCwd, { recursive: true }); + const emitted: SessionOutboundMessage[] = []; + + await handlePaseoWorktreeArchiveRequest( + { + paseoHome: path.join(tempDir, ".paseo"), + github: createGitHubServiceStub(), + workspaceGitService: { + getSnapshot: vi.fn(async () => null), + listWorktrees: vi.fn(async () => []), + }, + agentManager: { + listAgents: () => [], + archiveAgent: vi.fn(async () => ({ archivedAt: new Date().toISOString() })), + archiveSnapshot: vi.fn(async () => ({})), + }, + agentStorage: createAgentStorageStub(), + findWorkspaceIdForCwd: vi.fn(async () => workspaceId), + listActiveWorkspaces: vi.fn(async () => [ + { workspaceId, cwd: workspaceCwd, kind: "local_checkout" as const }, + ]), + archiveWorkspaceRecord: vi.fn(async () => { + throw new Error("registry archive failed"); + }), + emit: (message) => emitted.push(message), + emitWorkspaceUpdatesForWorkspaceIds: vi.fn(async () => {}), + markWorkspaceArchiving: vi.fn(), + clearWorkspaceArchiving: vi.fn(), + killTerminalsForWorkspace: vi.fn(async () => {}), + sessionLogger: createLogger(), + }, + { + type: "paseo_worktree_archive_request", + requestId: "req-archive-failure", + worktreePath: workspaceCwd, + repoRoot: workspaceCwd, + workspaceId, + scope: "workspace", + }, + ); + + expect( + emitted.find((message) => message.type === "paseo_worktree_archive_response"), + ).toMatchObject({ + payload: { + success: false, + error: { message: "Failed to archive one or more workspaces" }, + }, + }); + }); + test("archives every active workspace on the directory and removes it", async () => { const { tempDir, repoDir } = createGitRepo(); cleanupPaths.push(tempDir); From e2937e792969038b1b701e79abfbfe017bb635da Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 20:34:40 -0400 Subject: [PATCH 08/43] docs: declare v0.2.5 lifecycle patch --- patches/local-patches.json | 88 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 patches/local-patches.json diff --git a/patches/local-patches.json b/patches/local-patches.json new file mode 100644 index 0000000000..16187811d5 --- /dev/null +++ b/patches/local-patches.json @@ -0,0 +1,88 @@ +{ + "schemaVersion": 1, + "patches": [ + { + "id": "paseo-v025-worktree-lifecycle", + "category": "stable-core-worktree-lifecycle", + "base": { + "tag": "v0.2.5", + "commit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b" + }, + "candidate": { + "branch": "review-v025-worktree-lifecycle-upstream", + "baseCommit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b", + "receipt": "patches/receipts/paseo-v025-worktree-lifecycle.json" + }, + "sources": [ + "packages/protocol/src/messages.ts", + "packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts", + "packages/server/src/server/agent/create-agent/create.ts", + "packages/server/src/server/agent/tools/paseo-tools.ts", + "packages/server/src/server/bootstrap.ts", + "packages/server/src/server/session.ts", + "packages/server/src/server/workspace-archive-service.ts", + "packages/server/src/server/workspace-lifecycle-coordinator.ts", + "packages/server/src/server/workspace-registry.ts", + "packages/server/src/server/worktree-bootstrap.ts", + "packages/server/src/server/worktree-session.ts", + "packages/server/src/server/worktree/commands.ts", + "packages/server/src/utils/worktree-metadata.ts", + "packages/server/src/utils/worktree-setup-output.ts", + "packages/server/src/utils/worktree.ts" + ], + "tests": [ + "packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts", + "packages/server/src/server/agent/create-agent/create.test.ts", + "packages/server/src/server/agent/mcp-server.test.ts", + "packages/server/src/server/session.workspace-git-watch.test.ts", + "packages/server/src/server/session.workspaces.test.ts", + "packages/server/src/server/workspace-archive-service.test.ts", + "packages/server/src/server/worktree-bootstrap.test.ts", + "packages/server/src/server/worktree-session.test.ts", + "packages/server/src/utils/worktree-setup-output.test.ts", + "packages/server/src/utils/worktree.posix.test.ts" + ], + "testCases": [ + "setup reservations prevent archive from outrunning worktree creation", + "existing-workspace agent and terminal attachment are fenced against archive", + "MCP worktree creation releases setup reservations on every pre-agent failure", + "archive failures propagate through command, websocket, MCP, and normal session surfaces", + "normal archive responses require the persisted archived timestamp", + "failed physical cleanup remains retryable after the workspace record is archived", + "replacement worktree incarnations archive independently of stale cleanup intents", + "auto-archive retries after transient failure and cancellation observes rejection", + "setup output is bounded while preserving the final diagnostic tail" + ], + "verification": { + "receipt": "patches/receipts/paseo-v025-worktree-lifecycle.json", + "completedAt": "2026-08-01T00:34:00Z", + "commands": [ + "npm run build --workspace packages/protocol", + "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", + "cd packages/server && npx vitest run src/server/workspace-archive-service.test.ts src/server/worktree-bootstrap.test.ts src/server/worktree-session.test.ts src/utils/worktree-setup-output.test.ts src/utils/worktree.posix.test.ts", + "npm test --workspace packages/protocol", + "npm run build --workspace packages/server", + "npm run typecheck --workspace packages/server", + "npm run typecheck --workspace packages/protocol", + "npm run lint -- --deny-warnings", + "npx oxfmt --check ", + "git diff --check", + "jq -e '.schemaVersion == 1 and (.patches | length == 1) and .patches[0].id == \"paseo-v025-worktree-lifecycle\"' patches/local-patches.json" + ], + "result": { + "focusedServerTests": "359 passed, 5 skipped", + "protocolTests": "468 passed", + "serverBuild": "passed", + "serverAndProtocolTypechecks": "passed", + "lint": "0 warnings, 0 errors", + "formatDiffAndManifest": "passed" + } + }, + "upstreamable": true, + "upstreamCondition": "Submit independently while worktree setup, ownership attachment, archive, cleanup retry, and auto-archive are not coordinated by one durable lifecycle boundary.", + "dropCondition": "Remove when the latest stable release provides equivalent setup and ownership fencing, explicit archive failure propagation, incarnation-safe retryable cleanup, bounded setup output, and retryable observable auto-archive semantics.", + "deployedReceipt": null, + "deployedAt": null + } + ] +} From bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 20:35:02 -0400 Subject: [PATCH 09/43] docs: record lifecycle verification receipt --- .../paseo-v025-worktree-lifecycle.json | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 patches/receipts/paseo-v025-worktree-lifecycle.json diff --git a/patches/receipts/paseo-v025-worktree-lifecycle.json b/patches/receipts/paseo-v025-worktree-lifecycle.json new file mode 100644 index 0000000000..6efe4b2909 --- /dev/null +++ b/patches/receipts/paseo-v025-worktree-lifecycle.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 1, + "patchId": "paseo-v025-worktree-lifecycle", + "base": { + "tag": "v0.2.5", + "commit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b" + }, + "candidate": { + "branch": "review-v025-worktree-lifecycle-upstream", + "commit": "e2937e792969038b1b701e79abfbfe017bb635da", + "tree": "af3297ff70ecb52c78ce0d305a622162049cc4a8" + }, + "verification": { + "commit": "e2937e792969038b1b701e79abfbfe017bb635da", + "completedAt": "2026-08-01T00:34:00Z", + "commands": [ + "npm run build --workspace packages/protocol", + "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", + "cd packages/server && npx vitest run src/server/workspace-archive-service.test.ts src/server/worktree-bootstrap.test.ts src/server/worktree-session.test.ts src/utils/worktree-setup-output.test.ts src/utils/worktree.posix.test.ts", + "npm test --workspace packages/protocol", + "npm run build --workspace packages/server", + "npm run typecheck --workspace packages/server", + "npm run typecheck --workspace packages/protocol", + "npm run lint -- --deny-warnings", + "npx oxfmt --check ", + "git diff --check", + "jq -e '.schemaVersion == 1 and (.patches | length == 1) and .patches[0].id == \"paseo-v025-worktree-lifecycle\"' patches/local-patches.json" + ], + "result": { + "focusedServerTests": "359 passed, 5 skipped", + "protocolTests": "468 passed", + "serverBuild": "passed", + "serverAndProtocolTypechecks": "passed", + "lint": "0 warnings, 0 errors", + "formatDiffAndManifest": "passed" + } + } +} From 832508fe7b2221c33549a54b5588dff7667bf065 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 20:40:27 -0400 Subject: [PATCH 10/43] docs: record current-main lifecycle convergence --- ...-v025-worktree-lifecycle-current-main.json | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 patches/receipts/paseo-v025-worktree-lifecycle-current-main.json diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json new file mode 100644 index 0000000000..2181828742 --- /dev/null +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -0,0 +1,41 @@ +{ + "schemaVersion": 1, + "patchId": "paseo-v025-worktree-lifecycle", + "stableCandidate": { + "baseCommit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b", + "headCommit": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", + "verificationCommit": "e2937e792969038b1b701e79abfbfe017bb635da", + "verificationTree": "af3297ff70ecb52c78ce0d305a622162049cc4a8" + }, + "currentMainIntegration": { + "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", + "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", + "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", + "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", + "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" + }, + "verification": { + "commit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", + "completedAt": "2026-08-01T00:39:55Z", + "commands": [ + "npm run build --workspace packages/protocol", + "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", + "cd packages/server && npx vitest run src/server/workspace-archive-service.test.ts src/server/worktree-bootstrap.test.ts src/server/worktree-session.test.ts src/utils/worktree-setup-output.test.ts src/utils/worktree.posix.test.ts src/utils/checkout-git.test.ts src/utils/checkout-git.commits.test.ts src/server/worktree-branch-name-generator.test.ts", + "npm test --workspace packages/protocol", + "npm run build --workspace packages/server", + "npm run typecheck --workspace packages/server", + "npm run typecheck --workspace packages/protocol", + "npm run lint -- --deny-warnings", + "npx oxfmt --check ", + "git diff --cached --check -- packages/server/src/utils/worktree-metadata.ts" + ], + "result": { + "focusedAndBroadServerTests": "526 passed, 6 skipped", + "protocolTests": "470 passed", + "serverBuild": "passed", + "serverAndProtocolTypechecks": "passed", + "lint": "0 warnings, 0 errors", + "lifecycleFormatAndConflictDiff": "passed" + } + } +} From 551bdd371d82171177387e32ec5ca0278135050b Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 20:42:30 -0400 Subject: [PATCH 11/43] fix: fail unresolved workspace archives --- .../src/server/worktree-session.test.ts | 53 +++++++++++++++++++ .../server/src/server/worktree/commands.ts | 13 ++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index ea62651ab5..b2beeae70d 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -2012,6 +2012,59 @@ describe("handlePaseoWorktreeArchiveRequest worktree scope", () => { }); }); + test("reports failure when an archive path no longer resolves to a workspace", async () => { + const { tempDir } = createGitRepo(); + cleanupPaths.push(tempDir); + const workspaceCwd = path.join(tempDir, "unresolved-workspace"); + mkdirSync(workspaceCwd, { recursive: true }); + const emitted: SessionOutboundMessage[] = []; + + await handlePaseoWorktreeArchiveRequest( + { + paseoHome: path.join(tempDir, ".paseo"), + github: createGitHubServiceStub(), + workspaceGitService: { + getSnapshot: vi.fn(async () => null), + listWorktrees: vi.fn(async () => []), + }, + agentManager: { + listAgents: () => [], + archiveAgent: vi.fn(async () => ({ archivedAt: new Date().toISOString() })), + archiveSnapshot: vi.fn(async () => ({})), + }, + agentStorage: createAgentStorageStub(), + findWorkspaceIdForCwd: vi.fn(async () => null), + listActiveWorkspaces: vi.fn(async () => []), + archiveWorkspaceRecord: vi.fn(async () => {}), + emit: (message) => emitted.push(message), + emitWorkspaceUpdatesForWorkspaceIds: vi.fn(async () => {}), + markWorkspaceArchiving: vi.fn(), + clearWorkspaceArchiving: vi.fn(), + killTerminalsForWorkspace: vi.fn(async () => {}), + sessionLogger: createLogger(), + }, + { + type: "paseo_worktree_archive_request", + requestId: "req-unresolved-workspace", + worktreePath: workspaceCwd, + repoRoot: workspaceCwd, + scope: "workspace", + }, + ); + + expect( + emitted.find((message) => message.type === "paseo_worktree_archive_response"), + ).toMatchObject({ + payload: { + success: false, + error: { + code: "UNKNOWN", + message: `Workspace not found for archive target: ${workspaceCwd}`, + }, + }, + }); + }); + test("archives every active workspace on the directory and removes it", async () => { const { tempDir, repoDir } = createGitRepo(); cleanupPaths.push(tempDir); diff --git a/packages/server/src/server/worktree/commands.ts b/packages/server/src/server/worktree/commands.ts index edd6b55966..6bfc9c7168 100644 --- a/packages/server/src/server/worktree/commands.ts +++ b/packages/server/src/server/worktree/commands.ts @@ -115,7 +115,7 @@ export type ArchiveCommandResult = } | { ok: false; - code: "NOT_ALLOWED"; + code: "NOT_ALLOWED" | "UNKNOWN"; message: string; removedAgents: []; }; @@ -171,15 +171,12 @@ export async function archiveCommand( input.workspaceId ?? (await resolveWorkspaceIdAtPath(dependencies, targetPath)); if (!workspaceId) { - dependencies.sessionLogger?.warn( - { targetPath }, - "Could not resolve workspace for archive; skipping", - ); + dependencies.sessionLogger?.warn({ targetPath }, "Could not resolve workspace for archive"); return { - ok: true, + ok: false, + code: "UNKNOWN", + message: `Workspace not found for archive target: ${targetPath}`, removedAgents: [], - removedDirectory: false, - cleanupPending: false, }; } From 0858d4b662dc7e6bb2baec75ad2751a0151abf05 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 20:42:59 -0400 Subject: [PATCH 12/43] docs: close archive-response review gap --- patches/local-patches.json | 5 +++++ .../paseo-v025-worktree-lifecycle-current-main.json | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index 16187811d5..44535ae094 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -47,6 +47,7 @@ "existing-workspace agent and terminal attachment are fenced against archive", "MCP worktree creation releases setup reservations on every pre-agent failure", "archive failures propagate through command, websocket, MCP, and normal session surfaces", + "workspace archive by unresolved path fails instead of reporting a successful no-op", "normal archive responses require the persisted archived timestamp", "failed physical cleanup remains retryable after the workspace record is archived", "replacement worktree incarnations archive independently of stale cleanup intents", @@ -78,6 +79,10 @@ "formatDiffAndManifest": "passed" } }, + "currentMainIntegration": { + "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", + "receipt": "patches/receipts/paseo-v025-worktree-lifecycle-current-main.json" + }, "upstreamable": true, "upstreamCondition": "Submit independently while worktree setup, ownership attachment, archive, cleanup retry, and auto-archive are not coordinated by one durable lifecycle boundary.", "dropCondition": "Remove when the latest stable release provides equivalent setup and ownership fencing, explicit archive failure propagation, incarnation-safe retryable cleanup, bounded setup output, and retryable observable auto-archive semantics.", diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index 2181828742..483c72e7e1 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,12 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", + "headCommit": "551bdd371d82171177387e32ec5ca0278135050b", + "headTree": "7f0eda65ef47fafa3e1adacb08caa8d1000bcd47", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", - "completedAt": "2026-08-01T00:39:55Z", + "commit": "551bdd371d82171177387e32ec5ca0278135050b", + "completedAt": "2026-08-01T00:42:37Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -26,11 +28,12 @@ "npm run typecheck --workspace packages/server", "npm run typecheck --workspace packages/protocol", "npm run lint -- --deny-warnings", + "cd packages/server && npx vitest run src/server/worktree-session.test.ts", "npx oxfmt --check ", "git diff --cached --check -- packages/server/src/utils/worktree-metadata.ts" ], "result": { - "focusedAndBroadServerTests": "526 passed, 6 skipped", + "focusedAndBroadServerTests": "526 passed, 6 skipped; final archive-response suite 31 passed", "protocolTests": "470 passed", "serverBuild": "passed", "serverAndProtocolTypechecks": "passed", From d5743f9c37765b118d8ae78a2ae6c853ed796b65 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:22:38 -0400 Subject: [PATCH 13/43] fix: close workspace lifecycle ownership gaps --- .../server/agent/create-agent/create.test.ts | 32 +++++++ .../src/server/agent/create-agent/create.ts | 24 ++--- .../src/server/agent/mcp-server.test.ts | 32 +++++++ .../src/server/agent/tools/paseo-tools.ts | 42 ++++++--- .../archive-if-safe.test.ts | 45 ++++++++++ .../auto-archive-on-merge/archive-if-safe.ts | 3 + packages/server/src/server/bootstrap.ts | 7 ++ packages/server/src/server/session.ts | 89 +++++++++++-------- .../src/server/session.workspaces.test.ts | 4 + .../server/workspace-archive-service.test.ts | 87 ++++++++++++++++++ .../src/server/workspace-archive-service.ts | 24 ++--- .../server/workspace-lifecycle-coordinator.ts | 17 ++++ .../src/server/worktree-session.test.ts | 52 +++++++++++ .../server/src/server/worktree/commands.ts | 11 +++ .../terminal-session-controller.test.ts | 33 ++++++- .../terminal/terminal-session-controller.ts | 36 ++++++-- 16 files changed, 452 insertions(+), 86 deletions(-) diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index ea515d4a3f..10d3048eb2 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -333,6 +333,38 @@ test("agent creation holds workspace ownership until the agent is attached", asy archiveReservation.release(); }); +test("agent creation rejects an ownership attach after workspace archive", async () => { + const createAgent = vi.fn(); + const requireActiveWorkspaceForOwnership = vi.fn(async () => { + throw new Error("Workspace not found: ws-archived"); + }); + + await expect( + createAgentCommand( + { + agentManager: { createAgent } as unknown as AgentManager, + agentStorage: {} as AgentStorage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + lifecycleCoordinator: new WorkspaceLifecycleCoordinator(), + requireActiveWorkspaceForOwnership, + }, + { + kind: "session", + config: { provider: "codex", cwd: "/tmp/paseo-create-archived" }, + workspaceId: "ws-archived", + labels: {}, + provisionalTitle: null, + firstAgentContext: {}, + buildSessionConfig: async (config) => ({ sessionConfig: config }), + }, + ), + ).rejects.toThrow("Workspace not found: ws-archived"); + + expect(requireActiveWorkspaceForOwnership).toHaveBeenCalledWith("ws-archived"); + expect(createAgent).not.toHaveBeenCalled(); +}); + test("session create applies the resolved mode from the provider create config", async () => { const snapshot = { id: "agent-1", diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index 13c4e927fc..2094148c33 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -51,6 +51,7 @@ export interface CreateAgentCommandDependencies { // Mints a fresh directory workspace for a cwd and returns its id. ensureWorkspaceForCreate?: EnsureWorkspaceForCreate; lifecycleCoordinator?: WorkspaceLifecycleCoordinator; + requireActiveWorkspaceForOwnership?: (workspaceId: string) => Promise; } export type EnsureWorkspaceForCreate = ( @@ -187,26 +188,19 @@ export async function createAgentCommand( let snapshot: ManagedAgent; const lifecycleCoordinator = dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; - let ownershipReservation; + const workspaceId = requireResolvedWorkspaceId(resolved.createOptions.workspaceId); try { - ownershipReservation = lifecycleCoordinator.reserveWorkspaceOwnershipMutation( - requireResolvedWorkspaceId(resolved.createOptions.workspaceId), - ); - } catch (error) { - resolved.setupContinuation?.releaseWithoutStarting(); - throw error; - } - try { - snapshot = await dependencies.agentManager.createAgent( - resolved.config, - undefined, - resolved.createOptions, + snapshot = await lifecycleCoordinator.runWorkspaceOwnershipMutation( + workspaceId, + async () => { + await dependencies.requireActiveWorkspaceForOwnership?.(workspaceId); + }, + () => + dependencies.agentManager.createAgent(resolved.config, undefined, resolved.createOptions), ); } catch (error) { resolved.setupContinuation?.releaseWithoutStarting(); throw error; - } finally { - ownershipReservation.release(); } resolved.setupContinuation?.startAfterAgentCreate({ diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 8ff3d05045..026f2e119c 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -297,6 +297,13 @@ it("holds workspace ownership while an MCP terminal is being attached", async () }), providerSnapshotManager: createProviderSnapshotManagerStub().manager, ensureWorkspaceForCreate: async () => "ws-terminal-owned", + listActiveWorkspaces: async () => [ + { + workspaceId: "ws-terminal-owned", + cwd: "/tmp/terminal-owned", + kind: "local_checkout", + }, + ], lifecycleCoordinator, logger: createTestLogger(), }); @@ -324,6 +331,31 @@ it("holds workspace ownership while an MCP terminal is being attached", async () archiveReservation.release(); }); +it("rejects an MCP terminal attach after workspace archive", async () => { + const { agentManager, agentStorage } = createTestDeps(); + const createTerminal = vi.fn(); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + terminalManager: createTerminalManagerStub({ + createTerminal: createTerminal as unknown as TerminalManager["createTerminal"], + }), + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + ensureWorkspaceForCreate: async () => "ws-terminal-archived", + listActiveWorkspaces: async () => [], + lifecycleCoordinator: new WorkspaceLifecycleCoordinator(), + logger: createTestLogger(), + }); + + await expect( + registeredTool(server, "create_terminal").handler({ + cwd: "/tmp/terminal-archived", + name: "Archived terminal", + }), + ).rejects.toThrow("Workspace not found: ws-terminal-archived"); + expect(createTerminal).not.toHaveBeenCalled(); +}); + type ProviderSnapshotManagerStub = ReturnType; interface ConfigureProviderEntry { diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index 5cd45fa3b4..4e4cd4466a 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -148,6 +148,21 @@ export interface PaseoToolHostDependencies { logger: Logger; } +async function requireActiveWorkspaceForOwnership( + options: Pick, + workspaceId: string, +): Promise { + if (!options.listActiveWorkspaces && !options.workspaceRegistry) return; + const workspace = options.listActiveWorkspaces + ? (await options.listActiveWorkspaces()).find( + (candidate) => candidate.workspaceId === workspaceId, + ) + : await options.workspaceRegistry?.get(workspaceId); + if (!workspace || ("archivedAt" in workspace && workspace.archivedAt)) { + throw new Error(`Workspace not found: ${workspaceId}`); + } +} + function parseTimestamp(value: string | null | undefined): number { if (!value) { return 0; @@ -1447,6 +1462,8 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase providerSnapshotManager, lifecycleCoordinator: options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator, + requireActiveWorkspaceForOwnership: (workspaceId) => + requireActiveWorkspaceForOwnership(options, workspaceId), createPaseoWorktree: options.createPaseoWorktree, ...(options.ensureWorkspaceForCreate ? { ensureWorkspaceForCreate: options.ensureWorkspaceForCreate } @@ -2382,19 +2399,18 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase const resolvedCwd = resolveScopedCwd(cwd, { required: true }); const workspaceId = await resolveTerminalWorkspaceId(resolvedCwd); - const ownershipReservation = ( - options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator - ).reserveWorkspaceOwnershipMutation(workspaceId); - let terminal; - try { - terminal = await terminalManager.createTerminal({ - cwd: resolvedCwd, - workspaceId, - ...(name?.trim() ? { name: name.trim() } : {}), - }); - } finally { - ownershipReservation.release(); - } + const lifecycleCoordinator = + options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const terminal = await lifecycleCoordinator.runWorkspaceOwnershipMutation( + workspaceId, + () => requireActiveWorkspaceForOwnership(options, workspaceId), + () => + terminalManager.createTerminal({ + cwd: resolvedCwd, + workspaceId, + ...(name?.trim() ? { name: name.trim() } : {}), + }), + ); return { content: [], diff --git a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts index 9f83c1ce41..2505052706 100644 --- a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts +++ b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts @@ -16,6 +16,7 @@ import type { WorkspaceGitRuntimeSnapshot } from "../workspace-git-service.js"; import { createWorktree, type WorktreeConfig } from "../../utils/worktree.js"; import type { ForgeService } from "../../../services/forge-service.js"; import type { StoredAgentRecord } from "../agent/agent-storage.js"; +import { createPersistedWorkspaceRecord } from "../workspace-registry.js"; const CWD = "/tmp/paseo/worktrees/repo/branch"; const PASEO_HOME = "/tmp/paseo"; @@ -103,6 +104,11 @@ function createHarness(overrides?: { agentManager: {} as AutoArchiveArchiveOptions["agentManager"], agentStorage: {} as AutoArchiveArchiveOptions["agentStorage"], terminalManager: {} as AutoArchiveArchiveOptions["terminalManager"], + workspaceRegistry: { + get: vi.fn(async () => null), + list: vi.fn(async () => []), + update: vi.fn(async () => null), + }, findWorkspaceIdForCwd: vi.fn(async () => "ws-auto-archive"), listActiveWorkspaces: vi.fn(async () => []), archiveWorkspaceRecord: vi.fn(), @@ -259,6 +265,26 @@ function createRealOutcomeHarness(input: { archivedWorkspaceIds: Set; }) { const active = [...input.activeWorkspaces]; + const now = new Date().toISOString(); + const workspaceRecords = new Map( + active.map((workspace) => [ + workspace.workspaceId, + createPersistedWorkspaceRecord({ + workspaceId: workspace.workspaceId, + projectId: "project-auto-archive", + cwd: workspace.cwd, + kind: workspace.kind, + displayName: workspace.workspaceId, + ...(workspace.worktreeRoot ? { worktreeRoot: workspace.worktreeRoot } : {}), + ...(workspace.isPaseoOwnedWorktree !== undefined + ? { isPaseoOwnedWorktree: workspace.isPaseoOwnedWorktree } + : {}), + ...(workspace.mainRepoRoot ? { mainRepoRoot: workspace.mainRepoRoot } : {}), + createdAt: now, + updatedAt: now, + }), + ]), + ); const logger = pino({ level: "silent" }); vi.spyOn(logger, "info").mockImplementation(() => undefined); vi.spyOn(logger, "warn").mockImplementation(() => undefined); @@ -310,6 +336,17 @@ function createRealOutcomeHarness(input: { listDirectories: () => [], getTerminals: vi.fn().mockResolvedValue([]), } as unknown as AutoArchiveArchiveOptions["terminalManager"], + workspaceRegistry: { + get: async (workspaceId) => workspaceRecords.get(workspaceId) ?? null, + list: async () => Array.from(workspaceRecords.values()), + update: async (workspaceId, updater) => { + const existing = workspaceRecords.get(workspaceId); + if (!existing) return null; + const updated = updater(existing); + workspaceRecords.set(workspaceId, updated); + return updated; + }, + }, findWorkspaceIdForCwd: async (cwd: string) => { const match = active.find((workspace) => workspace.cwd === cwd); return match?.workspaceId ?? null; @@ -318,6 +355,13 @@ function createRealOutcomeHarness(input: { active.filter((workspace) => !input.archivedWorkspaceIds.has(workspace.workspaceId)), archiveWorkspaceRecord: async (workspaceId: string) => { input.archivedWorkspaceIds.add(workspaceId); + const existing = workspaceRecords.get(workspaceId); + if (existing) { + workspaceRecords.set(workspaceId, { + ...existing, + archivedAt: new Date().toISOString(), + }); + } const index = active.findIndex((workspace) => workspace.workspaceId === workspaceId); if (index !== -1) { active.splice(index, 1); @@ -483,6 +527,7 @@ describe("archiveIfSafe", () => { expect.objectContaining({ paseoHome: PASEO_HOME, workspaceGitService: harness.options.workspaceGitService, + workspaceRegistry: harness.options.workspaceRegistry, }), { scope: { kind: "workspace", workspaceId: "ws-auto-archive" }, diff --git a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts index 126f615bc8..077e81a721 100644 --- a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts +++ b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts @@ -15,6 +15,7 @@ import type { } from "../workspace-git-service.js"; import type { ForgeService } from "../../services/forge-service.js"; import type { TerminalManager } from "../../terminal/terminal-manager.js"; +import type { WorkspaceRegistry } from "../workspace-registry.js"; import { isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; export interface AutoArchiveArchiveOptions { @@ -26,6 +27,7 @@ export interface AutoArchiveArchiveOptions { agentManager: AgentManager; agentStorage: AgentStorage; terminalManager: TerminalManager; + workspaceRegistry: Pick; findWorkspaceIdForCwd: (cwd: string) => Promise; listActiveWorkspaces: () => Promise; archiveWorkspaceRecord: (workspaceId: string) => Promise; @@ -123,6 +125,7 @@ export async function archiveIfSafe(input: { findWorkspaceIdForCwd: options.findWorkspaceIdForCwd, listActiveWorkspaces: options.listActiveWorkspaces, archiveWorkspaceRecord: options.archiveWorkspaceRecord, + workspaceRegistry: options.workspaceRegistry, emitWorkspaceUpdatesForWorkspaceIds: options.emitWorkspaceUpdatesForWorkspaceIds, markWorkspaceArchiving: options.markWorkspaceArchiving, clearWorkspaceArchiving: options.clearWorkspaceArchiving, diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 85620f52fd..a08ea0552e 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -980,6 +980,7 @@ export async function createPaseoDaemon( agentManager, agentStorage, terminalManager, + workspaceRegistry, logger, findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, listActiveWorkspaces: listActiveWorkspacesExternal, @@ -1048,6 +1049,12 @@ export async function createPaseoDaemon( providerSnapshotManager, createPaseoWorktree: createPaseoWorktreeForTools, ensureWorkspaceForCreate: ensureWorkspaceForCreateAndBroadcastExternal, + requireActiveWorkspaceForOwnership: async (workspaceId) => { + const workspace = await workspaceRegistry.get(workspaceId); + if (!workspace || workspace.archivedAt) { + throw new Error(`Workspace not found: ${workspaceId}`); + } + }, }; const createAgent = (input: Parameters[1]) => createAgentCommand(createAgentCommandDependencies, input); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index a1a853cd94..1f9fd783f0 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -167,7 +167,6 @@ import { DownloadTokenStore } from "./file-download/token-store.js"; import { PushTokenStore } from "./push/token-store.js"; import { archivePersistedWorkspaceRecord, - archiveWorkspaceContents, requireActiveWorkspaceForArchive, } from "./workspace-archive-service.js"; import type { ServiceProxySubsystem } from "./service-proxy.js"; @@ -896,6 +895,7 @@ export class Session { isPathWithinRoot: (rootPath, candidatePath) => this.isPathWithinRoot(rootPath, candidatePath), sessionLogger: this.sessionLogger, listTerminalWorkspaceRefs: () => this.listActiveWorkspaceRefs(), + lifecycleCoordinator: defaultWorkspaceLifecycleCoordinator, clientSupportsWrapReflow: () => this.clientCapabilities.has(CLIENT_CAPS.terminalReflowableSnapshot), getClientBufferedAmount: () => this.getTransportBufferedAmount(), @@ -2744,48 +2744,55 @@ export class Session { const projectWorkspaces = (await this.workspaceRegistry.list()).filter( (workspace) => workspace.projectId === resolvedProjectId, ); - const activeWorkspaceIds = projectWorkspaces - .filter((workspace) => !workspace.archivedAt) + const workspaceIdsToArchive = projectWorkspaces + .filter((workspace) => !workspace.archivedAt || workspace.cleanupPending) .map((workspace) => workspace.workspaceId); - if (activeWorkspaceIds.length > 0) { - this.markWorkspaceArchiving(activeWorkspaceIds, new Date().toISOString()); - await this.emitWorkspaceUpdatesForWorkspaceIds(activeWorkspaceIds); - } - const removedWorkspaceIds: string[] = []; - try { - for (const workspaceId of activeWorkspaceIds) { - await archiveWorkspaceContents( - { - agentManager: this.agentManager, - agentStorage: this.agentStorage, - killTerminalsForWorkspace: (id) => - this.terminalController.killTerminalsForWorkspace(id), - sessionLogger: this.sessionLogger, - }, - workspaceId, - ); - await this.archiveWorkspaceRecord(workspaceId); - removedWorkspaceIds.push(workspaceId); - } - - await this.projectRegistry.remove(resolvedProjectId); - await removeProjectCustomIcon({ - paseoHome: this.paseoHome, - projectId: resolvedProjectId, - }).catch((error) => { - this.sessionLogger.warn( - { err: error, projectId: resolvedProjectId }, - "Failed to clean up removed project icon", - ); - }); - } finally { - if (activeWorkspaceIds.length > 0) { - this.clearWorkspaceArchiving(activeWorkspaceIds); - } + for (const workspaceId of workspaceIdsToArchive) { + await archiveByScope( + { + paseoHome: this.paseoHome, + paseoWorktreesBaseRoot: this.worktreesRoot, + github: this.github, + workspaceGitService: this.workspaceGitService, + agentManager: this.agentManager, + agentStorage: this.agentStorage, + findWorkspaceIdForCwd: (cwd) => this.findWorkspaceIdForCwd(cwd), + listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), + archiveWorkspaceRecord: (id) => this.archiveWorkspaceRecord(id), + workspaceRegistry: this.workspaceRegistry, + emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds) => + this.emitWorkspaceUpdatesForWorkspaceIds(workspaceIds), + markWorkspaceArchiving: (workspaceIds, archivingAt) => + this.markWorkspaceArchiving(workspaceIds, archivingAt), + clearWorkspaceArchiving: (workspaceIds) => this.clearWorkspaceArchiving(workspaceIds), + killTerminalsForWorkspace: (id) => + this.terminalController.killTerminalsForWorkspace(id), + sessionLogger: this.sessionLogger, + }, + { + scope: { kind: "workspace", workspaceId }, + requestId, + }, + ); + removedWorkspaceIds.push(workspaceId); } + await this.projectRegistry.remove(resolvedProjectId); + await removeProjectCustomIcon({ + paseoHome: this.paseoHome, + projectId: resolvedProjectId, + }).catch((error) => { + this.sessionLogger.warn( + { + err: error, + projectId: resolvedProjectId, + }, + "Failed to clean up removed project icon", + ); + }); + const updateIds = removedWorkspaceIds.length > 0 ? removedWorkspaceIds @@ -3116,6 +3123,12 @@ export class Session { worktreesRoot: this.worktreesRoot, providerSnapshotManager: this.providerSnapshotManager, lifecycleCoordinator: defaultWorkspaceLifecycleCoordinator, + requireActiveWorkspaceForOwnership: async (workspaceId) => { + const workspace = await this.workspaceRegistry.get(workspaceId); + if (!workspace || workspace.archivedAt) { + throw new Error(`Workspace not found: ${workspaceId}`); + } + }, }, { kind: "session", diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index 81aa626e58..603e1bd19b 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -74,6 +74,7 @@ import { type PersistedWorkspaceRecord, type WorkspaceMutation, } from "./workspace-registry.js"; +import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; const REPO_CWD = path.resolve("/tmp/repo"); const UNREGISTERED_CWD = path.resolve("/tmp/unregistered"); @@ -3449,6 +3450,9 @@ test("project.remove.request archives active workspaces and removes the project workspaces.get(workspaceId) ?? null; session.workspaceRegistry.list = async () => Array.from(workspaces.values()); session.workspaceRegistry.archive = async (workspaceId: string, archivedAt: string) => { + expect(() => + defaultWorkspaceLifecycleCoordinator.reserveWorkspaceOwnershipMutation(workspaceId), + ).toThrow(`Workspace ${workspaceId} is being archived`); const existing = workspaces.get(workspaceId); if (!existing) return; workspaces.set(workspaceId, { ...existing, updatedAt: archivedAt, archivedAt }); diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 4019fe9bb4..ae20b1d100 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -1008,6 +1008,93 @@ describe("archiveByScope", () => { expect((await registry.get(workspaceId))?.cleanupPending).toBeNull(); }); + test("a partial worktree retry preserves every sibling teardown obligation", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "partial-sibling-retry"); + const nestedCwd = path.join(worktree.worktreePath, "packages", "app"); + mkdirSync(nestedCwd, { recursive: true }); + const workspaceA = "ws-partial-retry-a"; + const workspaceB = "ws-partial-retry-b"; + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + for (const [workspaceId, cwd] of [ + [workspaceA, worktree.worktreePath], + [workspaceB, nestedCwd], + ] as const) { + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId, + projectId: "project-partial-retry", + cwd, + kind: "worktree", + displayName: workspaceId, + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + } + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId: workspaceA, + cwd: worktree.worktreePath, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + { + workspaceId: workspaceB, + cwd: nestedCwd, + kind: "worktree", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + ], + }); + deps.workspaceRegistry = registry; + const archiveActiveRecord = deps.archiveWorkspaceRecord; + let rejectWorkspaceBOnce = true; + deps.archiveWorkspaceRecord = async (workspaceId: string) => { + if (workspaceId === workspaceB && rejectWorkspaceBOnce) { + rejectWorkspaceBOnce = false; + throw new Error("registry archive failed once"); + } + await archiveActiveRecord(workspaceId); + await registry.archive(workspaceId, new Date().toISOString()); + }; + + await expect( + archiveByScope(deps, { + scope: { kind: "worktree", targetPath: worktree.worktreePath }, + requestId: "req-partial-retry-first", + }), + ).rejects.toThrow("Failed to archive one or more workspaces"); + expect((await registry.get(workspaceA))?.cleanupPending).not.toBeNull(); + expect((await registry.get(workspaceB))?.archivedAt).toBeNull(); + expect(existsSync(worktree.worktreePath)).toBe(true); + + const retry = await archiveByScope(deps, { + scope: { kind: "worktree", targetPath: worktree.worktreePath }, + requestId: "req-partial-retry-second", + }); + + expect(retry.removedDirectory).toBe(true); + expect((await registry.get(workspaceA))?.cleanupPending).toBeNull(); + expect((await registry.get(workspaceB))?.cleanupPending).toBeNull(); + expect(existsSync(worktree.worktreePath)).toBe(false); + }); + test("workspace teardown failure keeps the record active and prevents recursive deletion", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 176f473025..1b79e85fa0 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -327,21 +327,23 @@ async function resolveWorktreeArchiveTarget( (workspace) => workspace.mainRepoRoot, )?.mainRepoRoot; let teardownTargets: ArchiveTarget["teardownTargets"]; - if (targetWorkspaces.length > 0) { + if (targetWorkspaces.length > 0 || (pendingRecords && pendingRecords.length > 0)) { + const activeWorkspaceIds = new Set(targetWorkspaces.map((workspace) => workspace.workspaceId)); teardownTargets = targetWorkspaces.map((workspace) => ({ workspaceId: workspace.workspaceId, cwd: workspace.cwd, })); - } else if (pendingRecords && pendingRecords.length > 0) { - teardownTargets = pendingRecords.flatMap((workspace) => { - if (!workspace.cleanupPending) return []; - return [ - { - workspaceId: workspace.workspaceId, - cwd: workspace.cleanupPending.teardownCwd, - }, - ]; - }); + teardownTargets.push( + ...(pendingRecords ?? []).flatMap((workspace) => { + if (!workspace.cleanupPending || activeWorkspaceIds.has(workspace.workspaceId)) return []; + return [ + { + workspaceId: workspace.workspaceId, + cwd: workspace.cleanupPending.teardownCwd, + }, + ]; + }), + ); } else { teardownTargets = [{ workspaceId: null, cwd: targetPath }]; } diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts index fe6f6a0733..a12da10291 100644 --- a/packages/server/src/server/workspace-lifecycle-coordinator.ts +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -84,6 +84,23 @@ export class WorkspaceLifecycleCoordinator { return { release }; } + async runWorkspaceOwnershipMutation( + workspaceId: string, + requireActiveWorkspace: () => Promise, + operation: () => Promise, + ): Promise { + const reservation = this.reserveWorkspaceOwnershipMutation(workspaceId); + try { + // Validate after acquiring the reservation. An archive either already owns + // the workspace (and reservation fails), or must now wait for this check and + // mutation to finish. This also rejects attaches after archive completion. + await requireActiveWorkspace(); + return await operation(); + } finally { + reservation.release(); + } + } + reserveWorkspaceArchive(workspaceIds: Iterable): WorkspaceArchiveReservation { const ownedWorkspaceIds = new Set(); const add = (additionalWorkspaceIds: Iterable) => { diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index b2beeae70d..e91b615618 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -2065,6 +2065,58 @@ describe("handlePaseoWorktreeArchiveRequest worktree scope", () => { }); }); + test("reports failure for an explicit unknown workspace id", async () => { + const { tempDir } = createGitRepo(); + cleanupPaths.push(tempDir); + const emitted: SessionOutboundMessage[] = []; + const archiveWorkspaceRecord = vi.fn(async () => {}); + + await handlePaseoWorktreeArchiveRequest( + { + paseoHome: path.join(tempDir, ".paseo"), + github: createGitHubServiceStub(), + workspaceGitService: { + getSnapshot: vi.fn(async () => null), + listWorktrees: vi.fn(async () => []), + }, + agentManager: { + listAgents: () => [], + archiveAgent: vi.fn(async () => ({ archivedAt: new Date().toISOString() })), + archiveSnapshot: vi.fn(async () => ({})), + }, + agentStorage: createAgentStorageStub(), + findWorkspaceIdForCwd: vi.fn(async () => null), + listActiveWorkspaces: vi.fn(async () => []), + archiveWorkspaceRecord, + emit: (message) => emitted.push(message), + emitWorkspaceUpdatesForWorkspaceIds: vi.fn(async () => {}), + markWorkspaceArchiving: vi.fn(), + clearWorkspaceArchiving: vi.fn(), + killTerminalsForWorkspace: vi.fn(async () => {}), + sessionLogger: createLogger(), + }, + { + type: "paseo_worktree_archive_request", + requestId: "req-explicit-unknown", + workspaceId: "ws-explicit-unknown", + scope: "workspace", + }, + ); + + expect(archiveWorkspaceRecord).not.toHaveBeenCalled(); + expect( + emitted.find((message) => message.type === "paseo_worktree_archive_response"), + ).toMatchObject({ + payload: { + success: false, + error: { + code: "UNKNOWN", + message: "Workspace not found: ws-explicit-unknown", + }, + }, + }); + }); + test("archives every active workspace on the directory and removes it", async () => { const { tempDir, repoDir } = createGitRepo(); cleanupPaths.push(tempDir); diff --git a/packages/server/src/server/worktree/commands.ts b/packages/server/src/server/worktree/commands.ts index 6bfc9c7168..4a5ad48d2a 100644 --- a/packages/server/src/server/worktree/commands.ts +++ b/packages/server/src/server/worktree/commands.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { getPaseoWorktreesRoot, isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; import { archiveByScope, + requireActiveWorkspaceForArchive, resolveWorkspaceIdAtPath, type ArchiveDependencies, type ArchiveScope, @@ -126,6 +127,16 @@ export async function archiveCommand( ): Promise { const scope = input.scope ?? "workspace"; if (scope === "workspace" && input.workspaceId) { + try { + await requireActiveWorkspaceForArchive(dependencies, input.workspaceId); + } catch { + return { + ok: false, + code: "UNKNOWN", + message: `Workspace not found: ${input.workspaceId}`, + removedAgents: [], + }; + } const result = await archiveByScope(dependencies, { scope: { kind: "workspace", workspaceId: input.workspaceId }, requestId: input.requestId, diff --git a/packages/server/src/terminal/terminal-session-controller.test.ts b/packages/server/src/terminal/terminal-session-controller.test.ts index 81c3a15d6c..4846dbe84e 100644 --- a/packages/server/src/terminal/terminal-session-controller.test.ts +++ b/packages/server/src/terminal/terminal-session-controller.test.ts @@ -297,7 +297,7 @@ describe("terminal-session-controller legacy terminal creation", () => { hasBinaryChannel: () => true, isPathWithinRoot: isSameOrDescendantPath, sessionLogger: createLogger(), - listTerminalWorkspaceRefs: async () => [], + listTerminalWorkspaceRefs: async () => [{ workspaceId: "ws-1", cwd: "/work/repo" }], }); await controller.dispatch({ @@ -312,6 +312,37 @@ describe("terminal-session-controller legacy terminal creation", () => { expect.objectContaining({ cwd: "/work/repo", workspaceId: "ws-1", rows: 55, cols: 136 }), ); }); + + test("rejects a desktop terminal attach after its workspace was archived", async () => { + const outboundMessages: SessionOutboundMessage[] = []; + const createTerminal = vi.fn(); + const controller = new TerminalSessionController({ + terminalManager: { createTerminal } as unknown as TerminalManager, + emit: (message) => outboundMessages.push(message), + emitBinary: vi.fn(), + hasBinaryChannel: () => true, + isPathWithinRoot: isSameOrDescendantPath, + sessionLogger: createLogger(), + listTerminalWorkspaceRefs: async () => [], + }); + + await controller.dispatch({ + type: "create_terminal_request", + cwd: "/work/repo", + workspaceId: "ws-archived", + requestId: "req-archived", + }); + + expect(createTerminal).not.toHaveBeenCalled(); + expect(outboundMessages).toContainEqual({ + type: "create_terminal_response", + payload: { + terminal: null, + error: "Workspace not found: ws-archived", + requestId: "req-archived", + }, + }); + }); }); async function flushMicrotasks(): Promise { diff --git a/packages/server/src/terminal/terminal-session-controller.ts b/packages/server/src/terminal/terminal-session-controller.ts index dd7d053d62..9b175dff7a 100644 --- a/packages/server/src/terminal/terminal-session-controller.ts +++ b/packages/server/src/terminal/terminal-session-controller.ts @@ -14,6 +14,10 @@ import type { UnsubscribeTerminalsRequest, } from "../server/messages.js"; import { killTerminalsForWorkspace as killWorkspaceTerminals } from "../server/workspace-archive-service.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "../server/workspace-lifecycle-coordinator.js"; import { TerminalStreamOpcode, decodeTerminalResizePayload, @@ -69,6 +73,7 @@ export interface TerminalSessionControllerOptions { isPathWithinRoot: (rootPath: string, candidatePath: string) => boolean; sessionLogger: pino.Logger; listTerminalWorkspaceRefs?: () => Promise; + lifecycleCoordinator?: WorkspaceLifecycleCoordinator; listTerminalWorkspaceRoots?: () => Promise; // Whether the connected client can reflow restored snapshots. When true the // daemon attaches per-row soft-wrap flags to snapshots; otherwise it omits them @@ -128,6 +133,7 @@ export class TerminalSessionController { private readonly sessionLogger: pino.Logger; private readonly listTerminalWorkspaceRefs: () => Promise; private readonly listTerminalWorkspaceRoots: () => Promise; + private readonly lifecycleCoordinator: WorkspaceLifecycleCoordinator; private readonly clientSupportsWrapReflow: () => boolean; private readonly getClientBufferedAmount: () => number | null; @@ -153,6 +159,8 @@ export class TerminalSessionController { this.isPathWithinRoot = options.isPathWithinRoot; this.sessionLogger = options.sessionLogger; this.listTerminalWorkspaceRefs = options.listTerminalWorkspaceRefs ?? (async () => []); + this.lifecycleCoordinator = + options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; this.listTerminalWorkspaceRoots = options.listTerminalWorkspaceRoots ?? (async () => (await this.listTerminalWorkspaceRefs()).map((workspace) => workspace.cwd)); @@ -543,15 +551,27 @@ export class TerminalSessionController { return; } - const session = await this.terminalManager.createTerminal({ - cwd: msg.cwd, + const session = await this.lifecycleCoordinator.runWorkspaceOwnershipMutation( workspaceId, - name: msg.name, - command: msg.command, - args: msg.args, - rows: msg.size?.rows, - cols: msg.size?.cols, - }); + async () => { + const workspace = (await this.listTerminalWorkspaceRefs()).find( + (candidate) => candidate.workspaceId === workspaceId, + ); + if (!workspace) { + throw new Error(`Workspace not found: ${workspaceId}`); + } + }, + () => + this.terminalManager!.createTerminal({ + cwd: msg.cwd, + workspaceId, + name: msg.name, + command: msg.command, + args: msg.args, + rows: msg.size?.rows, + cols: msg.size?.cols, + }), + ); this.ensureExitSubscription(session); this.emit({ type: "create_terminal_response", From d3530db862e96ecb7cd9c3410b772ba7423c1367 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:23:22 -0400 Subject: [PATCH 14/43] docs: record lifecycle ownership repair proof --- patches/local-patches.json | 8 +++++++ ...-v025-worktree-lifecycle-current-main.json | 21 ++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index 44535ae094..b701144568 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -18,6 +18,7 @@ "packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts", "packages/server/src/server/agent/create-agent/create.ts", "packages/server/src/server/agent/tools/paseo-tools.ts", + "packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts", "packages/server/src/server/bootstrap.ts", "packages/server/src/server/session.ts", "packages/server/src/server/workspace-archive-service.ts", @@ -26,6 +27,7 @@ "packages/server/src/server/worktree-bootstrap.ts", "packages/server/src/server/worktree-session.ts", "packages/server/src/server/worktree/commands.ts", + "packages/server/src/terminal/terminal-session-controller.ts", "packages/server/src/utils/worktree-metadata.ts", "packages/server/src/utils/worktree-setup-output.ts", "packages/server/src/utils/worktree.ts" @@ -34,22 +36,28 @@ "packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts", "packages/server/src/server/agent/create-agent/create.test.ts", "packages/server/src/server/agent/mcp-server.test.ts", + "packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts", "packages/server/src/server/session.workspace-git-watch.test.ts", "packages/server/src/server/session.workspaces.test.ts", "packages/server/src/server/workspace-archive-service.test.ts", "packages/server/src/server/worktree-bootstrap.test.ts", "packages/server/src/server/worktree-session.test.ts", + "packages/server/src/terminal/terminal-session-controller.test.ts", "packages/server/src/utils/worktree-setup-output.test.ts", "packages/server/src/utils/worktree.posix.test.ts" ], "testCases": [ "setup reservations prevent archive from outrunning worktree creation", "existing-workspace agent and terminal attachment are fenced against archive", + "agent, MCP terminal, and desktop terminal attachment reject archived workspace ownership", + "project removal uses the same lifecycle archive boundary as direct workspace removal", "MCP worktree creation releases setup reservations on every pre-agent failure", "archive failures propagate through command, websocket, MCP, and normal session surfaces", "workspace archive by unresolved path fails instead of reporting a successful no-op", "normal archive responses require the persisted archived timestamp", "failed physical cleanup remains retryable after the workspace record is archived", + "partial multi-workspace retries preserve every archived sibling teardown obligation", + "merge auto-archive receives the durable workspace cleanup registry", "replacement worktree incarnations archive independently of stale cleanup intents", "auto-archive retries after transient failure and cancellation observes rejection", "setup output is bounded while preserving the final diagnostic tail" diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index 483c72e7e1..45b48e54c2 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,14 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "551bdd371d82171177387e32ec5ca0278135050b", - "headTree": "7f0eda65ef47fafa3e1adacb08caa8d1000bcd47", + "headCommit": "d5743f9c37765b118d8ae78a2ae6c853ed796b65", + "headTree": "d0cb749501fc5ba10ace42c4dae0d3a05b12f227", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "551bdd371d82171177387e32ec5ca0278135050b", - "completedAt": "2026-08-01T00:42:37Z", + "commit": "d5743f9c37765b118d8ae78a2ae6c853ed796b65", + "completedAt": "2026-08-01T01:22:49Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -29,8 +29,12 @@ "npm run typecheck --workspace packages/protocol", "npm run lint -- --deny-warnings", "cd packages/server && npx vitest run src/server/worktree-session.test.ts", - "npx oxfmt --check ", - "git diff --cached --check -- packages/server/src/utils/worktree-metadata.ts" + "cd packages/server && vitest run src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/terminal/terminal-session-controller.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/session.workspaces.test.ts --fileParallelism=false", + "npm run build --workspace packages/server", + "npm run typecheck --workspace packages/server", + "oxlint <16 changed lifecycle source and test files>", + "oxfmt --check <16 changed lifecycle source and test files>", + "git diff --check" ], "result": { "focusedAndBroadServerTests": "526 passed, 6 skipped; final archive-response suite 31 passed", @@ -38,7 +42,10 @@ "serverBuild": "passed", "serverAndProtocolTypechecks": "passed", "lint": "0 warnings, 0 errors", - "lifecycleFormatAndConflictDiff": "passed" + "lifecycleFormatAndConflictDiff": "passed", + "ownershipAndRetryRepairTests": "317 passed, 4 skipped; final ownership subset 144 passed", + "repairServerBuildAndTypecheck": "passed", + "allWorkspaceTypecheck": "unchanged app errors: DaemonClient lacks setProjectIcon/getProjectIcon on this stable-derived branch" } } } From 48646138bdfed7f40cd9ab50397f99763ba91e68 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:28:53 -0400 Subject: [PATCH 15/43] fix: keep auto-archive outcomes truthful --- .../archive-if-safe.test.ts | 56 ++++++++++++++++++- .../auto-archive-on-merge/archive-if-safe.ts | 21 ++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts index 2505052706..73495093f7 100644 --- a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts +++ b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts @@ -110,7 +110,9 @@ function createHarness(overrides?: { update: vi.fn(async () => null), }, findWorkspaceIdForCwd: vi.fn(async () => "ws-auto-archive"), - listActiveWorkspaces: vi.fn(async () => []), + listActiveWorkspaces: vi.fn(async () => [ + { workspaceId: "ws-auto-archive", cwd: CWD, kind: "worktree" }, + ]), archiveWorkspaceRecord: vi.fn(), markWorkspaceArchiving: vi.fn(), clearWorkspaceArchiving: vi.fn(), @@ -123,6 +125,7 @@ function createHarness(overrides?: { archivedAgentIds: [], archivedWorkspaceIds: [], removedDirectory: false, + cleanupPendingWorkspaceIds: [], }) satisfies ArchiveResult), ) as unknown as ArchiveIfSafeDependencies["archiveByScope"]; const resolveWorkspaceIdAtPath = vi.fn( @@ -509,6 +512,57 @@ describe("archiveIfSafe", () => { expect(harness.inFlight.has(CWD)).toBe(false); }); + test("does not report success for an inactive resolved workspace without cleanup", async () => { + const harness = createHarness(); + const now = new Date().toISOString(); + const archived = createPersistedWorkspaceRecord({ + workspaceId: "ws-auto-archive", + projectId: "project-auto-archive", + cwd: CWD, + kind: "worktree", + displayName: "Archived", + createdAt: now, + updatedAt: now, + archivedAt: now, + }); + harness.options.listActiveWorkspaces = vi.fn(async () => []); + harness.options.workspaceRegistry.get = vi.fn(async () => archived); + + await runArchiveIfSafe(harness); + + expect(harness.deps.archiveByScope).not.toHaveBeenCalled(); + expect(harness.log.info).not.toHaveBeenCalledWith( + { cwd: CWD }, + "Auto-archived worktree after PR merge", + ); + expect(harness.log.warn).toHaveBeenCalledWith( + { cwd: CWD, workspaceId: "ws-auto-archive" }, + "Auto-archive resolved an inactive workspace without pending cleanup; skipping", + ); + }); + + test("does not report success while durable cleanup remains pending", async () => { + const harness = createHarness({ + archiveByScope: async () => ({ + archivedAgentIds: [], + archivedWorkspaceIds: ["ws-auto-archive"], + removedDirectory: false, + cleanupPendingWorkspaceIds: ["ws-auto-archive"], + }), + }); + + await runArchiveIfSafe(harness); + + expect(harness.log.info).not.toHaveBeenCalledWith( + { cwd: CWD }, + "Auto-archived worktree after PR merge", + ); + expect(harness.log.warn).toHaveBeenCalledWith( + { err: expect.any(Error), cwd: CWD }, + "Auto-archive after merge failed", + ); + }); + test("archives a clean Paseo-owned worktree after merge", async () => { const harness = createHarness(); diff --git a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts index 077e81a721..baf1f4cbb7 100644 --- a/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts +++ b/packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts @@ -114,7 +114,21 @@ export async function archiveIfSafe(input: { return; } - await deps.archiveByScope( + const activeWorkspace = (await options.listActiveWorkspaces()).find( + (workspace) => workspace.workspaceId === workspaceId, + ); + if (!activeWorkspace) { + const persistedWorkspace = await options.workspaceRegistry.get(workspaceId); + if (!persistedWorkspace?.cleanupPending) { + log.warn( + { cwd, workspaceId }, + "Auto-archive resolved an inactive workspace without pending cleanup; skipping", + ); + return; + } + } + + const result = await deps.archiveByScope( { paseoHome: options.paseoHome, paseoWorktreesBaseRoot: options.paseoWorktreesBaseRoot, @@ -144,6 +158,11 @@ export async function archiveIfSafe(input: { requestId: "auto-archive-on-merge", }, ); + if (result.cleanupPendingWorkspaceIds.length > 0) { + throw new Error( + `Auto-archive cleanup remains pending for: ${result.cleanupPendingWorkspaceIds.join(", ")}`, + ); + } log.info({ cwd }, "Auto-archived worktree after PR merge"); } catch (error) { log.warn({ err: error, cwd }, "Auto-archive after merge failed"); From bd90c39ea5214937dc2b4d6328780b3127d67db1 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:29:28 -0400 Subject: [PATCH 16/43] docs: record truthful auto-archive proof --- patches/local-patches.json | 1 + .../paseo-v025-worktree-lifecycle-current-main.json | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index b701144568..97c211c9ad 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -58,6 +58,7 @@ "failed physical cleanup remains retryable after the workspace record is archived", "partial multi-workspace retries preserve every archived sibling teardown obligation", "merge auto-archive receives the durable workspace cleanup registry", + "merge auto-archive rejects stale resolved workspaces and never reports pending cleanup as success", "replacement worktree incarnations archive independently of stale cleanup intents", "auto-archive retries after transient failure and cancellation observes rejection", "setup output is bounded while preserving the final diagnostic tail" diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index 45b48e54c2..ee8cd3d9e6 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,14 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "d5743f9c37765b118d8ae78a2ae6c853ed796b65", - "headTree": "d0cb749501fc5ba10ace42c4dae0d3a05b12f227", + "headCommit": "48646138bdfed7f40cd9ab50397f99763ba91e68", + "headTree": "a946790495412748cd629375ca4150a8b8acf739", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "d5743f9c37765b118d8ae78a2ae6c853ed796b65", - "completedAt": "2026-08-01T01:22:49Z", + "commit": "48646138bdfed7f40cd9ab50397f99763ba91e68", + "completedAt": "2026-08-01T01:28:59Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -30,6 +30,7 @@ "npm run lint -- --deny-warnings", "cd packages/server && npx vitest run src/server/worktree-session.test.ts", "cd packages/server && vitest run src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/terminal/terminal-session-controller.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/session.workspaces.test.ts --fileParallelism=false", + "cd packages/server && vitest run src/server/auto-archive-on-merge/archive-if-safe.test.ts --fileParallelism=false", "npm run build --workspace packages/server", "npm run typecheck --workspace packages/server", "oxlint <16 changed lifecycle source and test files>", @@ -45,6 +46,7 @@ "lifecycleFormatAndConflictDiff": "passed", "ownershipAndRetryRepairTests": "317 passed, 4 skipped; final ownership subset 144 passed", "repairServerBuildAndTypecheck": "passed", + "autoArchiveTruthfulnessTests": "16 passed", "allWorkspaceTypecheck": "unchanged app errors: DaemonClient lacks setProjectIcon/getProjectIcon on this stable-derived branch" } } From 2ee289672030ed1e85cc7e1078e5f3b8cf7e0a44 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:32:14 -0400 Subject: [PATCH 17/43] fix: close archive target ownership set --- .../server/workspace-archive-service.test.ts | 57 +++++++++++++++++++ .../src/server/workspace-archive-service.ts | 30 ++++++---- 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index ae20b1d100..d7a2546d25 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -280,6 +280,63 @@ describe("archiveByScope", () => { }); }); + test("waits for ownership on a sibling discovered during final target refresh", async () => { + const { tempDir } = createGitRepo(); + const workspaceCwd = path.join(tempDir, "shared-checkout"); + mkdirSync(workspaceCwd); + const workspaceA = "ws-refresh-closure-a"; + const workspaceB = "ws-refresh-closure-b"; + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + const ownershipReservation = lifecycleCoordinator.reserveWorkspaceOwnershipMutation(workspaceB); + const deps = createArchiveDeps({ + paseoHome: path.join(tempDir, ".paseo"), + activeWorkspaces: [{ workspaceId: workspaceA, cwd: workspaceCwd, kind: "local_checkout" }], + }); + deps.lifecycleCoordinator = lifecycleCoordinator; + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + const listActiveWorkspaces = deps.listActiveWorkspaces; + let activeReadCount = 0; + deps.listActiveWorkspaces = async () => { + activeReadCount += 1; + if (activeReadCount === 2) { + deps.activeWorkspaces.push({ + workspaceId: workspaceB, + cwd: workspaceCwd, + kind: "local_checkout", + }); + } + return listActiveWorkspaces(); + }; + let markLateOwnershipWaitStarted: (() => void) | undefined; + const lateOwnershipWaitStarted = new Promise((resolveWait) => { + markLateOwnershipWaitStarted = resolveWait; + }); + const waitForWorkspaceOwnershipMutations = + lifecycleCoordinator.waitForWorkspaceOwnershipMutations.bind(lifecycleCoordinator); + vi.spyOn(lifecycleCoordinator, "waitForWorkspaceOwnershipMutations").mockImplementation( + async (workspaceIds) => { + if (Array.from(workspaceIds).includes(workspaceB)) { + markLateOwnershipWaitStarted?.(); + } + await waitForWorkspaceOwnershipMutations(workspaceIds); + }, + ); + + const archiveTask = archiveByScope(deps, { + scope: { kind: "worktree", targetPath: workspaceCwd }, + requestId: "req-refresh-closure", + }); + + await lateOwnershipWaitStarted; + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + ownershipReservation.release(); + + await expect(archiveTask).resolves.toMatchObject({ + archivedWorkspaceIds: expect.arrayContaining([workspaceA, workspaceB]), + }); + expect(deps.archiveWorkspaceRecord).toHaveBeenCalledTimes(2); + }); + test("coalesces simultaneous archive requests for the same backing directory", async () => { const { tempDir } = createGitRepo(); const workspaceId = "ws-simultaneous"; diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 1b79e85fa0..44a00585a5 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -148,16 +148,26 @@ export async function archiveByScope( initialTarget.workspaceIds, ); const archiveOperation = async () => { - await lifecycleCoordinator.waitForWorkspaceSetups(initialTarget.workspaceIds); - await lifecycleCoordinator.waitForWorkspaceOwnershipMutations(initialTarget.workspaceIds); - let refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); - archiveReservation.add(refreshedTarget.workspaceIds); - await lifecycleCoordinator.waitForWorkspaceSetups(refreshedTarget.workspaceIds); - await lifecycleCoordinator.waitForWorkspaceOwnershipMutations(refreshedTarget.workspaceIds); - refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); - archiveReservation.add(refreshedTarget.workspaceIds); - const target = mergeArchiveTargets(initialTarget, refreshedTarget); - return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); + const closureWorkspaceIds = new Set(initialTarget.workspaceIds); + let target = initialTarget; + while (true) { + await lifecycleCoordinator.waitForWorkspaceSetups(closureWorkspaceIds); + await lifecycleCoordinator.waitForWorkspaceOwnershipMutations(closureWorkspaceIds); + + const refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); + archiveReservation.add(refreshedTarget.workspaceIds); + target = mergeArchiveTargets(target, refreshedTarget); + + const newlyDiscoveredWorkspaceIds = refreshedTarget.workspaceIds.filter( + (workspaceId) => !closureWorkspaceIds.has(workspaceId), + ); + if (newlyDiscoveredWorkspaceIds.length === 0) { + return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); + } + for (const workspaceId of newlyDiscoveredWorkspaceIds) { + closureWorkspaceIds.add(workspaceId); + } + } }; try { const mutationRoot = From 8834fd7f81c2b1d979362d11cceedd85cec5a08a Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:34:19 -0400 Subject: [PATCH 18/43] fix: preserve archive registry failures --- .../src/server/workspace-archive-service.ts | 9 ++- .../src/server/worktree-session.test.ts | 55 +++++++++++++++++++ .../server/src/server/worktree/commands.ts | 4 +- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 44a00585a5..31100d868a 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -79,6 +79,13 @@ export interface ArchiveResult { cleanupPendingWorkspaceIds: string[]; } +export class WorkspaceArchiveTargetNotFoundError extends Error { + constructor(workspaceId: string) { + super(`Workspace not found: ${workspaceId}`); + this.name = "WorkspaceArchiveTargetNotFoundError"; + } +} + export interface ArchiveByScopeRequest { scope: ArchiveScope; requestId: string; @@ -98,7 +105,7 @@ export async function requireActiveWorkspaceForArchive( if (persistedWorkspace?.cleanupPending) { return persistedWorkspace; } - throw new Error(`Workspace not found: ${workspaceId}`); + throw new WorkspaceArchiveTargetNotFoundError(workspaceId); } interface BackingDirectory { diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index e91b615618..8fb3c7f355 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -2117,6 +2117,61 @@ describe("handlePaseoWorktreeArchiveRequest worktree scope", () => { }); }); + test("preserves registry failures while validating an explicit workspace id", async () => { + const { tempDir } = createGitRepo(); + cleanupPaths.push(tempDir); + const emitted: SessionOutboundMessage[] = []; + const registryFailure = new Error("workspace registry read failed"); + + await handlePaseoWorktreeArchiveRequest( + { + paseoHome: path.join(tempDir, ".paseo"), + github: createGitHubServiceStub(), + workspaceGitService: { + getSnapshot: vi.fn(async () => null), + listWorktrees: vi.fn(async () => []), + }, + agentManager: { + listAgents: () => [], + archiveAgent: vi.fn(async () => ({ archivedAt: new Date().toISOString() })), + archiveSnapshot: vi.fn(async () => ({})), + }, + agentStorage: createAgentStorageStub(), + findWorkspaceIdForCwd: vi.fn(async () => null), + listActiveWorkspaces: vi.fn(async () => []), + workspaceRegistry: { + get: vi.fn(async () => { + throw registryFailure; + }), + list: vi.fn(async () => []), + update: vi.fn(async () => null), + }, + archiveWorkspaceRecord: vi.fn(async () => {}), + emit: (message) => emitted.push(message), + emitWorkspaceUpdatesForWorkspaceIds: vi.fn(async () => {}), + markWorkspaceArchiving: vi.fn(), + clearWorkspaceArchiving: vi.fn(), + killTerminalsForWorkspace: vi.fn(async () => {}), + sessionLogger: createLogger(), + }, + { + type: "paseo_worktree_archive_request", + requestId: "req-registry-failure", + workspaceId: "ws-registry-failure", + scope: "workspace", + }, + ); + + expect( + emitted.find((message) => message.type === "paseo_worktree_archive_response"), + ).toMatchObject({ + payload: { + success: false, + error: { message: "workspace registry read failed" }, + }, + }); + }); + test("archives every active workspace on the directory and removes it", async () => { const { tempDir, repoDir } = createGitRepo(); cleanupPaths.push(tempDir); diff --git a/packages/server/src/server/worktree/commands.ts b/packages/server/src/server/worktree/commands.ts index 4a5ad48d2a..e8f93ed4ff 100644 --- a/packages/server/src/server/worktree/commands.ts +++ b/packages/server/src/server/worktree/commands.ts @@ -5,6 +5,7 @@ import { archiveByScope, requireActiveWorkspaceForArchive, resolveWorkspaceIdAtPath, + WorkspaceArchiveTargetNotFoundError, type ArchiveDependencies, type ArchiveScope, } from "../workspace-archive-service.js"; @@ -129,7 +130,8 @@ export async function archiveCommand( if (scope === "workspace" && input.workspaceId) { try { await requireActiveWorkspaceForArchive(dependencies, input.workspaceId); - } catch { + } catch (error) { + if (!(error instanceof WorkspaceArchiveTargetNotFoundError)) throw error; return { ok: false, code: "UNKNOWN", From 7e9b8bf54fb2735a1ca5551a4362940fb01d2869 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:37:15 -0400 Subject: [PATCH 19/43] fix: keep automated cleanup retryable --- .../create-agent-lifecycle-dispatch.test.ts | 35 +++++++++++++ .../agent/create-agent-lifecycle-dispatch.ts | 9 +++- packages/server/src/server/bootstrap.ts | 52 +++++++++++-------- packages/server/src/server/session.ts | 7 ++- .../src/server/workspace-archive-service.ts | 20 +++++++ 5 files changed, 97 insertions(+), 26 deletions(-) diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index e3af01335c..1babbf2d04 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -6,6 +6,7 @@ import { CreateAgentLifecycleDispatch, registerAgentAutoArchive, } from "./create-agent-lifecycle-dispatch.js"; +import { requireArchiveCleanupComplete } from "../workspace-archive-service.js"; class AgentLifecycleEvents { private readonly listeners = new Set(); @@ -75,6 +76,40 @@ test("auto-archive retries a later terminal event after a transient failure", as expect(agents.listenerCount()).toBe(0); }); +test("auto-archive retries when physical workspace cleanup remains pending", async () => { + const agentId = "agent-auto-archive-cleanup-pending"; + const agents = new AgentLifecycleEvents(); + const archive = vi.fn(async () => { + const cleanupPendingWorkspaceIds = + archive.mock.calls.length === 1 ? ["ws-cleanup-pending"] : []; + requireArchiveCleanupComplete( + { + archivedAgentIds: [], + archivedWorkspaceIds: ["ws-cleanup-pending"], + removedDirectory: cleanupPendingWorkspaceIds.length === 0, + cleanupPendingWorkspaceIds, + }, + "Auto-created worktree archive", + ); + }); + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive, + }); + + agents.completeTurn(agentId); + await Promise.resolve(); + await Promise.resolve(); + expect(archive).toHaveBeenCalledTimes(1); + expect(agents.listenerCount()).toBe(1); + + agents.completeTurn(agentId); + await registration.cancel(); + expect(archive).toHaveBeenCalledTimes(2); + expect(agents.listenerCount()).toBe(0); +}); + test("auto-archive cancellation observes an in-flight archive rejection", async () => { const agentId = "agent-auto-archive-cancel-failure"; const agents = new AgentLifecycleEvents(); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 20b1039b6a..7a64da23eb 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -3,7 +3,11 @@ import type pino from "pino"; import type { ForgeService } from "../../services/forge-service.js"; import { isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; -import { archiveByScope, type ActiveWorkspaceRef } from "../workspace-archive-service.js"; +import { + archiveByScope, + requireArchiveCleanupComplete, + type ActiveWorkspaceRef, +} from "../workspace-archive-service.js"; import type { CreatePaseoWorktreeWorkflowFn, CreatePaseoWorktreeWorkflowResult, @@ -209,7 +213,7 @@ export class CreateAgentLifecycleDispatch { throw new Error("Auto-created worktree is not a Paseo-owned worktree"); } - await archiveByScope( + const archiveResult = await archiveByScope( { paseoHome: this.dependencies.paseoHome, paseoWorktreesBaseRoot: this.dependencies.worktreesRoot, @@ -232,6 +236,7 @@ export class CreateAgentLifecycleDispatch { requestId: randomUUID(), }, ); + requireArchiveCleanupComplete(archiveResult, "Auto-created worktree archive"); if (options.agentId) { this.dependencies.emitAgentRemove(options.agentId); diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index a08ea0552e..d12086c289 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -152,6 +152,7 @@ import { archiveByScope, archivePersistedWorkspaceRecord, killTerminalsForWorkspace, + requireArchiveCleanupComplete, type ActiveWorkspaceRef, } from "./workspace-archive-service.js"; import { setupAutoArchiveOnMerge } from "./auto-archive-on-merge/index.js"; @@ -1058,27 +1059,33 @@ export async function createPaseoDaemon( }; const createAgent = (input: Parameters[1]) => createAgentCommand(createAgentCommandDependencies, input); - const archiveWorkspaceByIdExternal = (workspaceId: string, requestId: string) => - archiveByScope( - { - paseoHome: config.paseoHome, - paseoWorktreesBaseRoot: config.worktreesRoot, - github, - workspaceGitService, - agentManager, - agentStorage, - findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, - listActiveWorkspaces: listActiveWorkspacesExternal, - archiveWorkspaceRecord: archiveWorkspaceRecordExternal, - emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, - markWorkspaceArchiving: markWorkspaceArchivingExternal, - clearWorkspaceArchiving: clearWorkspaceArchivingExternal, - killTerminalsForWorkspace: (workspaceIdToKill) => - killTerminalsForWorkspace({ terminalManager, sessionLogger: logger }, workspaceIdToKill), - workspaceRegistry, - sessionLogger: logger, - }, - { scope: { kind: "workspace", workspaceId }, requestId }, + const archiveWorkspaceByIdExternal = async (workspaceId: string, requestId: string) => + requireArchiveCleanupComplete( + await archiveByScope( + { + paseoHome: config.paseoHome, + paseoWorktreesBaseRoot: config.worktreesRoot, + github, + workspaceGitService, + agentManager, + agentStorage, + findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, + listActiveWorkspaces: listActiveWorkspacesExternal, + archiveWorkspaceRecord: archiveWorkspaceRecordExternal, + emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, + markWorkspaceArchiving: markWorkspaceArchivingExternal, + clearWorkspaceArchiving: clearWorkspaceArchivingExternal, + killTerminalsForWorkspace: (workspaceIdToKill) => + killTerminalsForWorkspace( + { terminalManager, sessionLogger: logger }, + workspaceIdToKill, + ), + workspaceRegistry, + sessionLogger: logger, + }, + { scope: { kind: "workspace", workspaceId }, requestId }, + ), + "Workspace archive", ); const hubAgentLifecycle = new CreateAgentLifecycleDispatch({ paseoHome: config.paseoHome, @@ -1169,7 +1176,7 @@ export async function createPaseoDaemon( return result; }; const archiveScheduleWorkspaceExternal = async (workspaceId: string) => { - await archiveByScope( + const archiveResult = await archiveByScope( { paseoHome: config.paseoHome, paseoWorktreesBaseRoot: config.worktreesRoot, @@ -1199,6 +1206,7 @@ export async function createPaseoDaemon( requestId: "schedule-run-finish", }, ); + requireArchiveCleanupComplete(archiveResult, "Schedule workspace archive"); }; const scheduleService = new ScheduleService({ paseoHome: config.paseoHome, diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 1f9fd783f0..7676e2f37c 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -167,6 +167,7 @@ import { DownloadTokenStore } from "./file-download/token-store.js"; import { PushTokenStore } from "./push/token-store.js"; import { archivePersistedWorkspaceRecord, + requireArchiveCleanupComplete, requireActiveWorkspaceForArchive, } from "./workspace-archive-service.js"; import type { ServiceProxySubsystem } from "./service-proxy.js"; @@ -2750,7 +2751,7 @@ export class Session { const removedWorkspaceIds: string[] = []; for (const workspaceId of workspaceIdsToArchive) { - await archiveByScope( + const archiveResult = await archiveByScope( { paseoHome: this.paseoHome, paseoWorktreesBaseRoot: this.worktreesRoot, @@ -2776,6 +2777,7 @@ export class Session { requestId, }, ); + requireArchiveCleanupComplete(archiveResult, "Project workspace archive"); removedWorkspaceIds.push(workspaceId); } @@ -5843,7 +5845,7 @@ export class Session { request.workspaceId, ); - await archiveByScope( + const archiveResult = await archiveByScope( { paseoHome: this.paseoHome, paseoWorktreesBaseRoot: this.worktreesRoot, @@ -5869,6 +5871,7 @@ export class Session { requestId: request.requestId, }, ); + requireArchiveCleanupComplete(archiveResult, "Workspace archive"); const archivedWorkspace = await this.workspaceRegistry.get(request.workspaceId); if (!archivedWorkspace?.archivedAt) { diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 31100d868a..c990587bc8 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -86,6 +86,26 @@ export class WorkspaceArchiveTargetNotFoundError extends Error { } } +export class WorkspaceCleanupPendingError extends Error { + readonly workspaceIds: readonly string[]; + + constructor(operation: string, workspaceIds: readonly string[]) { + super(`${operation} left cleanup pending for: ${workspaceIds.join(", ")}`); + this.name = "WorkspaceCleanupPendingError"; + this.workspaceIds = [...workspaceIds]; + } +} + +export function requireArchiveCleanupComplete( + result: ArchiveResult, + operation: string, +): ArchiveResult { + if (result.cleanupPendingWorkspaceIds.length > 0) { + throw new WorkspaceCleanupPendingError(operation, result.cleanupPendingWorkspaceIds); + } + return result; +} + export interface ArchiveByScopeRequest { scope: ArchiveScope; requestId: string; From 4a548641cc4087589e0abfbdb9625caab8b5b3da Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:38:15 -0400 Subject: [PATCH 20/43] docs: record complete lifecycle repair proof --- patches/local-patches.json | 3 +++ ...paseo-v025-worktree-lifecycle-current-main.json | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index 97c211c9ad..2aa08c152a 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -54,11 +54,14 @@ "MCP worktree creation releases setup reservations on every pre-agent failure", "archive failures propagate through command, websocket, MCP, and normal session surfaces", "workspace archive by unresolved path fails instead of reporting a successful no-op", + "workspace archive preserves registry read failures instead of rewriting them as not found", "normal archive responses require the persisted archived timestamp", "failed physical cleanup remains retryable after the workspace record is archived", "partial multi-workspace retries preserve every archived sibling teardown obligation", + "archive target discovery reaches closure and waits ownership for late-discovered siblings", "merge auto-archive receives the durable workspace cleanup registry", "merge auto-archive rejects stale resolved workspaces and never reports pending cleanup as success", + "created-worktree, schedule, project, and normal workspace automation reject pending physical cleanup for retry", "replacement worktree incarnations archive independently of stale cleanup intents", "auto-archive retries after transient failure and cancellation observes rejection", "setup output is bounded while preserving the final diagnostic tail" diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index ee8cd3d9e6..f1bff1ee8b 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,14 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "48646138bdfed7f40cd9ab50397f99763ba91e68", - "headTree": "a946790495412748cd629375ca4150a8b8acf739", + "headCommit": "7e9b8bf54fb2735a1ca5551a4362940fb01d2869", + "headTree": "ec2ee9a9d60b58aeb5f2c8a460ac1970e5c38d1f", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "48646138bdfed7f40cd9ab50397f99763ba91e68", - "completedAt": "2026-08-01T01:28:59Z", + "commit": "7e9b8bf54fb2735a1ca5551a4362940fb01d2869", + "completedAt": "2026-08-01T01:37:33Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -31,6 +31,9 @@ "cd packages/server && npx vitest run src/server/worktree-session.test.ts", "cd packages/server && vitest run src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/terminal/terminal-session-controller.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/session.workspaces.test.ts --fileParallelism=false", "cd packages/server && vitest run src/server/auto-archive-on-merge/archive-if-safe.test.ts --fileParallelism=false", + "cd packages/server && vitest run src/server/workspace-archive-service.test.ts --fileParallelism=false", + "cd packages/server && vitest run src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts --fileParallelism=false", + "cd packages/server && vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts src/server/session.workspaces.test.ts --fileParallelism=false", "npm run build --workspace packages/server", "npm run typecheck --workspace packages/server", "oxlint <16 changed lifecycle source and test files>", @@ -47,6 +50,9 @@ "ownershipAndRetryRepairTests": "317 passed, 4 skipped; final ownership subset 144 passed", "repairServerBuildAndTypecheck": "passed", "autoArchiveTruthfulnessTests": "16 passed", + "archiveTargetClosureTests": "26 passed; combined auto-archive and closure proof 42 passed", + "archiveFailureTruthfulnessTests": "75 passed", + "automatedCleanupRetryTests": "182 passed, 4 skipped", "allWorkspaceTypecheck": "unchanged app errors: DaemonClient lacks setProjectIcon/getProjectIcon on this stable-derived branch" } } From 48b53ce71d850f065a22fb1a66cca4f6d4ec66ab Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:57:19 -0400 Subject: [PATCH 21/43] fix: retry pending workspace cleanup autonomously --- .../create-agent-lifecycle-dispatch.test.ts | 20 +- .../agent/create-agent-lifecycle-dispatch.ts | 67 +++++-- packages/server/src/server/bootstrap.ts | 40 ++++ .../server/workspace-archive-service.test.ts | 100 ++++++++++ .../workspace-cleanup-retry-service.test.ts | 120 ++++++++++++ .../server/workspace-cleanup-retry-service.ts | 180 ++++++++++++++++++ 6 files changed, 496 insertions(+), 31 deletions(-) create mode 100644 packages/server/src/server/workspace-cleanup-retry-service.test.ts create mode 100644 packages/server/src/server/workspace-cleanup-retry-service.ts diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index 1babbf2d04..9902f955ed 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -51,7 +51,7 @@ test("auto-archive self-releases once and later cancellation waits harmlessly", expect(agents.listenerCount()).toBe(0); }); -test("auto-archive retries a later terminal event after a transient failure", async () => { +test("auto-archive retries autonomously after a transient failure", async () => { const agentId = "agent-auto-archive-retry"; const agents = new AgentLifecycleEvents(); const archive = vi @@ -62,15 +62,12 @@ test("auto-archive retries a later terminal event after a transient failure", as agentManager: agents, agentId, archive, + retryBaseMs: 0, + retryMaxMs: 0, }); agents.completeTurn(agentId); - await Promise.resolve(); - await Promise.resolve(); - expect(archive).toHaveBeenCalledTimes(1); - expect(agents.listenerCount()).toBe(1); - - agents.completeTurn(agentId); + await vi.waitFor(() => expect(archive).toHaveBeenCalledTimes(2)); await registration.cancel(); expect(archive).toHaveBeenCalledTimes(2); expect(agents.listenerCount()).toBe(0); @@ -96,15 +93,12 @@ test("auto-archive retries when physical workspace cleanup remains pending", asy agentManager: agents, agentId, archive, + retryBaseMs: 0, + retryMaxMs: 0, }); agents.completeTurn(agentId); - await Promise.resolve(); - await Promise.resolve(); - expect(archive).toHaveBeenCalledTimes(1); - expect(agents.listenerCount()).toBe(1); - - agents.completeTurn(agentId); + await vi.waitFor(() => expect(archive).toHaveBeenCalledTimes(2)); await registration.cancel(); expect(archive).toHaveBeenCalledTimes(2); expect(agents.listenerCount()).toBe(0); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 7a64da23eb..90b29e0f93 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -248,18 +248,66 @@ export function registerAgentAutoArchive(input: { agentManager: AgentLifecycleEvents; agentId: string; archive: () => Promise; + retryBaseMs?: number; + retryMaxMs?: number; }): LifecycleRegistration { let unsubscribe: (() => void) | null = null; let archiveTask: Promise | null = null; + let retryTimer: ReturnType | null = null; let lastArchiveFailure: unknown | null = null; + let consecutiveFailures = 0; + let canceled = false; + const retryBaseMs = input.retryBaseMs ?? 1_000; + const retryMaxMs = input.retryMaxMs ?? 60_000; const release = () => { if (!unsubscribe) return; const subscribed = unsubscribe; unsubscribe = null; subscribed(); }; + const scheduleRetry = () => { + if (canceled || retryTimer) return; + const delay = Math.min(retryMaxMs, retryBaseMs * 2 ** Math.max(0, consecutiveFailures - 1)); + retryTimer = setTimeout(() => { + retryTimer = null; + attemptArchive(); + }, delay); + (retryTimer as unknown as { unref?: () => void }).unref?.(); + }; + const attemptArchive = () => { + if (canceled || archiveTask) return; + const task = input.archive(); + archiveTask = task; + void task.then( + () => { + if (archiveTask !== task) return; + archiveTask = null; + lastArchiveFailure = null; + consecutiveFailures = 0; + if (retryTimer) { + clearTimeout(retryTimer); + retryTimer = null; + } + release(); + return undefined; + }, + (error: unknown) => { + if (archiveTask !== task) return; + archiveTask = null; + lastArchiveFailure = error; + consecutiveFailures += 1; + scheduleRetry(); + return undefined; + }, + ); + }; const registration: LifecycleRegistration = { async cancel() { + canceled = true; + if (retryTimer) { + clearTimeout(retryTimer); + retryTimer = null; + } release(); const pendingArchive = archiveTask; if (pendingArchive) { @@ -280,24 +328,7 @@ export function registerAgentAutoArchive(input: { ) { return; } - if (archiveTask) return; - const task = input.archive(); - archiveTask = task; - void task.then( - () => { - if (archiveTask !== task) return; - archiveTask = null; - lastArchiveFailure = null; - release(); - return undefined; - }, - (error: unknown) => { - if (archiveTask !== task) return; - archiveTask = null; - lastArchiveFailure = error; - return undefined; - }, - ); + attemptArchive(); }, { agentId: input.agentId, replayState: false }, ); diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index d12086c289..51fc39d943 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -155,6 +155,7 @@ import { requireArchiveCleanupComplete, type ActiveWorkspaceRef, } from "./workspace-archive-service.js"; +import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service.js"; import { setupAutoArchiveOnMerge } from "./auto-archive-on-merge/index.js"; import { wrapSessionMessage, type SessionOutboundMessage } from "./messages.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; @@ -1087,6 +1088,44 @@ export async function createPaseoDaemon( ), "Workspace archive", ); + const retryPendingWorktreeCleanupExternal = async (targetPath: string): Promise => { + requireArchiveCleanupComplete( + await archiveByScope( + { + paseoHome: config.paseoHome, + paseoWorktreesBaseRoot: config.worktreesRoot, + github, + workspaceGitService, + agentManager, + agentStorage, + findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, + listActiveWorkspaces: listActiveWorkspacesExternal, + archiveWorkspaceRecord: archiveWorkspaceRecordExternal, + emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal, + markWorkspaceArchiving: markWorkspaceArchivingExternal, + clearWorkspaceArchiving: clearWorkspaceArchivingExternal, + killTerminalsForWorkspace: (workspaceIdToKill) => + killTerminalsForWorkspace( + { terminalManager, sessionLogger: logger }, + workspaceIdToKill, + ), + workspaceRegistry, + sessionLogger: logger, + }, + { + scope: { kind: "worktree", targetPath }, + requestId: `cleanup-retry:${randomUUID()}`, + }, + ), + "Workspace cleanup retry", + ); + }; + const workspaceCleanupRetryService = new WorkspaceCleanupRetryService({ + workspaceRegistry, + retryWorktreeCleanup: retryPendingWorktreeCleanupExternal, + logger, + }); + await workspaceCleanupRetryService.start(); const hubAgentLifecycle = new CreateAgentLifecycleDispatch({ paseoHome: config.paseoHome, worktreesRoot: config.worktreesRoot, @@ -1619,6 +1658,7 @@ export async function createPaseoDaemon( const stop = async () => { await hubRelationships.stop(); + await workspaceCleanupRetryService.stop(); workspaceReconciliation.dispose(); scriptHealthMonitor.stop(); // Freeze both ingress and registration before taking the agent closure snapshot. diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index d7a2546d25..dcd5d7278b 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import type { ForgeService } from "../services/forge-service.js"; import { createRealpathAwarePathMatcher } from "../utils/path.js"; import { createWorktree, type WorktreeConfig } from "../utils/worktree.js"; +import { readPaseoWorktreeIncarnationId } from "../utils/worktree-metadata.js"; import type { ManagedAgent } from "./agent/agent-manager.js"; import type { AgentStorage, StoredAgentRecord } from "./agent/agent-storage.js"; import type { WorkspaceGitService } from "./workspace-git-service.js"; @@ -17,8 +18,10 @@ import { type ArchiveDependencies, type ArchiveResult, requireActiveWorkspaceForArchive, + requireArchiveCleanupComplete, resolveWorkspaceIdAtPath, } from "./workspace-archive-service.js"; +import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service.js"; import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { createPersistedWorkspaceRecord, @@ -1152,6 +1155,103 @@ describe("archiveByScope", () => { expect(existsSync(worktree.worktreePath)).toBe(false); }); + test("startup retry completes archived schedule cleanup without losing sibling teardown obligations", async () => { + const { tempDir, repoDir } = createGitRepo(); + const nestedRelative = path.join("packages", "app"); + const sourceNested = path.join(repoDir, nestedRelative); + mkdirSync(sourceNested, { recursive: true }); + writeFileSync( + path.join(repoDir, "paseo.json"), + JSON.stringify({ + worktree: { + teardown: [ + "node -e \"require('fs').writeFileSync(process.env.PASEO_SOURCE_CHECKOUT_PATH+'/retry-root.log','root')\"", + ], + }, + }), + ); + writeFileSync( + path.join(sourceNested, "paseo.json"), + JSON.stringify({ + worktree: { + teardown: [ + "node -e \"require('fs').writeFileSync(process.env.PASEO_SOURCE_CHECKOUT_PATH+'/retry-nested.log','nested')\"", + ], + }, + }), + ); + execFileSync("git", ["add", "."], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "retry teardown"], { + cwd: repoDir, + stdio: "pipe", + }); + + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "schedule-cleanup-retry"); + const nestedCwd = path.join(worktree.worktreePath, nestedRelative); + const incarnationId = readPaseoWorktreeIncarnationId(worktree.worktreePath); + expect(incarnationId).toBeTruthy(); + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + for (const [workspaceId, teardownCwd] of [ + ["ws-completed-schedule-root", worktree.worktreePath], + ["ws-completed-schedule-nested", nestedCwd], + ] as const) { + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId, + projectId: "project-completed-schedule-cleanup", + cwd: teardownCwd, + kind: "worktree", + displayName: workspaceId, + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + archivedAt: timestamp, + cleanupPending: { + directoryPath: worktree.worktreePath, + teardownCwd, + mainRepoRoot: repoDir, + paseoWorktreesRoot: null, + worktreeIncarnationId: incarnationId, + }, + }), + ); + } + + const deps = createArchiveDeps({ paseoHome, activeWorkspaces: [] }); + deps.workspaceRegistry = registry; + const service = new WorkspaceCleanupRetryService({ + workspaceRegistry: registry, + retryWorktreeCleanup: async (targetPath) => { + requireArchiveCleanupComplete( + await archiveByScope(deps, { + scope: { kind: "worktree", targetPath }, + requestId: "startup-schedule-cleanup-retry", + }), + "Startup schedule cleanup retry", + ); + }, + logger: createLogger(), + idlePollMs: 60_000, + }); + + await service.start(); + await service.stop(); + + expect(readFileSync(path.join(repoDir, "retry-root.log"), "utf8")).toBe("root"); + expect(readFileSync(path.join(repoDir, "retry-nested.log"), "utf8")).toBe("nested"); + expect(existsSync(worktree.worktreePath)).toBe(false); + expect((await registry.get("ws-completed-schedule-root"))?.cleanupPending).toBeNull(); + expect((await registry.get("ws-completed-schedule-nested"))?.cleanupPending).toBeNull(); + }); + test("workspace teardown failure keeps the record active and prevents recursive deletion", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); diff --git a/packages/server/src/server/workspace-cleanup-retry-service.test.ts b/packages/server/src/server/workspace-cleanup-retry-service.test.ts new file mode 100644 index 0000000000..0cb3107237 --- /dev/null +++ b/packages/server/src/server/workspace-cleanup-retry-service.test.ts @@ -0,0 +1,120 @@ +import { afterEach, expect, test, vi } from "vitest"; + +import { createTestLogger } from "../test-utils/test-logger.js"; +import { + findWorkspaceCleanupRetryTargets, + WorkspaceCleanupRetryService, +} from "./workspace-cleanup-retry-service.js"; +import { createPersistedWorkspaceRecord } from "./workspace-registry.js"; + +afterEach(() => { + vi.useRealTimers(); +}); + +function pendingWorkspace(input: { + workspaceId: string; + directoryPath: string; + incarnationId: string | null; + archived?: boolean; +}) { + const timestamp = "2026-07-31T00:00:00.000Z"; + return createPersistedWorkspaceRecord({ + workspaceId: input.workspaceId, + projectId: "project-cleanup-retry", + cwd: input.directoryPath, + kind: "worktree", + displayName: input.workspaceId, + createdAt: timestamp, + updatedAt: timestamp, + archivedAt: input.archived === false ? null : timestamp, + cleanupPending: { + directoryPath: input.directoryPath, + teardownCwd: input.directoryPath, + mainRepoRoot: "/repo", + paseoWorktreesRoot: "/worktrees", + worktreeIncarnationId: input.incarnationId, + }, + }); +} + +test("groups archived cleanup by directory and requires one verified incarnation", () => { + const targets = findWorkspaceCleanupRetryTargets([ + pendingWorkspace({ + workspaceId: "ws-a", + directoryPath: "/worktrees/shared", + incarnationId: "inc-shared", + }), + pendingWorkspace({ + workspaceId: "ws-b", + directoryPath: "/worktrees/shared", + incarnationId: "inc-shared", + }), + pendingWorkspace({ + workspaceId: "ws-legacy", + directoryPath: "/worktrees/legacy", + incarnationId: null, + }), + pendingWorkspace({ + workspaceId: "ws-conflict-a", + directoryPath: "/worktrees/conflict", + incarnationId: "inc-a", + }), + pendingWorkspace({ + workspaceId: "ws-conflict-b", + directoryPath: "/worktrees/conflict", + incarnationId: "inc-b", + }), + pendingWorkspace({ + workspaceId: "ws-active", + directoryPath: "/worktrees/active", + incarnationId: "inc-active", + archived: false, + }), + ]); + + expect(targets).toEqual([ + { + directoryPath: "/worktrees/shared", + worktreeIncarnationId: "inc-shared", + workspaceIds: ["ws-a", "ws-b"], + }, + ]); +}); + +test("bounds each retry cycle and cancels the scheduled backoff on stop", async () => { + vi.useFakeTimers(); + const workspaces = [ + pendingWorkspace({ + workspaceId: "ws-first", + directoryPath: "/worktrees/first", + incarnationId: "inc-first", + }), + pendingWorkspace({ + workspaceId: "ws-second", + directoryPath: "/worktrees/second", + incarnationId: "inc-second", + }), + ]; + const retryWorktreeCleanup = vi.fn(async () => { + throw new Error("still busy"); + }); + const service = new WorkspaceCleanupRetryService({ + workspaceRegistry: { + list: async () => workspaces, + subscribeToMutations: () => () => undefined, + }, + retryWorktreeCleanup, + logger: createTestLogger(), + retryBaseMs: 10, + retryMaxMs: 10, + idlePollMs: 100, + maxTargetsPerCycle: 1, + }); + + await service.start(); + expect(retryWorktreeCleanup).toHaveBeenCalledTimes(1); + + await service.stop(); + await vi.advanceTimersByTimeAsync(1_000); + expect(retryWorktreeCleanup).toHaveBeenCalledTimes(1); +}); diff --git a/packages/server/src/server/workspace-cleanup-retry-service.ts b/packages/server/src/server/workspace-cleanup-retry-service.ts new file mode 100644 index 0000000000..ba8e4ae301 --- /dev/null +++ b/packages/server/src/server/workspace-cleanup-retry-service.ts @@ -0,0 +1,180 @@ +import { resolve } from "node:path"; + +import type { Logger } from "pino"; + +import type { PersistedWorkspaceRecord, WorkspaceRegistry } from "./workspace-registry.js"; + +const DEFAULT_IDLE_POLL_MS = 30_000; +const DEFAULT_RETRY_BASE_MS = 1_000; +const DEFAULT_RETRY_MAX_MS = 60_000; +const DEFAULT_MAX_TARGETS_PER_CYCLE = 8; + +interface WorkspaceCleanupRetryTarget { + directoryPath: string; + worktreeIncarnationId: string; + workspaceIds: string[]; +} + +export interface WorkspaceCleanupRetryServiceOptions { + workspaceRegistry: Pick; + retryWorktreeCleanup: (targetPath: string) => Promise; + logger: Logger; + idlePollMs?: number; + retryBaseMs?: number; + retryMaxMs?: number; + maxTargetsPerCycle?: number; +} + +export class WorkspaceCleanupRetryService { + private readonly logger: Logger; + private readonly idlePollMs: number; + private readonly retryBaseMs: number; + private readonly retryMaxMs: number; + private readonly maxTargetsPerCycle: number; + private timer: ReturnType | null = null; + private activeCycle: Promise | null = null; + private unsubscribe: (() => void) | null = null; + private started = false; + private stopped = false; + private consecutiveFailedCycles = 0; + private wakeRequested = false; + + constructor(private readonly options: WorkspaceCleanupRetryServiceOptions) { + this.logger = options.logger.child({ module: "workspace-cleanup-retry-service" }); + this.idlePollMs = options.idlePollMs ?? DEFAULT_IDLE_POLL_MS; + this.retryBaseMs = options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS; + this.retryMaxMs = options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS; + this.maxTargetsPerCycle = options.maxTargetsPerCycle ?? DEFAULT_MAX_TARGETS_PER_CYCLE; + } + + async start(): Promise { + if (this.started) return; + this.started = true; + this.stopped = false; + this.unsubscribe = + this.options.workspaceRegistry.subscribeToMutations?.((mutation) => { + if (mutation.workspace?.cleanupPending) { + this.requestWake(); + } + }) ?? null; + await this.runCycle(); + } + + async stop(): Promise { + this.stopped = true; + this.started = false; + this.wakeRequested = false; + this.unsubscribe?.(); + this.unsubscribe = null; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + await this.activeCycle; + } + + private requestWake(): void { + if (!this.started || this.stopped) return; + if (this.activeCycle) { + this.wakeRequested = true; + return; + } + this.schedule(0); + } + + private schedule(delayMs: number): void { + if (!this.started || this.stopped) return; + if (this.timer) clearTimeout(this.timer); + this.timer = setTimeout(() => { + this.timer = null; + void this.runCycle(); + }, delayMs); + (this.timer as unknown as { unref?: () => void }).unref?.(); + } + + private async runCycle(): Promise { + if (!this.started || this.stopped || this.activeCycle) return; + const task = this.executeCycle(); + this.activeCycle = task; + try { + await task; + } finally { + if (this.activeCycle === task) this.activeCycle = null; + if (this.started && !this.stopped) { + if (this.wakeRequested) { + this.wakeRequested = false; + this.schedule(0); + } else { + const delay = + this.consecutiveFailedCycles === 0 + ? this.idlePollMs + : Math.min( + this.retryMaxMs, + this.retryBaseMs * 2 ** (this.consecutiveFailedCycles - 1), + ); + this.schedule(delay); + } + } + } + } + + private async executeCycle(): Promise { + let targets: WorkspaceCleanupRetryTarget[]; + try { + targets = findWorkspaceCleanupRetryTargets(await this.options.workspaceRegistry.list()); + } catch (error) { + this.consecutiveFailedCycles += 1; + this.logger.warn({ err: error }, "Failed to list pending workspace cleanup"); + return; + } + + let failed = false; + for (const target of targets.slice(0, this.maxTargetsPerCycle)) { + try { + await this.options.retryWorktreeCleanup(target.directoryPath); + } catch (error) { + failed = true; + this.logger.warn( + { + err: error, + directoryPath: target.directoryPath, + workspaceIds: target.workspaceIds, + worktreeIncarnationId: target.worktreeIncarnationId, + }, + "Pending workspace cleanup retry failed", + ); + } + } + if (targets.length > this.maxTargetsPerCycle) { + this.wakeRequested = true; + } + this.consecutiveFailedCycles = failed ? this.consecutiveFailedCycles + 1 : 0; + } +} + +export function findWorkspaceCleanupRetryTargets( + workspaces: readonly PersistedWorkspaceRecord[], +): WorkspaceCleanupRetryTarget[] { + const byDirectory = new Map(); + for (const workspace of workspaces) { + if (!workspace.archivedAt || !workspace.cleanupPending) continue; + const directoryPath = resolve(workspace.cleanupPending.directoryPath); + const group = byDirectory.get(directoryPath) ?? []; + group.push(workspace); + byDirectory.set(directoryPath, group); + } + + const targets: WorkspaceCleanupRetryTarget[] = []; + for (const [directoryPath, group] of byDirectory) { + const incarnationIds = new Set( + group.map((workspace) => workspace.cleanupPending?.worktreeIncarnationId ?? null), + ); + if (incarnationIds.size !== 1 || incarnationIds.has(null)) continue; + targets.push({ + directoryPath, + worktreeIncarnationId: [...incarnationIds][0]!, + workspaceIds: group.map((workspace) => workspace.workspaceId).sort(), + }); + } + return targets.sort((left, right) => left.directoryPath.localeCompare(right.directoryPath)); +} From 4cf635d4e2c1925921bf663169914f40921d5f20 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 21:58:07 -0400 Subject: [PATCH 22/43] docs: record autonomous cleanup retry proof --- patches/local-patches.json | 7 +++++++ .../paseo-v025-worktree-lifecycle-current-main.json | 12 +++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index 2aa08c152a..44468a669b 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -22,6 +22,7 @@ "packages/server/src/server/bootstrap.ts", "packages/server/src/server/session.ts", "packages/server/src/server/workspace-archive-service.ts", + "packages/server/src/server/workspace-cleanup-retry-service.ts", "packages/server/src/server/workspace-lifecycle-coordinator.ts", "packages/server/src/server/workspace-registry.ts", "packages/server/src/server/worktree-bootstrap.ts", @@ -40,6 +41,7 @@ "packages/server/src/server/session.workspace-git-watch.test.ts", "packages/server/src/server/session.workspaces.test.ts", "packages/server/src/server/workspace-archive-service.test.ts", + "packages/server/src/server/workspace-cleanup-retry-service.test.ts", "packages/server/src/server/worktree-bootstrap.test.ts", "packages/server/src/server/worktree-session.test.ts", "packages/server/src/terminal/terminal-session-controller.test.ts", @@ -62,6 +64,9 @@ "merge auto-archive receives the durable workspace cleanup registry", "merge auto-archive rejects stale resolved workspaces and never reports pending cleanup as success", "created-worktree, schedule, project, and normal workspace automation reject pending physical cleanup for retry", + "persisted cleanup is retried autonomously at startup and after mutation without another agent event", + "cleanup retries group sibling obligations by incarnation-bound worktree path before deletion", + "cleanup retry cycles use bounded work with capped backoff and cancellable timers", "replacement worktree incarnations archive independently of stale cleanup intents", "auto-archive retries after transient failure and cancellation observes rejection", "setup output is bounded while preserving the final diagnostic tail" @@ -93,6 +98,8 @@ }, "currentMainIntegration": { "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", + "verifiedHeadCommit": "48b53ce71d850f065a22fb1a66cca4f6d4ec66ab", + "verifiedHeadTree": "335c1308797ad77810ecccd4cb0f73708b7c6979", "receipt": "patches/receipts/paseo-v025-worktree-lifecycle-current-main.json" }, "upstreamable": true, diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index f1bff1ee8b..1145b16432 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,14 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "7e9b8bf54fb2735a1ca5551a4362940fb01d2869", - "headTree": "ec2ee9a9d60b58aeb5f2c8a460ac1970e5c38d1f", + "headCommit": "48b53ce71d850f065a22fb1a66cca4f6d4ec66ab", + "headTree": "335c1308797ad77810ecccd4cb0f73708b7c6979", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "7e9b8bf54fb2735a1ca5551a4362940fb01d2869", - "completedAt": "2026-08-01T01:37:33Z", + "commit": "48b53ce71d850f065a22fb1a66cca4f6d4ec66ab", + "completedAt": "2026-08-01T01:57:31Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -34,6 +34,7 @@ "cd packages/server && vitest run src/server/workspace-archive-service.test.ts --fileParallelism=false", "cd packages/server && vitest run src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts --fileParallelism=false", "cd packages/server && vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts src/server/session.workspaces.test.ts --fileParallelism=false", + "vitest run packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts packages/server/src/server/agent/create-agent/create.test.ts packages/server/src/server/agent/mcp-server.test.ts packages/server/src/terminal/terminal-session-controller.test.ts packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts packages/server/src/server/workspace-archive-service.test.ts packages/server/src/server/workspace-cleanup-retry-service.test.ts packages/server/src/server/worktree-session.test.ts packages/server/src/server/session.workspaces.test.ts packages/server/src/server/schedule/service.test.ts --fileParallelism=false", "npm run build --workspace packages/server", "npm run typecheck --workspace packages/server", "oxlint <16 changed lifecycle source and test files>", @@ -52,7 +53,8 @@ "autoArchiveTruthfulnessTests": "16 passed", "archiveTargetClosureTests": "26 passed; combined auto-archive and closure proof 42 passed", "archiveFailureTruthfulnessTests": "75 passed", - "automatedCleanupRetryTests": "182 passed, 4 skipped", + "automatedCleanupRetryTests": "387 passed, 4 skipped", + "autonomousCleanupOwner": "startup and mutation-driven retry; incarnation-grouped worktree scope; bounded cycle and capped backoff; cancellable timers", "allWorkspaceTypecheck": "unchanged app errors: DaemonClient lacks setProjectIcon/getProjectIcon on this stable-derived branch" } } From 4bc441a334c61c17fd15f6e5936d1a4f6ce8b915 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:05:18 -0400 Subject: [PATCH 23/43] fix: keep cleanup retry off daemon readiness path --- .../server/workspace-archive-service.test.ts | 1 + .../workspace-cleanup-retry-service.test.ts | 44 +++++++++++++++++-- .../server/workspace-cleanup-retry-service.ts | 23 ++++++++-- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index dcd5d7278b..ba584208d1 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -1243,6 +1243,7 @@ describe("archiveByScope", () => { }); await service.start(); + await vi.waitFor(() => expect(existsSync(worktree.worktreePath)).toBe(false)); await service.stop(); expect(readFileSync(path.join(repoDir, "retry-root.log"), "utf8")).toBe("root"); diff --git a/packages/server/src/server/workspace-cleanup-retry-service.test.ts b/packages/server/src/server/workspace-cleanup-retry-service.test.ts index 0cb3107237..474b1ab17c 100644 --- a/packages/server/src/server/workspace-cleanup-retry-service.test.ts +++ b/packages/server/src/server/workspace-cleanup-retry-service.test.ts @@ -81,7 +81,7 @@ test("groups archived cleanup by directory and requires one verified incarnation ]); }); -test("bounds each retry cycle and cancels the scheduled backoff on stop", async () => { +test("rotates bounded batches and preserves backoff after a failed target", async () => { vi.useFakeTimers(); const workspaces = [ pendingWorkspace({ @@ -95,8 +95,10 @@ test("bounds each retry cycle and cancels the scheduled backoff on stop", async incarnationId: "inc-second", }), ]; - const retryWorktreeCleanup = vi.fn(async () => { - throw new Error("still busy"); + const attempts: string[] = []; + const retryWorktreeCleanup = vi.fn(async (targetPath: string) => { + attempts.push(targetPath); + if (targetPath === "/worktrees/first") throw new Error("still busy"); }); const service = new WorkspaceCleanupRetryService({ workspaceRegistry: { @@ -112,9 +114,43 @@ test("bounds each retry cycle and cancels the scheduled backoff on stop", async }); await service.start(); - expect(retryWorktreeCleanup).toHaveBeenCalledTimes(1); + await vi.advanceTimersToNextTimerAsync(); + expect(attempts).toEqual(["/worktrees/first"]); + + await vi.advanceTimersByTimeAsync(9); + expect(attempts).toEqual(["/worktrees/first"]); + await vi.advanceTimersByTimeAsync(1); + expect(attempts).toContain("/worktrees/second"); + + await vi.advanceTimersToNextTimerAsync(); + expect(attempts.filter((path) => path === "/worktrees/first")).toHaveLength(2); await service.stop(); await vi.advanceTimersByTimeAsync(1_000); + expect(retryWorktreeCleanup).toHaveBeenCalledTimes(attempts.length); +}); + +test("hung cleanup does not block startup or shutdown", async () => { + vi.useFakeTimers(); + const never = new Promise(() => undefined); + const retryWorktreeCleanup = vi.fn(() => never); + const service = new WorkspaceCleanupRetryService({ + workspaceRegistry: { + list: async () => [ + pendingWorkspace({ + workspaceId: "ws-hung", + directoryPath: "/worktrees/hung", + incarnationId: "inc-hung", + }), + ], + subscribeToMutations: () => () => undefined, + }, + retryWorktreeCleanup, + logger: createTestLogger(), + }); + + await expect(service.start()).resolves.toBeUndefined(); + await vi.advanceTimersToNextTimerAsync(); expect(retryWorktreeCleanup).toHaveBeenCalledTimes(1); + await expect(service.stop()).resolves.toBeUndefined(); }); diff --git a/packages/server/src/server/workspace-cleanup-retry-service.ts b/packages/server/src/server/workspace-cleanup-retry-service.ts index ba8e4ae301..34e43f9f4b 100644 --- a/packages/server/src/server/workspace-cleanup-retry-service.ts +++ b/packages/server/src/server/workspace-cleanup-retry-service.ts @@ -38,6 +38,7 @@ export class WorkspaceCleanupRetryService { private stopped = false; private consecutiveFailedCycles = 0; private wakeRequested = false; + private nextTargetDirectoryPath: string | null = null; constructor(private readonly options: WorkspaceCleanupRetryServiceOptions) { this.logger = options.logger.child({ module: "workspace-cleanup-retry-service" }); @@ -57,7 +58,7 @@ export class WorkspaceCleanupRetryService { this.requestWake(); } }) ?? null; - await this.runCycle(); + this.schedule(0); } async stop(): Promise { @@ -70,7 +71,6 @@ export class WorkspaceCleanupRetryService { clearTimeout(this.timer); this.timer = null; } - await this.activeCycle; } private requestWake(): void { @@ -128,8 +128,13 @@ export class WorkspaceCleanupRetryService { return; } + const rotatedTargets = rotateCleanupTargets(targets, this.nextTargetDirectoryPath); + const batch = rotatedTargets.slice(0, this.maxTargetsPerCycle); + const nextTarget = rotatedTargets[batch.length] ?? null; + this.nextTargetDirectoryPath = nextTarget?.directoryPath ?? null; + let failed = false; - for (const target of targets.slice(0, this.maxTargetsPerCycle)) { + for (const target of batch) { try { await this.options.retryWorktreeCleanup(target.directoryPath); } catch (error) { @@ -145,13 +150,23 @@ export class WorkspaceCleanupRetryService { ); } } - if (targets.length > this.maxTargetsPerCycle) { + if (!failed && nextTarget) { this.wakeRequested = true; } this.consecutiveFailedCycles = failed ? this.consecutiveFailedCycles + 1 : 0; } } +function rotateCleanupTargets( + targets: readonly WorkspaceCleanupRetryTarget[], + nextDirectoryPath: string | null, +): WorkspaceCleanupRetryTarget[] { + if (!nextDirectoryPath) return [...targets]; + const index = targets.findIndex((target) => target.directoryPath === nextDirectoryPath); + if (index <= 0) return [...targets]; + return [...targets.slice(index), ...targets.slice(0, index)]; +} + export function findWorkspaceCleanupRetryTargets( workspaces: readonly PersistedWorkspaceRecord[], ): WorkspaceCleanupRetryTarget[] { From 2985ba3cf7846c12d9a1c2fd571adab757030f93 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:05:48 -0400 Subject: [PATCH 24/43] docs: attest non-blocking cleanup retry --- patches/local-patches.json | 4 ++-- .../paseo-v025-worktree-lifecycle-current-main.json | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index 44468a669b..baa1443e66 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -98,8 +98,8 @@ }, "currentMainIntegration": { "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", - "verifiedHeadCommit": "48b53ce71d850f065a22fb1a66cca4f6d4ec66ab", - "verifiedHeadTree": "335c1308797ad77810ecccd4cb0f73708b7c6979", + "verifiedHeadCommit": "4bc441a334c61c17fd15f6e5936d1a4f6ce8b915", + "verifiedHeadTree": "1cf3144fc288f9dc398bc3954846a4e6bd85004f", "receipt": "patches/receipts/paseo-v025-worktree-lifecycle-current-main.json" }, "upstreamable": true, diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index 1145b16432..f96cd13ea7 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,14 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "48b53ce71d850f065a22fb1a66cca4f6d4ec66ab", - "headTree": "335c1308797ad77810ecccd4cb0f73708b7c6979", + "headCommit": "4bc441a334c61c17fd15f6e5936d1a4f6ce8b915", + "headTree": "1cf3144fc288f9dc398bc3954846a4e6bd85004f", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "48b53ce71d850f065a22fb1a66cca4f6d4ec66ab", - "completedAt": "2026-08-01T01:57:31Z", + "commit": "4bc441a334c61c17fd15f6e5936d1a4f6ce8b915", + "completedAt": "2026-08-01T02:05:35Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -53,8 +53,8 @@ "autoArchiveTruthfulnessTests": "16 passed", "archiveTargetClosureTests": "26 passed; combined auto-archive and closure proof 42 passed", "archiveFailureTruthfulnessTests": "75 passed", - "automatedCleanupRetryTests": "387 passed, 4 skipped", - "autonomousCleanupOwner": "startup and mutation-driven retry; incarnation-grouped worktree scope; bounded cycle and capped backoff; cancellable timers", + "automatedCleanupRetryTests": "388 passed, 4 skipped", + "autonomousCleanupOwner": "non-blocking startup and shutdown; startup and mutation-driven retry; rotating incarnation-grouped worktree batches; bounded cycle and capped backoff; cancellable timers", "allWorkspaceTypecheck": "unchanged app errors: DaemonClient lacks setProjectIcon/getProjectIcon on this stable-derived branch" } } From 5e17fcf769cb855326683413cdcd9b3cbe82ba43 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:18:13 -0400 Subject: [PATCH 25/43] fix: fence and cancel cleanup retries --- packages/server/src/server/bootstrap.ts | 11 ++- .../server/workspace-archive-service.test.ts | 95 ++++++++++++++++++- .../src/server/workspace-archive-service.ts | 95 ++++++++++++++++++- .../workspace-cleanup-retry-service.test.ts | 20 +++- .../server/workspace-cleanup-retry-service.ts | 46 +++++---- .../server/src/utils/worktree.posix.test.ts | 40 ++++++++ packages/server/src/utils/worktree.ts | 21 ++++ 7 files changed, 300 insertions(+), 28 deletions(-) diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 51fc39d943..034031f309 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -153,6 +153,7 @@ import { archivePersistedWorkspaceRecord, killTerminalsForWorkspace, requireArchiveCleanupComplete, + retryPendingWorkspaceCleanup, type ActiveWorkspaceRef, } from "./workspace-archive-service.js"; import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service.js"; @@ -1088,9 +1089,11 @@ export async function createPaseoDaemon( ), "Workspace archive", ); - const retryPendingWorktreeCleanupExternal = async (targetPath: string): Promise => { + const retryPendingWorktreeCleanupExternal: ConstructorParameters< + typeof WorkspaceCleanupRetryService + >[0]["retryWorktreeCleanup"] = async (target, signal) => { requireArchiveCleanupComplete( - await archiveByScope( + await retryPendingWorkspaceCleanup( { paseoHome: config.paseoHome, paseoWorktreesBaseRoot: config.worktreesRoot, @@ -1113,8 +1116,10 @@ export async function createPaseoDaemon( sessionLogger: logger, }, { - scope: { kind: "worktree", targetPath }, + directoryPath: target.directoryPath, + worktreeIncarnationId: target.worktreeIncarnationId, requestId: `cleanup-retry:${randomUUID()}`, + signal, }, ), "Workspace cleanup retry", diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index ba584208d1..2583cb5450 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -19,6 +19,7 @@ import { type ArchiveResult, requireActiveWorkspaceForArchive, requireArchiveCleanupComplete, + retryPendingWorkspaceCleanup, resolveWorkspaceIdAtPath, } from "./workspace-archive-service.js"; import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service.js"; @@ -1229,11 +1230,13 @@ describe("archiveByScope", () => { deps.workspaceRegistry = registry; const service = new WorkspaceCleanupRetryService({ workspaceRegistry: registry, - retryWorktreeCleanup: async (targetPath) => { + retryWorktreeCleanup: async (target, signal) => { requireArchiveCleanupComplete( - await archiveByScope(deps, { - scope: { kind: "worktree", targetPath }, + await retryPendingWorkspaceCleanup(deps, { + directoryPath: target.directoryPath, + worktreeIncarnationId: target.worktreeIncarnationId, requestId: "startup-schedule-cleanup-retry", + signal, }), "Startup schedule cleanup retry", ); @@ -1243,7 +1246,10 @@ describe("archiveByScope", () => { }); await service.start(); - await vi.waitFor(() => expect(existsSync(worktree.worktreePath)).toBe(false)); + await vi.waitFor(async () => { + expect((await registry.get("ws-completed-schedule-root"))?.cleanupPending).toBeNull(); + expect((await registry.get("ws-completed-schedule-nested"))?.cleanupPending).toBeNull(); + }); await service.stop(); expect(readFileSync(path.join(repoDir, "retry-root.log"), "utf8")).toBe("root"); @@ -1253,6 +1259,87 @@ describe("archiveByScope", () => { expect((await registry.get("ws-completed-schedule-nested"))?.cleanupPending).toBeNull(); }); + test("cleanup-only retry never archives a live replacement at a reused path", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const replacement = await createPaseoOwnedWorktree(repoDir, paseoHome, "reused-cleanup-path"); + const replacementIncarnation = readPaseoWorktreeIncarnationId(replacement.worktreePath); + expect(replacementIncarnation).toBeTruthy(); + const staleWorkspaceId = "ws-stale-cleanup-owner"; + const replacementWorkspaceId = "ws-live-replacement-owner"; + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + await registry.initialize(); + const timestamp = new Date().toISOString(); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId: staleWorkspaceId, + projectId: "project-reused-cleanup-path", + cwd: replacement.worktreePath, + kind: "worktree", + displayName: "Stale cleanup owner", + worktreeRoot: replacement.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + archivedAt: timestamp, + cleanupPending: { + directoryPath: replacement.worktreePath, + teardownCwd: replacement.worktreePath, + mainRepoRoot: repoDir, + paseoWorktreesRoot: null, + worktreeIncarnationId: "incarnation-before-replacement", + }, + }), + ); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId: replacementWorkspaceId, + projectId: "project-reused-cleanup-path", + cwd: replacement.worktreePath, + kind: "worktree", + displayName: "Live replacement owner", + worktreeRoot: replacement.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + }), + ); + const deps = createArchiveDeps({ + paseoHome, + activeWorkspaces: [ + { + workspaceId: replacementWorkspaceId, + cwd: replacement.worktreePath, + kind: "worktree", + worktreeRoot: replacement.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }, + ], + }); + deps.workspaceRegistry = registry; + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + deps.killTerminalsForWorkspace = vi.fn(async () => undefined); + + const result = await retryPendingWorkspaceCleanup(deps, { + directoryPath: replacement.worktreePath, + worktreeIncarnationId: "incarnation-before-replacement", + requestId: "stale-cleanup-live-replacement", + }); + + expect(result.removedDirectory).toBe(false); + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + expect(deps.killTerminalsForWorkspace).not.toHaveBeenCalled(); + expect(existsSync(replacement.worktreePath)).toBe(true); + expect((await registry.get(replacementWorkspaceId))?.archivedAt).toBeNull(); + expect((await registry.get(staleWorkspaceId))?.cleanupPending).toBeNull(); + }); + test("workspace teardown failure keeps the record active and prevents recursive deletion", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index c990587bc8..96bad44eda 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -111,6 +111,13 @@ export interface ArchiveByScopeRequest { requestId: string; } +export interface PendingWorkspaceCleanupRetryRequest { + directoryPath: string; + worktreeIncarnationId: string; + requestId: string; + signal?: AbortSignal; +} + export async function requireActiveWorkspaceForArchive( dependencies: Pick, workspaceId: string, @@ -209,6 +216,82 @@ export async function archiveByScope( }); } +// Retries only the physical teardown intent already persisted on archived +// workspace records. Unlike normal worktree-scope archive, this never archives +// active workspace owners that may now occupy a reused path. +export async function retryPendingWorkspaceCleanup( + dependencies: ArchiveDependencies, + request: PendingWorkspaceCleanupRetryRequest, +): Promise { + const workspaceRegistry = dependencies.workspaceRegistry; + if (!workspaceRegistry) { + return { + archivedAgentIds: [], + archivedWorkspaceIds: [], + removedDirectory: false, + cleanupPendingWorkspaceIds: [], + }; + } + const lifecycleCoordinator = + dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const directoryPath = resolve(request.directoryPath); + const operationKey = `cleanup:${directoryPath}:${request.worktreeIncarnationId}`; + + return lifecycleCoordinator.runArchive(operationKey, async () => { + const matchesDirectory = createRealpathAwarePathMatcher(directoryPath); + const pendingRecords = (await workspaceRegistry.list()).filter( + (workspace) => + workspace.archivedAt !== null && + workspace.cleanupPending !== null && + workspace.cleanupPending.worktreeIncarnationId === request.worktreeIncarnationId && + matchesDirectory(workspace.cleanupPending.directoryPath), + ); + const firstPending = pendingRecords[0]?.cleanupPending ?? null; + if (!firstPending) { + return { + archivedAgentIds: [], + archivedWorkspaceIds: [], + removedDirectory: false, + cleanupPendingWorkspaceIds: [], + }; + } + const target: ArchiveTarget = { + backing: { + path: firstPending.directoryPath, + isPaseoOwnedWorktree: true, + mainRepoRoot: firstPending.mainRepoRoot, + paseoWorktreesRoot: firstPending.paseoWorktreesRoot, + }, + teardownTargets: pendingRecords.map((workspace) => ({ + workspaceId: workspace.workspaceId, + cwd: workspace.cleanupPending!.teardownCwd, + })), + workspaceIds: [], + }; + const retry = async (): Promise => { + if (request.signal?.aborted) throw new Error("Workspace cleanup retry canceled"); + const removedDirectory = await maybeRemoveDirectory( + dependencies, + lifecycleCoordinator, + request, + target, + [], + request.signal, + ); + return { + archivedAgentIds: [], + archivedWorkspaceIds: [], + removedDirectory, + cleanupPendingWorkspaceIds: await getCleanupPendingWorkspaceIds(dependencies, target), + }; + }; + const mutationRoot = await resolveWorktreeMutationRoot(dependencies, target.backing); + return mutationRoot + ? lifecycleCoordinator.runWorktreeMutationExclusive(mutationRoot, retry) + : retry(); + }); +} + async function resolveWorktreeMutationRoot( dependencies: Pick, backing: BackingDirectory | null, @@ -506,6 +589,7 @@ async function maybeRemoveDirectory( request: Pick, target: ArchiveTarget, archivedWorkspaceIds: string[], + signal?: AbortSignal, ): Promise { const backing = target.backing; if (!backing?.isPaseoOwnedWorktree) { @@ -513,7 +597,13 @@ async function maybeRemoveDirectory( } return lifecycleCoordinator.runDirectoryExclusive(backing.path, async () => { - return maybeRemoveDirectoryExclusive(dependencies, request, target, archivedWorkspaceIds); + return maybeRemoveDirectoryExclusive( + dependencies, + request, + target, + archivedWorkspaceIds, + signal, + ); }); } @@ -522,6 +612,7 @@ async function maybeRemoveDirectoryExclusive( request: Pick, target: ArchiveTarget, archivedWorkspaceIds: string[], + signal?: AbortSignal, ): Promise { const backing = target.backing; if (!backing) return false; @@ -575,6 +666,7 @@ async function maybeRemoveDirectoryExclusive( worktreePath: backing.path, teardownCwd, repoRootPath: backing.mainRepoRoot ?? undefined, + signal, }); } } catch (error) { @@ -621,6 +713,7 @@ async function maybeRemoveDirectoryExclusive( worktreesRoot: backing.paseoWorktreesRoot ?? undefined, paseoHome: dependencies.paseoHome, worktreesBaseRoot: dependencies.paseoWorktreesBaseRoot, + signal, }); dependencies.github.invalidate({ cwd: backing.path }); await clearPendingCleanup(dependencies, pendingCleanupTargets); diff --git a/packages/server/src/server/workspace-cleanup-retry-service.test.ts b/packages/server/src/server/workspace-cleanup-retry-service.test.ts index 474b1ab17c..5fc8edbbb0 100644 --- a/packages/server/src/server/workspace-cleanup-retry-service.test.ts +++ b/packages/server/src/server/workspace-cleanup-retry-service.test.ts @@ -96,7 +96,7 @@ test("rotates bounded batches and preserves backoff after a failed target", asyn }), ]; const attempts: string[] = []; - const retryWorktreeCleanup = vi.fn(async (targetPath: string) => { + const retryWorktreeCleanup = vi.fn(async ({ directoryPath: targetPath }) => { attempts.push(targetPath); if (targetPath === "/worktrees/first") throw new Error("still busy"); }); @@ -130,10 +130,21 @@ test("rotates bounded batches and preserves backoff after a failed target", asyn expect(retryWorktreeCleanup).toHaveBeenCalledTimes(attempts.length); }); -test("hung cleanup does not block startup or shutdown", async () => { +test("shutdown cancels and owns an active cleanup cycle", async () => { vi.useFakeTimers(); - const never = new Promise(() => undefined); - const retryWorktreeCleanup = vi.fn(() => never); + let aborted = false; + const retryWorktreeCleanup = vi.fn((_target, signal: AbortSignal) => { + return new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => { + aborted = true; + reject(new Error("cleanup canceled")); + }, + { once: true }, + ); + }); + }); const service = new WorkspaceCleanupRetryService({ workspaceRegistry: { list: async () => [ @@ -153,4 +164,5 @@ test("hung cleanup does not block startup or shutdown", async () => { await vi.advanceTimersToNextTimerAsync(); expect(retryWorktreeCleanup).toHaveBeenCalledTimes(1); await expect(service.stop()).resolves.toBeUndefined(); + expect(aborted).toBe(true); }); diff --git a/packages/server/src/server/workspace-cleanup-retry-service.ts b/packages/server/src/server/workspace-cleanup-retry-service.ts index 34e43f9f4b..b0b7fed56b 100644 --- a/packages/server/src/server/workspace-cleanup-retry-service.ts +++ b/packages/server/src/server/workspace-cleanup-retry-service.ts @@ -9,7 +9,7 @@ const DEFAULT_RETRY_BASE_MS = 1_000; const DEFAULT_RETRY_MAX_MS = 60_000; const DEFAULT_MAX_TARGETS_PER_CYCLE = 8; -interface WorkspaceCleanupRetryTarget { +export interface WorkspaceCleanupRetryTarget { directoryPath: string; worktreeIncarnationId: string; workspaceIds: string[]; @@ -17,7 +17,7 @@ interface WorkspaceCleanupRetryTarget { export interface WorkspaceCleanupRetryServiceOptions { workspaceRegistry: Pick; - retryWorktreeCleanup: (targetPath: string) => Promise; + retryWorktreeCleanup: (target: WorkspaceCleanupRetryTarget, signal: AbortSignal) => Promise; logger: Logger; idlePollMs?: number; retryBaseMs?: number; @@ -39,6 +39,7 @@ export class WorkspaceCleanupRetryService { private consecutiveFailedCycles = 0; private wakeRequested = false; private nextTargetDirectoryPath: string | null = null; + private activeAbortController: AbortController | null = null; constructor(private readonly options: WorkspaceCleanupRetryServiceOptions) { this.logger = options.logger.child({ module: "workspace-cleanup-retry-service" }); @@ -71,6 +72,10 @@ export class WorkspaceCleanupRetryService { clearTimeout(this.timer); this.timer = null; } + const activeCycle = this.activeCycle; + this.activeAbortController?.abort(); + this.activeAbortController = null; + await activeCycle; } private requestWake(): void { @@ -132,23 +137,32 @@ export class WorkspaceCleanupRetryService { const batch = rotatedTargets.slice(0, this.maxTargetsPerCycle); const nextTarget = rotatedTargets[batch.length] ?? null; this.nextTargetDirectoryPath = nextTarget?.directoryPath ?? null; + const abortController = new AbortController(); + this.activeAbortController = abortController; let failed = false; - for (const target of batch) { - try { - await this.options.retryWorktreeCleanup(target.directoryPath); - } catch (error) { - failed = true; - this.logger.warn( - { - err: error, - directoryPath: target.directoryPath, - workspaceIds: target.workspaceIds, - worktreeIncarnationId: target.worktreeIncarnationId, - }, - "Pending workspace cleanup retry failed", - ); + try { + for (const target of batch) { + if (abortController.signal.aborted) break; + try { + await this.options.retryWorktreeCleanup(target, abortController.signal); + } catch (error) { + failed = true; + if (!abortController.signal.aborted) { + this.logger.warn( + { + err: error, + directoryPath: target.directoryPath, + workspaceIds: target.workspaceIds, + worktreeIncarnationId: target.worktreeIncarnationId, + }, + "Pending workspace cleanup retry failed", + ); + } + } } + } finally { + if (this.activeAbortController === abortController) this.activeAbortController = null; } if (!failed && nextTarget) { this.wakeRequested = true; diff --git a/packages/server/src/utils/worktree.posix.test.ts b/packages/server/src/utils/worktree.posix.test.ts index fb29290572..f1656af161 100644 --- a/packages/server/src/utils/worktree.posix.test.ts +++ b/packages/server/src/utils/worktree.posix.test.ts @@ -1352,5 +1352,45 @@ describe.skipIf(isPlatform("win32"))("worktree POSIX-only", () => { expect(existsSync(created.worktreePath)).toBe(true); expect(existsSync(join(repoDir, "teardown-start.log"))).toBe(true); }); + + it("cancels a running teardown process tree before deleting the worktree", async () => { + writeFileSync( + join(repoDir, "paseo.json"), + JSON.stringify({ + worktree: { + teardown: ['node -e "setInterval(() => {}, 1000)"'], + }, + }), + ); + execFileSync("git", ["add", "paseo.json"], { cwd: repoDir }); + execFileSync( + "git", + ["-c", "commit.gpgsign=false", "commit", "-m", "add cancellable teardown"], + { cwd: repoDir }, + ); + const created = await createLegacyWorktreeForTest({ + branchName: "teardown-cancel-branch", + cwd: repoDir, + baseBranch: "main", + worktreeSlug: "teardown-cancel-test", + paseoHome, + }); + const controller = new AbortController(); + const abortTimer = setTimeout(() => controller.abort(), 100); + + try { + await expect( + deletePaseoWorktree({ + cwd: repoDir, + worktreePath: created.worktreePath, + paseoHome, + signal: controller.signal, + }), + ).rejects.toThrow("Worktree teardown command failed"); + } finally { + clearTimeout(abortTimer); + } + expect(existsSync(created.worktreePath)).toBe(true); + }); }); }); diff --git a/packages/server/src/utils/worktree.ts b/packages/server/src/utils/worktree.ts index 6321268246..079bd57cb0 100644 --- a/packages/server/src/utils/worktree.ts +++ b/packages/server/src/utils/worktree.ts @@ -30,6 +30,7 @@ import { } from "./worktree-metadata.js"; import { runGitCommand } from "./run-git-command.js"; import { spawnProcess } from "./spawn.js"; +import { terminateWithTreeKill } from "./tree-kill.js"; import { resolvePaseoHome } from "../server/paseo-home.js"; import { createExternalProcessEnv } from "../server/paseo-env.js"; import { parseGitRevParsePath, resolveGitRevParsePath } from "./git-rev-parse-path.js"; @@ -422,6 +423,7 @@ async function execSetupCommandStreamed(options: { total: number; maxOutputBytes: number; onEvent?: (event: WorktreeSetupCommandProgressEvent) => void; + signal?: AbortSignal; }): Promise { return new Promise((resolvePromise) => { const startedAt = Date.now(); @@ -455,6 +457,7 @@ async function execSetupCommandStreamed(options: { return; } settled = true; + options.signal?.removeEventListener("abort", abortCommand); const combinedBytes = stdoutAccumulator.totalBytes + stderrAccumulator.totalBytes; const stdoutOutputBytes = combinedBytes === 0 @@ -500,6 +503,15 @@ async function execSetupCommandStreamed(options: { shell: false, stdio: ["ignore", "pipe", "pipe"], }); + const abortCommand = () => { + emitOutput("stderr", "Worktree lifecycle command canceled"); + void terminateWithTreeKill(child, { + gracefulTimeoutMs: 1_000, + forceTimeoutMs: 1_000, + }).finally(() => finish(null)); + }; + options.signal?.addEventListener("abort", abortCommand, { once: true }); + if (options.signal?.aborted) abortCommand(); child.stdout?.on("data", (chunk: Buffer | string) => { emitOutput("stdout", chunk.toString()); @@ -709,6 +721,7 @@ export async function runWorktreeTeardownCommands(options: { teardownCwd?: string; branchName?: string; repoRootPath?: string; + signal?: AbortSignal; }): Promise { const teardownCwd = options.teardownCwd ?? options.worktreePath; if (getRealpathAwareRelativePath(options.worktreePath, teardownCwd) === null) { @@ -749,6 +762,7 @@ export async function runWorktreeTeardownCommands(options: { index: index + 1, total: teardownCommands.length, maxOutputBytes, + signal: options.signal, }); results.push(result); @@ -1066,6 +1080,7 @@ export interface DeletePaseoWorktreeOptions { worktreesRoot?: string; paseoHome?: string; worktreesBaseRoot?: string; + signal?: AbortSignal; } export async function deletePaseoWorktree({ @@ -1076,6 +1091,7 @@ export async function deletePaseoWorktree({ worktreesRoot, paseoHome, worktreesBaseRoot, + signal, }: DeletePaseoWorktreeOptions): Promise { if (!worktreePath && !worktreeSlug) { throw new Error("worktreePath or worktreeSlug is required"); @@ -1115,10 +1131,15 @@ export async function deletePaseoWorktree({ await runWorktreeTeardownCommands({ worktreePath: resolvedWorktree, teardownCwd, + signal, }); } } + if (signal?.aborted) { + throw new Error("Worktree deletion canceled"); + } + if (cwd) { try { await runGitCommand(["worktree", "remove", resolvedWorktree, "--force"], { From 322039e7a2b568be05c42c449e0aa89dfb7c77a6 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:19:15 -0400 Subject: [PATCH 26/43] docs: attest cleanup-only retry repair --- patches/local-patches.json | 6 ++++-- ...-v025-worktree-lifecycle-current-main.json | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index baa1443e66..12c83fe76a 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -67,6 +67,8 @@ "persisted cleanup is retried autonomously at startup and after mutation without another agent event", "cleanup retries group sibling obligations by incarnation-bound worktree path before deletion", "cleanup retry cycles use bounded work with capped backoff and cancellable timers", + "cleanup retries use an exact cleanup-only path and incarnation boundary and cannot archive a live replacement at a reused path", + "shutdown aborts and awaits the active cleanup cycle and terminates a hanging teardown subprocess tree before returning", "replacement worktree incarnations archive independently of stale cleanup intents", "auto-archive retries after transient failure and cancellation observes rejection", "setup output is bounded while preserving the final diagnostic tail" @@ -98,8 +100,8 @@ }, "currentMainIntegration": { "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", - "verifiedHeadCommit": "4bc441a334c61c17fd15f6e5936d1a4f6ce8b915", - "verifiedHeadTree": "1cf3144fc288f9dc398bc3954846a4e6bd85004f", + "verifiedHeadCommit": "5e17fcf769cb855326683413cdcd9b3cbe82ba43", + "verifiedHeadTree": "e83f37f06eeb44d5b87ff30d5ca18c0d25288f5b", "receipt": "patches/receipts/paseo-v025-worktree-lifecycle-current-main.json" }, "upstreamable": true, diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index f96cd13ea7..b9c4d9e78a 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,14 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "4bc441a334c61c17fd15f6e5936d1a4f6ce8b915", - "headTree": "1cf3144fc288f9dc398bc3954846a4e6bd85004f", + "headCommit": "5e17fcf769cb855326683413cdcd9b3cbe82ba43", + "headTree": "e83f37f06eeb44d5b87ff30d5ca18c0d25288f5b", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "4bc441a334c61c17fd15f6e5936d1a4f6ce8b915", - "completedAt": "2026-08-01T02:05:35Z", + "commit": "5e17fcf769cb855326683413cdcd9b3cbe82ba43", + "completedAt": "2026-08-01T02:18:00Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -39,6 +39,12 @@ "npm run typecheck --workspace packages/server", "oxlint <16 changed lifecycle source and test files>", "oxfmt --check <16 changed lifecycle source and test files>", + "git diff --check", + "vitest run packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts packages/server/src/server/agent/create-agent/create.test.ts packages/server/src/server/agent/mcp-server.test.ts packages/server/src/terminal/terminal-session-controller.test.ts packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts packages/server/src/server/workspace-archive-service.test.ts packages/server/src/server/workspace-cleanup-retry-service.test.ts packages/server/src/server/worktree-session.test.ts packages/server/src/server/session.workspaces.test.ts packages/server/src/server/schedule/service.test.ts packages/server/src/utils/worktree.posix.test.ts --fileParallelism=false", + "npm run build --workspace packages/server", + "npm run typecheck --workspace packages/server", + "oxlint <7 cleanup retry source and test files>", + "oxfmt --check <7 cleanup retry source and test files>", "git diff --check" ], "result": { @@ -55,6 +61,11 @@ "archiveFailureTruthfulnessTests": "75 passed", "automatedCleanupRetryTests": "388 passed, 4 skipped", "autonomousCleanupOwner": "non-blocking startup and shutdown; startup and mutation-driven retry; rotating incarnation-grouped worktree batches; bounded cycle and capped backoff; cancellable timers", + "cleanupOnlyRetryRepairTests": "434 passed, 5 skipped across 11 lifecycle files", + "cleanupOnlyRetryBoundary": "exact persisted directory and incarnation only; never invokes normal workspace or agent archive scope; stale cleanup cannot archive a live replacement at a reused path", + "cleanupRetryShutdown": "stop aborts and awaits the owned active cycle; hanging teardown subprocess trees are terminated before shutdown completes", + "cleanupOnlyRetryBuildAndTypecheck": "passed", + "cleanupOnlyRetryLintAndFormat": "0 warnings, 0 errors; formatting and diff checks passed", "allWorkspaceTypecheck": "unchanged app errors: DaemonClient lacks setProjectIcon/getProjectIcon on this stable-derived branch" } } From 5c70d522437ea0c02c1e769ee5f9975b8077811e Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:31:17 -0400 Subject: [PATCH 27/43] fix: serialize workspace creation with cleanup --- packages/server/src/server/session.ts | 1 + .../workspace-provisioning-service.test.ts | 4 + .../workspace-provisioning-service.ts | 100 ++++++++++++----- .../server/workspace-archive-service.test.ts | 104 ++++++++++++++++++ 4 files changed, 184 insertions(+), 25 deletions(-) diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 7676e2f37c..eafcb248da 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -751,6 +751,7 @@ export class Session { projectRegistry: this.projectRegistry, workspaceGitService: this.workspaceGitService, logger: this.sessionLogger, + isDirectory: (targetPath) => this.filesystem.isDirectory(targetPath), }); this.workspaceRecovery = createWorkspaceRecoveryService({ paseoHome: this.paseoHome, diff --git a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts index 8c08093f22..87a375d9da 100644 --- a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts +++ b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts @@ -83,6 +83,7 @@ beforeEach(async () => { projectRegistry, workspaceGitService: gitService(), logger, + isDirectory: async () => true, }); }); @@ -175,6 +176,7 @@ test("persists manual worktree ownership separately from its workspace kind", as mainRepoRoot, }), }), + isDirectory: async () => true, }); const workspace = await manualWorktreeProvisioning.findOrCreateWorkspaceForDirectory(cwd); @@ -237,6 +239,7 @@ test("reopening archived exact-root records restores the fresh Git project", asy mainRepoRoot: null, }), }), + isDirectory: async () => true, }); const reopened = await archivedProvisioning.ensureWorkspaceRecordUnarchived(workspace); @@ -276,6 +279,7 @@ test("uses one workspace snapshot when reopening an archived workspace", async ( workspaceRegistry: snapshotRegistry, projectRegistry, workspaceGitService: gitService(), + isDirectory: async () => true, }); const reopened = await snapshotProvisioning.findOrCreateWorkspaceForDirectory(repo); diff --git a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts index 61a25b96bd..87d8ca6542 100644 --- a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts +++ b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts @@ -1,3 +1,4 @@ +import { stat } from "node:fs/promises"; import { basename, resolve } from "node:path"; import type { Logger } from "pino"; import { @@ -16,6 +17,10 @@ import type { WorkspaceGitService } from "../../workspace-git-service.js"; import type { CreatePaseoWorktreeWorkflowResult } from "../../worktree-session.js"; import { deriveProjectKey } from "../../project-key.js"; import { areEquivalentPaths, createRealpathAwarePathMatcher } from "../../../utils/path.js"; +import { + defaultWorkspaceLifecycleCoordinator, + type WorkspaceLifecycleCoordinator, +} from "../../workspace-lifecycle-coordinator.js"; export interface ResolveOrCreateWorkspaceIdInput { createdWorktree: CreatePaseoWorktreeWorkflowResult | null; @@ -90,8 +95,20 @@ export function createWorkspaceProvisioningService(deps: { projectRegistry: ProjectRegistry; workspaceGitService: Pick; logger: Logger; + lifecycleCoordinator?: Pick; + isDirectory?: (path: string) => Promise; }): WorkspaceProvisioningService { const { serverId, workspaceRegistry, projectRegistry, workspaceGitService, logger } = deps; + const lifecycleCoordinator = deps.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + const isDirectory = + deps.isDirectory ?? + (async (targetPath: string) => { + try { + return (await stat(targetPath)).isDirectory(); + } catch { + return false; + } + }); async function runInImportWorkspace( input: ImportWorkspaceInput, @@ -189,6 +206,18 @@ export function createWorkspaceProvisioningService(deps: { context?: { expectsInitialAgent?: boolean }, ): Promise { const normalizedCwd = resolve(cwd); + return lifecycleCoordinator.runDirectoryExclusive(normalizedCwd, async () => { + await requireDirectoryAfterLifecycleWait(normalizedCwd); + return createWorkspaceForDirectoryUnlocked(normalizedCwd, title, projectId, context); + }); + } + + async function createWorkspaceForDirectoryUnlocked( + normalizedCwd: string, + title?: string | null, + projectId?: string, + context?: { expectsInitialAgent?: boolean }, + ): Promise { const checkout = await workspaceGitService.getCheckout(normalizedCwd); const project = projectId ? await refreshProjectKind(await requireActiveProject(projectId), normalizedCwd, checkout) @@ -207,6 +236,12 @@ export function createWorkspaceProvisioningService(deps: { return workspace; } + async function requireDirectoryAfterLifecycleWait(cwd: string): Promise { + if (!(await isDirectory(cwd))) { + throw new Error(`Working directory does not exist or is not a directory: ${cwd}`); + } + } + async function createWorkspaceForWorktree( input: CreateWorktreeWorkspaceInput, ): Promise { @@ -284,31 +319,36 @@ export function createWorkspaceProvisioningService(deps: { async function findOrCreateWorkspaceForDirectory(cwd: string): Promise { const normalizedCwd = resolve(cwd); - const workspaces = await workspaceRegistry.list(); - const active = workspaces - .filter( - (workspace) => !workspace.archivedAt && areEquivalentPaths(workspace.cwd, normalizedCwd), - ) - .sort( - (left, right) => - Date.parse(left.createdAt) - Date.parse(right.createdAt) || - left.workspaceId.localeCompare(right.workspaceId), - )[0]; - if (active) return refreshWorkspaceRecord(active); - const archived = workspaces - .filter( - (workspace) => workspace.archivedAt && areEquivalentPaths(workspace.cwd, normalizedCwd), - ) - .sort( - (left, right) => - Date.parse(left.createdAt) - Date.parse(right.createdAt) || - left.workspaceId.localeCompare(right.workspaceId), - )[0]; - if (archived) { - const project = await projectRegistry.get(archived.projectId); - if (project && !project.archivedAt) return ensureWorkspaceRecordUnarchived(archived); - } - return createWorkspaceForDirectory(normalizedCwd); + return lifecycleCoordinator.runDirectoryExclusive(normalizedCwd, async () => { + await requireDirectoryAfterLifecycleWait(normalizedCwd); + const workspaces = await workspaceRegistry.list(); + const active = workspaces + .filter( + (workspace) => !workspace.archivedAt && areEquivalentPaths(workspace.cwd, normalizedCwd), + ) + .sort( + (left, right) => + Date.parse(left.createdAt) - Date.parse(right.createdAt) || + left.workspaceId.localeCompare(right.workspaceId), + )[0]; + if (active) return refreshWorkspaceRecord(active); + const archived = workspaces + .filter( + (workspace) => workspace.archivedAt && areEquivalentPaths(workspace.cwd, normalizedCwd), + ) + .sort( + (left, right) => + Date.parse(left.createdAt) - Date.parse(right.createdAt) || + left.workspaceId.localeCompare(right.workspaceId), + )[0]; + if (archived) { + const project = await projectRegistry.get(archived.projectId); + if (project && !project.archivedAt) { + return ensureWorkspaceRecordUnarchivedUnlocked(archived); + } + } + return createWorkspaceForDirectoryUnlocked(normalizedCwd); + }); } async function resolveOrCreateWorkspaceIdForCreateAgent( @@ -325,6 +365,16 @@ export function createWorkspaceProvisioningService(deps: { async function ensureWorkspaceRecordUnarchived( workspace: PersistedWorkspaceRecord, + ): Promise { + const normalizedCwd = resolve(workspace.cwd); + return lifecycleCoordinator.runDirectoryExclusive(normalizedCwd, async () => { + await requireDirectoryAfterLifecycleWait(normalizedCwd); + return ensureWorkspaceRecordUnarchivedUnlocked(workspace); + }); + } + + async function ensureWorkspaceRecordUnarchivedUnlocked( + workspace: PersistedWorkspaceRecord, ): Promise { const project = await projectRegistry.get(workspace.projectId); if (!project) throw new Error(`Unknown project: ${workspace.projectId}`); diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 2583cb5450..af2e133046 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -26,8 +26,10 @@ import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service. import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { createPersistedWorkspaceRecord, + FileBackedProjectRegistry, FileBackedWorkspaceRegistry, } from "./workspace-registry.js"; +import { createWorkspaceProvisioningService } from "./session/workspace-provisioning/workspace-provisioning-service.js"; const cleanupPaths: string[] = []; @@ -1340,6 +1342,108 @@ describe("archiveByScope", () => { expect((await registry.get(staleWorkspaceId))?.cleanupPending).toBeNull(); }); + test("directory workspace creation waits for cleanup and revalidates after final owner read", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "cleanup-create-race"); + const incarnationId = readPaseoWorktreeIncarnationId(worktree.worktreePath); + expect(incarnationId).toBeTruthy(); + const registry = new FileBackedWorkspaceRegistry( + path.join(tempDir, "workspaces.json"), + createLogger(), + ); + const projectRegistry = new FileBackedProjectRegistry( + path.join(tempDir, "projects.json"), + createLogger(), + ); + await registry.initialize(); + await projectRegistry.initialize(); + const timestamp = new Date().toISOString(); + await registry.upsert( + createPersistedWorkspaceRecord({ + workspaceId: "ws-cleanup-create-race-stale", + projectId: "project-cleanup-create-race", + cwd: worktree.worktreePath, + kind: "worktree", + displayName: "Archived cleanup owner", + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + createdAt: timestamp, + updatedAt: timestamp, + archivedAt: timestamp, + cleanupPending: { + directoryPath: worktree.worktreePath, + teardownCwd: worktree.worktreePath, + mainRepoRoot: repoDir, + paseoWorktreesRoot: null, + worktreeIncarnationId: incarnationId, + }, + }), + ); + + const lifecycleCoordinator = new WorkspaceLifecycleCoordinator(); + let resolveFinalOwnerReadStarted: (() => void) | null = null; + const finalOwnerReadStarted = new Promise((resolvePromise) => { + resolveFinalOwnerReadStarted = resolvePromise; + }); + let allowFinalOwnerRead: (() => void) | null = null; + const finalOwnerReadAllowed = new Promise((resolvePromise) => { + allowFinalOwnerRead = resolvePromise; + }); + let ownerReads = 0; + const deps = createArchiveDeps({ paseoHome, activeWorkspaces: [] }); + deps.lifecycleCoordinator = lifecycleCoordinator; + deps.workspaceRegistry = registry; + deps.listActiveWorkspaces = vi.fn(async () => { + ownerReads += 1; + if (ownerReads === 2) { + resolveFinalOwnerReadStarted?.(); + await finalOwnerReadAllowed; + } + return []; + }); + const cleanup = retryPendingWorkspaceCleanup(deps, { + directoryPath: worktree.worktreePath, + worktreeIncarnationId: incarnationId!, + requestId: "cleanup-create-race", + }); + await finalOwnerReadStarted; + + const provisioning = createWorkspaceProvisioningService({ + workspaceRegistry: registry, + projectRegistry, + workspaceGitService: { + getCheckout: async (cwd) => ({ + cwd, + isGit: true, + currentBranch: "cleanup-create-race", + remoteUrl: null, + worktreeRoot: worktree.worktreePath, + isPaseoOwnedWorktree: true, + mainRepoRoot: repoDir, + }), + peekSnapshot: () => null, + }, + logger: createLogger(), + lifecycleCoordinator, + }); + let creationSettled = false; + const creation = provisioning.createWorkspaceForDirectory(worktree.worktreePath).finally(() => { + creationSettled = true; + }); + await Promise.resolve(); + expect(creationSettled).toBe(false); + + allowFinalOwnerRead?.(); + await expect(cleanup).resolves.toMatchObject({ removedDirectory: true }); + await expect(creation).rejects.toThrow("Working directory does not exist"); + expect(existsSync(worktree.worktreePath)).toBe(false); + expect((await registry.list()).filter((workspace) => workspace.archivedAt === null)).toEqual( + [], + ); + }); + test("workspace teardown failure keeps the record active and prevents recursive deletion", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); From ad3728d20d546716b5d744914be957dfde279010 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:32:22 -0400 Subject: [PATCH 28/43] docs: attest cleanup creation exclusion --- patches/local-patches.json | 7 +++++-- ...-v025-worktree-lifecycle-current-main.json | 20 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index 12c83fe76a..7fad704b31 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -21,6 +21,7 @@ "packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts", "packages/server/src/server/bootstrap.ts", "packages/server/src/server/session.ts", + "packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts", "packages/server/src/server/workspace-archive-service.ts", "packages/server/src/server/workspace-cleanup-retry-service.ts", "packages/server/src/server/workspace-lifecycle-coordinator.ts", @@ -40,6 +41,7 @@ "packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts", "packages/server/src/server/session.workspace-git-watch.test.ts", "packages/server/src/server/session.workspaces.test.ts", + "packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts", "packages/server/src/server/workspace-archive-service.test.ts", "packages/server/src/server/workspace-cleanup-retry-service.test.ts", "packages/server/src/server/worktree-bootstrap.test.ts", @@ -68,6 +70,7 @@ "cleanup retries group sibling obligations by incarnation-bound worktree path before deletion", "cleanup retry cycles use bounded work with capped backoff and cancellable timers", "cleanup retries use an exact cleanup-only path and incarnation boundary and cannot archive a live replacement at a reused path", + "directory workspace creation and unarchive share cleanup exclusion and revalidate the directory after waiting, preventing a replacement workspace from being minted during deletion", "shutdown aborts and awaits the active cleanup cycle and terminates a hanging teardown subprocess tree before returning", "replacement worktree incarnations archive independently of stale cleanup intents", "auto-archive retries after transient failure and cancellation observes rejection", @@ -100,8 +103,8 @@ }, "currentMainIntegration": { "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", - "verifiedHeadCommit": "5e17fcf769cb855326683413cdcd9b3cbe82ba43", - "verifiedHeadTree": "e83f37f06eeb44d5b87ff30d5ca18c0d25288f5b", + "verifiedHeadCommit": "5c70d522437ea0c02c1e769ee5f9975b8077811e", + "verifiedHeadTree": "1b4b0a140a483504225e043bfdd5cf1f26e7f875", "receipt": "patches/receipts/paseo-v025-worktree-lifecycle-current-main.json" }, "upstreamable": true, diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index b9c4d9e78a..86d53dc5d1 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,14 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "5e17fcf769cb855326683413cdcd9b3cbe82ba43", - "headTree": "e83f37f06eeb44d5b87ff30d5ca18c0d25288f5b", + "headCommit": "5c70d522437ea0c02c1e769ee5f9975b8077811e", + "headTree": "1b4b0a140a483504225e043bfdd5cf1f26e7f875", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "5e17fcf769cb855326683413cdcd9b3cbe82ba43", - "completedAt": "2026-08-01T02:18:00Z", + "commit": "5c70d522437ea0c02c1e769ee5f9975b8077811e", + "completedAt": "2026-08-01T02:31:40Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -45,6 +45,12 @@ "npm run typecheck --workspace packages/server", "oxlint <7 cleanup retry source and test files>", "oxfmt --check <7 cleanup retry source and test files>", + "git diff --check", + "vitest run <14 lifecycle, provisioning, and affected call-site test files> --fileParallelism=false", + "npm run build --workspace packages/server", + "npm run typecheck --workspace packages/server", + "oxlint <4 cleanup/create exclusion source and test files>", + "oxfmt --check <4 cleanup/create exclusion source and test files>", "git diff --check" ], "result": { @@ -66,7 +72,11 @@ "cleanupRetryShutdown": "stop aborts and awaits the owned active cycle; hanging teardown subprocess trees are terminated before shutdown completes", "cleanupOnlyRetryBuildAndTypecheck": "passed", "cleanupOnlyRetryLintAndFormat": "0 warnings, 0 errors; formatting and diff checks passed", - "allWorkspaceTypecheck": "unchanged app errors: DaemonClient lacks setProjectIcon/getProjectIcon on this stable-derived branch" + "cleanupCreateExclusionTests": "509 passed, 5 skipped across 14 lifecycle, provisioning, and affected call-site files", + "cleanupCreateExclusionBoundary": "directory workspace creation and unarchive share cleanup directory exclusion and revalidate directory existence after waiting; deterministic final-owner-read race leaves no replacement record for a deleted worktree", + "cleanupCreateExclusionBuildAndTypecheck": "passed; repository pre-commit all-workspace typecheck also passed", + "cleanupCreateExclusionLintAndFormat": "0 warnings, 0 errors; formatting and diff checks passed", + "allWorkspaceTypecheck": "passed in repository pre-commit hook" } } } From 759ed14f0b24499f0a3f161cf64c920ac3393c4b Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:39:21 -0400 Subject: [PATCH 29/43] fix: key lifecycle locks by filesystem identity --- .../server/workspace-archive-service.test.ts | 20 +++++++++++--- .../server/workspace-lifecycle-coordinator.ts | 26 +++++++++++-------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index af2e133046..1e12eb96a4 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -1,5 +1,13 @@ import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import pino, { type Logger } from "pino"; @@ -1346,6 +1354,8 @@ describe("archiveByScope", () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); const worktree = await createPaseoOwnedWorktree(repoDir, paseoHome, "cleanup-create-race"); + const equivalentWorktreePath = path.join(tempDir, "cleanup-create-race-alias"); + symlinkSync(worktree.worktreePath, equivalentWorktreePath, "dir"); const incarnationId = readPaseoWorktreeIncarnationId(worktree.worktreePath); expect(incarnationId).toBeTruthy(); const registry = new FileBackedWorkspaceRegistry( @@ -1429,9 +1439,11 @@ describe("archiveByScope", () => { lifecycleCoordinator, }); let creationSettled = false; - const creation = provisioning.createWorkspaceForDirectory(worktree.worktreePath).finally(() => { - creationSettled = true; - }); + const creation = provisioning + .createWorkspaceForDirectory(equivalentWorktreePath) + .finally(() => { + creationSettled = true; + }); await Promise.resolve(); expect(creationSettled).toBe(false); diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts index a12da10291..0cd1ce96f3 100644 --- a/packages/server/src/server/workspace-lifecycle-coordinator.ts +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -1,3 +1,5 @@ +import { normalizePathForIdentity } from "../utils/path.js"; + export class WorkspaceLifecycleCoordinator { private readonly setupTasks = new Map>(); private readonly setupTasksByDirectory = new Map>>(); @@ -9,21 +11,22 @@ export class WorkspaceLifecycleCoordinator { trackWorkspaceSetup(workspaceId: string, task: Promise, directoryPath?: string): void { this.setupTasks.set(workspaceId, task); - const directoryTasks = directoryPath - ? (this.setupTasksByDirectory.get(directoryPath) ?? new Set>()) + const directoryKey = directoryPath ? normalizePathForIdentity(directoryPath) : null; + const directoryTasks = directoryKey + ? (this.setupTasksByDirectory.get(directoryKey) ?? new Set>()) : null; - if (directoryPath && directoryTasks) { + if (directoryKey && directoryTasks) { directoryTasks.add(task); - this.setupTasksByDirectory.set(directoryPath, directoryTasks); + this.setupTasksByDirectory.set(directoryKey, directoryTasks); } const clearTask = () => { if (this.setupTasks.get(workspaceId) === task) { this.setupTasks.delete(workspaceId); } - if (directoryPath && directoryTasks) { + if (directoryKey && directoryTasks) { directoryTasks.delete(task); if (directoryTasks.size === 0) { - this.setupTasksByDirectory.delete(directoryPath); + this.setupTasksByDirectory.delete(directoryKey); } } }; @@ -158,21 +161,22 @@ export class WorkspaceLifecycleCoordinator { } runDirectoryExclusive(directoryPath: string, operation: () => Promise): Promise { - const previous = this.directoryOperations.get(directoryPath) ?? Promise.resolve(); + const directoryKey = normalizePathForIdentity(directoryPath); + const previous = this.directoryOperations.get(directoryKey) ?? Promise.resolve(); const task = previous .catch(() => undefined) .then(async () => { while (true) { - const setupTasks = Array.from(this.setupTasksByDirectory.get(directoryPath) ?? []); + const setupTasks = Array.from(this.setupTasksByDirectory.get(directoryKey) ?? []); if (setupTasks.length === 0) break; await Promise.allSettled(setupTasks); } return operation(); }); - this.directoryOperations.set(directoryPath, task); + this.directoryOperations.set(directoryKey, task); const clearTask = () => { - if (this.directoryOperations.get(directoryPath) === task) { - this.directoryOperations.delete(directoryPath); + if (this.directoryOperations.get(directoryKey) === task) { + this.directoryOperations.delete(directoryKey); } }; void task.then(clearTask, clearTask); From 7fa508cfd9ec689868af9b680a5389d4fee8dd89 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:40:05 -0400 Subject: [PATCH 30/43] docs: attest filesystem-identity lifecycle locks --- patches/local-patches.json | 5 +++-- ...o-v025-worktree-lifecycle-current-main.json | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/patches/local-patches.json b/patches/local-patches.json index 7fad704b31..d6857e92fa 100644 --- a/patches/local-patches.json +++ b/patches/local-patches.json @@ -71,6 +71,7 @@ "cleanup retry cycles use bounded work with capped backoff and cancellable timers", "cleanup retries use an exact cleanup-only path and incarnation boundary and cannot archive a live replacement at a reused path", "directory workspace creation and unarchive share cleanup exclusion and revalidate the directory after waiting, preventing a replacement workspace from being minted during deletion", + "directory setup tracking and cleanup exclusion use realpath-aware filesystem identity so symlink and macOS path aliases cannot bypass the lifecycle lock", "shutdown aborts and awaits the active cleanup cycle and terminates a hanging teardown subprocess tree before returning", "replacement worktree incarnations archive independently of stale cleanup intents", "auto-archive retries after transient failure and cancellation observes rejection", @@ -103,8 +104,8 @@ }, "currentMainIntegration": { "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", - "verifiedHeadCommit": "5c70d522437ea0c02c1e769ee5f9975b8077811e", - "verifiedHeadTree": "1b4b0a140a483504225e043bfdd5cf1f26e7f875", + "verifiedHeadCommit": "759ed14f0b24499f0a3f161cf64c920ac3393c4b", + "verifiedHeadTree": "602929fb3aa5106fecf438ab0b4d3570d7daa62b", "receipt": "patches/receipts/paseo-v025-worktree-lifecycle-current-main.json" }, "upstreamable": true, diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index 86d53dc5d1..6bab45ca2e 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -11,14 +11,14 @@ "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "5c70d522437ea0c02c1e769ee5f9975b8077811e", - "headTree": "1b4b0a140a483504225e043bfdd5cf1f26e7f875", + "headCommit": "759ed14f0b24499f0a3f161cf64c920ac3393c4b", + "headTree": "602929fb3aa5106fecf438ab0b4d3570d7daa62b", "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" }, "verification": { - "commit": "5c70d522437ea0c02c1e769ee5f9975b8077811e", - "completedAt": "2026-08-01T02:31:40Z", + "commit": "759ed14f0b24499f0a3f161cf64c920ac3393c4b", + "completedAt": "2026-08-01T02:39:35Z", "commands": [ "npm run build --workspace packages/protocol", "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", @@ -51,6 +51,12 @@ "npm run typecheck --workspace packages/server", "oxlint <4 cleanup/create exclusion source and test files>", "oxfmt --check <4 cleanup/create exclusion source and test files>", + "git diff --check", + "vitest run <14 lifecycle, provisioning, and affected call-site test files> --fileParallelism=false", + "npm run build --workspace packages/server", + "npm run typecheck --workspace packages/server", + "oxlint packages/server/src/server/workspace-lifecycle-coordinator.ts packages/server/src/server/workspace-archive-service.test.ts", + "oxfmt --check packages/server/src/server/workspace-lifecycle-coordinator.ts packages/server/src/server/workspace-archive-service.test.ts", "git diff --check" ], "result": { @@ -76,6 +82,10 @@ "cleanupCreateExclusionBoundary": "directory workspace creation and unarchive share cleanup directory exclusion and revalidate directory existence after waiting; deterministic final-owner-read race leaves no replacement record for a deleted worktree", "cleanupCreateExclusionBuildAndTypecheck": "passed; repository pre-commit all-workspace typecheck also passed", "cleanupCreateExclusionLintAndFormat": "0 warnings, 0 errors; formatting and diff checks passed", + "filesystemIdentityLockTests": "509 passed, 5 skipped across 14 lifecycle, provisioning, and affected call-site files", + "filesystemIdentityLockBoundary": "directory setup tracking and exclusion keys use realpath-aware filesystem identity; the final-owner-read race uses a symlink spelling and proves equivalent-path creation remains blocked before rejecting after cleanup deletion", + "filesystemIdentityBuildAndTypecheck": "passed; repository pre-commit all-workspace typecheck also passed", + "filesystemIdentityLintAndFormat": "0 warnings, 0 errors; formatting and diff checks passed", "allWorkspaceTypecheck": "passed in repository pre-commit hook" } } From 260fa3174ddf0174cc5d1d467ee19b94b619432f Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Fri, 31 Jul 2026 22:42:56 -0400 Subject: [PATCH 31/43] docs: record exact lifecycle test command --- .../receipts/paseo-v025-worktree-lifecycle-current-main.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json index 6bab45ca2e..4b20ea4de8 100644 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json @@ -46,13 +46,13 @@ "oxlint <7 cleanup retry source and test files>", "oxfmt --check <7 cleanup retry source and test files>", "git diff --check", - "vitest run <14 lifecycle, provisioning, and affected call-site test files> --fileParallelism=false", + "vitest run packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts packages/server/src/server/agent/create-agent/create.test.ts packages/server/src/server/agent/mcp-server.test.ts packages/server/src/terminal/terminal-session-controller.test.ts packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts packages/server/src/server/workspace-archive-service.test.ts packages/server/src/server/workspace-cleanup-retry-service.test.ts packages/server/src/server/worktree-session.test.ts packages/server/src/server/session.workspaces.test.ts packages/server/src/server/schedule/service.test.ts packages/server/src/server/loop-service.test.ts packages/server/src/server/paseo-worktree-service.test.ts packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts packages/server/src/utils/worktree.posix.test.ts --fileParallelism=false", "npm run build --workspace packages/server", "npm run typecheck --workspace packages/server", "oxlint <4 cleanup/create exclusion source and test files>", "oxfmt --check <4 cleanup/create exclusion source and test files>", "git diff --check", - "vitest run <14 lifecycle, provisioning, and affected call-site test files> --fileParallelism=false", + "vitest run packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts packages/server/src/server/agent/create-agent/create.test.ts packages/server/src/server/agent/mcp-server.test.ts packages/server/src/terminal/terminal-session-controller.test.ts packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts packages/server/src/server/workspace-archive-service.test.ts packages/server/src/server/workspace-cleanup-retry-service.test.ts packages/server/src/server/worktree-session.test.ts packages/server/src/server/session.workspaces.test.ts packages/server/src/server/schedule/service.test.ts packages/server/src/server/loop-service.test.ts packages/server/src/server/paseo-worktree-service.test.ts packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts packages/server/src/utils/worktree.posix.test.ts --fileParallelism=false", "npm run build --workspace packages/server", "npm run typecheck --workspace packages/server", "oxlint packages/server/src/server/workspace-lifecycle-coordinator.ts packages/server/src/server/workspace-archive-service.test.ts", From bb956ee4c7640c39d14a7984c1695650fa57a8bb Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 06:55:35 +0100 Subject: [PATCH 32/43] fix(server): drain create lifecycle recovery on shutdown --- .../create-agent-lifecycle-dispatch.test.ts | 187 +++++++++++++++++- .../agent/create-agent-lifecycle-dispatch.ts | 138 +++++++++++-- packages/server/src/server/bootstrap.ts | 6 + 3 files changed, 308 insertions(+), 23 deletions(-) diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index 9902f955ed..0ec4237e8d 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -6,7 +6,10 @@ import { CreateAgentLifecycleDispatch, registerAgentAutoArchive, } from "./create-agent-lifecycle-dispatch.js"; -import { requireArchiveCleanupComplete } from "../workspace-archive-service.js"; +import { + requireArchiveCleanupComplete, + WorkspaceCleanupPendingError, +} from "../workspace-archive-service.js"; class AgentLifecycleEvents { private readonly listeners = new Set(); @@ -30,6 +33,50 @@ class AgentLifecycleEvents { } } +class LifecycleRearmEvents { + private readonly listeners = new Set<() => void>(); + + subscribe(listener: () => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + publish(): void { + for (const listener of this.listeners) listener(); + } + + listenerCount(): number { + return this.listeners.size; + } +} + +function createLifecycleDispatch( + agents: AgentLifecycleEvents, + archiveAgentForClose: (agentId: string) => Promise, +): CreateAgentLifecycleDispatch { + const dependencies = { + paseoHome: "/tmp/paseo", + agentManager: agents, + agentStorage: {}, + github: {}, + workspaceGitService: {}, + createPaseoWorktreeWorkflow: vi.fn(), + archiveAgentForClose, + findWorkspaceIdForCwd: async () => null, + listActiveWorkspaces: async () => [], + archiveWorkspaceRecord: async () => undefined, + workspaceRegistry: {}, + emit: () => undefined, + emitAgentRemove: () => undefined, + emitWorkspaceUpdatesForWorkspaceIds: async () => undefined, + markWorkspaceArchiving: () => undefined, + clearWorkspaceArchiving: () => undefined, + killTerminalsForWorkspace: async () => undefined, + logger: createTestLogger(), + } as unknown as ConstructorParameters[0]; + return new CreateAgentLifecycleDispatch(dependencies); +} + test("auto-archive self-releases once and later cancellation waits harmlessly", async () => { const agentId = "4a7e2521-286d-4ad5-af35-e091c55302e3"; const agents = new AgentLifecycleEvents(); @@ -73,9 +120,11 @@ test("auto-archive retries autonomously after a transient failure", async () => expect(agents.listenerCount()).toBe(0); }); -test("auto-archive retries when physical workspace cleanup remains pending", async () => { +test("cleanup pending waits for a lifecycle rearm without a retry loop", async () => { + vi.useFakeTimers(); const agentId = "agent-auto-archive-cleanup-pending"; const agents = new AgentLifecycleEvents(); + const rearmEvents = new LifecycleRearmEvents(); const archive = vi.fn(async () => { const cleanupPendingWorkspaceIds = archive.mock.calls.length === 1 ? ["ws-cleanup-pending"] : []; @@ -93,15 +142,138 @@ test("auto-archive retries when physical workspace cleanup remains pending", asy agentManager: agents, agentId, archive, - retryBaseMs: 0, - retryMaxMs: 0, + shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), + subscribeToRearm: (rearm) => rearmEvents.subscribe(rearm), + }); + + try { + agents.completeTurn(agentId); + await vi.advanceTimersByTimeAsync(60_000); + + expect(archive).toHaveBeenCalledTimes(1); + expect(rearmEvents.listenerCount()).toBe(1); + + rearmEvents.publish(); + await expect(registration.settled).resolves.toBe("completed"); + + expect(archive).toHaveBeenCalledTimes(2); + expect(agents.listenerCount()).toBe(0); + expect(rearmEvents.listenerCount()).toBe(0); + } finally { + await registration.cancel().catch(() => undefined); + vi.useRealTimers(); + } +}); + +test("a registry mutation during cleanup failure cannot be lost before rearm waiting", async () => { + const agentId = "agent-auto-archive-concurrent-rearm"; + const agents = new AgentLifecycleEvents(); + const rearmEvents = new LifecycleRearmEvents(); + let archiveCount = 0; + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive: async () => { + archiveCount += 1; + if (archiveCount === 1) { + rearmEvents.publish(); + throw new WorkspaceCleanupPendingError("Auto-created worktree archive", [ + "ws-cleanup-pending", + ]); + } + }, + shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), + subscribeToRearm: (rearm) => rearmEvents.subscribe(rearm), }); agents.completeTurn(agentId); - await vi.waitFor(() => expect(archive).toHaveBeenCalledTimes(2)); - await registration.cancel(); - expect(archive).toHaveBeenCalledTimes(2); + await expect(registration.settled).resolves.toBe("completed"); + + expect(archiveCount).toBe(2); expect(agents.listenerCount()).toBe(0); + expect(rearmEvents.listenerCount()).toBe(0); +}); + +test("lifecycle shutdown cancels listeners and truthfully awaits an active archive", async () => { + const agentId = "agent-auto-archive-shutdown-wait"; + const agents = new AgentLifecycleEvents(); + let markArchiveStarted!: () => void; + let releaseArchive!: () => void; + const archiveStarted = new Promise((resolve) => { + markArchiveStarted = resolve; + }); + const archiveFinished = new Promise((resolve) => { + releaseArchive = resolve; + }); + const dispatch = createLifecycleDispatch(agents, async () => { + markArchiveStarted(); + await archiveFinished; + }); + const registration = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + agents.completeTurn(agentId); + await archiveStarted; + + let shutdownSettled = false; + const shutdown = dispatch.shutdown({ timeoutMs: 5_000 }).then((result) => { + shutdownSettled = true; + return result; + }); + await Promise.resolve(); + + expect(shutdownSettled).toBe(false); + expect(agents.listenerCount()).toBe(0); + + releaseArchive(); + await expect(shutdown).resolves.toEqual({ completed: true, pendingAgentIds: [] }); + await expect(registration.settled).resolves.toBe("completed"); +}); + +test("lifecycle shutdown reports its active archive at the deadline", async () => { + vi.useFakeTimers(); + const agentId = "agent-auto-archive-shutdown-deadline"; + const agents = new AgentLifecycleEvents(); + let markArchiveStarted!: () => void; + let releaseArchive!: () => void; + const archiveStarted = new Promise((resolve) => { + markArchiveStarted = resolve; + }); + const archiveFinished = new Promise((resolve) => { + releaseArchive = resolve; + }); + const dispatch = createLifecycleDispatch(agents, async () => { + markArchiveStarted(); + await archiveFinished; + }); + const registration = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + + try { + agents.completeTurn(agentId); + await archiveStarted; + + const shutdown = dispatch.shutdown({ timeoutMs: 100 }); + await vi.advanceTimersByTimeAsync(100); + + await expect(shutdown).resolves.toEqual({ completed: false, pendingAgentIds: [agentId] }); + expect(agents.listenerCount()).toBe(0); + + releaseArchive(); + await expect(registration.settled).resolves.toBe("completed"); + await expect(dispatch.shutdown({ timeoutMs: 100 })).resolves.toEqual({ + completed: true, + pendingAgentIds: [], + }); + } finally { + releaseArchive(); + vi.useRealTimers(); + } }); test("auto-archive cancellation observes an in-flight archive rejection", async () => { @@ -131,6 +303,7 @@ test("lifecycle dispatch does not permanently suppress an agent after archive fa const dependencies = { agentManager: agents, archiveAgentForClose, + workspaceRegistry: {}, logger: createTestLogger(), } as unknown as ConstructorParameters[0]; const dispatch = new CreateAgentLifecycleDispatch(dependencies); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 90b29e0f93..f6f3293af5 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -2,11 +2,13 @@ import { randomUUID } from "node:crypto"; import type pino from "pino"; import type { ForgeService } from "../../services/forge-service.js"; +import { withTimeout } from "../../utils/promise-timeout.js"; import { isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; import { archiveByScope, requireArchiveCleanupComplete, type ActiveWorkspaceRef, + WorkspaceCleanupPendingError, } from "../workspace-archive-service.js"; import type { CreatePaseoWorktreeWorkflowFn, @@ -34,7 +36,7 @@ interface CreateAgentLifecycleDispatchDependencies { findWorkspaceIdForCwd: (cwd: string) => Promise; listActiveWorkspaces: () => Promise; archiveWorkspaceRecord: (workspaceId: string) => Promise; - workspaceRegistry: Pick; + workspaceRegistry: Pick; emit: (message: SessionOutboundMessage) => void; emitAgentRemove: (agentId: string) => void; emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds: Iterable) => Promise; @@ -45,6 +47,7 @@ interface CreateAgentLifecycleDispatchDependencies { } export interface LifecycleRegistration { + readonly settled: Promise<"completed" | "cancelled">; cancel(): Promise; } @@ -52,7 +55,15 @@ interface AgentLifecycleEvents { subscribe(callback: AgentSubscriber, options?: SubscribeOptions): () => void; } -const inactiveRegistration: LifecycleRegistration = { cancel: async () => undefined }; +const inactiveRegistration: LifecycleRegistration = { + settled: Promise.resolve("cancelled"), + cancel: async () => undefined, +}; + +export interface LifecycleDispatchShutdownResult { + completed: boolean; + pendingAgentIds: string[]; +} type AutoArchiveTarget = | { kind: "agent-only" } @@ -60,6 +71,8 @@ type AutoArchiveTarget = export class CreateAgentLifecycleDispatch { private readonly autoArchiveTasks = new Map>(); + private readonly autoArchiveRegistrations = new Map(); + private shuttingDown = false; constructor(private readonly dependencies: CreateAgentLifecycleDispatchDependencies) {} @@ -162,11 +175,57 @@ export class CreateAgentLifecycleDispatch { agentId: string, target: AutoArchiveTarget, ): LifecycleRegistration { - return registerAgentAutoArchive({ + if (this.shuttingDown) return inactiveRegistration; + const existing = this.autoArchiveRegistrations.get(agentId); + if (existing) return existing; + + const subscribeToMutations = this.dependencies.workspaceRegistry.subscribeToMutations?.bind( + this.dependencies.workspaceRegistry, + ); + const registration = registerAgentAutoArchive({ agentManager: this.dependencies.agentManager, agentId, archive: () => this.autoArchiveAgentOnce(agentId, target), + shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), + subscribeToRearm: subscribeToMutations + ? (rearm) => subscribeToMutations(() => rearm()) + : undefined, }); + this.autoArchiveRegistrations.set(agentId, registration); + void registration.settled.then(() => { + if (this.autoArchiveRegistrations.get(agentId) === registration) { + this.autoArchiveRegistrations.delete(agentId); + } + return undefined; + }); + return registration; + } + + async shutdown(options?: { timeoutMs?: number }): Promise { + this.shuttingDown = true; + const registrations = Array.from(this.autoArchiveRegistrations.entries()); + const cancellation = Promise.allSettled( + registrations.map(([, registration]) => registration.cancel()), + ).then(async () => { + await Promise.allSettled(Array.from(this.autoArchiveTasks.values())); + return undefined; + }); + try { + await withTimeout( + cancellation, + options?.timeoutMs ?? 10_000, + "Timed out shutting down create-agent lifecycle registrations", + ); + } catch (error) { + this.dependencies.logger.error( + { err: error, agentIds: registrations.map(([registeredAgentId]) => registeredAgentId) }, + "Create-agent lifecycle shutdown remains incomplete", + ); + } + const pendingAgentIds = Array.from( + new Set([...this.autoArchiveRegistrations.keys(), ...this.autoArchiveTasks.keys()]), + ); + return { completed: pendingAgentIds.length === 0, pendingAgentIds }; } private async autoArchiveAgentOnce(agentId: string, target: AutoArchiveTarget): Promise { @@ -250,20 +309,38 @@ export function registerAgentAutoArchive(input: { archive: () => Promise; retryBaseMs?: number; retryMaxMs?: number; + shouldRetry?: (error: unknown) => boolean; + subscribeToRearm?: (rearm: () => void) => () => void; }): LifecycleRegistration { let unsubscribe: (() => void) | null = null; + let unsubscribeRearm: (() => void) | null = null; let archiveTask: Promise | null = null; let retryTimer: ReturnType | null = null; let lastArchiveFailure: unknown | null = null; let consecutiveFailures = 0; + let mutationVersion = 0; + let awaitingRearm = false; + let terminalObserved = false; let canceled = false; + let settleRegistration!: (result: "completed" | "cancelled") => void; + let registrationSettled = false; + const settled = new Promise<"completed" | "cancelled">((resolve) => { + settleRegistration = resolve; + }); const retryBaseMs = input.retryBaseMs ?? 1_000; const retryMaxMs = input.retryMaxMs ?? 60_000; + const settle = (result: "completed" | "cancelled") => { + if (registrationSettled) return; + registrationSettled = true; + settleRegistration(result); + }; const release = () => { - if (!unsubscribe) return; - const subscribed = unsubscribe; + const subscribedToAgent = unsubscribe; unsubscribe = null; - subscribed(); + subscribedToAgent?.(); + const subscribedToRearm = unsubscribeRearm; + unsubscribeRearm = null; + subscribedToRearm?.(); }; const scheduleRetry = () => { if (canceled || retryTimer) return; @@ -274,9 +351,20 @@ export function registerAgentAutoArchive(input: { }, delay); (retryTimer as unknown as { unref?: () => void }).unref?.(); }; - const attemptArchive = () => { + if (input.subscribeToRearm) { + unsubscribeRearm = input.subscribeToRearm(() => { + mutationVersion += 1; + if (!canceled && terminalObserved && awaitingRearm && !archiveTask) { + awaitingRearm = false; + consecutiveFailures = 0; + attemptArchive(); + } + }); + } + function attemptArchive(): void { if (canceled || archiveTask) return; - const task = input.archive(); + const attemptMutationVersion = mutationVersion; + const task = Promise.resolve().then(input.archive); archiveTask = task; void task.then( () => { @@ -289,6 +377,7 @@ export function registerAgentAutoArchive(input: { retryTimer = null; } release(); + settle("completed"); return undefined; }, (error: unknown) => { @@ -296,25 +385,41 @@ export function registerAgentAutoArchive(input: { archiveTask = null; lastArchiveFailure = error; consecutiveFailures += 1; - scheduleRetry(); + const retryAllowed = input.shouldRetry?.(error) ?? true; + if (!retryAllowed) { + awaitingRearm = true; + if (!canceled && terminalObserved && mutationVersion !== attemptMutationVersion) { + awaitingRearm = false; + consecutiveFailures = 0; + attemptArchive(); + } + } else { + scheduleRetry(); + } return undefined; }, ); - }; + } const registration: LifecycleRegistration = { + settled, async cancel() { canceled = true; + awaitingRearm = false; if (retryTimer) { clearTimeout(retryTimer); retryTimer = null; } release(); - const pendingArchive = archiveTask; - if (pendingArchive) { - await pendingArchive; - } - if (lastArchiveFailure) { - throw lastArchiveFailure; + try { + const pendingArchive = archiveTask; + if (pendingArchive) { + await pendingArchive; + } + if (lastArchiveFailure) { + throw lastArchiveFailure; + } + } finally { + settle("cancelled"); } }, }; @@ -328,6 +433,7 @@ export function registerAgentAutoArchive(input: { ) { return; } + terminalObserved = true; attemptArchive(); }, { agentId: input.agentId, replayState: false }, diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 034031f309..7df048cfc3 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -1669,6 +1669,12 @@ export async function createPaseoDaemon( // Freeze both ingress and registration before taking the agent closure snapshot. wsServer?.prepareForShutdown(); agentManager.prepareForShutdown(); + const lifecycleShutdown = await hubAgentLifecycle.shutdown({ timeoutMs: 10_000 }); + if (!lifecycleShutdown.completed) { + throw new Error( + `Create-agent lifecycle shutdown remains incomplete for agents: ${lifecycleShutdown.pendingAgentIds.join(", ")}`, + ); + } await closeAllAgents(logger, agentManager); await agentManager.flushForShutdown().catch(() => undefined); detachAgentStoragePersistence(); From f128a34dc601fd5f09bc83d85d3d4ce796994e7d Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 07:59:08 +0100 Subject: [PATCH 33/43] fix(server): own create lifecycle through shutdown --- .../create-agent-lifecycle-dispatch.test.ts | 95 ++++++++++++- .../agent/create-agent-lifecycle-dispatch.ts | 129 +++++++----------- .../server/src/server/bootstrap.smoke.test.ts | 15 ++ packages/server/src/server/bootstrap.ts | 104 +++++++++----- packages/server/src/server/daemon-worker.ts | 7 +- packages/server/src/server/session.test.ts | 2 + packages/server/src/server/session.ts | 83 +++++++---- .../session.workspace-git-watch.test.ts | 6 +- ...on.workspace-resolution-invariants.test.ts | 6 +- .../src/server/session.workspaces.test.ts | 10 ++ .../snapshot-mutation-ownership.test.ts | 6 +- .../src/server/test-utils/session-stubs.ts | 10 ++ .../websocket-server.browser-tools.test.ts | 2 + .../websocket-server.notifications.test.ts | 2 + .../websocket-server.relay-reconnect.test.ts | 2 + ...cket-server.terminal-notifications.test.ts | 2 + .../server/src/server/websocket-server.ts | 5 + .../server/src/server/wire-compat.test.ts | 6 +- 18 files changed, 337 insertions(+), 155 deletions(-) diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index 0ec4237e8d..73802889de 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -34,15 +34,15 @@ class AgentLifecycleEvents { } class LifecycleRearmEvents { - private readonly listeners = new Set<() => void>(); + private readonly listeners = new Set<(workspaceId: string) => void>(); - subscribe(listener: () => void): () => void { + subscribe(listener: (workspaceId: string) => void): () => void { this.listeners.add(listener); return () => this.listeners.delete(listener); } - publish(): void { - for (const listener of this.listeners) listener(); + publish(workspaceId = "ws-cleanup-pending"): void { + for (const listener of this.listeners) listener(workspaceId); } listenerCount(): number { @@ -60,7 +60,6 @@ function createLifecycleDispatch( agentStorage: {}, github: {}, workspaceGitService: {}, - createPaseoWorktreeWorkflow: vi.fn(), archiveAgentForClose, findWorkspaceIdForCwd: async () => null, listActiveWorkspaces: async () => [], @@ -144,6 +143,8 @@ test("cleanup pending waits for a lifecycle rearm without a retry loop", async ( archive, shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), subscribeToRearm: (rearm) => rearmEvents.subscribe(rearm), + rearmKeysForError: (error) => + error instanceof WorkspaceCleanupPendingError ? error.workspaceIds : [], }); try { @@ -184,6 +185,8 @@ test("a registry mutation during cleanup failure cannot be lost before rearm wai }, shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), subscribeToRearm: (rearm) => rearmEvents.subscribe(rearm), + rearmKeysForError: (error) => + error instanceof WorkspaceCleanupPendingError ? error.workspaceIds : [], }); agents.completeTurn(agentId); @@ -194,6 +197,38 @@ test("a registry mutation during cleanup failure cannot be lost before rearm wai expect(rearmEvents.listenerCount()).toBe(0); }); +test("cleanup pending ignores unrelated workspace mutations", async () => { + const agentId = "agent-auto-archive-filtered-rearm"; + const agents = new AgentLifecycleEvents(); + const rearmEvents = new LifecycleRearmEvents(); + const archive = vi + .fn<() => Promise>() + .mockRejectedValueOnce( + new WorkspaceCleanupPendingError("Auto-created worktree archive", ["ws-relevant"]), + ) + .mockResolvedValueOnce(undefined); + const registration = registerAgentAutoArchive({ + agentManager: agents, + agentId, + archive, + shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), + subscribeToRearm: (rearm) => rearmEvents.subscribe(rearm), + rearmKeysForError: (error) => + error instanceof WorkspaceCleanupPendingError ? error.workspaceIds : [], + }); + + agents.completeTurn(agentId); + await vi.waitFor(() => expect(archive).toHaveBeenCalledTimes(1)); + + rearmEvents.publish("ws-unrelated"); + await Promise.resolve(); + expect(archive).toHaveBeenCalledTimes(1); + + rearmEvents.publish("ws-relevant"); + await expect(registration.settled).resolves.toBe("completed"); + expect(archive).toHaveBeenCalledTimes(2); +}); + test("lifecycle shutdown cancels listeners and truthfully awaits an active archive", async () => { const agentId = "agent-auto-archive-shutdown-wait"; const agents = new AgentLifecycleEvents(); @@ -232,6 +267,29 @@ test("lifecycle shutdown cancels listeners and truthfully awaits an active archi await expect(registration.settled).resolves.toBe("completed"); }); +test("one dispatcher keeps one lifecycle registration across repeated callers", async () => { + const agentId = "agent-shared-lifecycle-registration"; + const agents = new AgentLifecycleEvents(); + const dispatch = createLifecycleDispatch(agents, async () => undefined); + + const first = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + const second = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + + expect(second).toBe(first); + expect(agents.listenerCount()).toBe(1); + + await first.cancel(); + expect(agents.listenerCount()).toBe(0); +}); + test("lifecycle shutdown reports its active archive at the deadline", async () => { vi.useFakeTimers(); const agentId = "agent-auto-archive-shutdown-deadline"; @@ -290,6 +348,33 @@ test("auto-archive cancellation observes an in-flight archive rejection", async agents.completeTurn(agentId); await expect(registration.cancel()).rejects.toBe(failure); + await expect(registration.settled).resolves.toBe("unresolved"); + expect(agents.listenerCount()).toBe(0); +}); + +test("lifecycle shutdown reports cleanup-pending cancellation as unresolved", async () => { + const agentId = "agent-auto-archive-shutdown-unresolved"; + const agents = new AgentLifecycleEvents(); + const failure = new WorkspaceCleanupPendingError("Auto-created worktree archive", [ + "ws-cleanup-pending", + ]); + const dispatch = createLifecycleDispatch(agents, async () => { + throw failure; + }); + const registration = dispatch.registerAutoArchiveIfRequested({ + autoArchive: true, + agentId, + createdWorktree: null, + }); + + agents.completeTurn(agentId); + await vi.waitFor(() => expect(agents.listenerCount()).toBe(1)); + + await expect(dispatch.shutdown()).resolves.toEqual({ + completed: false, + pendingAgentIds: [agentId], + }); + await expect(registration.settled).resolves.toBe("unresolved"); expect(agents.listenerCount()).toBe(0); }); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index f6f3293af5..185ebd7824 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -10,17 +10,10 @@ import { type ActiveWorkspaceRef, WorkspaceCleanupPendingError, } from "../workspace-archive-service.js"; -import type { - CreatePaseoWorktreeWorkflowFn, - CreatePaseoWorktreeWorkflowResult, -} from "../worktree-session.js"; +import type { CreatePaseoWorktreeWorkflowResult } from "../worktree-session.js"; import type { WorkspaceGitService } from "../workspace-git-service.js"; import type { WorkspaceRegistry } from "../workspace-registry.js"; -import type { - CreateAgentWorktreeTarget, - FirstAgentContext, - SessionOutboundMessage, -} from "../messages.js"; +import type { SessionOutboundMessage } from "../messages.js"; import type { AgentManager, AgentSubscriber, SubscribeOptions } from "./agent-manager.js"; import type { AgentStorage } from "./agent-storage.js"; @@ -31,7 +24,6 @@ interface CreateAgentLifecycleDispatchDependencies { agentStorage: AgentStorage; github: ForgeService; workspaceGitService: WorkspaceGitService; - createPaseoWorktreeWorkflow: CreatePaseoWorktreeWorkflowFn; archiveAgentForClose: (agentId: string) => Promise; findWorkspaceIdForCwd: (cwd: string) => Promise; listActiveWorkspaces: () => Promise; @@ -47,7 +39,7 @@ interface CreateAgentLifecycleDispatchDependencies { } export interface LifecycleRegistration { - readonly settled: Promise<"completed" | "cancelled">; + readonly settled: Promise<"completed" | "cancelled" | "unresolved">; cancel(): Promise; } @@ -76,22 +68,6 @@ export class CreateAgentLifecycleDispatch { constructor(private readonly dependencies: CreateAgentLifecycleDispatchDependencies) {} - async createWorktreeForRequest(input: { - cwd: string; - target: CreateAgentWorktreeTarget | undefined; - firstAgentContext: FirstAgentContext; - hasLegacyGitOptions: boolean; - }): Promise { - if (input.target && input.hasLegacyGitOptions) { - throw new Error("create_agent_request worktree cannot be combined with git options"); - } - if (!input.target) { - return null; - } - - return this.createWorktreeForTarget(input.cwd, input.target, input.firstAgentContext); - } - registerAutoArchiveIfRequested(input: { autoArchive: boolean | undefined; agentId: string; @@ -130,47 +106,6 @@ export class CreateAgentLifecycleDispatch { }); } - private async createWorktreeForTarget( - cwd: string, - target: CreateAgentWorktreeTarget, - firstAgentContext: FirstAgentContext, - ): Promise { - const baseInput = { - cwd, - firstAgentContext, - runSetup: false, - paseoHome: this.dependencies.paseoHome, - worktreesRoot: this.dependencies.worktreesRoot, - } as const; - - switch (target.mode) { - case "branch-off": - return this.dependencies.createPaseoWorktreeWorkflow( - { - ...baseInput, - worktreeSlug: target.newBranch, - action: "branch-off", - ...(target.base ? { refName: target.base } : {}), - }, - target.base ? { resolveDefaultBranch: async () => target.base! } : undefined, - ); - case "checkout-branch": - return this.dependencies.createPaseoWorktreeWorkflow({ - ...baseInput, - action: "checkout", - refName: target.branch, - }); - case "checkout-pr": - return this.dependencies.createPaseoWorktreeWorkflow({ - ...baseInput, - action: "checkout", - githubPrNumber: target.prNumber, - }); - default: - throw new Error("Unsupported create_agent_request worktree target"); - } - } - private registerAutoArchiveOnTerminalState( agentId: string, target: AutoArchiveTarget, @@ -188,8 +123,10 @@ export class CreateAgentLifecycleDispatch { archive: () => this.autoArchiveAgentOnce(agentId, target), shouldRetry: (error) => !(error instanceof WorkspaceCleanupPendingError), subscribeToRearm: subscribeToMutations - ? (rearm) => subscribeToMutations(() => rearm()) + ? (rearm) => subscribeToMutations((mutation) => rearm(mutation.workspaceId)) : undefined, + rearmKeysForError: (error) => + error instanceof WorkspaceCleanupPendingError ? error.workspaceIds : [], }); this.autoArchiveRegistrations.set(agentId, registration); void registration.settled.then(() => { @@ -206,12 +143,13 @@ export class CreateAgentLifecycleDispatch { const registrations = Array.from(this.autoArchiveRegistrations.entries()); const cancellation = Promise.allSettled( registrations.map(([, registration]) => registration.cancel()), - ).then(async () => { + ).then(async (results) => { await Promise.allSettled(Array.from(this.autoArchiveTasks.values())); - return undefined; + return results; }); + let cancellationResults: PromiseSettledResult[] | null = null; try { - await withTimeout( + cancellationResults = await withTimeout( cancellation, options?.timeoutMs ?? 10_000, "Timed out shutting down create-agent lifecycle registrations", @@ -222,8 +160,17 @@ export class CreateAgentLifecycleDispatch { "Create-agent lifecycle shutdown remains incomplete", ); } + const unresolvedAgentIds = cancellationResults + ? registrations.flatMap(([agentId], index) => + cancellationResults[index]?.status === "rejected" ? [agentId] : [], + ) + : registrations.map(([agentId]) => agentId); const pendingAgentIds = Array.from( - new Set([...this.autoArchiveRegistrations.keys(), ...this.autoArchiveTasks.keys()]), + new Set([ + ...unresolvedAgentIds, + ...this.autoArchiveRegistrations.keys(), + ...this.autoArchiveTasks.keys(), + ]), ); return { completed: pendingAgentIds.length === 0, pendingAgentIds }; } @@ -310,7 +257,8 @@ export function registerAgentAutoArchive(input: { retryBaseMs?: number; retryMaxMs?: number; shouldRetry?: (error: unknown) => boolean; - subscribeToRearm?: (rearm: () => void) => () => void; + subscribeToRearm?: (rearm: (key: string) => void) => () => void; + rearmKeysForError?: (error: unknown) => readonly string[]; }): LifecycleRegistration { let unsubscribe: (() => void) | null = null; let unsubscribeRearm: (() => void) | null = null; @@ -319,17 +267,18 @@ export function registerAgentAutoArchive(input: { let lastArchiveFailure: unknown | null = null; let consecutiveFailures = 0; let mutationVersion = 0; + const mutationVersionByKey = new Map(); let awaitingRearm = false; let terminalObserved = false; let canceled = false; - let settleRegistration!: (result: "completed" | "cancelled") => void; + let settleRegistration!: (result: "completed" | "cancelled" | "unresolved") => void; let registrationSettled = false; - const settled = new Promise<"completed" | "cancelled">((resolve) => { + const settled = new Promise<"completed" | "cancelled" | "unresolved">((resolve) => { settleRegistration = resolve; }); const retryBaseMs = input.retryBaseMs ?? 1_000; const retryMaxMs = input.retryMaxMs ?? 60_000; - const settle = (result: "completed" | "cancelled") => { + const settle = (result: "completed" | "cancelled" | "unresolved") => { if (registrationSettled) return; registrationSettled = true; settleRegistration(result); @@ -352,9 +301,19 @@ export function registerAgentAutoArchive(input: { (retryTimer as unknown as { unref?: () => void }).unref?.(); }; if (input.subscribeToRearm) { - unsubscribeRearm = input.subscribeToRearm(() => { + unsubscribeRearm = input.subscribeToRearm((key) => { mutationVersion += 1; - if (!canceled && terminalObserved && awaitingRearm && !archiveTask) { + mutationVersionByKey.set(key, mutationVersion); + const rearmKeys = lastArchiveFailure + ? (input.rearmKeysForError?.(lastArchiveFailure) ?? null) + : null; + if ( + !canceled && + terminalObserved && + awaitingRearm && + !archiveTask && + (rearmKeys === null || rearmKeys.includes(key)) + ) { awaitingRearm = false; consecutiveFailures = 0; attemptArchive(); @@ -388,7 +347,11 @@ export function registerAgentAutoArchive(input: { const retryAllowed = input.shouldRetry?.(error) ?? true; if (!retryAllowed) { awaitingRearm = true; - if (!canceled && terminalObserved && mutationVersion !== attemptMutationVersion) { + const rearmKeys = input.rearmKeysForError?.(error) ?? null; + const relevantMutationObserved = rearmKeys + ? rearmKeys.some((key) => (mutationVersionByKey.get(key) ?? 0) > attemptMutationVersion) + : mutationVersion !== attemptMutationVersion; + if (!canceled && terminalObserved && relevantMutationObserved) { awaitingRearm = false; consecutiveFailures = 0; attemptArchive(); @@ -413,13 +376,15 @@ export function registerAgentAutoArchive(input: { try { const pendingArchive = archiveTask; if (pendingArchive) { - await pendingArchive; + await pendingArchive.catch(() => undefined); } if (lastArchiveFailure) { throw lastArchiveFailure; } - } finally { settle("cancelled"); + } catch (error) { + settle("unresolved"); + throw error; } }, }; diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 3509453322..5871e17c57 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -342,6 +342,21 @@ describe("paseo daemon bootstrap", () => { } }); + test("continues teardown after a shutdown step fails", async () => { + const daemonHandle = await createTestPaseoDaemon(); + const failure = new Error("service proxy stop failed"); + vi.spyOn(daemonHandle.daemon.serviceProxy, "stopStandalone").mockRejectedValueOnce(failure); + + try { + await expect(daemonHandle.daemon.stop()).rejects.toThrow( + "One or more daemon shutdown steps failed", + ); + await expect(fetch(`http://127.0.0.1:${daemonHandle.port}/api/health`)).rejects.toThrow(); + } finally { + await daemonHandle.close(); + } + }); + test("standalone listener exposes services only", async () => { const standalonePort = await findFreePort(); const upstream = http.createServer((_req, res) => { diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 7df048cfc3..04da7c5588 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -443,10 +443,14 @@ export interface PaseoDaemon { scriptRuntimeStore: WorkspaceScriptRuntimeStore; browserToolsBroker: BrowserToolsBroker; start(): Promise; - stop(): Promise; + stop(options?: PaseoDaemonStopOptions): Promise; getListenTarget(): ListenTarget | null; } +export interface PaseoDaemonStopOptions { + deadlineAt?: number; +} + export interface PaseoDaemonDependencies { hubRelationshipRemote?: HubRelationshipRemote; hubRelationshipClock?: HubRelationshipClock; @@ -1138,7 +1142,6 @@ export async function createPaseoDaemon( agentStorage, github, workspaceGitService, - createPaseoWorktreeWorkflow: createPaseoWorktreeForTools, archiveAgentForClose: (agentId) => archiveAgentCommand({ agentManager, agentStorage, logger }, agentId), findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, @@ -1562,6 +1565,7 @@ export async function createPaseoDaemon( relayConfig: dependencies.serverFeatureOverrides?.relayConfig, }, workspaceAutoName, + hubAgentLifecycle, config.auth, speechService, terminalManager, @@ -1661,33 +1665,47 @@ export async function createPaseoDaemon( } }; - const stop = async () => { - await hubRelationships.stop(); - await workspaceCleanupRetryService.stop(); - workspaceReconciliation.dispose(); - scriptHealthMonitor.stop(); + const stop = async (options?: PaseoDaemonStopOptions) => { + const errors: unknown[] = []; + const attempt = async (label: string, action: () => void | Promise): Promise => { + try { + await action(); + } catch (error) { + errors.push(error); + logger.error({ err: error }, `Daemon shutdown step failed: ${label}`); + } + }; + + await attempt("hub relationships", () => hubRelationships.stop()); + await attempt("workspace cleanup retry service", () => workspaceCleanupRetryService.stop()); + await attempt("workspace reconciliation", () => workspaceReconciliation.dispose()); + await attempt("script health monitor", () => scriptHealthMonitor.stop()); // Freeze both ingress and registration before taking the agent closure snapshot. - wsServer?.prepareForShutdown(); - agentManager.prepareForShutdown(); - const lifecycleShutdown = await hubAgentLifecycle.shutdown({ timeoutMs: 10_000 }); - if (!lifecycleShutdown.completed) { - throw new Error( - `Create-agent lifecycle shutdown remains incomplete for agents: ${lifecycleShutdown.pendingAgentIds.join(", ")}`, - ); - } - await closeAllAgents(logger, agentManager); - await agentManager.flushForShutdown().catch(() => undefined); - detachAgentStoragePersistence(); - await agentStorage.flush().catch(() => undefined); - await providerSnapshotManager.shutdown(); - terminalManager.killAll(); - speechService.stop(); - await scheduleService.stop().catch(() => undefined); - await relayRuntime?.stop().catch(() => undefined); - if (wsServer) { - await wsServer.close(); - } - await serviceProxy.stopStandalone(); + await attempt("WebSocket ingress", () => wsServer?.prepareForShutdown()); + await attempt("agent registration", () => agentManager.prepareForShutdown()); + await attempt("create-agent lifecycle", async () => { + const remainingMs = + options?.deadlineAt === undefined + ? 10_000 + : Math.max(0, options.deadlineAt - Date.now() - 2_000); + const lifecycleShutdown = await hubAgentLifecycle.shutdown({ timeoutMs: remainingMs }); + if (!lifecycleShutdown.completed) { + throw new Error( + `Create-agent lifecycle shutdown remains incomplete for agents: ${lifecycleShutdown.pendingAgentIds.join(", ")}`, + ); + } + }); + await attempt("agents", () => closeAllAgents(logger, agentManager)); + await attempt("agent persistence", () => agentManager.flushForShutdown()); + await attempt("agent storage detachment", () => detachAgentStoragePersistence()); + await attempt("agent storage", () => agentStorage.flush()); + await attempt("provider snapshots", () => providerSnapshotManager.shutdown()); + await attempt("terminals", () => terminalManager.killAll()); + await attempt("speech", () => speechService.stop()); + await attempt("schedules", () => scheduleService.stop()); + await attempt("relay", () => relayRuntime?.stop()); + await attempt("WebSocket server", () => wsServer?.close()); + await attempt("service proxy", () => serviceProxy.stopStandalone()); // Force-drop remaining sockets so httpServer.close() resolves promptly. // We've already closed wsServer (which sent ws-layer close frames) and // stopped every other service, so anything still attached is a TCP @@ -1695,13 +1713,22 @@ export async function createPaseoDaemon( // upgraded WS sockets in the closing handshake, or HTTP keep-alive // sockets in CLOSE_WAIT). closeIdleConnections() does not catch // upgraded sockets, so we use closeAllConnections() here. - httpServer.closeAllConnections(); - await new Promise((resolve) => { - httpServer.close(() => resolve()); - }); + await attempt("HTTP connections", () => httpServer.closeAllConnections()); + await attempt( + "HTTP server", + () => + new Promise((resolve) => { + httpServer.close(() => resolve()); + }), + ); // Clean up socket files - if (listenTarget.type === "socket" && existsSync(listenTarget.path)) { - unlinkSync(listenTarget.path); + await attempt("socket file", () => { + if (listenTarget.type === "socket" && existsSync(listenTarget.path)) { + unlinkSync(listenTarget.path); + } + }); + if (errors.length > 0) { + throw new AggregateError(errors, "One or more daemon shutdown steps failed"); } }; @@ -1721,13 +1748,20 @@ export async function createPaseoDaemon( async function closeAllAgents(logger: Logger, agentManager: AgentManager): Promise { const agents = agentManager.listAgents(); - await Promise.all( + const results = await Promise.allSettled( agents.map(async (agent) => { try { await agentManager.closeAgent(agent.id); } catch (err) { logger.error({ err, agentId: agent.id }, "Failed to close agent"); + throw err; } }), ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length > 0) { + throw new AggregateError(failures, "Failed to close one or more agents"); + } } diff --git a/packages/server/src/server/daemon-worker.ts b/packages/server/src/server/daemon-worker.ts index 4656523ef7..74eac3f9ba 100644 --- a/packages/server/src/server/daemon-worker.ts +++ b/packages/server/src/server/daemon-worker.ts @@ -33,6 +33,8 @@ interface BootstrapResult { config: ReturnType; } +const SHUTDOWN_TIMEOUT_MS = 10_000; + function isPidAlive(pid: number): boolean { try { process.kill(pid, 0); @@ -142,13 +144,14 @@ async function main() { ); shutdownPromise = (async () => { + const deadlineAt = Date.now() + SHUTDOWN_TIMEOUT_MS; const forceExit = setTimeout(() => { logger.warn( { signal, reason, ...getProcessDiagnostics() }, "Forcing shutdown - HTTP server didn't close in time", ); process.exit(1); - }, 10000); + }, SHUTDOWN_TIMEOUT_MS); try { if (!daemon) { @@ -156,7 +159,7 @@ async function main() { clearTimeout(forceExit); return 1; } - await daemon.stop(); + await daemon.stop({ deadlineAt }); clearTimeout(forceExit); logger.info("Server closed"); return options?.successExitCode ?? 0; diff --git a/packages/server/src/server/session.test.ts b/packages/server/src/server/session.test.ts index 5677750165..66f163c686 100644 --- a/packages/server/src/server/session.test.ts +++ b/packages/server/src/server/session.test.ts @@ -41,6 +41,7 @@ import { asGitHubService, asWorkspaceGitService, asDaemonConfigStore, + createAgentLifecycleDispatchStub, createProviderSnapshotManagerStub, } from "./test-utils/session-stubs.js"; import { isPlatform } from "../test-utils/platform.js"; @@ -375,6 +376,7 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session { list: vi.fn().mockResolvedValue([]), ...options.agentStorage, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: { list: vi.fn().mockResolvedValue([]), get: vi.fn(), diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index eafcb248da..b18d8c1db4 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -8,6 +8,7 @@ import { serializeAgentStreamEvent, type AgentSnapshotPayload, type AgentAttachment, + type CreateAgentWorktreeTarget, type FirstAgentContext, type SessionInboundMessage, type SessionOutboundMessage, @@ -244,7 +245,7 @@ import { ProjectDirectoryRequestError, } from "./project-directory-service.js"; import { runGitCommand } from "../utils/run-git-command.js"; -import { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import { resolveWorktreeSourceCwd } from "./workspace-source.js"; // TODO: Remove once all app store clients are on >=0.1.45 and understand arbitrary provider strings. @@ -423,6 +424,7 @@ export interface SessionOptions { worktreesRoot?: string; agentManager: AgentManager; agentStorage: AgentStorage; + createAgentLifecycleDispatch: CreateAgentLifecycleDispatch; projectRegistry: ProjectRegistry; workspaceRegistry: WorkspaceRegistry; filesystem?: SessionFileSystem; @@ -668,6 +670,7 @@ export class Session { worktreesRoot, agentManager, agentStorage, + createAgentLifecycleDispatch, projectRegistry, workspaceRegistry, filesystem, @@ -913,31 +916,7 @@ export class Session { this.emitWorkspaceUpdateForWorkspaceId(workspaceId), logger: this.sessionLogger, }); - this.createAgentLifecycleDispatch = new CreateAgentLifecycleDispatch({ - paseoHome: this.paseoHome, - worktreesRoot: this.worktreesRoot, - agentManager: this.agentManager, - agentStorage: this.agentStorage, - github: this.github, - workspaceGitService: this.workspaceGitService, - createPaseoWorktreeWorkflow: (input, workflowOptions) => - this.createPaseoWorktreeWorkflow(input, workflowOptions), - archiveAgentForClose: (agentId) => this.archiveAgentForClose(agentId), - findWorkspaceIdForCwd: (cwd) => this.findWorkspaceIdForCwd(cwd), - listActiveWorkspaces: () => this.listActiveWorkspaceRefs(), - archiveWorkspaceRecord: (workspaceId) => this.archiveWorkspaceRecord(workspaceId), - workspaceRegistry: this.workspaceRegistry, - emit: (message) => this.emit(message), - emitAgentRemove: (agentId) => this.agentUpdates.removeAgent(agentId), - emitWorkspaceUpdatesForWorkspaceIds: (workspaceIds) => - this.emitWorkspaceUpdatesForWorkspaceIds(workspaceIds), - markWorkspaceArchiving: (workspaceIds, archivingAt) => - this.markWorkspaceArchiving(workspaceIds, archivingAt), - clearWorkspaceArchiving: (workspaceIds) => this.clearWorkspaceArchiving(workspaceIds), - killTerminalsForWorkspace: (workspaceId) => - this.terminalController.killTerminalsForWorkspace(workspaceId), - logger: this.sessionLogger, - }); + this.createAgentLifecycleDispatch = createAgentLifecycleDispatch; this.providerSnapshotManager = providerSnapshotManager; this.serviceProxy = serviceProxy ?? null; this.scriptRuntimeStore = scriptRuntimeStore ?? null; @@ -3100,7 +3079,7 @@ export class Session { ...(attachments && attachments.length > 0 ? { attachments } : {}), }; const workspacePromptTitle = resolveFirstAgentPromptTitle(firstAgentContext); - const createdWorktree = await this.createAgentLifecycleDispatch.createWorktreeForRequest({ + const createdWorktree = await this.createWorktreeForCreateAgentRequest({ cwd: config.cwd, target: worktree, firstAgentContext, @@ -5822,6 +5801,56 @@ export class Session { ); } + private async createWorktreeForCreateAgentRequest(input: { + cwd: string; + target: CreateAgentWorktreeTarget | undefined; + firstAgentContext: FirstAgentContext; + hasLegacyGitOptions: boolean; + }): Promise { + if (input.target && input.hasLegacyGitOptions) { + throw new Error("create_agent_request worktree cannot be combined with git options"); + } + if (!input.target) { + return null; + } + + const baseInput = { + cwd: input.cwd, + firstAgentContext: input.firstAgentContext, + runSetup: false, + paseoHome: this.paseoHome, + worktreesRoot: this.worktreesRoot, + } as const; + switch (input.target.mode) { + case "branch-off": { + const base = input.target.base; + return this.createPaseoWorktreeWorkflow( + { + ...baseInput, + worktreeSlug: input.target.newBranch, + action: "branch-off", + ...(base ? { refName: base } : {}), + }, + base ? { resolveDefaultBranch: async () => base } : undefined, + ); + } + case "checkout-branch": + return this.createPaseoWorktreeWorkflow({ + ...baseInput, + action: "checkout", + refName: input.target.branch, + }); + case "checkout-pr": + return this.createPaseoWorktreeWorkflow({ + ...baseInput, + action: "checkout", + githubPrNumber: input.target.prNumber, + }); + default: + throw new Error("Unsupported create_agent_request worktree target"); + } + } + private async handleWorkspaceSetupStatusRequest( request: Extract, ): Promise { diff --git a/packages/server/src/server/session.workspace-git-watch.test.ts b/packages/server/src/server/session.workspace-git-watch.test.ts index 9d119f3136..07ce64efe5 100644 --- a/packages/server/src/server/session.workspace-git-watch.test.ts +++ b/packages/server/src/server/session.workspace-git-watch.test.ts @@ -5,7 +5,10 @@ import { createBranchChangeRouteHandler } from "./script-route-branch-handler.js import { createServiceProxySubsystem, type ServiceProxySubsystem } from "./service-proxy.js"; import { Session, type SessionOptions } from "./session.js"; import { asInternals, createStub } from "./test-utils/class-mocks.js"; -import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { + createAgentLifecycleDispatchStub, + createProviderSnapshotManagerStub, +} from "./test-utils/session-stubs.js"; import { createTestLogger } from "../test-utils/test-logger.js"; import { WorkspaceScriptRuntimeStore } from "./workspace-script-runtime-store.js"; import type { @@ -202,6 +205,7 @@ function createSessionForWorkspaceGitWatchTests(options?: { list: async () => [], get: async () => null, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: createStub({ subscribeToMutations: () => () => {}, initialize: async () => {}, diff --git a/packages/server/src/server/session.workspace-resolution-invariants.test.ts b/packages/server/src/server/session.workspace-resolution-invariants.test.ts index fdc3a8aa42..0d2c474dae 100644 --- a/packages/server/src/server/session.workspace-resolution-invariants.test.ts +++ b/packages/server/src/server/session.workspace-resolution-invariants.test.ts @@ -12,7 +12,10 @@ import { Session, type SessionOptions } from "./session.js"; import type { SessionOutboundMessage } from "@getpaseo/protocol/messages"; import { createNoopWorkspaceGitService } from "./test-utils/workspace-git-service-stub.js"; import { asInternals, createStub } from "./test-utils/class-mocks.js"; -import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { + createAgentLifecycleDispatchStub, + createProviderSnapshotManagerStub, +} from "./test-utils/session-stubs.js"; import { createPersistedProjectRecord, createPersistedWorkspaceRecord, @@ -108,6 +111,7 @@ function createHarness(input: { list: async () => [], get: async () => null, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: createStub({ subscribeToMutations: () => () => {}, initialize: async () => {}, diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index 603e1bd19b..aa48d95e03 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -60,6 +60,7 @@ import { asDaemonConfigStore, asTerminalManager, asSessionInternals, + createAgentLifecycleDispatchStub, createProviderSnapshotManagerStub, isSessionOutboundMessage, filterByType, @@ -656,6 +657,7 @@ function createSessionForWorkspaceTests( : null, upsert: async () => {}, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: options.projectRegistry ?? { initialize: async () => {}, existsOnDisk: async () => true, @@ -831,6 +833,7 @@ test("create_agent_request keeps requested child cwd when grouped under an exist paseoHome: path.join(workdir, "paseo-home"), agentManager, agentStorage, + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry, workspaceRegistry, chatService: asChatService(), @@ -985,6 +988,7 @@ test("create_agent_request launches from an exact subdirectory in a created work paseoHome: path.join(workdir, "paseo-home"), agentManager, agentStorage, + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry, workspaceRegistry, chatService: asChatService(), @@ -1124,6 +1128,7 @@ test("create_agent_request does not title an existing workspace from the agent p paseoHome: path.join(workdir, "paseo-home"), agentManager, agentStorage, + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry, workspaceRegistry, chatService: asChatService(), @@ -1418,6 +1423,7 @@ test("archive emits an authoritative agent_update upsert for subscribed clients" Object.assign(archivedRecord, record); }, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: (() => { const proj = createPersistedProjectRecord({ projectId: "proj-1", @@ -1780,6 +1786,7 @@ test("close_items_request archives agents and kills terminals in one batch", asy return archivedRecord; }, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: (() => { const proj = createPersistedProjectRecord({ projectId: "proj-close", @@ -1967,6 +1974,7 @@ test("close_items_request archives stored agents that are not currently loaded", }, upsert: upsertStoredRecord, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: (() => { const proj = createPersistedProjectRecord({ projectId: "proj-stored", @@ -2116,6 +2124,7 @@ test("close_items_request continues after an archive failure", async () => { return goodRecord; }, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: (() => { const proj = createPersistedProjectRecord({ projectId: "proj-err", @@ -3199,6 +3208,7 @@ test("workspace update stream keeps persisted workspace visible after agents sto list: async () => [], get: async () => null, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: { initialize: async () => {}, existsOnDisk: async () => true, diff --git a/packages/server/src/server/snapshot-mutation-ownership.test.ts b/packages/server/src/server/snapshot-mutation-ownership.test.ts index 3733643484..d3fde990f7 100644 --- a/packages/server/src/server/snapshot-mutation-ownership.test.ts +++ b/packages/server/src/server/snapshot-mutation-ownership.test.ts @@ -7,7 +7,10 @@ import { Session } from "./session.js"; import type { SessionOptions } from "./session.js"; import { createTestPaseoDaemon } from "./test-utils/paseo-daemon.js"; import { asInternals, createStub } from "./test-utils/class-mocks.js"; -import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { + createAgentLifecycleDispatchStub, + createProviderSnapshotManagerStub, +} from "./test-utils/session-stubs.js"; interface SessionInternals { archiveAgentForClose(agentId: string): Promise<{ archivedAt: string }>; @@ -118,6 +121,7 @@ describe("snapshot mutation ownership boundary", () => { applySnapshot: directStorageWrite, upsert: directStorageWrite, }), + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: createStub({ subscribeToMutations: () => () => {}, initialize: async () => {}, diff --git a/packages/server/src/server/test-utils/session-stubs.ts b/packages/server/src/server/test-utils/session-stubs.ts index f7ddbdd95d..1d9071dbf4 100644 --- a/packages/server/src/server/test-utils/session-stubs.ts +++ b/packages/server/src/server/test-utils/session-stubs.ts @@ -42,6 +42,16 @@ export function asAgentStorage(stub: { return createStub(stub); } +export function createAgentLifecycleDispatchStub(): SessionOptions["createAgentLifecycleDispatch"] { + return createStub({ + registerAutoArchiveIfRequested: () => ({ + settled: Promise.resolve("cancelled"), + cancel: async () => {}, + }), + cleanupCreatedWorktreeAfterFailedAgentCreate: async () => {}, + }); +} + export function asDownloadTokenStore(): SessionOptions["downloadTokenStore"] { return createStub({}); } diff --git a/packages/server/src/server/websocket-server.browser-tools.test.ts b/packages/server/src/server/websocket-server.browser-tools.test.ts index 4ee596c420..b1ab6c8825 100644 --- a/packages/server/src/server/websocket-server.browser-tools.test.ts +++ b/packages/server/src/server/websocket-server.browser-tools.test.ts @@ -13,6 +13,7 @@ import { afterEach, describe, expect, it } from "vitest"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import { BrowserToolsBroker } from "./browser-tools/broker.js"; import type { CheckoutDiffManager } from "./checkout-diff-manager.js"; import type { FileBackedChatService } from "./chat/chat-service.js"; @@ -299,6 +300,7 @@ function createVoiceAssistantWebSocketServer(params: { null, { allowedOrigins: new Set(["*"]) }, createWorkspaceAutoNameStub(), + createStub({}), undefined, undefined, undefined, diff --git a/packages/server/src/server/websocket-server.notifications.test.ts b/packages/server/src/server/websocket-server.notifications.test.ts index f662a044a9..f7f982c9e1 100644 --- a/packages/server/src/server/websocket-server.notifications.test.ts +++ b/packages/server/src/server/websocket-server.notifications.test.ts @@ -3,6 +3,7 @@ import type { Server as HTTPServer } from "http"; import type pino from "pino"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { DaemonConfigStore } from "./daemon-config-store.js"; import type { FileBackedChatService } from "./chat/chat-service.js"; @@ -117,6 +118,7 @@ function createServer(agentManagerOverrides?: Record) { null, { allowedOrigins: new Set() }, createWorkspaceAutoNameStub(), + createStub({}), undefined, undefined, undefined, diff --git a/packages/server/src/server/websocket-server.relay-reconnect.test.ts b/packages/server/src/server/websocket-server.relay-reconnect.test.ts index 6a6cef196d..52d285cff7 100644 --- a/packages/server/src/server/websocket-server.relay-reconnect.test.ts +++ b/packages/server/src/server/websocket-server.relay-reconnect.test.ts @@ -3,6 +3,7 @@ import type { Server as HTTPServer } from "http"; import type pino from "pino"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { DaemonConfigStore } from "./daemon-config-store.js"; import type { FileBackedChatService } from "./chat/chat-service.js"; @@ -259,6 +260,7 @@ function createServer(options?: { null, { allowedOrigins: new Set() }, createWorkspaceAutoNameStub(), + createStub({}), undefined, speechReadiness ? { diff --git a/packages/server/src/server/websocket-server.terminal-notifications.test.ts b/packages/server/src/server/websocket-server.terminal-notifications.test.ts index 7d3cba294f..1443079f24 100644 --- a/packages/server/src/server/websocket-server.terminal-notifications.test.ts +++ b/packages/server/src/server/websocket-server.terminal-notifications.test.ts @@ -3,6 +3,7 @@ import type { Server as HTTPServer } from "http"; import type pino from "pino"; import type { AgentManager } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { DaemonConfigStore } from "./daemon-config-store.js"; import type { FileBackedChatService } from "./chat/chat-service.js"; @@ -148,6 +149,7 @@ function createServer(terminalManager: TerminalManager, workspaceRegistry?: Work null, { allowedOrigins: new Set() }, createWorkspaceAutoNameStub(), + createStub({}), undefined, undefined, terminalManager, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 6c5886fed9..550ef7c707 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -6,6 +6,7 @@ import { randomUUID } from "node:crypto"; import { monitorEventLoopDelay } from "node:perf_hooks"; import type { AgentManager, AgentMetricsSnapshot } from "./agent/agent-manager.js"; import type { AgentStorage } from "./agent/agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "./agent/create-agent-lifecycle-dispatch.js"; import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import type pino from "pino"; @@ -506,6 +507,7 @@ export class VoiceAssistantWebSocketServer { private readonly daemonRuntimeConfig: DaemonRuntimeConfig | undefined; private readonly agentManager: AgentManager; private readonly agentStorage: AgentStorage; + private readonly createAgentLifecycleDispatch: CreateAgentLifecycleDispatch; private readonly projectRegistry: ProjectRegistry; private readonly workspaceRegistry: WorkspaceRegistry; private readonly chatService: FileBackedChatService; @@ -571,6 +573,7 @@ export class VoiceAssistantWebSocketServer { mcpBaseUrl: string | null, wsConfig: WebSocketServerConfig, workspaceAutoName: WorkspaceAutoName, + createAgentLifecycleDispatch: CreateAgentLifecycleDispatch, auth?: DaemonAuthConfig, speech?: SpeechService | null, terminalManager?: TerminalManager | null, @@ -617,6 +620,7 @@ export class VoiceAssistantWebSocketServer { this.hubRelationships = hubRelationships ?? null; this.agentManager = agentManager; this.agentStorage = agentStorage; + this.createAgentLifecycleDispatch = createAgentLifecycleDispatch; this.projectRegistry = projectRegistry ?? createNoopProjectRegistry(); this.workspaceRegistry = workspaceRegistry ?? createNoopWorkspaceRegistry(); const requiredServices = requireWebSocketServices({ @@ -1325,6 +1329,7 @@ export class VoiceAssistantWebSocketServer { worktreesRoot: this.worktreesRoot, agentManager: this.agentManager, agentStorage: this.agentStorage, + createAgentLifecycleDispatch: this.createAgentLifecycleDispatch, projectRegistry: this.projectRegistry, workspaceRegistry: this.workspaceRegistry, chatService: this.chatService, diff --git a/packages/server/src/server/wire-compat.test.ts b/packages/server/src/server/wire-compat.test.ts index f5956e97eb..094c006abb 100644 --- a/packages/server/src/server/wire-compat.test.ts +++ b/packages/server/src/server/wire-compat.test.ts @@ -14,7 +14,10 @@ import { WSHelloMessageSchema, } from "@getpaseo/protocol/messages"; import { Session, type SessionOptions } from "./session.js"; -import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { + createAgentLifecycleDispatchStub, + createProviderSnapshotManagerStub, +} from "./test-utils/session-stubs.js"; import type { AgentTimelineRow } from "./agent/agent-manager.js"; import { handleCreatePaseoWorktreeRequest } from "./worktree-session.js"; import { createPersistedProjectRecord } from "./workspace-registry.js"; @@ -249,6 +252,7 @@ function createSessionForWireCompatTest(options?: { paseoHome: "/tmp/paseo-home", agentManager: new InMemoryAgentManager(rows) as unknown as SessionOptions["agentManager"], agentStorage: new EmptyAgentStorage() as unknown as SessionOptions["agentStorage"], + createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), projectRegistry: new EmptyProjectRegistry() as unknown as SessionOptions["projectRegistry"], workspaceRegistry: new EmptyWorkspaceRegistry() as unknown as SessionOptions["workspaceRegistry"], From cf87f9561fbb38a724db54578f44672737f7cab6 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 09:31:11 +0100 Subject: [PATCH 34/43] fix(server): make auto-archive shutdown durable --- .../server/src/server/agent/agent-manager.ts | 13 +- .../src/server/agent/agent-storage.test.ts | 35 ++ .../server/src/server/agent/agent-storage.ts | 109 +++-- .../create-agent-lifecycle-dispatch.test.ts | 158 ++++++- .../agent/create-agent-lifecycle-dispatch.ts | 86 ++-- .../server/agent/create-agent/create.test.ts | 52 +++ .../src/server/agent/create-agent/create.ts | 9 +- .../server/src/server/bootstrap.smoke.test.ts | 40 ++ packages/server/src/server/bootstrap.ts | 80 +++- packages/server/src/server/session.ts | 27 +- .../src/server/test-utils/paseo-daemon.ts | 4 + .../src/server/test-utils/session-stubs.ts | 4 + .../src/server/workspace-archive-service.ts | 212 ++++----- .../workspace-lifecycle-coordinator.test.ts | 40 ++ .../server/workspace-lifecycle-coordinator.ts | 91 +++- .../server/src/utils/run-git-command.test.ts | 38 ++ packages/server/src/utils/run-git-command.ts | 403 ++++++++++-------- packages/server/src/utils/worktree.ts | 49 ++- patches/local-patches.json | 118 ----- ...-v025-worktree-lifecycle-current-main.json | 92 ---- .../paseo-v025-worktree-lifecycle.json | 38 -- 21 files changed, 1067 insertions(+), 631 deletions(-) create mode 100644 packages/server/src/server/workspace-lifecycle-coordinator.test.ts delete mode 100644 patches/local-patches.json delete mode 100644 patches/receipts/paseo-v025-worktree-lifecycle-current-main.json delete mode 100644 patches/receipts/paseo-v025-worktree-lifecycle.json diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 72fe1a7ec8..476ad0de23 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -44,7 +44,7 @@ import { type ListImportableSessionsOptions, } from "./agent-sdk-types.js"; import { buildArchivedAgentRecord, type ArchivedStoredAgentRecord } from "./agent-archive.js"; -import type { StoredAgentRecord, AgentStorage } from "./agent-storage.js"; +import type { AutoArchiveObligation, StoredAgentRecord, AgentStorage } from "./agent-storage.js"; import type { AgentOwner } from "./agent-owner.js"; import { InMemoryAgentTimelineStore, @@ -236,6 +236,7 @@ export interface CreateAgentOptions { // undefined is an explicit decision: the agent never appears in the sidebar. workspaceId: string | undefined; owner?: AgentOwner; + autoArchiveObligation?: AutoArchiveObligation; } export interface AgentManagerOptions { @@ -1037,6 +1038,7 @@ export class AgentManager { initialTitle: options.initialTitle, workspaceId: options.workspaceId, owner: options.owner, + autoArchiveObligation: options.autoArchiveObligation, }); } @@ -1061,6 +1063,7 @@ export class AgentManager { labels?: Record; workspaceId?: string; owner?: AgentOwner; + autoArchiveObligation?: AutoArchiveObligation; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { @@ -2676,6 +2679,7 @@ export class AgentManager { publishWhenReady?: boolean; workspaceId?: string; owner?: AgentOwner; + autoArchiveObligation?: AutoArchiveObligation; }, ): Promise { let registered = false; @@ -2716,6 +2720,7 @@ export class AgentManager { this.assertAgentRegistrationActive(managed); await this.persistSnapshot(managed, { title: initialPersistedTitle, + autoArchiveObligation: options?.autoArchiveObligation, }); this.assertAgentRegistrationActive(managed); if (!options?.publishWhenReady) { @@ -3057,7 +3062,11 @@ export class AgentManager { private async persistSnapshot( agent: ManagedAgent, - options?: { title?: string | null; internal?: boolean }, + options?: { + title?: string | null; + internal?: boolean; + autoArchiveObligation?: AutoArchiveObligation; + }, ): Promise { if (!this.registry) { return; diff --git a/packages/server/src/server/agent/agent-storage.test.ts b/packages/server/src/server/agent/agent-storage.test.ts index a9b1ab762f..36107c4168 100644 --- a/packages/server/src/server/agent/agent-storage.test.ts +++ b/packages/server/src/server/agent/agent-storage.test.ts @@ -192,6 +192,41 @@ describe("AgentStorage", () => { expect(persisted.config?.extra?.claude).toMatchObject({ maxThinkingTokens: 1024 }); }); + test("persists auto-archive obligations across restart", async () => { + await storage.applySnapshot(createManagedAgent({ id: "agent-auto-archive" }), { + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-auto-archive" }, + }, + }); + + const reloaded = new AgentStorage(storagePath, logger); + await expect(reloaded.get("agent-auto-archive")).resolves.toMatchObject({ + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-auto-archive" }, + }, + }); + }); + + test("serialized snapshots cannot overwrite a pending auto-archive obligation", async () => { + const agent = createManagedAgent({ id: "agent-auto-archive-race" }); + await storage.applySnapshot(agent, { + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + }); + + const pending = storage.update(agent.id, (record) => ({ + ...record, + autoArchiveObligation: { phase: "pending", target: { kind: "agent" } }, + })); + const snapshot = storage.applySnapshot({ ...agent, updatedAt: new Date("2025-02-01") }); + await Promise.all([pending, snapshot]); + + await expect(storage.get(agent.id)).resolves.toMatchObject({ + autoArchiveObligation: { phase: "pending", target: { kind: "agent" } }, + }); + }); + test("applySnapshot stores and reloads featureValues when present", async () => { await storage.applySnapshot( createManagedAgent({ diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 5e504cee64..831cc11699 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -33,6 +33,16 @@ const PERSISTENCE_HANDLE_SCHEMA = z .nullable() .optional(); +const AUTO_ARCHIVE_TARGET_SCHEMA = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("agent") }), + z.object({ kind: z.literal("workspace"), workspaceId: z.string() }), +]); + +const AUTO_ARCHIVE_OBLIGATION_SCHEMA = z.object({ + phase: z.enum(["armed", "pending"]), + target: AUTO_ARCHIVE_TARGET_SCHEMA, +}); + const STORED_AGENT_SCHEMA = z.object({ id: z.string(), provider: z.string(), @@ -66,6 +76,7 @@ const STORED_AGENT_SCHEMA = z.object({ internal: z.boolean().optional(), archivedAt: z.string().nullable().optional(), owner: AgentOwnerSchema.optional(), + autoArchiveObligation: AUTO_ARCHIVE_OBLIGATION_SCHEMA.optional(), }); export type SerializableAgentConfig = Pick< @@ -80,6 +91,7 @@ export type SerializableAgentConfig = Pick< >; export type StoredAgentRecord = z.infer; +export type AutoArchiveObligation = z.infer; export function parseStoredAgentRecord(value: unknown): StoredAgentRecord { return STORED_AGENT_SCHEMA.parse(value); } @@ -127,6 +139,35 @@ export class AgentStorage { await this.queueRecordWrite(record); } + async update( + agentId: string, + mutation: (record: StoredAgentRecord) => StoredAgentRecord, + ): Promise { + await this.load(); + const previousWrite = this.pendingWrites.get(agentId) ?? Promise.resolve(); + const operation = (async () => { + await Promise.allSettled([previousWrite]); + const current = this.cache.get(agentId); + if (!current) { + throw new Error(`Agent ${agentId} not found`); + } + const next = mutation(current); + if (!this.deleting.has(agentId)) { + await this.writeRecord(next); + } + return next; + })(); + const tracked = operation.then( + () => undefined, + () => undefined, + ); + void tracked.finally(() => { + if (this.pendingWrites.get(agentId) === tracked) this.pendingWrites.delete(agentId); + }); + this.pendingWrites.set(agentId, tracked); + return operation; + } + private queueRecordWrite(record: StoredAgentRecord): Promise { const agentId = record.id; const prev = this.pendingWrites.get(agentId) ?? Promise.resolve(); @@ -204,38 +245,50 @@ export class AgentStorage { async applySnapshot( agent: ManagedAgent, - options?: { title?: string | null; internal?: boolean }, + options?: { + title?: string | null; + internal?: boolean; + autoArchiveObligation?: AutoArchiveObligation; + }, ): Promise { await this.load(); - await this.waitForPendingWrite(agent.id); - const existing = (await this.get(agent.id)) ?? null; - const hasTitleOverride = - options !== undefined && Object.prototype.hasOwnProperty.call(options, "title"); - const hasInternalOverride = - options !== undefined && Object.prototype.hasOwnProperty.call(options, "internal"); - const record = toStoredAgentRecord(agent, { - title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), - createdAt: existing?.createdAt, - internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), + const previousWrite = this.pendingWrites.get(agent.id) ?? Promise.resolve(); + const operation = (async () => { + await Promise.allSettled([previousWrite]); + const existing = this.cache.get(agent.id) ?? null; + const hasTitleOverride = + options !== undefined && Object.prototype.hasOwnProperty.call(options, "title"); + const hasInternalOverride = + options !== undefined && Object.prototype.hasOwnProperty.call(options, "internal"); + const hasAutoArchiveOverride = + options !== undefined && + Object.prototype.hasOwnProperty.call(options, "autoArchiveObligation"); + const record = toStoredAgentRecord(agent, { + title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), + createdAt: existing?.createdAt, + internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), + }); + record.archivedAt = existing?.archivedAt; + record.autoArchiveObligation = hasAutoArchiveOverride + ? options?.autoArchiveObligation + : existing?.autoArchiveObligation; + if (!this.deleting.has(agent.id)) { + await this.writeRecord(record); + } + })(); + const tracked = operation.then( + () => undefined, + () => undefined, + ); + void tracked.finally(() => { + if (this.pendingWrites.get(agent.id) === tracked) this.pendingWrites.delete(agent.id); }); - - // Preserve soft-delete/archive status across snapshot flushes. - // `archivedAt` is not part of the ManagedAgent snapshot, so a naive projection - // would wipe it during normal persistence (including on daemon restart). - if (existing && existing.archivedAt !== undefined) { - record.archivedAt = existing.archivedAt; - } - await this.upsert(record); + this.pendingWrites.set(agent.id, tracked); + await operation; } async setTitle(agentId: string, title: string): Promise { - await this.load(); - await this.waitForPendingWrite(agentId); - const record = await this.get(agentId); - if (!record) { - throw new Error(`Agent ${agentId} not found`); - } - await this.upsert({ ...record, title }); + await this.update(agentId, (record) => ({ ...record, title })); } async flush(): Promise { @@ -386,10 +439,6 @@ export class AgentStorage { } this.daemonExecutionKeysByAgentId.delete(agentId); } - - private async waitForPendingWrite(agentId: string): Promise { - await (this.pendingWrites.get(agentId) ?? Promise.resolve()).catch(() => undefined); - } } function projectDirNameFromCwd(cwd: string): string { diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index 73802889de..19436169a2 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -10,6 +10,7 @@ import { requireArchiveCleanupComplete, WorkspaceCleanupPendingError, } from "../workspace-archive-service.js"; +import type { StoredAgentRecord } from "./agent-storage.js"; class AgentLifecycleEvents { private readonly listeners = new Set(); @@ -53,14 +54,31 @@ class LifecycleRearmEvents { function createLifecycleDispatch( agents: AgentLifecycleEvents, archiveAgentForClose: (agentId: string) => Promise, + options?: { + records?: Map; + archiveWorkspaceForClose?: (workspaceId: string, signal?: AbortSignal) => Promise; + }, ): CreateAgentLifecycleDispatch { + const records = options?.records ?? new Map(); const dependencies = { paseoHome: "/tmp/paseo", agentManager: agents, - agentStorage: {}, + agentStorage: { + list: async () => Array.from(records.values()), + update: async ( + agentId: string, + mutation: (record: StoredAgentRecord) => StoredAgentRecord, + ) => { + const current = records.get(agentId) ?? ({ id: agentId } as StoredAgentRecord); + const next = mutation(current); + records.set(agentId, next); + return next; + }, + }, github: {}, workspaceGitService: {}, archiveAgentForClose, + archiveWorkspaceForClose: options?.archiveWorkspaceForClose ?? (async () => undefined), findWorkspaceIdForCwd: async () => null, listActiveWorkspaces: async () => [], archiveWorkspaceRecord: async () => undefined, @@ -334,6 +352,39 @@ test("lifecycle shutdown reports its active archive at the deadline", async () = } }); +test("lifecycle shutdown aborts an active workspace archive", async () => { + const agentId = "agent-auto-archive-workspace-shutdown"; + const agents = new AgentLifecycleEvents(); + let markArchiveStarted = () => {}; + const archiveStarted = new Promise((resolve) => { + markArchiveStarted = resolve; + }); + let receivedSignal: AbortSignal | undefined; + const dispatch = createLifecycleDispatch(agents, async () => undefined, { + archiveWorkspaceForClose: async (_workspaceId, signal) => { + receivedSignal = signal; + markArchiveStarted(); + await new Promise((_resolve, reject) => { + signal?.addEventListener("abort", () => reject(new Error("workspace archive canceled")), { + once: true, + }); + }); + }, + }); + dispatch.registerAutoArchive({ + agentId, + obligation: { phase: "armed", target: { kind: "workspace", workspaceId: "ws-shutdown" } }, + }); + agents.completeTurn(agentId); + await archiveStarted; + + await expect(dispatch.shutdown()).resolves.toEqual({ + completed: false, + pendingAgentIds: [agentId], + }); + expect(receivedSignal?.aborted).toBe(true); +}); + test("auto-archive cancellation observes an in-flight archive rejection", async () => { const agentId = "agent-auto-archive-cancel-failure"; const agents = new AgentLifecycleEvents(); @@ -385,13 +436,7 @@ test("lifecycle dispatch does not permanently suppress an agent after archive fa .fn<(agentId: string) => Promise>() .mockRejectedValueOnce(new Error("transient close failure")) .mockResolvedValueOnce(undefined); - const dependencies = { - agentManager: agents, - archiveAgentForClose, - workspaceRegistry: {}, - logger: createTestLogger(), - } as unknown as ConstructorParameters[0]; - const dispatch = new CreateAgentLifecycleDispatch(dependencies); + const dispatch = createLifecycleDispatch(agents, archiveAgentForClose); const registration = dispatch.registerAutoArchiveIfRequested({ autoArchive: true, agentId, @@ -406,3 +451,100 @@ test("lifecycle dispatch does not permanently suppress an agent after archive fa expect(archiveAgentForClose).toHaveBeenCalledTimes(2); expect(agents.listenerCount()).toBe(0); }); + +test("startup completes an armed agent-only auto-archive obligation", async () => { + const agentId = "agent-restart-armed"; + const records = new Map([ + [ + agentId, + { + id: agentId, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + } as StoredAgentRecord, + ], + ]); + const archiveAgentForClose = vi.fn(async () => undefined); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), archiveAgentForClose, { + records, + }); + + await dispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => expect(archiveAgentForClose).toHaveBeenCalledWith(agentId)); + + expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined(); +}); + +test("startup completes a persisted workspace auto-archive obligation", async () => { + const agentId = "agent-restart-workspace"; + const records = new Map([ + [ + agentId, + { + id: agentId, + autoArchiveObligation: { + phase: "pending", + target: { kind: "workspace", workspaceId: "ws-restart" }, + }, + } as StoredAgentRecord, + ], + ]); + const archiveWorkspaceForClose = vi.fn(async () => undefined); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + records, + archiveWorkspaceForClose, + }); + + await dispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => + expect(archiveWorkspaceForClose).toHaveBeenCalledWith("ws-restart", expect.any(AbortSignal)), + ); + + expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined(); +}); + +test.each([ + ["workspace cleanup failure", new WorkspaceCleanupPendingError("archive", ["ws-restart"])], + ["failure before cleanup-pending persistence", new Error("git removal failed early")], +])("startup retains and retries a workspace obligation after %s", async (_name, failure) => { + const agentId = "agent-restart-workspace-failure"; + const records = new Map([ + [ + agentId, + { + id: agentId, + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-restart" }, + }, + } as StoredAgentRecord, + ], + ]); + const firstArchive = vi.fn(async () => { + throw failure; + }); + const firstDispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + records, + archiveWorkspaceForClose: firstArchive, + }); + + await firstDispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => expect(firstArchive).toHaveBeenCalledOnce()); + await vi.waitFor(() => + expect(records.get(agentId)?.autoArchiveObligation).toEqual({ + phase: "pending", + target: { kind: "workspace", workspaceId: "ws-restart" }, + }), + ); + + const retryArchive = vi.fn(async () => undefined); + const restartedDispatch = createLifecycleDispatch( + new AgentLifecycleEvents(), + async () => undefined, + { records, archiveWorkspaceForClose: retryArchive }, + ); + await restartedDispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => + expect(retryArchive).toHaveBeenCalledWith("ws-restart", expect.any(AbortSignal)), + ); + expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined(); +}); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 185ebd7824..66a1d53e59 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -15,7 +15,7 @@ import type { WorkspaceGitService } from "../workspace-git-service.js"; import type { WorkspaceRegistry } from "../workspace-registry.js"; import type { SessionOutboundMessage } from "../messages.js"; import type { AgentManager, AgentSubscriber, SubscribeOptions } from "./agent-manager.js"; -import type { AgentStorage } from "./agent-storage.js"; +import type { AgentStorage, AutoArchiveObligation } from "./agent-storage.js"; interface CreateAgentLifecycleDispatchDependencies { paseoHome: string; @@ -25,6 +25,7 @@ interface CreateAgentLifecycleDispatchDependencies { github: ForgeService; workspaceGitService: WorkspaceGitService; archiveAgentForClose: (agentId: string) => Promise; + archiveWorkspaceForClose: (workspaceId: string, signal?: AbortSignal) => Promise; findWorkspaceIdForCwd: (cwd: string) => Promise; listActiveWorkspaces: () => Promise; archiveWorkspaceRecord: (workspaceId: string) => Promise; @@ -57,13 +58,10 @@ export interface LifecycleDispatchShutdownResult { pendingAgentIds: string[]; } -type AutoArchiveTarget = - | { kind: "agent-only" } - | { kind: "created-worktree"; result: CreatePaseoWorktreeWorkflowResult }; - export class CreateAgentLifecycleDispatch { private readonly autoArchiveTasks = new Map>(); private readonly autoArchiveRegistrations = new Map(); + private readonly shutdownController = new AbortController(); private shuttingDown = false; constructor(private readonly dependencies: CreateAgentLifecycleDispatchDependencies) {} @@ -77,10 +75,36 @@ export class CreateAgentLifecycleDispatch { return inactiveRegistration; } - return this.registerAutoArchiveOnTerminalState( - input.agentId, - toAutoArchiveTarget(input.createdWorktree), - ); + return this.registerAutoArchive({ + agentId: input.agentId, + obligation: { + phase: "armed", + target: input.createdWorktree + ? { kind: "workspace", workspaceId: input.createdWorktree.workspace.workspaceId } + : { kind: "agent" }, + }, + }); + } + + registerAutoArchive(input: { + agentId: string; + obligation: AutoArchiveObligation; + }): LifecycleRegistration { + if (input.obligation.phase === "pending") { + void this.autoArchiveAgentOnce(input.agentId, input.obligation.target).catch(() => undefined); + return inactiveRegistration; + } + return this.registerAutoArchiveOnTerminalState(input.agentId, input.obligation.target); + } + + async recoverPersistedAutoArchives(): Promise { + const records = await this.dependencies.agentStorage.list(); + for (const record of records) { + if (!record.autoArchiveObligation) continue; + void this.autoArchiveAgentOnce(record.id, record.autoArchiveObligation.target).catch( + () => undefined, + ); + } } async cleanupCreatedWorktreeAfterFailedAgentCreate(input: { @@ -108,7 +132,7 @@ export class CreateAgentLifecycleDispatch { private registerAutoArchiveOnTerminalState( agentId: string, - target: AutoArchiveTarget, + target: AutoArchiveObligation["target"], ): LifecycleRegistration { if (this.shuttingDown) return inactiveRegistration; const existing = this.autoArchiveRegistrations.get(agentId); @@ -140,6 +164,7 @@ export class CreateAgentLifecycleDispatch { async shutdown(options?: { timeoutMs?: number }): Promise { this.shuttingDown = true; + this.shutdownController.abort(); const registrations = Array.from(this.autoArchiveRegistrations.entries()); const cancellation = Promise.allSettled( registrations.map(([, registration]) => registration.cancel()), @@ -175,7 +200,10 @@ export class CreateAgentLifecycleDispatch { return { completed: pendingAgentIds.length === 0, pendingAgentIds }; } - private async autoArchiveAgentOnce(agentId: string, target: AutoArchiveTarget): Promise { + private async autoArchiveAgentOnce( + agentId: string, + target: AutoArchiveObligation["target"], + ): Promise { const existingTask = this.autoArchiveTasks.get(agentId); if (existingTask) return existingTask; @@ -193,16 +221,25 @@ export class CreateAgentLifecycleDispatch { } } - private async runAutoArchiveAgent(agentId: string, target: AutoArchiveTarget): Promise { - if (target.kind === "created-worktree") { - await this.archiveAutoCreatedWorktree({ - agentId, - createdWorktree: target.result, - }); - return; + private async runAutoArchiveAgent( + agentId: string, + target: AutoArchiveObligation["target"], + ): Promise { + await this.dependencies.agentStorage.update(agentId, (record) => ({ + ...record, + autoArchiveObligation: { phase: "pending", target }, + })); + if (target.kind === "workspace") { + await this.dependencies.archiveWorkspaceForClose( + target.workspaceId, + this.shutdownController.signal, + ); + } else { + await this.dependencies.archiveAgentForClose(agentId); } - - await this.dependencies.archiveAgentForClose(agentId); + await this.dependencies.agentStorage.update(agentId, (record) => { + return { ...record, autoArchiveObligation: undefined }; + }); } private async archiveAutoCreatedWorktree(options: { @@ -240,6 +277,7 @@ export class CreateAgentLifecycleDispatch { { scope: { kind: "workspace", workspaceId: createdWorktree.workspace.workspaceId }, requestId: randomUUID(), + signal: this.shutdownController.signal, }, ); requireArchiveCleanupComplete(archiveResult, "Auto-created worktree archive"); @@ -405,11 +443,3 @@ export function registerAgentAutoArchive(input: { ); return registration; } - -function toAutoArchiveTarget( - createdWorktree: CreatePaseoWorktreeWorkflowResult | null, -): AutoArchiveTarget { - return createdWorktree - ? { kind: "created-worktree", result: createdWorktree } - : { kind: "agent-only" }; -} diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index 10d3048eb2..f954b243d8 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -84,6 +84,58 @@ test("session create forwards clientMessageId to the initial prompt run options" }); }); +test("session create persists and arms auto-archive before starting the initial prompt", async () => { + const order: string[] = []; + const snapshot = { + id: "agent-auto-archive-gap", + provider: "codex", + cwd: "/tmp/paseo-create-test", + runtimeInfo: null, + } as ManagedAgent; + const createAgent = vi.fn(async (_config, _agentId, options) => { + expect(options.autoArchiveObligation).toEqual({ + phase: "armed", + target: { kind: "agent" }, + }); + order.push("persisted"); + return snapshot; + }); + const streamAgent = vi.fn(() => { + order.push("prompt"); + return (async function* noop() {})(); + }); + + await createAgentCommand( + { + agentManager: { + createAgent, + getAgent: vi.fn(() => snapshot), + tryRunOutOfBand: vi.fn(() => false), + hasInFlightRun: vi.fn(() => false), + streamAgent, + waitForAgentRunStart: vi.fn(async () => undefined), + } as unknown as AgentManager, + agentStorage: {} as AgentStorage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + }, + { + kind: "session", + config: { provider: "codex", cwd: "/tmp/paseo-create-test" }, + workspaceId: "ws-create-test", + initialPrompt: "finish immediately", + labels: {}, + provisionalTitle: null, + firstAgentContext: {}, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + onCreated: () => order.push("armed"), + buildSessionConfig: async (config) => ({ sessionConfig: config }), + }, + ); + + expect(order).toEqual(["persisted", "armed", "prompt"]); +}); + test("session create validates the requested mode against the provider's modes", async () => { const snapshot = { id: "agent-1", diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index 2094148c33..d2df5bb49c 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -13,7 +13,7 @@ import type { import type { AgentAttachment, FirstAgentContext, GitSetupOptions } from "../../messages.js"; import type { AgentManager, CreateAgentOptions, ManagedAgent } from "../agent-manager.js"; import type { AgentPromptInput, AgentRunOptions, AgentSessionConfig } from "../agent-sdk-types.js"; -import type { AgentStorage } from "../agent-storage.js"; +import type { AgentStorage, AutoArchiveObligation } from "../agent-storage.js"; import type { AgentOwner } from "../agent-owner.js"; import type { ProviderSnapshotManager } from "../provider-snapshot-manager.js"; import { setupFinishNotification, startCreatedAgentInitialPrompt } from "../agent-prompt.js"; @@ -74,6 +74,8 @@ export interface CreateAgentFromSessionInput { env?: Record; provisionalTitle: string | null; firstAgentContext: FirstAgentContext; + autoArchiveObligation?: AutoArchiveObligation; + onCreated?: (created: { agentId: string }) => void; buildSessionConfig: ( config: AgentSessionConfig, gitOptions?: GitSetupOptions, @@ -210,7 +212,9 @@ export async function createAgentCommand( let liveSnapshot = snapshot; let initialPromptStarted = false; let initialPromptError: unknown | null = null; - if (input.kind === "mcp") { + if (input.kind === "session") { + input.onCreated?.({ agentId: snapshot.id }); + } else { input.onCreated?.({ agentId: snapshot.id, createdWorktree: resolved.createdWorktree ?? null }); } if (resolved.prompt !== undefined) { @@ -306,6 +310,7 @@ async function resolveSessionCreateAgent( // agent belongs to that workspace, not the source one. createdWorkspaceId // is the freshly created worktree's workspace. workspaceId: requireResolvedWorkspaceId(workspaceId), + autoArchiveObligation: input.autoArchiveObligation, }, prompt: hasPromptContent ? prompt : undefined, runOptions, diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 5871e17c57..4be92b7a77 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -357,6 +357,46 @@ describe("paseo daemon bootstrap", () => { } }); + test("a blocked cleanup stop cannot prevent later teardown before the outer deadline", async () => { + let markCleanupStopStarted = () => {}; + const cleanupStopStarted = new Promise((resolve) => { + markCleanupStopStarted = resolve; + }); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + dependencies: { + workspaceCleanupRetryService: { + start: async () => undefined, + stop: () => { + markCleanupStopStarted(); + return new Promise(() => undefined); + }, + }, + }, + }); + const killAll = vi.spyOn(daemonHandle.daemon.terminalManager, "killAll"); + const outerDeadlineAt = Date.now() + 1_000; + + try { + const stopPromise = daemonHandle.daemon.stop({ deadlineAt: outerDeadlineAt }); + await cleanupStopStarted; + await expect(stopPromise).rejects.toMatchObject({ + errors: expect.arrayContaining([ + expect.objectContaining({ name: "DaemonShutdownDeadlineError" }), + ]), + }); + + expect(killAll).toHaveBeenCalledOnce(); + expect(Date.now()).toBeLessThan(outerDeadlineAt); + } finally { + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + ]); + } + }); + test("standalone listener exposes services only", async () => { const standalonePort = await findFreePort(); const upstream = http.createServer((_req, res) => { diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 04da7c5588..e1b0441f8f 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -217,6 +217,8 @@ import { DaemonExecutions } from "./hub/daemon-executions.js"; const MAX_MCP_DEBUG_BATCH_ITEMS = 10; const REDACTED_LOG_VALUE = "[redacted]"; +const DEFAULT_DAEMON_SHUTDOWN_TIMEOUT_MS = 10_000; +const FORCE_EXIT_SHUTDOWN_RESERVE_MS = 250; const DOWNLOAD_OPEN_FLAGS = process.platform === "win32" ? constants.O_RDONLY : constants.O_RDONLY | constants.O_NOFOLLOW; @@ -451,17 +453,53 @@ export interface PaseoDaemonStopOptions { deadlineAt?: number; } +export interface WorkspaceCleanupRetryLifecycle { + start(): Promise; + stop(): Promise; +} + export interface PaseoDaemonDependencies { hubRelationshipRemote?: HubRelationshipRemote; hubRelationshipClock?: HubRelationshipClock; hubRelationshipRetryPolicy?: HubRelationshipRetryPolicy; createHubDaemonId?: () => string; + workspaceCleanupRetryService?: WorkspaceCleanupRetryLifecycle; serverFeatureOverrides?: { daemonStatusRpc?: boolean; relayConfig?: boolean; }; } +class DaemonShutdownDeadlineError extends Error { + constructor(label: string) { + super(`Daemon shutdown deadline reached during: ${label}`); + this.name = "DaemonShutdownDeadlineError"; + } +} + +function waitForShutdownStep( + task: Promise, + signal: AbortSignal, + label: string, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const finish = (callback: () => void) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + callback(); + }; + const onAbort = () => finish(() => reject(new DaemonShutdownDeadlineError(label))); + signal.addEventListener("abort", onAbort, { once: true }); + void task.then( + () => finish(resolve), + (error) => finish(() => reject(error)), + ); + if (signal.aborted) onAbort(); + }); +} + function createBootstrapManagedProcessRegistry( config: Pick, logger: Logger, @@ -1065,7 +1103,11 @@ export async function createPaseoDaemon( }; const createAgent = (input: Parameters[1]) => createAgentCommand(createAgentCommandDependencies, input); - const archiveWorkspaceByIdExternal = async (workspaceId: string, requestId: string) => + const archiveWorkspaceByIdExternal = async ( + workspaceId: string, + requestId: string, + signal?: AbortSignal, + ) => requireArchiveCleanupComplete( await archiveByScope( { @@ -1089,7 +1131,7 @@ export async function createPaseoDaemon( workspaceRegistry, sessionLogger: logger, }, - { scope: { kind: "workspace", workspaceId }, requestId }, + { scope: { kind: "workspace", workspaceId }, requestId, signal }, ), "Workspace archive", ); @@ -1129,11 +1171,13 @@ export async function createPaseoDaemon( "Workspace cleanup retry", ); }; - const workspaceCleanupRetryService = new WorkspaceCleanupRetryService({ - workspaceRegistry, - retryWorktreeCleanup: retryPendingWorktreeCleanupExternal, - logger, - }); + const workspaceCleanupRetryService = + dependencies.workspaceCleanupRetryService ?? + new WorkspaceCleanupRetryService({ + workspaceRegistry, + retryWorktreeCleanup: retryPendingWorktreeCleanupExternal, + logger, + }); await workspaceCleanupRetryService.start(); const hubAgentLifecycle = new CreateAgentLifecycleDispatch({ paseoHome: config.paseoHome, @@ -1144,6 +1188,8 @@ export async function createPaseoDaemon( workspaceGitService, archiveAgentForClose: (agentId) => archiveAgentCommand({ agentManager, agentStorage, logger }, agentId), + archiveWorkspaceForClose: (workspaceId, signal) => + archiveWorkspaceByIdExternal(workspaceId, randomUUID(), signal), findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, listActiveWorkspaces: listActiveWorkspacesExternal, archiveWorkspaceRecord: archiveWorkspaceRecordExternal, @@ -1276,6 +1322,7 @@ export async function createPaseoDaemon( logger.info({ elapsed: elapsed() }, "Schedule service initialized"); logger.info({ elapsed: elapsed() }, "Loading persisted agent registry"); const persistedRecords = await agentStorage.list(); + await hubAgentLifecycle.recoverPersistedAutoArchives(); logger.info( { elapsed: elapsed() }, `Agent registry loaded (${persistedRecords.length} record${persistedRecords.length === 1 ? "" : "s"}); agents will initialize on demand`, @@ -1667,9 +1714,20 @@ export async function createPaseoDaemon( const stop = async (options?: PaseoDaemonStopOptions) => { const errors: unknown[] = []; + const outerDeadlineAt = options?.deadlineAt ?? Date.now() + DEFAULT_DAEMON_SHUTDOWN_TIMEOUT_MS; + const shutdownDeadlineAt = options?.deadlineAt + ? Math.max(Date.now(), outerDeadlineAt - FORCE_EXIT_SHUTDOWN_RESERVE_MS) + : outerDeadlineAt; + const shutdownController = new AbortController(); + const deadlineTimer = setTimeout( + () => shutdownController.abort(), + Math.max(0, shutdownDeadlineAt - Date.now()), + ); + deadlineTimer.unref(); const attempt = async (label: string, action: () => void | Promise): Promise => { + const task = Promise.resolve().then(action); try { - await action(); + await waitForShutdownStep(task, shutdownController.signal, label); } catch (error) { errors.push(error); logger.error({ err: error }, `Daemon shutdown step failed: ${label}`); @@ -1684,10 +1742,7 @@ export async function createPaseoDaemon( await attempt("WebSocket ingress", () => wsServer?.prepareForShutdown()); await attempt("agent registration", () => agentManager.prepareForShutdown()); await attempt("create-agent lifecycle", async () => { - const remainingMs = - options?.deadlineAt === undefined - ? 10_000 - : Math.max(0, options.deadlineAt - Date.now() - 2_000); + const remainingMs = Math.max(0, shutdownDeadlineAt - Date.now()); const lifecycleShutdown = await hubAgentLifecycle.shutdown({ timeoutMs: remainingMs }); if (!lifecycleShutdown.completed) { throw new Error( @@ -1727,6 +1782,7 @@ export async function createPaseoDaemon( unlinkSync(listenTarget.path); } }); + clearTimeout(deadlineTimer); if (errors.length > 0) { throw new AggregateError(errors, "One or more daemon shutdown steps failed"); } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index b18d8c1db4..ff32bba825 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -112,7 +112,7 @@ import { type AgentRunOptions, type AgentSessionConfig, } from "./agent/agent-sdk-types.js"; -import type { StoredAgentRecord } from "./agent/agent-storage.js"; +import type { AutoArchiveObligation, StoredAgentRecord } from "./agent/agent-storage.js"; import type { AgentStorage } from "./agent/agent-storage.js"; import { ImportSessionsRequestError, @@ -3096,6 +3096,15 @@ export class Session { throw new Error(`Working directory does not exist or is not a directory: ${resolvedCwd}`); } + const autoArchiveObligation: AutoArchiveObligation | undefined = + autoArchive === true + ? { + phase: "armed", + target: createdWorktree + ? { kind: "workspace", workspaceId: createdWorktree.workspace.workspaceId } + : { kind: "agent" }, + } + : undefined; const { snapshot, liveSnapshot } = await createAgentCommand( { agentManager: this.agentManager, @@ -3127,11 +3136,20 @@ export class Session { env, provisionalTitle, firstAgentContext, + autoArchiveObligation, + onCreated: ({ agentId }) => { + createdAgentId = agentId; + if (autoArchiveObligation) { + this.createAgentLifecycleDispatch.registerAutoArchive({ + agentId, + obligation: autoArchiveObligation, + }); + } + }, buildSessionConfig: (sessionConfig, gitOptions, legacyWorktreeName, ctx) => this.buildAgentSessionConfig(sessionConfig, gitOptions, legacyWorktreeName, ctx), }, ); - createdAgentId = snapshot.id; await this.agentUpdates.forwardLiveAgent(snapshot); if (resolvedIntent.createdDirectoryWorkspace && trimmedPrompt) { this.workspaceAutoName.scheduleForDirectory( @@ -3143,11 +3161,6 @@ export class Session { { currentSelection: this.getFocusedAgentSelectionForCwd(resolvedIntent.config.cwd) }, ); } - this.createAgentLifecycleDispatch.registerAutoArchiveIfRequested({ - autoArchive, - agentId: snapshot.id, - createdWorktree, - }); if (requestId) { const agentPayload = await this.buildAgentPayload(liveSnapshot); this.emit({ diff --git a/packages/server/src/server/test-utils/paseo-daemon.ts b/packages/server/src/server/test-utils/paseo-daemon.ts index ed4708ec7b..64f8884410 100644 --- a/packages/server/src/server/test-utils/paseo-daemon.ts +++ b/packages/server/src/server/test-utils/paseo-daemon.ts @@ -6,6 +6,7 @@ import pino from "pino"; import { createPaseoDaemon, type PaseoDaemonConfig, + type PaseoDaemonDependencies, type PaseoOpenAIConfig, type PaseoSpeechConfig, } from "../bootstrap.js"; @@ -45,6 +46,7 @@ interface TestPaseoDaemonOptions { serviceProxy?: PaseoDaemonConfig["serviceProxy"]; webUi?: PaseoDaemonConfig["webUi"]; trustedProxies?: PaseoDaemonConfig["trustedProxies"]; + dependencies?: PaseoDaemonDependencies; } export interface TestPaseoDaemon { @@ -95,7 +97,9 @@ export async function createTestPaseoDaemon( const { config, paseoHomeRoot, paseoHome, staticDir } = await prepareTestDaemonConfig(options); const logger = options.logger ?? pino({ level: "silent" }); const daemon = await createPaseoDaemon(config, logger, { + ...options.dependencies, serverFeatureOverrides: { + ...options.dependencies?.serverFeatureOverrides, daemonStatusRpc: options.daemonStatusRpcCapability, relayConfig: options.relayConfigCapability, }, diff --git a/packages/server/src/server/test-utils/session-stubs.ts b/packages/server/src/server/test-utils/session-stubs.ts index 1d9071dbf4..ff76e04f1c 100644 --- a/packages/server/src/server/test-utils/session-stubs.ts +++ b/packages/server/src/server/test-utils/session-stubs.ts @@ -44,6 +44,10 @@ export function asAgentStorage(stub: { export function createAgentLifecycleDispatchStub(): SessionOptions["createAgentLifecycleDispatch"] { return createStub({ + registerAutoArchive: () => ({ + settled: Promise.resolve("cancelled"), + cancel: async () => {}, + }), registerAutoArchiveIfRequested: () => ({ settled: Promise.resolve("cancelled"), cancel: async () => {}, diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 96bad44eda..7c2beccff8 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -109,6 +109,7 @@ export function requireArchiveCleanupComplete( export interface ArchiveByScopeRequest { scope: ArchiveScope; requestId: string; + signal?: AbortSignal; } export interface PendingWorkspaceCleanupRetryRequest { @@ -176,44 +177,55 @@ export async function archiveByScope( ? `workspace:${request.scope.workspaceId}` : `worktree:${resolve(request.scope.targetPath)}`; - return lifecycleCoordinator.runArchive(operationKey, async () => { - const initialTarget = await resolveArchiveTarget(dependencies, request.scope); - const archiveReservation = lifecycleCoordinator.reserveWorkspaceArchive( - initialTarget.workspaceIds, - ); - const archiveOperation = async () => { - const closureWorkspaceIds = new Set(initialTarget.workspaceIds); - let target = initialTarget; - while (true) { - await lifecycleCoordinator.waitForWorkspaceSetups(closureWorkspaceIds); - await lifecycleCoordinator.waitForWorkspaceOwnershipMutations(closureWorkspaceIds); - - const refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); - archiveReservation.add(refreshedTarget.workspaceIds); - target = mergeArchiveTargets(target, refreshedTarget); - - const newlyDiscoveredWorkspaceIds = refreshedTarget.workspaceIds.filter( - (workspaceId) => !closureWorkspaceIds.has(workspaceId), - ); - if (newlyDiscoveredWorkspaceIds.length === 0) { - return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); - } - for (const workspaceId of newlyDiscoveredWorkspaceIds) { - closureWorkspaceIds.add(workspaceId); + return lifecycleCoordinator.runArchive( + operationKey, + async () => { + const initialTarget = await resolveArchiveTarget(dependencies, request.scope); + const archiveReservation = lifecycleCoordinator.reserveWorkspaceArchive( + initialTarget.workspaceIds, + ); + const archiveOperation = async () => { + const closureWorkspaceIds = new Set(initialTarget.workspaceIds); + let target = initialTarget; + while (true) { + await lifecycleCoordinator.waitForWorkspaceSetups(closureWorkspaceIds, request.signal); + await lifecycleCoordinator.waitForWorkspaceOwnershipMutations( + closureWorkspaceIds, + request.signal, + ); + + const refreshedTarget = await resolveArchiveTarget(dependencies, request.scope); + archiveReservation.add(refreshedTarget.workspaceIds); + target = mergeArchiveTargets(target, refreshedTarget); + + const newlyDiscoveredWorkspaceIds = refreshedTarget.workspaceIds.filter( + (workspaceId) => !closureWorkspaceIds.has(workspaceId), + ); + if (newlyDiscoveredWorkspaceIds.length === 0) { + return archiveResolvedTarget(dependencies, lifecycleCoordinator, request, target); + } + for (const workspaceId of newlyDiscoveredWorkspaceIds) { + closureWorkspaceIds.add(workspaceId); + } } + }; + try { + const mutationRoot = + (await resolveWorktreeMutationRoot(dependencies, initialTarget.backing)) ?? + (request.scope.kind === "worktree" ? dirname(resolve(request.scope.targetPath)) : null); + return await (mutationRoot + ? lifecycleCoordinator.runWorktreeMutationExclusive( + mutationRoot, + archiveOperation, + request.signal, + ) + : archiveOperation()); + } finally { + archiveReservation.release(); } - }; - try { - const mutationRoot = - (await resolveWorktreeMutationRoot(dependencies, initialTarget.backing)) ?? - (request.scope.kind === "worktree" ? dirname(resolve(request.scope.targetPath)) : null); - return await (mutationRoot - ? lifecycleCoordinator.runWorktreeMutationExclusive(mutationRoot, archiveOperation) - : archiveOperation()); - } finally { - archiveReservation.release(); - } - }); + }, + request.signal, + ); } // Retries only the physical teardown intent already persisted on archived @@ -237,59 +249,63 @@ export async function retryPendingWorkspaceCleanup( const directoryPath = resolve(request.directoryPath); const operationKey = `cleanup:${directoryPath}:${request.worktreeIncarnationId}`; - return lifecycleCoordinator.runArchive(operationKey, async () => { - const matchesDirectory = createRealpathAwarePathMatcher(directoryPath); - const pendingRecords = (await workspaceRegistry.list()).filter( - (workspace) => - workspace.archivedAt !== null && - workspace.cleanupPending !== null && - workspace.cleanupPending.worktreeIncarnationId === request.worktreeIncarnationId && - matchesDirectory(workspace.cleanupPending.directoryPath), - ); - const firstPending = pendingRecords[0]?.cleanupPending ?? null; - if (!firstPending) { - return { - archivedAgentIds: [], - archivedWorkspaceIds: [], - removedDirectory: false, - cleanupPendingWorkspaceIds: [], - }; - } - const target: ArchiveTarget = { - backing: { - path: firstPending.directoryPath, - isPaseoOwnedWorktree: true, - mainRepoRoot: firstPending.mainRepoRoot, - paseoWorktreesRoot: firstPending.paseoWorktreesRoot, - }, - teardownTargets: pendingRecords.map((workspace) => ({ - workspaceId: workspace.workspaceId, - cwd: workspace.cleanupPending!.teardownCwd, - })), - workspaceIds: [], - }; - const retry = async (): Promise => { - if (request.signal?.aborted) throw new Error("Workspace cleanup retry canceled"); - const removedDirectory = await maybeRemoveDirectory( - dependencies, - lifecycleCoordinator, - request, - target, - [], - request.signal, + return lifecycleCoordinator.runArchive( + operationKey, + async () => { + const matchesDirectory = createRealpathAwarePathMatcher(directoryPath); + const pendingRecords = (await workspaceRegistry.list()).filter( + (workspace) => + workspace.archivedAt !== null && + workspace.cleanupPending !== null && + workspace.cleanupPending.worktreeIncarnationId === request.worktreeIncarnationId && + matchesDirectory(workspace.cleanupPending.directoryPath), ); - return { - archivedAgentIds: [], - archivedWorkspaceIds: [], - removedDirectory, - cleanupPendingWorkspaceIds: await getCleanupPendingWorkspaceIds(dependencies, target), + const firstPending = pendingRecords[0]?.cleanupPending ?? null; + if (!firstPending) { + return { + archivedAgentIds: [], + archivedWorkspaceIds: [], + removedDirectory: false, + cleanupPendingWorkspaceIds: [], + }; + } + const target: ArchiveTarget = { + backing: { + path: firstPending.directoryPath, + isPaseoOwnedWorktree: true, + mainRepoRoot: firstPending.mainRepoRoot, + paseoWorktreesRoot: firstPending.paseoWorktreesRoot, + }, + teardownTargets: pendingRecords.map((workspace) => ({ + workspaceId: workspace.workspaceId, + cwd: workspace.cleanupPending!.teardownCwd, + })), + workspaceIds: [], }; - }; - const mutationRoot = await resolveWorktreeMutationRoot(dependencies, target.backing); - return mutationRoot - ? lifecycleCoordinator.runWorktreeMutationExclusive(mutationRoot, retry) - : retry(); - }); + const retry = async (): Promise => { + if (request.signal?.aborted) throw new Error("Workspace cleanup retry canceled"); + const removedDirectory = await maybeRemoveDirectory( + dependencies, + lifecycleCoordinator, + request, + target, + [], + request.signal, + ); + return { + archivedAgentIds: [], + archivedWorkspaceIds: [], + removedDirectory, + cleanupPendingWorkspaceIds: await getCleanupPendingWorkspaceIds(dependencies, target), + }; + }; + const mutationRoot = await resolveWorktreeMutationRoot(dependencies, target.backing); + return mutationRoot + ? lifecycleCoordinator.runWorktreeMutationExclusive(mutationRoot, retry, request.signal) + : retry(); + }, + request.signal, + ); } async function resolveWorktreeMutationRoot( @@ -596,15 +612,19 @@ async function maybeRemoveDirectory( return false; } - return lifecycleCoordinator.runDirectoryExclusive(backing.path, async () => { - return maybeRemoveDirectoryExclusive( - dependencies, - request, - target, - archivedWorkspaceIds, - signal, - ); - }); + return lifecycleCoordinator.runDirectoryExclusive( + backing.path, + async () => { + return maybeRemoveDirectoryExclusive( + dependencies, + request, + target, + archivedWorkspaceIds, + signal, + ); + }, + signal, + ); } async function maybeRemoveDirectoryExclusive( diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.test.ts b/packages/server/src/server/workspace-lifecycle-coordinator.test.ts new file mode 100644 index 0000000000..cf1dd9c286 --- /dev/null +++ b/packages/server/src/server/workspace-lifecycle-coordinator.test.ts @@ -0,0 +1,40 @@ +import { expect, test, vi } from "vitest"; + +import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; + +test("cancels a workspace setup wait without waiting for the setup to settle", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + const reservation = coordinator.reserveWorkspaceSetup("ws-blocked", "/worktrees/blocked"); + const controller = new AbortController(); + + const waiting = coordinator.waitForWorkspaceSetups(["ws-blocked"], controller.signal); + controller.abort(); + + await expect(waiting).rejects.toThrow("Workspace lifecycle operation canceled"); + reservation.release(); +}); + +test("cancels a queued worktree mutation before it starts", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + let releaseFirst = () => {}; + const firstBlocked = new Promise((resolve) => { + releaseFirst = resolve; + }); + const first = coordinator.runWorktreeMutationExclusive("/worktrees", () => firstBlocked); + const secondOperation = vi.fn(async () => undefined); + const controller = new AbortController(); + const second = coordinator.runWorktreeMutationExclusive( + "/worktrees", + secondOperation, + controller.signal, + ); + + controller.abort(); + + await expect(second).rejects.toThrow("Workspace lifecycle operation canceled"); + expect(secondOperation).not.toHaveBeenCalled(); + releaseFirst(); + await first; + await Promise.resolve(); + expect(secondOperation).not.toHaveBeenCalled(); +}); diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts index 0cd1ce96f3..56fbfc0e29 100644 --- a/packages/server/src/server/workspace-lifecycle-coordinator.ts +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -1,5 +1,33 @@ import { normalizePathForIdentity } from "../utils/path.js"; +function createCancellationError(message: string): Error { + const error = new Error(message); + error.name = "AbortError"; + return error; +} + +function waitForAbortable(task: Promise, signal: AbortSignal | undefined): Promise { + if (!signal) return task; + + return new Promise((resolve, reject) => { + let settled = false; + const finish = (callback: () => void) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + callback(); + }; + const onAbort = () => + finish(() => reject(createCancellationError("Workspace lifecycle operation canceled"))); + signal.addEventListener("abort", onAbort, { once: true }); + void task.then( + (value) => finish(() => resolve(value)), + (error) => finish(() => reject(error)), + ); + if (signal.aborted) onAbort(); + }); +} + export class WorkspaceLifecycleCoordinator { private readonly setupTasks = new Map>(); private readonly setupTasksByDirectory = new Map>>(); @@ -54,11 +82,17 @@ export class WorkspaceLifecycleCoordinator { }; } - async waitForWorkspaceSetups(workspaceIds: Iterable): Promise { + async waitForWorkspaceSetups( + workspaceIds: Iterable, + signal?: AbortSignal, + ): Promise { const tasks = Array.from(workspaceIds, (workspaceId) => this.setupTasks.get(workspaceId), ).filter((task): task is Promise => task !== undefined); - await Promise.allSettled(tasks); + await waitForAbortable( + Promise.allSettled(tasks).then(() => undefined), + signal, + ); } reserveWorkspaceOwnershipMutation(workspaceId: string): WorkspaceOwnershipMutationReservation { @@ -136,20 +170,31 @@ export class WorkspaceLifecycleCoordinator { }; } - async waitForWorkspaceOwnershipMutations(workspaceIds: Iterable): Promise { + async waitForWorkspaceOwnershipMutations( + workspaceIds: Iterable, + signal?: AbortSignal, + ): Promise { const tasks = Array.from(workspaceIds).flatMap((workspaceId) => Array.from(this.ownershipMutationTasks.get(workspaceId) ?? []), ); - await Promise.allSettled(tasks); + await waitForAbortable( + Promise.allSettled(tasks).then(() => undefined), + signal, + ); } - runArchive(key: string, operation: () => Promise): Promise { + runArchive(key: string, operation: () => Promise, signal?: AbortSignal): Promise { const existing = this.archiveOperations.get(key); if (existing) { - return existing as Promise; + return waitForAbortable(existing as Promise, signal); } - const task = Promise.resolve().then(operation); + const task = Promise.resolve().then(() => { + if (signal?.aborted) { + throw createCancellationError("Workspace archive canceled"); + } + return operation(); + }); this.archiveOperations.set(key, task); const clearTask = () => { if (this.archiveOperations.get(key) === task) { @@ -157,19 +202,26 @@ export class WorkspaceLifecycleCoordinator { } }; void task.then(clearTask, clearTask); - return task; + return waitForAbortable(task, signal); } - runDirectoryExclusive(directoryPath: string, operation: () => Promise): Promise { + runDirectoryExclusive( + directoryPath: string, + operation: () => Promise, + signal?: AbortSignal, + ): Promise { const directoryKey = normalizePathForIdentity(directoryPath); const previous = this.directoryOperations.get(directoryKey) ?? Promise.resolve(); const task = previous .catch(() => undefined) .then(async () => { + if (signal?.aborted) { + throw createCancellationError("Workspace directory operation canceled"); + } while (true) { const setupTasks = Array.from(this.setupTasksByDirectory.get(directoryKey) ?? []); if (setupTasks.length === 0) break; - await Promise.allSettled(setupTasks); + await waitForAbortable(Promise.allSettled(setupTasks), signal); } return operation(); }); @@ -180,12 +232,23 @@ export class WorkspaceLifecycleCoordinator { } }; void task.then(clearTask, clearTask); - return task; + return waitForAbortable(task, signal); } - runWorktreeMutationExclusive(worktreesRoot: string, operation: () => Promise): Promise { + runWorktreeMutationExclusive( + worktreesRoot: string, + operation: () => Promise, + signal?: AbortSignal, + ): Promise { const previous = this.worktreeMutationOperations.get(worktreesRoot) ?? Promise.resolve(); - const task = previous.catch(() => undefined).then(operation); + const task = previous + .catch(() => undefined) + .then(() => { + if (signal?.aborted) { + throw createCancellationError("Worktree mutation canceled"); + } + return operation(); + }); this.worktreeMutationOperations.set(worktreesRoot, task); const clearTask = () => { if (this.worktreeMutationOperations.get(worktreesRoot) === task) { @@ -193,7 +256,7 @@ export class WorkspaceLifecycleCoordinator { } }; void task.then(clearTask, clearTask); - return task; + return waitForAbortable(task, signal); } } diff --git a/packages/server/src/utils/run-git-command.test.ts b/packages/server/src/utils/run-git-command.test.ts index 371b0d3355..78b1c3b48d 100644 --- a/packages/server/src/utils/run-git-command.test.ts +++ b/packages/server/src/utils/run-git-command.test.ts @@ -230,6 +230,44 @@ describe("runGitCommand", () => { }); }); + it("cancels active and queued git commands without starting the queued process", async () => { + const { runGitCommand } = await loadRunGitCommand(1); + const activeController = new AbortController(); + const queuedController = new AbortController(); + + enqueueSpawnBehaviors({ delayMs: 5_000 }, { delayMs: 0 }); + const active = runGitCommand(["status"], { + cwd: process.cwd(), + signal: activeController.signal, + }); + await vi.waitFor(() => expect(fakeSpawnController.processes).toHaveLength(1)); + const queued = runGitCommand(["rev-parse", "--show-toplevel"], { + cwd: process.cwd(), + signal: queuedController.signal, + }); + + queuedController.abort(); + await expect(queued).rejects.toThrow("Git command canceled: git rev-parse --show-toplevel"); + expect(fakeSpawnController.processes).toHaveLength(1); + + activeController.abort(); + await expect(active).rejects.toThrow("Git command canceled: git status"); + expect(fakeSpawnController.processes[0]?.killSignals).toEqual(["SIGKILL"]); + await vi.waitFor(() => expect(fakeSpawnController.activeCount).toBe(0)); + expect(fakeSpawnController.processes).toHaveLength(1); + + const preAbortedController = new AbortController(); + preAbortedController.abort(); + await expect( + runGitCommand(["status", "--short"], { + cwd: process.cwd(), + signal: preAbortedController.signal, + }), + ).rejects.toThrow("Git command canceled: git status --short"); + await Promise.resolve(); + expect(fakeSpawnController.processes).toHaveLength(1); + }); + it("resolves truncated stdout, caps output, and kills the child process", async () => { const { runGitCommand } = await loadRunGitCommand(1); diff --git a/packages/server/src/utils/run-git-command.ts b/packages/server/src/utils/run-git-command.ts index 254c980462..a476c79e1f 100644 --- a/packages/server/src/utils/run-git-command.ts +++ b/packages/server/src/utils/run-git-command.ts @@ -24,6 +24,7 @@ export interface GitCommandOptions { timeout?: number; maxOutputBytes?: number; acceptExitCodes?: number[]; + signal?: AbortSignal; } export interface GitCommandResult { @@ -131,198 +132,228 @@ function getEnvOverlayKeys(envOverlay: ProcessEnvRecord | undefined): string[] { return Object.keys(envOverlay ?? {}).sort(); } +function createGitCancellationError(args: readonly string[]): Error { + const error = new Error(`Git command canceled: ${formatGitCommand(args)}`); + error.name = "AbortError"; + return error; +} + +function waitForGitCommand( + task: Promise, + signal: AbortSignal | undefined, + args: readonly string[], +): Promise { + if (!signal) return task; + + return new Promise((resolve, reject) => { + let settled = false; + const finish = (callback: () => void) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + callback(); + }; + const onAbort = () => finish(() => reject(createGitCancellationError(args))); + signal.addEventListener("abort", onAbort, { once: true }); + void task.then( + (value) => finish(() => resolve(value)), + (error) => finish(() => reject(error)), + ); + if (signal.aborted) onAbort(); + }); +} + export function runGitCommand( args: string[], options: GitCommandOptions, ): Promise { const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args)); - const promise = gitLimit( - () => - new Promise((resolve, reject) => { - gitRuntimeMetrics.start(runtimeMetric); - const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS; - const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; - const acceptExitCodes = options.acceptExitCodes ?? [0]; - const command = formatGitCommand(args); - const envOverlay = mergeEnvOverlays(options.env, options.envOverlay); - const startedAt = Date.now(); - const metricsState = beginGitCommandMetric(); - const logger = typeof options.logger?.trace === "function" ? options.logger : undefined; - const traceContext = logger - ? { - command: "git", - args, - cwd: options.cwd, - cwdExists: existsSync(options.cwd), - timeout, - maxOutputBytes, - acceptExitCodes, - envOverlayKeys: getEnvOverlayKeys(envOverlay), - } - : null; - - if (logger && traceContext) { - logger.trace(traceContext, "Spawning git command"); - } - - // `core.quotepath=false` makes git emit raw UTF-8 paths instead of - // octal-escaping non-ASCII bytes (e.g. `测试文件.txt` vs `"\346\265\213..."`). - const child = spawnProcess("git", ["-c", "core.quotepath=false", ...args], { + const promise = gitLimit(() => { + if (options.signal?.aborted) { + gitRuntimeMetrics.start(runtimeMetric); + gitRuntimeMetrics.finish(runtimeMetric, { success: false, timedOut: false }); + throw createGitCancellationError(args); + } + return new Promise((resolve, reject) => { + gitRuntimeMetrics.start(runtimeMetric); + const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS; + const maxOutputBytes = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES; + const acceptExitCodes = options.acceptExitCodes ?? [0]; + const command = formatGitCommand(args); + const envOverlay = mergeEnvOverlays(options.env, options.envOverlay); + const startedAt = Date.now(); + const metricsState = beginGitCommandMetric(); + const logger = typeof options.logger?.trace === "function" ? options.logger : undefined; + const traceContext = logger + ? { + command: "git", + args, + cwd: options.cwd, + cwdExists: existsSync(options.cwd), + timeout, + maxOutputBytes, + acceptExitCodes, + envOverlayKeys: getEnvOverlayKeys(envOverlay), + } + : null; + + if (logger && traceContext) { + logger.trace(traceContext, "Spawning git command"); + } + + // `core.quotepath=false` makes git emit raw UTF-8 paths instead of + // octal-escaping non-ASCII bytes (e.g. `测试文件.txt` vs `"\346\265\213..."`). + const child = spawnProcess("git", ["-c", "core.quotepath=false", ...args], { + cwd: options.cwd, + envOverlay, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + + let settled = false; + let metricFinished = false; + let truncated = false; + let stdoutBytes = 0; + let stderrBytes = 0; + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + + const settle = (callback: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + callback(); + }; + + const finishMetricOnce = (metric: GitCommandMetric, timedOut = false) => { + if (metricFinished) return; + metricFinished = true; + finishGitCommandMetric(metricsState, metric); + gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut }); + }; + + const timer = setTimeout(() => { + const error = new Error(`Git command timed out after ${timeout}ms: ${command}`); + child.kill("SIGKILL"); + finishMetricOnce( + { + args, + cwd: options.cwd, + startedAtMs: startedAt, + durationMs: Date.now() - startedAt, + exitCode: null, + signal: "SIGKILL", + success: false, + }, + true, + ); + settle(() => reject(error)); + }, timeout); + + const onAbort = () => { + child.kill("SIGKILL"); + finishMetricOnce({ + args, cwd: options.cwd, - envOverlay, - shell: false, - stdio: ["ignore", "pipe", "pipe"], + startedAtMs: startedAt, + durationMs: Date.now() - startedAt, + exitCode: null, + signal: "SIGKILL", + success: false, }); + settle(() => reject(createGitCancellationError(args))); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + if (options.signal?.aborted) onAbort(); - let settled = false; - let metricFinished = false; - let truncated = false; - let stdoutBytes = 0; - let stderrBytes = 0; - const stdoutChunks: Buffer[] = []; - const stderrChunks: Buffer[] = []; - - const settle = (callback: () => void) => { - if (settled) return; - settled = true; - clearTimeout(timer); - callback(); - }; + child.stdout!.on("data", (chunk: Buffer | string) => { + if (settled || truncated) return; - const finishMetricOnce = (metric: GitCommandMetric, timedOut = false) => { - if (metricFinished) return; - metricFinished = true; - finishGitCommandMetric(metricsState, metric); - gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut }); - }; + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const remainingBytes = maxOutputBytes - stdoutBytes; - const timer = setTimeout(() => { - const error = new Error(`Git command timed out after ${timeout}ms: ${command}`); + if (remainingBytes <= 0) { + truncated = true; child.kill("SIGKILL"); - finishMetricOnce( - { - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, - exitCode: null, - signal: "SIGKILL", - success: false, - }, - true, - ); - settle(() => reject(error)); - }, timeout); - - child.stdout!.on("data", (chunk: Buffer | string) => { - if (settled || truncated) return; - - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - const remainingBytes = maxOutputBytes - stdoutBytes; - - if (remainingBytes <= 0) { - truncated = true; - child.kill("SIGKILL"); - return; - } + return; + } - if (buffer.length > remainingBytes) { - stdoutChunks.push(buffer.subarray(0, remainingBytes)); - stdoutBytes += remainingBytes; - truncated = true; - child.kill("SIGKILL"); - return; - } + if (buffer.length > remainingBytes) { + stdoutChunks.push(buffer.subarray(0, remainingBytes)); + stdoutBytes += remainingBytes; + truncated = true; + child.kill("SIGKILL"); + return; + } - stdoutChunks.push(buffer); - stdoutBytes += buffer.length; - }); + stdoutChunks.push(buffer); + stdoutBytes += buffer.length; + }); - child.stderr!.on("data", (chunk: Buffer | string) => { - if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return; + child.stderr!.on("data", (chunk: Buffer | string) => { + if (settled || stderrBytes >= DEFAULT_STDERR_LIMIT) return; - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); - const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes; + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + const remainingBytes = DEFAULT_STDERR_LIMIT - stderrBytes; - if (buffer.length > remainingBytes) { - stderrChunks.push(buffer.subarray(0, remainingBytes)); - stderrBytes += remainingBytes; - return; - } + if (buffer.length > remainingBytes) { + stderrChunks.push(buffer.subarray(0, remainingBytes)); + stderrBytes += remainingBytes; + return; + } - stderrChunks.push(buffer); - stderrBytes += buffer.length; - }); + stderrChunks.push(buffer); + stderrBytes += buffer.length; + }); - child.on("error", (error) => { - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, - exitCode: null, - signal: null, - success: false, - }); - if (logger && traceContext) { - logger.trace( - { - ...traceContext, - err: error, - durationMs: Date.now() - startedAt, - }, - "Git command process error", - ); - } - settle(() => reject(error)); + child.on("error", (error) => { + finishMetricOnce({ + args, + cwd: options.cwd, + startedAtMs: startedAt, + durationMs: Date.now() - startedAt, + exitCode: null, + signal: null, + success: false, }); - - child.on("close", (exitCode, signal) => { - const result: GitCommandResult = { - stdout: Buffer.concat(stdoutChunks).toString("utf8"), - stderr: Buffer.concat(stderrChunks).toString("utf8"), - truncated, - exitCode, - signal, - }; - if (logger && traceContext) { - logger.trace( - { - ...traceContext, - durationMs: Date.now() - startedAt, - exitCode, - signal, - truncated, - stdoutBytes, - stderrBytes, - }, - "Git command closed", - ); - } - - if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) { - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, + if (logger && traceContext) { + logger.trace( + { + ...traceContext, + err: error, + durationMs: Date.now() - startedAt, + }, + "Git command process error", + ); + } + settle(() => reject(error)); + }); + + child.on("close", (exitCode, signal) => { + const result: GitCommandResult = { + stdout: Buffer.concat(stdoutChunks).toString("utf8"), + stderr: Buffer.concat(stderrChunks).toString("utf8"), + truncated, + exitCode, + signal, + }; + if (logger && traceContext) { + logger.trace( + { + ...traceContext, durationMs: Date.now() - startedAt, exitCode, signal, - success: false, - }); - const stderrPreview = result.stderr.trim() || "(no stderr)"; - const truncationNote = result.truncated ? " (stdout truncated)" : ""; - - settle(() => - reject( - new Error( - `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`, - ), - ), - ); - return; - } + truncated, + stdoutBytes, + stderrBytes, + }, + "Git command closed", + ); + } + if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) { finishMetricOnce({ args, cwd: options.cwd, @@ -330,17 +361,39 @@ export function runGitCommand( durationMs: Date.now() - startedAt, exitCode, signal, - success: true, + success: false, }); - settle(() => resolve(result)); + const stderrPreview = result.stderr.trim() || "(no stderr)"; + const truncationNote = result.truncated ? " (stdout truncated)" : ""; + + settle(() => + reject( + new Error( + `Git command failed: ${command}${truncationNote} (exit code: ${String(exitCode)}, signal: ${signal ?? "none"})\n${stderrPreview}`, + ), + ), + ); + return; + } + + finishMetricOnce({ + args, + cwd: options.cwd, + startedAtMs: startedAt, + durationMs: Date.now() - startedAt, + exitCode, + signal, + success: true, }); - }), - ); + settle(() => resolve(result)); + }); + }); + }); gitRuntimeMetrics.observeLimiter(gitLimit.activeCount, gitLimit.pendingCount); - return promise; + return waitForGitCommand(promise, options.signal, args); } -function formatGitCommand(args: string[]): string { +function formatGitCommand(args: readonly string[]): string { return ["git", ...args].join(" "); } diff --git a/packages/server/src/utils/worktree.ts b/packages/server/src/utils/worktree.ts index 079bd57cb0..dad72da818 100644 --- a/packages/server/src/utils/worktree.ts +++ b/packages/server/src/utils/worktree.ts @@ -1083,6 +1083,35 @@ export interface DeletePaseoWorktreeOptions { signal?: AbortSignal; } +function throwIfWorktreeDeletionCanceled(signal: AbortSignal | undefined): void { + if (!signal?.aborted) return; + const error = new Error("Worktree deletion canceled"); + error.name = "AbortError"; + throw error; +} + +async function waitForWorktreeDeletionRetry(delayMs: number, signal?: AbortSignal): Promise { + throwIfWorktreeDeletionCanceled(signal); + if (delayMs === 0) return; + + await new Promise((resolvePromise, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolvePromise(); + }, delayMs); + const onAbort = () => { + clearTimeout(timer); + reject( + Object.assign(new Error("Worktree deletion canceled"), { + name: "AbortError", + }), + ); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + }); +} + export async function deletePaseoWorktree({ cwd, worktreePath, @@ -1136,17 +1165,17 @@ export async function deletePaseoWorktree({ } } - if (signal?.aborted) { - throw new Error("Worktree deletion canceled"); - } + throwIfWorktreeDeletionCanceled(signal); if (cwd) { try { await runGitCommand(["worktree", "remove", resolvedWorktree, "--force"], { cwd, timeout: 120_000, + signal, }); } catch { + throwIfWorktreeDeletionCanceled(signal); // `git worktree remove` fails if the admin dir is already gone (e.g. a // prior archive attempt removed it before the working tree could be // fully cleaned up), or if the repo root has moved. Fall through to the @@ -1154,12 +1183,13 @@ export async function deletePaseoWorktree({ } } - await removeDirectoryWithRetries(resolvedWorktree); + await removeDirectoryWithRetries(resolvedWorktree, signal); if (cwd) { try { - await runGitCommand(["worktree", "prune"], { cwd, timeout: 30_000 }); + await runGitCommand(["worktree", "prune"], { cwd, timeout: 30_000, signal }); } catch { + throwIfWorktreeDeletionCanceled(signal); // not critical; git will prune lazily } } @@ -1198,7 +1228,8 @@ async function pathExists(path: string): Promise { } } -async function removeDirectoryWithRetries(path: string): Promise { +async function removeDirectoryWithRetries(path: string, signal?: AbortSignal): Promise { + throwIfWorktreeDeletionCanceled(signal); if (!(await pathExists(path))) { return; } @@ -1206,16 +1237,16 @@ async function removeDirectoryWithRetries(path: string): Promise { const delaysMs = [0, 100, 300, 700, 1500]; let lastError: unknown = null; for (const delay of delaysMs) { - if (delay > 0) { - await new Promise((resolvePromise) => setTimeout(resolvePromise, delay)); - } + await waitForWorktreeDeletionRetry(delay, signal); try { await rm(path, { recursive: true, force: true }); + throwIfWorktreeDeletionCanceled(signal); if (!(await pathExists(path))) { return; } lastError = new Error(`Directory still present after rm: ${path}`); } catch (error) { + throwIfWorktreeDeletionCanceled(signal); lastError = error; } } diff --git a/patches/local-patches.json b/patches/local-patches.json deleted file mode 100644 index d6857e92fa..0000000000 --- a/patches/local-patches.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "schemaVersion": 1, - "patches": [ - { - "id": "paseo-v025-worktree-lifecycle", - "category": "stable-core-worktree-lifecycle", - "base": { - "tag": "v0.2.5", - "commit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b" - }, - "candidate": { - "branch": "review-v025-worktree-lifecycle-upstream", - "baseCommit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b", - "receipt": "patches/receipts/paseo-v025-worktree-lifecycle.json" - }, - "sources": [ - "packages/protocol/src/messages.ts", - "packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts", - "packages/server/src/server/agent/create-agent/create.ts", - "packages/server/src/server/agent/tools/paseo-tools.ts", - "packages/server/src/server/auto-archive-on-merge/archive-if-safe.ts", - "packages/server/src/server/bootstrap.ts", - "packages/server/src/server/session.ts", - "packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts", - "packages/server/src/server/workspace-archive-service.ts", - "packages/server/src/server/workspace-cleanup-retry-service.ts", - "packages/server/src/server/workspace-lifecycle-coordinator.ts", - "packages/server/src/server/workspace-registry.ts", - "packages/server/src/server/worktree-bootstrap.ts", - "packages/server/src/server/worktree-session.ts", - "packages/server/src/server/worktree/commands.ts", - "packages/server/src/terminal/terminal-session-controller.ts", - "packages/server/src/utils/worktree-metadata.ts", - "packages/server/src/utils/worktree-setup-output.ts", - "packages/server/src/utils/worktree.ts" - ], - "tests": [ - "packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts", - "packages/server/src/server/agent/create-agent/create.test.ts", - "packages/server/src/server/agent/mcp-server.test.ts", - "packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts", - "packages/server/src/server/session.workspace-git-watch.test.ts", - "packages/server/src/server/session.workspaces.test.ts", - "packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts", - "packages/server/src/server/workspace-archive-service.test.ts", - "packages/server/src/server/workspace-cleanup-retry-service.test.ts", - "packages/server/src/server/worktree-bootstrap.test.ts", - "packages/server/src/server/worktree-session.test.ts", - "packages/server/src/terminal/terminal-session-controller.test.ts", - "packages/server/src/utils/worktree-setup-output.test.ts", - "packages/server/src/utils/worktree.posix.test.ts" - ], - "testCases": [ - "setup reservations prevent archive from outrunning worktree creation", - "existing-workspace agent and terminal attachment are fenced against archive", - "agent, MCP terminal, and desktop terminal attachment reject archived workspace ownership", - "project removal uses the same lifecycle archive boundary as direct workspace removal", - "MCP worktree creation releases setup reservations on every pre-agent failure", - "archive failures propagate through command, websocket, MCP, and normal session surfaces", - "workspace archive by unresolved path fails instead of reporting a successful no-op", - "workspace archive preserves registry read failures instead of rewriting them as not found", - "normal archive responses require the persisted archived timestamp", - "failed physical cleanup remains retryable after the workspace record is archived", - "partial multi-workspace retries preserve every archived sibling teardown obligation", - "archive target discovery reaches closure and waits ownership for late-discovered siblings", - "merge auto-archive receives the durable workspace cleanup registry", - "merge auto-archive rejects stale resolved workspaces and never reports pending cleanup as success", - "created-worktree, schedule, project, and normal workspace automation reject pending physical cleanup for retry", - "persisted cleanup is retried autonomously at startup and after mutation without another agent event", - "cleanup retries group sibling obligations by incarnation-bound worktree path before deletion", - "cleanup retry cycles use bounded work with capped backoff and cancellable timers", - "cleanup retries use an exact cleanup-only path and incarnation boundary and cannot archive a live replacement at a reused path", - "directory workspace creation and unarchive share cleanup exclusion and revalidate the directory after waiting, preventing a replacement workspace from being minted during deletion", - "directory setup tracking and cleanup exclusion use realpath-aware filesystem identity so symlink and macOS path aliases cannot bypass the lifecycle lock", - "shutdown aborts and awaits the active cleanup cycle and terminates a hanging teardown subprocess tree before returning", - "replacement worktree incarnations archive independently of stale cleanup intents", - "auto-archive retries after transient failure and cancellation observes rejection", - "setup output is bounded while preserving the final diagnostic tail" - ], - "verification": { - "receipt": "patches/receipts/paseo-v025-worktree-lifecycle.json", - "completedAt": "2026-08-01T00:34:00Z", - "commands": [ - "npm run build --workspace packages/protocol", - "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", - "cd packages/server && npx vitest run src/server/workspace-archive-service.test.ts src/server/worktree-bootstrap.test.ts src/server/worktree-session.test.ts src/utils/worktree-setup-output.test.ts src/utils/worktree.posix.test.ts", - "npm test --workspace packages/protocol", - "npm run build --workspace packages/server", - "npm run typecheck --workspace packages/server", - "npm run typecheck --workspace packages/protocol", - "npm run lint -- --deny-warnings", - "npx oxfmt --check ", - "git diff --check", - "jq -e '.schemaVersion == 1 and (.patches | length == 1) and .patches[0].id == \"paseo-v025-worktree-lifecycle\"' patches/local-patches.json" - ], - "result": { - "focusedServerTests": "359 passed, 5 skipped", - "protocolTests": "468 passed", - "serverBuild": "passed", - "serverAndProtocolTypechecks": "passed", - "lint": "0 warnings, 0 errors", - "formatDiffAndManifest": "passed" - } - }, - "currentMainIntegration": { - "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", - "verifiedHeadCommit": "759ed14f0b24499f0a3f161cf64c920ac3393c4b", - "verifiedHeadTree": "602929fb3aa5106fecf438ab0b4d3570d7daa62b", - "receipt": "patches/receipts/paseo-v025-worktree-lifecycle-current-main.json" - }, - "upstreamable": true, - "upstreamCondition": "Submit independently while worktree setup, ownership attachment, archive, cleanup retry, and auto-archive are not coordinated by one durable lifecycle boundary.", - "dropCondition": "Remove when the latest stable release provides equivalent setup and ownership fencing, explicit archive failure propagation, incarnation-safe retryable cleanup, bounded setup output, and retryable observable auto-archive semantics.", - "deployedReceipt": null, - "deployedAt": null - } - ] -} diff --git a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json b/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json deleted file mode 100644 index 4b20ea4de8..0000000000 --- a/patches/receipts/paseo-v025-worktree-lifecycle-current-main.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "schemaVersion": 1, - "patchId": "paseo-v025-worktree-lifecycle", - "stableCandidate": { - "baseCommit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b", - "headCommit": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", - "verificationCommit": "e2937e792969038b1b701e79abfbfe017bb635da", - "verificationTree": "af3297ff70ecb52c78ce0d305a622162049cc4a8" - }, - "currentMainIntegration": { - "upstreamMainCommit": "70ed70d36e2eabd3876a22931954319ec202c902", - "mergeCommit": "e5e5d607ea8e14dd874c3db4ce095634a9da7092", - "mergeTree": "529add05cfc1424c287d576f27d8194f37503c35", - "headCommit": "759ed14f0b24499f0a3f161cf64c920ac3393c4b", - "headTree": "602929fb3aa5106fecf438ab0b4d3570d7daa62b", - "firstParent": "bfdb6ed4a8668e05826effd3f8b1a18dbf74c9b2", - "secondParent": "70ed70d36e2eabd3876a22931954319ec202c902" - }, - "verification": { - "commit": "759ed14f0b24499f0a3f161cf64c920ac3393c4b", - "completedAt": "2026-08-01T02:39:35Z", - "commands": [ - "npm run build --workspace packages/protocol", - "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", - "cd packages/server && npx vitest run src/server/workspace-archive-service.test.ts src/server/worktree-bootstrap.test.ts src/server/worktree-session.test.ts src/utils/worktree-setup-output.test.ts src/utils/worktree.posix.test.ts src/utils/checkout-git.test.ts src/utils/checkout-git.commits.test.ts src/server/worktree-branch-name-generator.test.ts", - "npm test --workspace packages/protocol", - "npm run build --workspace packages/server", - "npm run typecheck --workspace packages/server", - "npm run typecheck --workspace packages/protocol", - "npm run lint -- --deny-warnings", - "cd packages/server && npx vitest run src/server/worktree-session.test.ts", - "cd packages/server && vitest run src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/terminal/terminal-session-controller.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/session.workspaces.test.ts --fileParallelism=false", - "cd packages/server && vitest run src/server/auto-archive-on-merge/archive-if-safe.test.ts --fileParallelism=false", - "cd packages/server && vitest run src/server/workspace-archive-service.test.ts --fileParallelism=false", - "cd packages/server && vitest run src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts --fileParallelism=false", - "cd packages/server && vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/workspace-archive-service.test.ts src/server/worktree-session.test.ts src/server/auto-archive-on-merge/archive-if-safe.test.ts src/server/session.workspaces.test.ts --fileParallelism=false", - "vitest run packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts packages/server/src/server/agent/create-agent/create.test.ts packages/server/src/server/agent/mcp-server.test.ts packages/server/src/terminal/terminal-session-controller.test.ts packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts packages/server/src/server/workspace-archive-service.test.ts packages/server/src/server/workspace-cleanup-retry-service.test.ts packages/server/src/server/worktree-session.test.ts packages/server/src/server/session.workspaces.test.ts packages/server/src/server/schedule/service.test.ts --fileParallelism=false", - "npm run build --workspace packages/server", - "npm run typecheck --workspace packages/server", - "oxlint <16 changed lifecycle source and test files>", - "oxfmt --check <16 changed lifecycle source and test files>", - "git diff --check", - "vitest run packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts packages/server/src/server/agent/create-agent/create.test.ts packages/server/src/server/agent/mcp-server.test.ts packages/server/src/terminal/terminal-session-controller.test.ts packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts packages/server/src/server/workspace-archive-service.test.ts packages/server/src/server/workspace-cleanup-retry-service.test.ts packages/server/src/server/worktree-session.test.ts packages/server/src/server/session.workspaces.test.ts packages/server/src/server/schedule/service.test.ts packages/server/src/utils/worktree.posix.test.ts --fileParallelism=false", - "npm run build --workspace packages/server", - "npm run typecheck --workspace packages/server", - "oxlint <7 cleanup retry source and test files>", - "oxfmt --check <7 cleanup retry source and test files>", - "git diff --check", - "vitest run packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts packages/server/src/server/agent/create-agent/create.test.ts packages/server/src/server/agent/mcp-server.test.ts packages/server/src/terminal/terminal-session-controller.test.ts packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts packages/server/src/server/workspace-archive-service.test.ts packages/server/src/server/workspace-cleanup-retry-service.test.ts packages/server/src/server/worktree-session.test.ts packages/server/src/server/session.workspaces.test.ts packages/server/src/server/schedule/service.test.ts packages/server/src/server/loop-service.test.ts packages/server/src/server/paseo-worktree-service.test.ts packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts packages/server/src/utils/worktree.posix.test.ts --fileParallelism=false", - "npm run build --workspace packages/server", - "npm run typecheck --workspace packages/server", - "oxlint <4 cleanup/create exclusion source and test files>", - "oxfmt --check <4 cleanup/create exclusion source and test files>", - "git diff --check", - "vitest run packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts packages/server/src/server/agent/create-agent/create.test.ts packages/server/src/server/agent/mcp-server.test.ts packages/server/src/terminal/terminal-session-controller.test.ts packages/server/src/server/auto-archive-on-merge/archive-if-safe.test.ts packages/server/src/server/workspace-archive-service.test.ts packages/server/src/server/workspace-cleanup-retry-service.test.ts packages/server/src/server/worktree-session.test.ts packages/server/src/server/session.workspaces.test.ts packages/server/src/server/schedule/service.test.ts packages/server/src/server/loop-service.test.ts packages/server/src/server/paseo-worktree-service.test.ts packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts packages/server/src/utils/worktree.posix.test.ts --fileParallelism=false", - "npm run build --workspace packages/server", - "npm run typecheck --workspace packages/server", - "oxlint packages/server/src/server/workspace-lifecycle-coordinator.ts packages/server/src/server/workspace-archive-service.test.ts", - "oxfmt --check packages/server/src/server/workspace-lifecycle-coordinator.ts packages/server/src/server/workspace-archive-service.test.ts", - "git diff --check" - ], - "result": { - "focusedAndBroadServerTests": "526 passed, 6 skipped; final archive-response suite 31 passed", - "protocolTests": "470 passed", - "serverBuild": "passed", - "serverAndProtocolTypechecks": "passed", - "lint": "0 warnings, 0 errors", - "lifecycleFormatAndConflictDiff": "passed", - "ownershipAndRetryRepairTests": "317 passed, 4 skipped; final ownership subset 144 passed", - "repairServerBuildAndTypecheck": "passed", - "autoArchiveTruthfulnessTests": "16 passed", - "archiveTargetClosureTests": "26 passed; combined auto-archive and closure proof 42 passed", - "archiveFailureTruthfulnessTests": "75 passed", - "automatedCleanupRetryTests": "388 passed, 4 skipped", - "autonomousCleanupOwner": "non-blocking startup and shutdown; startup and mutation-driven retry; rotating incarnation-grouped worktree batches; bounded cycle and capped backoff; cancellable timers", - "cleanupOnlyRetryRepairTests": "434 passed, 5 skipped across 11 lifecycle files", - "cleanupOnlyRetryBoundary": "exact persisted directory and incarnation only; never invokes normal workspace or agent archive scope; stale cleanup cannot archive a live replacement at a reused path", - "cleanupRetryShutdown": "stop aborts and awaits the owned active cycle; hanging teardown subprocess trees are terminated before shutdown completes", - "cleanupOnlyRetryBuildAndTypecheck": "passed", - "cleanupOnlyRetryLintAndFormat": "0 warnings, 0 errors; formatting and diff checks passed", - "cleanupCreateExclusionTests": "509 passed, 5 skipped across 14 lifecycle, provisioning, and affected call-site files", - "cleanupCreateExclusionBoundary": "directory workspace creation and unarchive share cleanup directory exclusion and revalidate directory existence after waiting; deterministic final-owner-read race leaves no replacement record for a deleted worktree", - "cleanupCreateExclusionBuildAndTypecheck": "passed; repository pre-commit all-workspace typecheck also passed", - "cleanupCreateExclusionLintAndFormat": "0 warnings, 0 errors; formatting and diff checks passed", - "filesystemIdentityLockTests": "509 passed, 5 skipped across 14 lifecycle, provisioning, and affected call-site files", - "filesystemIdentityLockBoundary": "directory setup tracking and exclusion keys use realpath-aware filesystem identity; the final-owner-read race uses a symlink spelling and proves equivalent-path creation remains blocked before rejecting after cleanup deletion", - "filesystemIdentityBuildAndTypecheck": "passed; repository pre-commit all-workspace typecheck also passed", - "filesystemIdentityLintAndFormat": "0 warnings, 0 errors; formatting and diff checks passed", - "allWorkspaceTypecheck": "passed in repository pre-commit hook" - } - } -} diff --git a/patches/receipts/paseo-v025-worktree-lifecycle.json b/patches/receipts/paseo-v025-worktree-lifecycle.json deleted file mode 100644 index 6efe4b2909..0000000000 --- a/patches/receipts/paseo-v025-worktree-lifecycle.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "schemaVersion": 1, - "patchId": "paseo-v025-worktree-lifecycle", - "base": { - "tag": "v0.2.5", - "commit": "6fc491e6220fba6543bbbe4bf1b1f58cfe59228b" - }, - "candidate": { - "branch": "review-v025-worktree-lifecycle-upstream", - "commit": "e2937e792969038b1b701e79abfbfe017bb635da", - "tree": "af3297ff70ecb52c78ce0d305a622162049cc4a8" - }, - "verification": { - "commit": "e2937e792969038b1b701e79abfbfe017bb635da", - "completedAt": "2026-08-01T00:34:00Z", - "commands": [ - "npm run build --workspace packages/protocol", - "cd packages/server && npx vitest run src/server/agent/create-agent-lifecycle-dispatch.test.ts src/server/agent/create-agent/create.test.ts src/server/agent/mcp-server.test.ts src/server/session.workspace-git-watch.test.ts src/server/session.workspaces.test.ts", - "cd packages/server && npx vitest run src/server/workspace-archive-service.test.ts src/server/worktree-bootstrap.test.ts src/server/worktree-session.test.ts src/utils/worktree-setup-output.test.ts src/utils/worktree.posix.test.ts", - "npm test --workspace packages/protocol", - "npm run build --workspace packages/server", - "npm run typecheck --workspace packages/server", - "npm run typecheck --workspace packages/protocol", - "npm run lint -- --deny-warnings", - "npx oxfmt --check ", - "git diff --check", - "jq -e '.schemaVersion == 1 and (.patches | length == 1) and .patches[0].id == \"paseo-v025-worktree-lifecycle\"' patches/local-patches.json" - ], - "result": { - "focusedServerTests": "359 passed, 5 skipped", - "protocolTests": "468 passed", - "serverBuild": "passed", - "serverAndProtocolTypechecks": "passed", - "lint": "0 warnings, 0 errors", - "formatDiffAndManifest": "passed" - } - } -} From 4a432854401afe7e085e387ad7a4c1adb18fe8b5 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 12:28:30 +0100 Subject: [PATCH 35/43] fix(server): recover interrupted auto-archive creation --- .../server/src/server/agent/agent-loading.ts | 6 +- .../server/src/server/agent/agent-manager.ts | 5 + .../src/server/agent/agent-storage.test.ts | 54 +++ .../server/src/server/agent/agent-storage.ts | 103 ++++- .../create-agent-lifecycle-dispatch.test.ts | 376 +++++++++++++++++- .../agent/create-agent-lifecycle-dispatch.ts | 156 +++++++- .../server/agent/create-agent/create.test.ts | 81 ++++ .../src/server/agent/create-agent/create.ts | 47 ++- .../server/src/server/bootstrap.smoke.test.ts | 36 ++ packages/server/src/server/bootstrap.ts | 3 + ...ate-agent-worktree-autoarchive.e2e.test.ts | 38 +- packages/server/src/server/session.ts | 71 +++- .../workspace-lifecycle-coordinator.test.ts | 64 +++ .../server/workspace-lifecycle-coordinator.ts | 19 + .../src/server/worktree-core.posix.test.ts | 52 ++- packages/server/src/server/worktree-core.ts | 6 + .../server/src/server/worktree-session.ts | 6 + packages/server/src/utils/worktree.ts | 59 ++- 18 files changed, 1132 insertions(+), 50 deletions(-) diff --git a/packages/server/src/server/agent/agent-loading.ts b/packages/server/src/server/agent/agent-loading.ts index f87f38911c..e273236842 100644 --- a/packages/server/src/server/agent/agent-loading.ts +++ b/packages/server/src/server/agent/agent-loading.ts @@ -109,7 +109,10 @@ export async function ensureAgentLoaded( handle, buildConfigOverrides(record), agentId, - extractTimestamps(record), + { + ...extractTimestamps(record), + autoArchiveObligation: record.autoArchiveObligation, + }, record.archivedAt ? { purpose: "history" } : undefined, ); deps.logger.info({ agentId, provider: record.provider }, "Agent resumed from persistence"); @@ -124,6 +127,7 @@ export async function ensureAgentLoaded( labels: record.labels, workspaceId: record.workspaceId, owner: record.owner, + autoArchiveObligation: record.autoArchiveObligation, }); deps.logger.info({ agentId, provider: record.provider }, "Agent created from stored config"); } diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 476ad0de23..eb825a47a8 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -1007,6 +1007,10 @@ export class AgentManager { return this.trackAgentRegistrationOperation(this.createAgentInternal(config, agentId, options)); } + allocateAgentId(): string { + return validateAgentId(this.idFactory(), "allocateAgentId"); + } + private async createAgentInternal( config: AgentSessionConfig, agentId: string | undefined, @@ -1083,6 +1087,7 @@ export class AgentManager { labels?: Record; workspaceId?: string; owner?: AgentOwner; + autoArchiveObligation?: AutoArchiveObligation; }, resumeOptions?: AgentResumeSessionOptions, ): Promise { diff --git a/packages/server/src/server/agent/agent-storage.test.ts b/packages/server/src/server/agent/agent-storage.test.ts index 36107c4168..c595577a95 100644 --- a/packages/server/src/server/agent/agent-storage.test.ts +++ b/packages/server/src/server/agent/agent-storage.test.ts @@ -209,6 +209,60 @@ describe("AgentStorage", () => { }); }); + test("persists pending creation intent separately from visible agents", async () => { + await storage.beginPendingAgentCreation("agent-pending-create"); + await storage.setPendingAgentCreationWorktree( + "agent-pending-create", + "/tmp/paseo/worktrees/project/feature-2", + { + worktreeIncarnationId: "4d2ce498-4c27-4ea2-8ed3-46720de7194e", + directoryIdentity: { device: "7", inode: "42" }, + metadataBaseRefName: "main", + }, + ); + + const reloaded = new AgentStorage(storagePath, logger); + await expect(reloaded.list()).resolves.toEqual([]); + await expect(reloaded.listPendingAgentCreations()).resolves.toEqual([ + { + agentId: "agent-pending-create", + createdAt: expect.any(String), + cleanupTarget: { + kind: "worktree", + targetPath: "/tmp/paseo/worktrees/project/feature-2", + worktreeIncarnationId: "4d2ce498-4c27-4ea2-8ed3-46720de7194e", + directoryIdentity: { device: "7", inode: "42" }, + metadataBaseRefName: "main", + }, + }, + ]); + + await reloaded.removePendingAgentCreation("agent-pending-create"); + await expect(reloaded.listPendingAgentCreations()).resolves.toEqual([]); + }); + + test.each(["armed", "pending"] as const)( + "explicit undefined preserves a persisted %s auto-archive obligation", + async (phase) => { + const agent = createManagedAgent({ id: `agent-auto-archive-${phase}` }); + await storage.applySnapshot(agent, { + autoArchiveObligation: { phase, target: { kind: "agent" } }, + }); + + await storage.applySnapshot( + { ...agent, updatedAt: new Date("2025-02-01") }, + { + autoArchiveObligation: undefined, + }, + ); + + const reloaded = new AgentStorage(storagePath, logger); + await expect(reloaded.get(agent.id)).resolves.toMatchObject({ + autoArchiveObligation: { phase, target: { kind: "agent" } }, + }); + }, + ); + test("serialized snapshots cannot overwrite a pending auto-archive obligation", async () => { const agent = createManagedAgent({ id: "agent-auto-archive-race" }); await storage.applySnapshot(agent, { diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 831cc11699..38c41f670d 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -43,6 +43,24 @@ const AUTO_ARCHIVE_OBLIGATION_SCHEMA = z.object({ target: AUTO_ARCHIVE_TARGET_SCHEMA, }); +const PENDING_AGENT_CREATION_SCHEMA = z.object({ + agentId: z.string(), + createdAt: z.string(), + cleanupTarget: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("agent") }), + z.object({ + kind: z.literal("worktree"), + targetPath: z.string(), + worktreeIncarnationId: z.string().uuid(), + directoryIdentity: z.object({ + device: z.string(), + inode: z.string(), + }), + metadataBaseRefName: z.string().min(1), + }), + ]), +}); + const STORED_AGENT_SCHEMA = z.object({ id: z.string(), provider: z.string(), @@ -92,6 +110,7 @@ export type SerializableAgentConfig = Pick< export type StoredAgentRecord = z.infer; export type AutoArchiveObligation = z.infer; +export type PendingAgentCreation = z.infer; export function parseStoredAgentRecord(value: unknown): StoredAgentRecord { return STORED_AGENT_SCHEMA.parse(value); } @@ -139,6 +158,57 @@ export class AgentStorage { await this.queueRecordWrite(record); } + async beginPendingAgentCreation(agentId: string): Promise { + const record: PendingAgentCreation = { + agentId, + createdAt: new Date().toISOString(), + cleanupTarget: { kind: "agent" }, + }; + await writeJsonFileAtomic(this.pendingCreationPath(agentId), record); + return record; + } + + async setPendingAgentCreationWorktree( + agentId: string, + targetPath: string, + reservation: { + worktreeIncarnationId: string; + directoryIdentity: { device: string; inode: string }; + metadataBaseRefName: string; + }, + ): Promise { + const record = await this.readPendingAgentCreation(agentId); + if (!record) { + throw new Error(`Pending agent creation ${agentId} not found`); + } + await writeJsonFileAtomic(this.pendingCreationPath(agentId), { + ...record, + cleanupTarget: { kind: "worktree", targetPath, ...reservation }, + } satisfies PendingAgentCreation); + } + + async listPendingAgentCreations(): Promise { + let entries: Dirent[]; + try { + entries = await fs.readdir(this.pendingCreationDir(), { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + const records = await Promise.all( + entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".intent")) + .map((entry) => + this.readPendingAgentCreationFile(path.join(this.pendingCreationDir(), entry.name)), + ), + ); + return records.filter((record): record is PendingAgentCreation => record !== null); + } + + async removePendingAgentCreation(agentId: string): Promise { + await fs.rm(this.pendingCreationPath(agentId), { force: true }); + } + async update( agentId: string, mutation: (record: StoredAgentRecord) => StoredAgentRecord, @@ -260,18 +330,14 @@ export class AgentStorage { options !== undefined && Object.prototype.hasOwnProperty.call(options, "title"); const hasInternalOverride = options !== undefined && Object.prototype.hasOwnProperty.call(options, "internal"); - const hasAutoArchiveOverride = - options !== undefined && - Object.prototype.hasOwnProperty.call(options, "autoArchiveObligation"); const record = toStoredAgentRecord(agent, { title: hasTitleOverride ? (options?.title ?? null) : (existing?.title ?? null), createdAt: existing?.createdAt, internal: hasInternalOverride ? options?.internal : (agent.internal ?? existing?.internal), }); record.archivedAt = existing?.archivedAt; - record.autoArchiveObligation = hasAutoArchiveOverride - ? options?.autoArchiveObligation - : existing?.autoArchiveObligation; + record.autoArchiveObligation = + options?.autoArchiveObligation ?? existing?.autoArchiveObligation; if (!this.deleting.has(agent.id)) { await this.writeRecord(record); } @@ -385,6 +451,31 @@ export class AgentStorage { return records; } + private pendingCreationDir(): string { + return path.join(this.baseDir, ".creating"); + } + + private pendingCreationPath(agentId: string): string { + return path.join(this.pendingCreationDir(), `${agentId}.intent`); + } + + private async readPendingAgentCreation(agentId: string): Promise { + return this.readPendingAgentCreationFile(this.pendingCreationPath(agentId)); + } + + private async readPendingAgentCreationFile( + filePath: string, + ): Promise { + try { + const value: unknown = JSON.parse(await fs.readFile(filePath, "utf8")); + return PENDING_AGENT_CREATION_SCHEMA.parse(value); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + this.logger.warn({ err: error, filePath }, "Ignoring invalid pending agent creation"); + return null; + } + } + private async readRecordFile(filePath: string): Promise { try { const content = await fs.readFile(filePath, "utf8"); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index 19436169a2..d67ec2a2a2 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -1,3 +1,6 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve as resolvePath } from "node:path"; import { expect, test, vi } from "vitest"; import type { AgentManagerEvent, AgentSubscriber } from "./agent-manager.js"; @@ -10,7 +13,28 @@ import { requireArchiveCleanupComplete, WorkspaceCleanupPendingError, } from "../workspace-archive-service.js"; -import type { StoredAgentRecord } from "./agent-storage.js"; +import type { PendingAgentCreation, StoredAgentRecord } from "./agent-storage.js"; +import type { PersistedWorkspaceRecord } from "../workspace-registry.js"; +import type { CreatePaseoWorktreeWorkflowResult } from "../worktree-session.js"; +import { readPaseoWorktreeMetadata } from "../../utils/worktree-metadata.js"; + +const PENDING_WORKTREE_INCARNATION_ID = "4d2ce498-4c27-4ea2-8ed3-46720de7194e"; + +function pendingWorktreeTarget( + targetPath: string, +): Extract { + const directoryStat = statSync(targetPath, { bigint: true }); + return { + kind: "worktree", + targetPath, + worktreeIncarnationId: PENDING_WORKTREE_INCARNATION_ID, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, + metadataBaseRefName: "main", + }; +} class AgentLifecycleEvents { private readonly listeners = new Set(); @@ -57,14 +81,24 @@ function createLifecycleDispatch( options?: { records?: Map; archiveWorkspaceForClose?: (workspaceId: string, signal?: AbortSignal) => Promise; + drainWorkspaceLifecycleOperations?: () => Promise; + getWorkspace?: (workspaceId: string) => Promise; + pendingCreations?: Map; + workspaces?: PersistedWorkspaceRecord[]; }, ): CreateAgentLifecycleDispatch { const records = options?.records ?? new Map(); + const pendingCreations = options?.pendingCreations ?? new Map(); const dependencies = { paseoHome: "/tmp/paseo", agentManager: agents, agentStorage: { list: async () => Array.from(records.values()), + get: async (agentId: string) => records.get(agentId) ?? null, + listPendingAgentCreations: async () => Array.from(pendingCreations.values()), + removePendingAgentCreation: async (agentId: string) => { + pendingCreations.delete(agentId); + }, update: async ( agentId: string, mutation: (record: StoredAgentRecord) => StoredAgentRecord, @@ -79,10 +113,22 @@ function createLifecycleDispatch( workspaceGitService: {}, archiveAgentForClose, archiveWorkspaceForClose: options?.archiveWorkspaceForClose ?? (async () => undefined), + drainWorkspaceLifecycleOperations: + options?.drainWorkspaceLifecycleOperations ?? (async () => undefined), findWorkspaceIdForCwd: async () => null, listActiveWorkspaces: async () => [], archiveWorkspaceRecord: async () => undefined, - workspaceRegistry: {}, + workspaceRegistry: { + get: + options?.getWorkspace ?? + (async (workspaceId: string) => + ({ + workspaceId, + archivedAt: "2025-01-01T00:00:00.000Z", + cleanupPending: null, + }) as PersistedWorkspaceRecord), + list: async () => options?.workspaces ?? [], + }, emit: () => undefined, emitAgentRemove: () => undefined, emitWorkspaceUpdatesForWorkspaceIds: async () => undefined, @@ -94,6 +140,224 @@ function createLifecycleDispatch( return new CreateAgentLifecycleDispatch(dependencies); } +test("startup removes a side-effect-free pending creation reservation", async () => { + const agentId = "agent-pending-no-side-effects"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: { kind: "agent" }, + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + + await dispatch.recoverPendingAgentCreations(); + + expect(pendingCreations.size).toBe(0); +}); + +test("startup leaves persisted agent cleanup to its durable obligation", async () => { + const agentId = "agent-pending-now-persisted"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: { + kind: "worktree", + targetPath: "/tmp/persisted-agent-worktree", + worktreeIncarnationId: PENDING_WORKTREE_INCARNATION_ID, + directoryIdentity: { device: "7", inode: "42" }, + metadataBaseRefName: "main", + }, + }, + ], + ]); + const records = new Map([ + [ + agentId, + { + id: agentId, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + } as StoredAgentRecord, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + records, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); +}); + +test("startup removes an exact pre-git worktree reservation without archiving", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-pre-git-reservation-")); + const agentId = "agent-pending-pre-git-reservation"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(false); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup retains and retries an orphan worktree pending creation", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-orphan-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const agentId = "agent-pending-orphan-worktree"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi + .spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ) + .mockRejectedValueOnce(new Error("physical archive failed")) + .mockResolvedValue(undefined); + + try { + await dispatch.recoverPendingAgentCreations(); + expect(pendingCreations.has(agentId)).toBe(true); + + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).toHaveBeenCalledTimes(2); + expect(archiveWorktreePath).toHaveBeenLastCalledWith(resolvePath(worktreePath)); + expect(pendingCreations.size).toBe(0); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup does not archive a replacement at a reused pending-creation path", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-replaced-worktree-")); + const staleTarget = pendingWorktreeTarget(worktreePath); + rmSync(worktreePath, { recursive: true, force: true }); + mkdirSync(worktreePath); + const agentId = "agent-pending-replaced-worktree"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: staleTarget, + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(statSync(worktreePath).isDirectory()).toBe(true); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup repairs metadata after git creation and before archive recovery", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-unrecorded-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const agentId = "agent-pending-metadata-gap"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi + .spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ) + .mockImplementation(async () => { + expect(readPaseoWorktreeMetadata(worktreePath)).toMatchObject({ + baseRefName: "main", + incarnationId: PENDING_WORKTREE_INCARNATION_ID, + }); + }); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).toHaveBeenCalledWith(resolvePath(worktreePath)); + expect(pendingCreations.size).toBe(0); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + test("auto-archive self-releases once and later cancellation waits harmlessly", async () => { const agentId = "4a7e2521-286d-4ad5-af35-e091c55302e3"; const agents = new AgentLifecycleEvents(); @@ -285,6 +549,69 @@ test("lifecycle shutdown cancels listeners and truthfully awaits an active archi await expect(registration.settled).resolves.toBe("completed"); }); +test("lifecycle shutdown joins failed-create cleanup that starts during drain", async () => { + const agents = new AgentLifecycleEvents(); + let markDrainStarted!: () => void; + let releaseDrain!: () => void; + const drainStarted = new Promise((resolve) => { + markDrainStarted = resolve; + }); + const drainFinished = new Promise((resolve) => { + releaseDrain = resolve; + }); + let drainCount = 0; + const dispatch = createLifecycleDispatch(agents, async () => undefined, { + drainWorkspaceLifecycleOperations: async () => { + drainCount += 1; + if (drainCount === 1) { + markDrainStarted(); + await drainFinished; + } + }, + }); + let markCleanupStarted!: () => void; + let releaseCleanup!: () => void; + const cleanupStarted = new Promise((resolve) => { + markCleanupStarted = resolve; + }); + const cleanupFinished = new Promise((resolve) => { + releaseCleanup = resolve; + }); + vi.spyOn( + dispatch as unknown as { + archiveAutoCreatedWorktree(options: unknown): Promise; + }, + "archiveAutoCreatedWorktree", + ).mockImplementation(async () => { + markCleanupStarted(); + await cleanupFinished; + }); + + let shutdownSettled = false; + const shutdown = dispatch.shutdown({ timeoutMs: 5_000 }).then((result) => { + shutdownSettled = true; + return result; + }); + await drainStarted; + const cleanup = dispatch.cleanupCreatedWorktreeAfterFailedAgentCreate({ + createdAgentId: null, + createdWorktree: { + created: true, + worktree: { worktreePath: "/tmp/paseo/worktrees/repo/failed-create" }, + } as CreatePaseoWorktreeWorkflowResult, + }); + await cleanupStarted; + releaseDrain(); + await Promise.resolve(); + + expect(shutdownSettled).toBe(false); + + releaseCleanup(); + await expect(cleanup).resolves.toBeUndefined(); + await expect(shutdown).resolves.toEqual({ completed: true, pendingAgentIds: [] }); + expect(drainCount).toBe(2); +}); + test("one dispatcher keeps one lifecycle registration across repeated callers", async () => { const agentId = "agent-shared-lifecycle-registration"; const agents = new AgentLifecycleEvents(); @@ -471,7 +798,7 @@ test("startup completes an armed agent-only auto-archive obligation", async () = await dispatch.recoverPersistedAutoArchives(); await vi.waitFor(() => expect(archiveAgentForClose).toHaveBeenCalledWith(agentId)); - expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined(); + await vi.waitFor(() => expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined()); }); test("startup completes a persisted workspace auto-archive obligation", async () => { @@ -499,9 +826,48 @@ test("startup completes a persisted workspace auto-archive obligation", async () expect(archiveWorkspaceForClose).toHaveBeenCalledWith("ws-restart", expect.any(AbortSignal)), ); - expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined(); + await vi.waitFor(() => expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined()); }); +test.each([ + ["missing", async () => null], + [ + "still active", + async (workspaceId: string) => + ({ workspaceId, archivedAt: null, cleanupPending: null }) as PersistedWorkspaceRecord, + ], + [ + "cleanup pending", + async (workspaceId: string) => + ({ + workspaceId, + archivedAt: "2025-01-01T00:00:00.000Z", + cleanupPending: { directoryPath: "/tmp/worktree" }, + }) as PersistedWorkspaceRecord, + ], +])( + "startup retains a workspace obligation when archive readback is %s", + async (_name, getWorkspace) => { + const agentId = "agent-restart-workspace-unverified"; + const obligation = { + phase: "pending", + target: { kind: "workspace", workspaceId: "ws-unverified" }, + } as const; + const records = new Map([ + [agentId, { id: agentId, autoArchiveObligation: obligation } as StoredAgentRecord], + ]); + const getWorkspaceSpy = vi.fn(getWorkspace); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + records, + getWorkspace: getWorkspaceSpy, + }); + + await dispatch.recoverPersistedAutoArchives(); + await vi.waitFor(() => expect(getWorkspaceSpy).toHaveBeenCalledWith("ws-unverified")); + expect(records.get(agentId)?.autoArchiveObligation).toEqual(obligation); + }, +); + test.each([ ["workspace cleanup failure", new WorkspaceCleanupPendingError("archive", ["ws-restart"])], ["failure before cleanup-pending persistence", new Error("git removal failed early")], @@ -546,5 +912,5 @@ test.each([ await vi.waitFor(() => expect(retryArchive).toHaveBeenCalledWith("ws-restart", expect.any(AbortSignal)), ); - expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined(); + await vi.waitFor(() => expect(records.get(agentId)?.autoArchiveObligation).toBeUndefined()); }); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 66a1d53e59..48be7ec38c 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -1,9 +1,16 @@ import { randomUUID } from "node:crypto"; +import { existsSync, statSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { resolve as resolvePath } from "node:path"; import type pino from "pino"; import type { ForgeService } from "../../services/forge-service.js"; import { withTimeout } from "../../utils/promise-timeout.js"; import { isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; +import { + readPaseoWorktreeMetadata, + writePaseoWorktreeMetadata, +} from "../../utils/worktree-metadata.js"; import { archiveByScope, requireArchiveCleanupComplete, @@ -26,6 +33,7 @@ interface CreateAgentLifecycleDispatchDependencies { workspaceGitService: WorkspaceGitService; archiveAgentForClose: (agentId: string) => Promise; archiveWorkspaceForClose: (workspaceId: string, signal?: AbortSignal) => Promise; + drainWorkspaceLifecycleOperations: () => Promise; findWorkspaceIdForCwd: (cwd: string) => Promise; listActiveWorkspaces: () => Promise; archiveWorkspaceRecord: (workspaceId: string) => Promise; @@ -61,6 +69,7 @@ export interface LifecycleDispatchShutdownResult { export class CreateAgentLifecycleDispatch { private readonly autoArchiveTasks = new Map>(); private readonly autoArchiveRegistrations = new Map(); + private readonly failedCreateCleanupTasks = new Map>(); private readonly shutdownController = new AbortController(); private shuttingDown = false; @@ -107,27 +116,128 @@ export class CreateAgentLifecycleDispatch { } } + async recoverPendingAgentCreations(): Promise { + const records = await this.dependencies.agentStorage.listPendingAgentCreations(); + for (const record of records) { + await this.recoverPendingAgentCreation(record.agentId); + } + } + + async recoverPendingAgentCreation(agentId: string): Promise { + const pending = (await this.dependencies.agentStorage.listPendingAgentCreations()).find( + (record) => record.agentId === agentId, + ); + if (!pending) return; + + if ( + (await this.dependencies.agentStorage.get(agentId)) || + pending.cleanupTarget.kind === "agent" + ) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + + const targetPath = resolvePath(pending.cleanupTarget.targetPath); + try { + if (!existsSync(targetPath)) { + const ownsActiveWorkspace = (await this.dependencies.workspaceRegistry.list()).some( + (workspace) => + !workspace.archivedAt && + resolvePath(workspace.worktreeRoot ?? workspace.cwd) === targetPath, + ); + if (!ownsActiveWorkspace) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + } + + if (existsSync(targetPath)) { + const targetStat = statSync(targetPath, { bigint: true }); + const currentDirectoryIdentity = { + device: targetStat.dev.toString(), + inode: targetStat.ino.toString(), + }; + if ( + currentDirectoryIdentity.device !== pending.cleanupTarget.directoryIdentity.device || + currentDirectoryIdentity.inode !== pending.cleanupTarget.directoryIdentity.inode + ) { + this.dependencies.logger.warn( + { agentId, worktreePath: targetPath }, + "Pending agent creation no longer owns its worktree path", + ); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + + const gitPath = resolvePath(targetPath, ".git"); + if (!existsSync(gitPath)) { + await rm(targetPath, { recursive: true, force: true }); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + const metadata = readPaseoWorktreeMetadata(targetPath); + if ( + metadata?.incarnationId && + metadata.incarnationId !== pending.cleanupTarget.worktreeIncarnationId + ) { + this.dependencies.logger.warn( + { agentId, worktreePath: targetPath }, + "Pending agent creation found a replacement worktree incarnation", + ); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + if (!metadata?.incarnationId) { + writePaseoWorktreeMetadata(targetPath, { + baseRefName: metadata?.baseRefName ?? pending.cleanupTarget.metadataBaseRefName, + incarnationId: pending.cleanupTarget.worktreeIncarnationId, + ...(metadata?.changeRequestLookupTarget + ? { changeRequestLookupTarget: metadata.changeRequestLookupTarget } + : {}), + }); + } + } + + await this.archiveWorktreePath(targetPath); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + } catch (error) { + this.dependencies.logger.warn( + { err: error, agentId, worktreePath: targetPath }, + "Failed to recover pending agent creation", + ); + } + } + async cleanupCreatedWorktreeAfterFailedAgentCreate(input: { createdWorktree: CreatePaseoWorktreeWorkflowResult | null; createdAgentId: string | null; }): Promise { const { createdWorktree, createdAgentId } = input; - if (!createdWorktree || createdAgentId) { + if (!createdWorktree?.created || createdAgentId) { return; } - await this.archiveAutoCreatedWorktree({ + const worktreePath = createdWorktree.worktree.worktreePath; + const existingTask = this.failedCreateCleanupTasks.get(worktreePath); + if (existingTask) return existingTask; + + const cleanupTask = this.archiveAutoCreatedWorktree({ agentId: null, createdWorktree, }).catch((archiveError) => { this.dependencies.logger.warn( - { - err: archiveError, - worktreePath: createdWorktree.worktree.worktreePath, - }, + { err: archiveError, worktreePath }, "Failed to clean up worktree after create_agent_request failed", ); }); + this.failedCreateCleanupTasks.set(worktreePath, cleanupTask); + try { + await cleanupTask; + } finally { + if (this.failedCreateCleanupTasks.get(worktreePath) === cleanupTask) { + this.failedCreateCleanupTasks.delete(worktreePath); + } + } } private registerAutoArchiveOnTerminalState( @@ -170,6 +280,9 @@ export class CreateAgentLifecycleDispatch { registrations.map(([, registration]) => registration.cancel()), ).then(async (results) => { await Promise.allSettled(Array.from(this.autoArchiveTasks.values())); + await this.dependencies.drainWorkspaceLifecycleOperations(); + await Promise.allSettled(Array.from(this.failedCreateCleanupTasks.values())); + await this.dependencies.drainWorkspaceLifecycleOperations(); return results; }); let cancellationResults: PromiseSettledResult[] | null = null; @@ -197,7 +310,13 @@ export class CreateAgentLifecycleDispatch { ...this.autoArchiveTasks.keys(), ]), ); - return { completed: pendingAgentIds.length === 0, pendingAgentIds }; + return { + completed: + cancellationResults !== null && + pendingAgentIds.length === 0 && + this.failedCreateCleanupTasks.size === 0, + pendingAgentIds, + }; } private async autoArchiveAgentOnce( @@ -234,6 +353,12 @@ export class CreateAgentLifecycleDispatch { target.workspaceId, this.shutdownController.signal, ); + const workspace = await this.dependencies.workspaceRegistry.get(target.workspaceId); + if (!workspace?.archivedAt || workspace.cleanupPending) { + throw new WorkspaceCleanupPendingError("Workspace auto-archive verification", [ + target.workspaceId, + ]); + } } else { await this.dependencies.archiveAgentForClose(agentId); } @@ -256,6 +381,14 @@ export class CreateAgentLifecycleDispatch { throw new Error("Auto-created worktree is not a Paseo-owned worktree"); } + await this.archiveWorktreePath(worktreePath, createdWorktree.workspace.workspaceId); + + if (options.agentId) { + this.dependencies.emitAgentRemove(options.agentId); + } + } + + private async archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise { const archiveResult = await archiveByScope( { paseoHome: this.dependencies.paseoHome, @@ -275,16 +408,13 @@ export class CreateAgentLifecycleDispatch { sessionLogger: this.dependencies.logger, }, { - scope: { kind: "workspace", workspaceId: createdWorktree.workspace.workspaceId }, + scope: workspaceId + ? { kind: "workspace", workspaceId } + : { kind: "worktree", targetPath: worktreePath }, requestId: randomUUID(), - signal: this.shutdownController.signal, }, ); requireArchiveCleanupComplete(archiveResult, "Auto-created worktree archive"); - - if (options.agentId) { - this.dependencies.emitAgentRemove(options.agentId); - } } } diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index f954b243d8..c99f8f55e3 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -136,6 +136,87 @@ test("session create persists and arms auto-archive before starting the initial expect(order).toEqual(["persisted", "armed", "prompt"]); }); +test("legacy worktree create keeps its journal through durable agent registration", async () => { + const workdir = mkdtempSync(join(tmpdir(), "create-agent-journal-test-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const agentManager = createRealAgentManager(storage); + const agentId = agentManager.allocateAgentId(); + const resolvedWorktreePath = join(workdir, "worktrees", "feature-2"); + const removalOrder: string[] = []; + const removePendingAgentCreation = storage.removePendingAgentCreation.bind(storage); + + try { + await storage.beginPendingAgentCreation(agentId); + vi.spyOn(storage, "removePendingAgentCreation").mockImplementation(async (removingAgentId) => { + expect(removingAgentId).toBe(agentId); + expect(await storage.get(agentId)).toMatchObject({ + id: agentId, + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-created-worktree" }, + }, + }); + removalOrder.push("registered"); + await removePendingAgentCreation(removingAgentId); + }); + + const { snapshot } = await createAgentCommand( + { + agentManager, + agentStorage: storage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + }, + { + kind: "session", + agentId, + config: { provider: "codex", cwd: workdir }, + workspaceId: "ws-source", + labels: {}, + provisionalTitle: null, + firstAgentContext: { attachments: [] }, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + buildSessionConfig: async (config, _git, _worktreeName, _context, onPath) => { + await onPath?.(resolvedWorktreePath, { + worktreeIncarnationId: "30704df3-6339-4c9c-8277-1416544ed7cc", + directoryIdentity: { device: "7", inode: "42" }, + metadataBaseRefName: "main", + }); + expect(await storage.listPendingAgentCreations()).toEqual([ + expect.objectContaining({ + agentId, + cleanupTarget: { + kind: "worktree", + targetPath: resolvedWorktreePath, + worktreeIncarnationId: "30704df3-6339-4c9c-8277-1416544ed7cc", + directoryIdentity: { device: "7", inode: "42" }, + metadataBaseRefName: "main", + }, + }), + ]); + removalOrder.push("worktree-resolved"); + mkdirSync(resolvedWorktreePath, { recursive: true }); + return { + sessionConfig: { ...config, cwd: resolvedWorktreePath }, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: () => undefined, + releaseWithoutStarting: () => undefined, + }, + createdWorkspaceId: "ws-created-worktree", + }; + }, + }, + ); + + expect(snapshot.id).toBe(agentId); + expect(removalOrder).toEqual(["worktree-resolved", "registered"]); + await expect(storage.listPendingAgentCreations()).resolves.toEqual([]); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("session create validates the requested mode against the provider's modes", async () => { const snapshot = { id: "agent-1", diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index d2df5bb49c..1048415db5 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -1,6 +1,7 @@ import type { Logger } from "pino"; import type { TerminalManager } from "../../../terminal/terminal-manager.js"; +import type { WorktreeCreationReservation } from "../../../utils/worktree.js"; import type { CreatePaseoWorktreeInput } from "../../paseo-worktree-service.js"; import { expandUserPath, resolvePathFromBase } from "../../path-utils.js"; import { toWorktreeRequestError } from "../../worktree-errors.js"; @@ -75,12 +76,17 @@ export interface CreateAgentFromSessionInput { provisionalTitle: string | null; firstAgentContext: FirstAgentContext; autoArchiveObligation?: AutoArchiveObligation; - onCreated?: (created: { agentId: string }) => void; + agentId?: string; + onCreated?: (created: { agentId: string; autoArchiveObligation?: AutoArchiveObligation }) => void; buildSessionConfig: ( config: AgentSessionConfig, gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise, ) => Promise; } @@ -198,7 +204,11 @@ export async function createAgentCommand( await dependencies.requireActiveWorkspaceForOwnership?.(workspaceId); }, () => - dependencies.agentManager.createAgent(resolved.config, undefined, resolved.createOptions), + dependencies.agentManager.createAgent( + resolved.config, + input.kind === "session" ? input.agentId : undefined, + resolved.createOptions, + ), ); } catch (error) { resolved.setupContinuation?.releaseWithoutStarting(); @@ -213,7 +223,18 @@ export async function createAgentCommand( let initialPromptStarted = false; let initialPromptError: unknown | null = null; if (input.kind === "session") { - input.onCreated?.({ agentId: snapshot.id }); + if (input.agentId) { + await dependencies.agentStorage.removePendingAgentCreation(input.agentId).catch((error) => { + dependencies.logger.warn( + { err: error, agentId: input.agentId }, + "Failed to remove completed agent creation journal", + ); + }); + } + input.onCreated?.({ + agentId: snapshot.id, + autoArchiveObligation: resolved.createOptions.autoArchiveObligation, + }); } else { input.onCreated?.({ agentId: snapshot.id, createdWorktree: resolved.createdWorktree ?? null }); } @@ -250,6 +271,7 @@ async function resolveSessionCreateAgent( input: CreateAgentFromSessionInput, ): Promise { const trimmedPrompt = input.initialPrompt?.trim(); + const pendingAgentId = input.agentId; const { sessionConfig: builtSessionConfig, setupContinuation, @@ -259,6 +281,14 @@ async function resolveSessionCreateAgent( input.git, input.worktreeName, input.firstAgentContext, + pendingAgentId + ? (worktreePath, reservation) => + dependencies.agentStorage.setPendingAgentCreationWorktree( + pendingAgentId, + worktreePath, + reservation, + ) + : undefined, ); // Validate the requested mode against the provider's modes for the resolved // cwd. The app remembers mode preferences globally, so a saved mode can be @@ -310,7 +340,16 @@ async function resolveSessionCreateAgent( // agent belongs to that workspace, not the source one. createdWorkspaceId // is the freshly created worktree's workspace. workspaceId: requireResolvedWorkspaceId(workspaceId), - autoArchiveObligation: input.autoArchiveObligation, + autoArchiveObligation: + input.autoArchiveObligation && setupContinuation + ? { + phase: input.autoArchiveObligation.phase, + target: { + kind: "workspace", + workspaceId: requireResolvedWorkspaceId(createdWorkspaceId), + }, + } + : input.autoArchiveObligation, }, prompt: hasPromptContent ? prompt : undefined, runOptions, diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 4be92b7a77..7b87c685fa 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -15,6 +15,7 @@ import { createTestAgentClients } from "./test-utils/fake-agent-client.js"; import { DaemonClient } from "./test-utils/daemon-client.js"; import { isPlatform } from "../test-utils/platform.js"; import { findFreePort } from "./service-proxy.js"; +import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import type { HubEnrollment, HubEnrollmentResult, @@ -357,6 +358,41 @@ describe("paseo daemon bootstrap", () => { } }); + test("shutdown waits for a real workspace operation after its abortable wrapper rejects", async () => { + const daemonHandle = await createTestPaseoDaemon(); + const controller = new AbortController(); + let releaseOperation = () => {}; + const physicalOperation = new Promise((resolve) => { + releaseOperation = resolve; + }); + const wrappedOperation = defaultWorkspaceLifecycleCoordinator.runArchive( + `bootstrap-drain-${daemonHandle.port}`, + () => physicalOperation, + controller.signal, + ); + let stopSettled = false; + + try { + await Promise.resolve(); + controller.abort(); + await expect(wrappedOperation).rejects.toThrow("Workspace lifecycle operation canceled"); + + const stopping = daemonHandle.daemon.stop().then(() => { + stopSettled = true; + return undefined; + }); + await Promise.resolve(); + expect(stopSettled).toBe(false); + + releaseOperation(); + await stopping; + expect(stopSettled).toBe(true); + } finally { + releaseOperation(); + await daemonHandle.close(); + } + }); + test("a blocked cleanup stop cannot prevent later teardown before the outer deadline", async () => { let markCleanupStopStarted = () => {}; const cleanupStopStarted = new Promise((resolve) => { diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index e1b0441f8f..76c8eceb6b 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -157,6 +157,7 @@ import { type ActiveWorkspaceRef, } from "./workspace-archive-service.js"; import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service.js"; +import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { setupAutoArchiveOnMerge } from "./auto-archive-on-merge/index.js"; import { wrapSessionMessage, type SessionOutboundMessage } from "./messages.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; @@ -1190,6 +1191,7 @@ export async function createPaseoDaemon( archiveAgentCommand({ agentManager, agentStorage, logger }, agentId), archiveWorkspaceForClose: (workspaceId, signal) => archiveWorkspaceByIdExternal(workspaceId, randomUUID(), signal), + drainWorkspaceLifecycleOperations: () => defaultWorkspaceLifecycleCoordinator.drain(), findWorkspaceIdForCwd: findWorkspaceIdForCwdExternal, listActiveWorkspaces: listActiveWorkspacesExternal, archiveWorkspaceRecord: archiveWorkspaceRecordExternal, @@ -1322,6 +1324,7 @@ export async function createPaseoDaemon( logger.info({ elapsed: elapsed() }, "Schedule service initialized"); logger.info({ elapsed: elapsed() }, "Loading persisted agent registry"); const persistedRecords = await agentStorage.list(); + await hubAgentLifecycle.recoverPendingAgentCreations(); await hubAgentLifecycle.recoverPersistedAutoArchives(); logger.info( { elapsed: elapsed() }, diff --git a/packages/server/src/server/session.create-agent-worktree-autoarchive.e2e.test.ts b/packages/server/src/server/session.create-agent-worktree-autoarchive.e2e.test.ts index 8507a935ef..3e6ff5870c 100644 --- a/packages/server/src/server/session.create-agent-worktree-autoarchive.e2e.test.ts +++ b/packages/server/src/server/session.create-agent-worktree-autoarchive.e2e.test.ts @@ -82,7 +82,8 @@ async function expectWorktreePresentInList(repoDir: string, worktreePath: string .poll( async () => { const listed = await ctx.client.getPaseoWorktreeList({ cwd: repoDir }); - return listed.worktrees.map((worktree) => worktree.worktreePath).includes(worktreePath); + const matchesWorktree = createRealpathAwarePathMatcher(worktreePath); + return listed.worktrees.some((worktree) => matchesWorktree(worktree.worktreePath)); }, { timeout: 5000, interval: 100 }, ) @@ -97,6 +98,7 @@ async function expectWorktreeListEmpty(repoDir: string): Promise { async function createAgentInBranchOffWorktree(options?: { autoArchive?: boolean; branchName?: string; + deferInitialPrompt?: boolean; repoDir?: string; }): Promise<{ repoDir: string; agentId: string; worktreePath: string }> { const repoDir = options?.repoDir ?? createGitRepo(); @@ -112,7 +114,7 @@ async function createAgentInBranchOffWorktree(options?: { base: "main", }, ...(options?.autoArchive !== undefined ? { autoArchive: options.autoArchive } : {}), - initialPrompt: "Say done.", + ...(options?.deferInitialPrompt ? {} : { initialPrompt: "Say done." }), }); return { repoDir, agentId: created.id, worktreePath: created.cwd }; } @@ -259,6 +261,30 @@ test("failed nested worktree creation cleans up the created workspace and backin await ctx.client.archivePaseoWorktree({ worktreePath: source.worktreePath }); }, 30000); +test("failed auto-archive agent creation leaves a reused worktree active", async () => { + const source = await createAgentInBranchOffWorktree({ branchName: "reused-failure-source" }); + await ctx.client.waitForFinish(source.agentId, 10000); + + await expect( + ctx.client.createAgent({ + config: { provider: "unknown-provider", cwd: source.repoDir }, + worktree: { + mode: "branch-off", + newBranch: "reused-failure-source", + base: "main", + }, + autoArchive: true, + initialPrompt: "This agent cannot be created.", + }), + ).rejects.toThrow(); + + await expectWorktreePresentInList(source.repoDir, source.worktreePath); + expect(existsSync(source.worktreePath)).toBe(true); + await expectAgentPresentInActiveList(source.agentId); + + await ctx.client.archivePaseoWorktree({ worktreePath: source.worktreePath }); +}, 30000); + test("create_agent_request with autoArchive archives only the agent when no worktree was created", async () => { const repoDir = createGitRepo(); const created = await ctx.client.createAgent({ @@ -335,9 +361,12 @@ test("archiving a created worktree removes the directory on last reference", asy }); test("auto-archiving a created worktree keeps the directory when a sibling workspace references it", async () => { - const created = await createAgentInBranchOffWorktree({ autoArchive: true }); + const created = await createAgentInBranchOffWorktree({ + autoArchive: true, + deferInitialPrompt: true, + }); - // Create a sibling workspace that shares the same backing directory. + // Register the sibling before the first terminal event can trigger auto-archive. const sibling = await ctx.client.createWorkspace({ source: { kind: "directory", path: created.worktreePath }, title: "sibling", @@ -346,6 +375,7 @@ test("auto-archiving a created worktree keeps the directory when a sibling works throw new Error(sibling.error ?? "Failed to create sibling workspace"); } + await ctx.client.sendMessage(created.agentId, "Say done."); await ctx.client.waitForFinish(created.agentId, 10000); await expectAgentAbsentFromActiveList(created.agentId); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index ff32bba825..ab1c2b252d 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -27,6 +27,7 @@ import type { TerminalManager, TerminalWorkspaceContributionChangedEvent, } from "../terminal/terminal-manager.js"; +import type { WorktreeCreationReservation } from "../utils/worktree.js"; import { TerminalSessionController } from "../terminal/terminal-session-controller.js"; import type { TerminalActivity } from "@getpaseo/protocol/terminal-activity"; import type { BinaryFrame } from "@getpaseo/protocol/binary-frames/index"; @@ -352,6 +353,14 @@ interface ResolvedSessionCreateAgentIntent { createdDirectoryWorkspace: boolean; } +interface PendingAgentCreationReservation { + agentId?: string; + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; +} + type FetchWorkspacesRequestMessage = Extract< SessionInboundMessage, { type: "fetch_workspaces_request" } @@ -3061,6 +3070,7 @@ export class Session { let createdWorktreeForCleanup: CreatePaseoWorktreeWorkflowResult | null = null; let createdAgentId: string | null = null; + let pendingCreationAgentId: string | undefined; try { const requestedCwd = resolve(config.cwd); const needsRequestedDirectory = @@ -3079,11 +3089,14 @@ export class Session { ...(attachments && attachments.length > 0 ? { attachments } : {}), }; const workspacePromptTitle = resolveFirstAgentPromptTitle(firstAgentContext); + const pendingCreation = await this.reservePendingAgentCreation(autoArchive); + pendingCreationAgentId = pendingCreation.agentId; const createdWorktree = await this.createWorktreeForCreateAgentRequest({ cwd: config.cwd, target: worktree, firstAgentContext, hasLegacyGitOptions: Boolean(git), + onWorktreePathResolved: pendingCreation.onWorktreePathResolved, }); createdWorktreeForCleanup = createdWorktree; const resolvedIntent = await this.resolveSessionCreateAgentIntent({ @@ -3137,17 +3150,19 @@ export class Session { provisionalTitle, firstAgentContext, autoArchiveObligation, - onCreated: ({ agentId }) => { + agentId: pendingCreationAgentId, + onCreated: ({ agentId, autoArchiveObligation: persistedObligation }) => { createdAgentId = agentId; - if (autoArchiveObligation) { - this.createAgentLifecycleDispatch.registerAutoArchive({ - agentId, - obligation: autoArchiveObligation, - }); - } + this.registerCreatedAgentAutoArchive(agentId, persistedObligation); }, - buildSessionConfig: (sessionConfig, gitOptions, legacyWorktreeName, ctx) => - this.buildAgentSessionConfig(sessionConfig, gitOptions, legacyWorktreeName, ctx), + buildSessionConfig: (sessionConfig, gitOptions, legacyWorktreeName, ctx, onPath) => + this.buildAgentSessionConfig( + sessionConfig, + gitOptions, + legacyWorktreeName, + ctx, + onPath, + ), }, ); await this.agentUpdates.forwardLiveAgent(snapshot); @@ -3183,6 +3198,7 @@ export class Session { createdWorktree: createdWorktreeForCleanup, createdAgentId, }); + await this.recoverPendingAgentCreation(pendingCreationAgentId); const wireError = toWorktreeWireError(error); this.sessionLogger.error({ err: error }, "Failed to create agent"); if (requestId) { @@ -3208,6 +3224,33 @@ export class Session { } } + private async reservePendingAgentCreation( + autoArchive: boolean | undefined, + ): Promise { + if (autoArchive !== true) return {}; + + const agentId = this.agentManager.allocateAgentId(); + await this.agentStorage.beginPendingAgentCreation(agentId); + return { + agentId, + onWorktreePathResolved: (worktreePath, reservation) => + this.agentStorage.setPendingAgentCreationWorktree(agentId, worktreePath, reservation), + }; + } + + private registerCreatedAgentAutoArchive( + agentId: string, + obligation: AutoArchiveObligation | undefined, + ): void { + if (!obligation) return; + this.createAgentLifecycleDispatch.registerAutoArchive({ agentId, obligation }); + } + + private async recoverPendingAgentCreation(agentId: string | undefined): Promise { + if (!agentId) return; + await this.createAgentLifecycleDispatch.recoverPendingAgentCreation(agentId); + } + private async resolveSessionCreateAgentIntent(input: { request: CreateAgentRequestMessage; createdWorktree: CreatePaseoWorktreeWorkflowResult | null; @@ -3558,6 +3601,10 @@ export class Session { gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise, ): Promise<{ sessionConfig: AgentSessionConfig; setupContinuation?: CreatePaseoWorktreeWorkflowResult["setupContinuation"]; @@ -3598,6 +3645,7 @@ export class Session { gitOptions, legacyWorktreeName, firstAgentContext, + onWorktreePathResolved, ); } @@ -5819,6 +5867,10 @@ export class Session { target: CreateAgentWorktreeTarget | undefined; firstAgentContext: FirstAgentContext; hasLegacyGitOptions: boolean; + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; }): Promise { if (input.target && input.hasLegacyGitOptions) { throw new Error("create_agent_request worktree cannot be combined with git options"); @@ -5833,6 +5885,7 @@ export class Session { runSetup: false, paseoHome: this.paseoHome, worktreesRoot: this.worktreesRoot, + onWorktreePathResolved: input.onWorktreePathResolved, } as const; switch (input.target.mode) { case "branch-off": { diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.test.ts b/packages/server/src/server/workspace-lifecycle-coordinator.test.ts index cf1dd9c286..94964dcea1 100644 --- a/packages/server/src/server/workspace-lifecycle-coordinator.test.ts +++ b/packages/server/src/server/workspace-lifecycle-coordinator.test.ts @@ -38,3 +38,67 @@ test("cancels a queued worktree mutation before it starts", async () => { await Promise.resolve(); expect(secondOperation).not.toHaveBeenCalled(); }); + +test("drain waits for the real archive after its abortable wrapper rejects", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + const controller = new AbortController(); + let releaseArchive = () => {}; + const physicalArchive = new Promise((resolve) => { + releaseArchive = resolve; + }); + const archive = coordinator.runArchive( + "workspace:ws-drain", + () => physicalArchive, + controller.signal, + ); + await Promise.resolve(); + + controller.abort(); + await expect(archive).rejects.toThrow("Workspace lifecycle operation canceled"); + + let drained = false; + const drain = coordinator.drain().then(() => { + drained = true; + return undefined; + }); + await Promise.resolve(); + expect(drained).toBe(false); + + releaseArchive(); + await drain; + expect(drained).toBe(true); +}); + +test("drain waits for workspace setup reservations", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + const reservation = coordinator.reserveWorkspaceSetup("ws-setup", "/worktrees/setup"); + + let drained = false; + const drain = coordinator.drain().then(() => { + drained = true; + return undefined; + }); + await Promise.resolve(); + expect(drained).toBe(false); + + reservation.release(); + await drain; + expect(drained).toBe(true); +}); + +test("drain waits for workspace ownership mutation reservations", async () => { + const coordinator = new WorkspaceLifecycleCoordinator(); + const reservation = coordinator.reserveWorkspaceOwnershipMutation("ws-ownership"); + + let drained = false; + const drain = coordinator.drain().then(() => { + drained = true; + return undefined; + }); + await Promise.resolve(); + expect(drained).toBe(false); + + reservation.release(); + await drain; + expect(drained).toBe(true); +}); diff --git a/packages/server/src/server/workspace-lifecycle-coordinator.ts b/packages/server/src/server/workspace-lifecycle-coordinator.ts index 56fbfc0e29..b6a8561120 100644 --- a/packages/server/src/server/workspace-lifecycle-coordinator.ts +++ b/packages/server/src/server/workspace-lifecycle-coordinator.ts @@ -183,6 +183,25 @@ export class WorkspaceLifecycleCoordinator { ); } + async drain(): Promise { + while (true) { + const tasks = new Set>([ + ...this.setupTasks.values(), + ...Array.from(this.setupTasksByDirectory.values()).flatMap((directoryTasks) => + Array.from(directoryTasks), + ), + ...this.archiveOperations.values(), + ...this.directoryOperations.values(), + ...this.worktreeMutationOperations.values(), + ...Array.from(this.ownershipMutationTasks.values()).flatMap((ownershipTasks) => + Array.from(ownershipTasks), + ), + ]); + if (tasks.size === 0) return; + await Promise.allSettled(tasks); + } + } + runArchive(key: string, operation: () => Promise, signal?: AbortSignal): Promise { const existing = this.archiveOperations.get(key); if (existing) { diff --git a/packages/server/src/server/worktree-core.posix.test.ts b/packages/server/src/server/worktree-core.posix.test.ts index 40bd983c7b..956d27854e 100644 --- a/packages/server/src/server/worktree-core.posix.test.ts +++ b/packages/server/src/server/worktree-core.posix.test.ts @@ -8,6 +8,7 @@ import { readFileSync, realpathSync, rmSync, + statSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -24,7 +25,7 @@ import { getPaseoWorktreeMetadataPath, readPaseoWorktreeMetadata, } from "../utils/worktree-metadata.js"; -import { UnknownBranchError } from "../utils/worktree.js"; +import { computeWorktreePath, UnknownBranchError } from "../utils/worktree.js"; import { createWorktreeCore as createCoreWorktree } from "./worktree-core.js"; import { isPlatform } from "../test-utils/platform.js"; @@ -420,6 +421,43 @@ describe.skipIf(isPlatform("win32"))("worktree-core POSIX-only", () => { expect(existsSync(result.worktree.worktreePath)).toBe(true); }); + test("publishes the collision-resolved path before creating the worktree", async () => { + const { tempDir, repoDir, paseoHome } = createGitRepo(); + cleanupPaths.push(tempDir); + const occupiedPath = await computeWorktreePath(repoDir, "collision", paseoHome); + mkdirSync(occupiedPath, { recursive: true }); + const resolvedPath = `${occupiedPath}-1`; + let observedPath: string | null = null; + let observedIncarnationId: string | null = null; + + const result = await createCoreWorktree( + { + cwd: repoDir, + worktreeSlug: "collision", + paseoHome, + runSetup: false, + onWorktreePathResolved: async (worktreePath, reservation) => { + observedPath = worktreePath; + observedIncarnationId = reservation.worktreeIncarnationId; + expect(worktreePath).toBe(resolvedPath); + expect(existsSync(worktreePath)).toBe(true); + const directoryStat = statSync(worktreePath, { bigint: true }); + expect(reservation.directoryIdentity).toEqual({ + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }); + expect(reservation.metadataBaseRefName).toBe("main"); + }, + }, + createCoreDeps(), + ); + + expect(observedPath).toBe(resolvedPath); + expect(result.worktree.worktreePath).toBe(resolvedPath); + expect(existsSync(resolvedPath)).toBe(true); + expect(readPaseoWorktreeMetadata(resolvedPath)?.incarnationId).toBe(observedIncarnationId); + }); + test("creates branch-off worktrees from origin main without tracking origin main", async () => { const { tempDir, repoDir, paseoHome } = createGitRepoWithOriginMain(); cleanupPaths.push(tempDir); @@ -1443,19 +1481,29 @@ describe.skipIf(isPlatform("win32"))("worktree-core POSIX-only", () => { const { tempDir, repoDir, paseoHome } = createGitRepo(); cleanupPaths.push(tempDir); const deps = createCoreDeps(); + const resolvedPaths: string[] = []; const first = await createCoreWorktree( { cwd: repoDir, worktreeSlug: "reused-worktree", paseoHome, runSetup: false }, deps, ); const second = await createCoreWorktree( - { cwd: repoDir, worktreeSlug: "reused-worktree", paseoHome, runSetup: false }, + { + cwd: repoDir, + worktreeSlug: "reused-worktree", + paseoHome, + runSetup: false, + onWorktreePathResolved: async (worktreePath) => { + resolvedPaths.push(worktreePath); + }, + }, deps, ); expect(first.created).toBe(true); expect(second.created).toBe(false); expect(second.worktree).toEqual(first.worktree); + expect(resolvedPaths).toEqual([]); }); // POSIX-only: Windows git worktree paths need separate canonicalization coverage. diff --git a/packages/server/src/server/worktree-core.ts b/packages/server/src/server/worktree-core.ts index 90d0788905..1d7a0a05e7 100644 --- a/packages/server/src/server/worktree-core.ts +++ b/packages/server/src/server/worktree-core.ts @@ -6,6 +6,7 @@ import { resolveExistingWorktreeForSlug, slugify, validateBranchSlug, + type WorktreeCreationReservation, type WorktreeConfig, } from "../utils/worktree.js"; import { @@ -29,6 +30,10 @@ export interface CreateWorktreeCoreInput { paseoHome?: string; worktreesRoot?: string; runSetup?: boolean; + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; } export interface CreateWorktreeCoreDeps { @@ -128,6 +133,7 @@ export async function createWorktreeCore( runSetup: input.runSetup ?? true, paseoHome: input.paseoHome, worktreesRoot: input.worktreesRoot, + onWorktreePathResolved: input.onWorktreePathResolved, }), intent, repoRoot, diff --git a/packages/server/src/server/worktree-session.ts b/packages/server/src/server/worktree-session.ts index f968929d7b..27dcb2520b 100644 --- a/packages/server/src/server/worktree-session.ts +++ b/packages/server/src/server/worktree-session.ts @@ -33,6 +33,7 @@ import { runWorktreeSetupCommands, slugify, validateBranchSlug, + type WorktreeCreationReservation, type WorktreeConfig, type WorktreeSetupCommandResult, WorktreeSetupError, @@ -221,6 +222,10 @@ export async function buildAgentSessionConfig( gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise, ): Promise<{ sessionConfig: AgentSessionConfig; setupContinuation?: AgentWorktreeSetupContinuation; @@ -258,6 +263,7 @@ export async function buildAgentSessionConfig( runSetup: false, paseoHome: dependencies.paseoHome, worktreesRoot: dependencies.worktreesRoot, + onWorktreePathResolved, }, { resolveDefaultBranch: normalized.baseBranch diff --git a/packages/server/src/utils/worktree.ts b/packages/server/src/utils/worktree.ts index dad72da818..7a22b66435 100644 --- a/packages/server/src/utils/worktree.ts +++ b/packages/server/src/utils/worktree.ts @@ -208,6 +208,19 @@ export interface CreateWorktreeOptions { runSetup: boolean; paseoHome?: string; worktreesRoot?: string; + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; +} + +export interface WorktreeCreationReservation { + worktreeIncarnationId: string; + directoryIdentity: { + device: string; + inode: string; + }; + metadataBaseRefName: string; } interface ResolveExistingWorktreeForSlugOptions { @@ -1268,17 +1281,51 @@ export const createWorktree = async ({ runSetup, paseoHome, worktreesRoot, + onWorktreePathResolved, }: CreateWorktreeOptions): Promise => { const sourcePlan = await resolveWorktreeSourcePlan({ cwd, source, desiredSlug: worktreeSlug }); let worktreePath = join(await getPaseoWorktreesRoot(cwd, paseoHome, worktreesRoot), worktreeSlug); mkdirSync(dirname(worktreePath), { recursive: true }); - // Also handle worktree path collision + // Also handle worktree path collision. When creation is journaled, reserve + // the directory so the journal can identify this exact path incarnation. let finalWorktreePath = worktreePath; let pathSuffix = 1; - while (existsSync(finalWorktreePath)) { - finalWorktreePath = `${worktreePath}-${pathSuffix}`; - pathSuffix++; + if (onWorktreePathResolved) { + while (true) { + try { + mkdirSync(finalWorktreePath); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + finalWorktreePath = `${worktreePath}-${pathSuffix}`; + pathSuffix++; + } + } + } else { + while (existsSync(finalWorktreePath)) { + finalWorktreePath = `${worktreePath}-${pathSuffix}`; + pathSuffix++; + } + } + + const normalizedWorktreePath = normalizePathForOwnership(finalWorktreePath); + const worktreeIncarnationId = randomUUID(); + if (onWorktreePathResolved) { + const directoryStat = statSync(normalizedWorktreePath, { bigint: true }); + try { + await onWorktreePathResolved(normalizedWorktreePath, { + worktreeIncarnationId, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, + metadataBaseRefName: sourcePlan.metadataBaseRefName, + }); + } catch (error) { + rmSync(normalizedWorktreePath, { recursive: true, force: true }); + throw error; + } } // Primitive owner for `git worktree add`; callers route through createWorktreeCore. @@ -1286,7 +1333,7 @@ export const createWorktree = async ({ cwd, timeout: 120_000, }); - worktreePath = normalizePathForOwnership(finalWorktreePath); + worktreePath = normalizedWorktreePath; if (sourcePlan.pushRemote) { await configureWorktreePushRemote({ @@ -1305,7 +1352,7 @@ export const createWorktree = async ({ writePaseoWorktreeMetadata(worktreePath, { baseRefName: sourcePlan.metadataBaseRefName, - incarnationId: randomUUID(), + incarnationId: worktreeIncarnationId, ...(sourcePlan.changeRequestLookupTarget ? { changeRequestLookupTarget: sourcePlan.changeRequestLookupTarget } : {}), From 2799e2e051988208d2dd477d80e2f5764b22ab5e Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 08:47:53 -0400 Subject: [PATCH 36/43] fix(server): join canceled git process cleanup --- .../workspace-cleanup-retry-service.test.ts | 39 +++++++-- .../server/src/utils/run-git-command.test.ts | 61 +++++++++---- packages/server/src/utils/run-git-command.ts | 85 +++++++++++-------- 3 files changed, 127 insertions(+), 58 deletions(-) diff --git a/packages/server/src/server/workspace-cleanup-retry-service.test.ts b/packages/server/src/server/workspace-cleanup-retry-service.test.ts index 5fc8edbbb0..47250d29b5 100644 --- a/packages/server/src/server/workspace-cleanup-retry-service.test.ts +++ b/packages/server/src/server/workspace-cleanup-retry-service.test.ts @@ -130,16 +130,26 @@ test("rotates bounded batches and preserves backoff after a failed target", asyn expect(retryWorktreeCleanup).toHaveBeenCalledTimes(attempts.length); }); -test("shutdown cancels and owns an active cleanup cycle", async () => { +test("shutdown joins canceled cleanup through process closure and releases once", async () => { vi.useFakeTimers(); - let aborted = false; + let abortCount = 0; + let closeProcess = () => {}; + const processClosed = new Promise((resolve) => { + closeProcess = resolve; + }); + const rejectAfterProcessClose = (reject: (error: Error) => void) => + processClosed.then(() => { + reject(new Error("cleanup canceled after process close")); + return undefined; + }); + const releaseSubscription = vi.fn(); const retryWorktreeCleanup = vi.fn((_target, signal: AbortSignal) => { return new Promise((_resolve, reject) => { signal.addEventListener( "abort", () => { - aborted = true; - reject(new Error("cleanup canceled")); + abortCount += 1; + void rejectAfterProcessClose(reject); }, { once: true }, ); @@ -154,7 +164,7 @@ test("shutdown cancels and owns an active cleanup cycle", async () => { incarnationId: "inc-hung", }), ], - subscribeToMutations: () => () => undefined, + subscribeToMutations: () => releaseSubscription, }, retryWorktreeCleanup, logger: createTestLogger(), @@ -163,6 +173,23 @@ test("shutdown cancels and owns an active cleanup cycle", async () => { await expect(service.start()).resolves.toBeUndefined(); await vi.advanceTimersToNextTimerAsync(); expect(retryWorktreeCleanup).toHaveBeenCalledTimes(1); + + let stopSettled = false; + const stopping = service.stop().then(() => { + stopSettled = true; + return undefined; + }); + await Promise.resolve(); + + expect(abortCount).toBe(1); + expect(stopSettled).toBe(false); + expect(releaseSubscription).toHaveBeenCalledOnce(); + + closeProcess(); + await expect(stopping).resolves.toBeUndefined(); + expect(stopSettled).toBe(true); + await expect(service.stop()).resolves.toBeUndefined(); - expect(aborted).toBe(true); + expect(abortCount).toBe(1); + expect(releaseSubscription).toHaveBeenCalledOnce(); }); diff --git a/packages/server/src/utils/run-git-command.test.ts b/packages/server/src/utils/run-git-command.test.ts index 78b1c3b48d..63476ae8ef 100644 --- a/packages/server/src/utils/run-git-command.test.ts +++ b/packages/server/src/utils/run-git-command.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; interface FakeSpawnBehavior { delayMs?: number; + deferCloseOnKill?: boolean; emitError?: Error; exitCode?: number | null; stderrData?: Buffer | string; @@ -69,6 +70,9 @@ class FakeChildProcess extends EventEmitter { this.killed = true; this.killSignals.push(signal); this.clearTimers(); + if (this.behavior.deferCloseOnKill) { + return true; + } this.schedule(() => { this.finishClose({ exitCode: null, @@ -78,6 +82,11 @@ class FakeChildProcess extends EventEmitter { return true; } + public closeKilledProcess(): void { + const signal = this.killSignals.at(-1) ?? null; + this.finishClose({ exitCode: null, signal }); + } + public dispose(): void { this.clearTimers(); this.closed = true; @@ -103,7 +112,10 @@ class FakeChildProcess extends EventEmitter { if (this.behavior.emitError) { this.schedule(() => { if (this.closed) return; - this.finishError(this.behavior.emitError); + this.emit("error", this.behavior.emitError); + this.schedule(() => { + this.finishClose({ exitCode: null, signal: null }); + }, 0); }, this.behavior.delayMs ?? 0); return; } @@ -131,15 +143,6 @@ class FakeChildProcess extends EventEmitter { this.emit("close", exitCode, signal); } - private finishError(error: Error): void { - if (this.closed) return; - - this.closed = true; - this.clearTimers(); - fakeSpawnController.activeCount -= 1; - this.emit("error", error); - } - private schedule(callback: () => void, delayMs: number): void { const timer = setTimeout(callback, delayMs); this.timers.push(timer); @@ -268,6 +271,36 @@ describe("runGitCommand", () => { expect(fakeSpawnController.processes).toHaveLength(1); }); + it("keeps an active cancellation pending until the killed process closes", async () => { + const { runGitCommand } = await loadRunGitCommand(1); + const controller = new AbortController(); + + enqueueSpawnBehaviors({ delayMs: 5_000, deferCloseOnKill: true }); + const command = runGitCommand(["worktree", "remove", "/tmp/example", "--force"], { + cwd: process.cwd(), + signal: controller.signal, + }); + const observedRejection = vi.fn(); + void command.catch(observedRejection); + await vi.waitFor(() => expect(fakeSpawnController.processes).toHaveLength(1)); + + controller.abort(); + await Promise.resolve(); + + expect(observedRejection).not.toHaveBeenCalled(); + expect(fakeSpawnController.activeCount).toBe(1); + expect(fakeSpawnController.processes[0]?.killSignals).toEqual(["SIGKILL"]); + + fakeSpawnController.processes[0]?.closeKilledProcess(); + + await expect(command).rejects.toMatchObject({ + name: "AbortError", + message: "Git command canceled: git worktree remove /tmp/example --force", + }); + expect(observedRejection).toHaveBeenCalledOnce(); + expect(fakeSpawnController.activeCount).toBe(0); + }); + it("resolves truncated stdout, caps output, and kills the child process", async () => { const { runGitCommand } = await loadRunGitCommand(1); @@ -290,17 +323,15 @@ describe("runGitCommand", () => { it("rejects process errors and frees the limiter for the next command", async () => { const { runGitCommand } = await loadRunGitCommand(1); + const spawnError = new Error("spawn exploded"); - enqueueSpawnBehaviors( - { emitError: new Error("spawn exploded") }, - { delayMs: 0, stdoutData: "ok" }, - ); + enqueueSpawnBehaviors({ emitError: spawnError }, { delayMs: 0, stdoutData: "ok" }); await expect( runGitCommand(["status"], { cwd: process.cwd(), }), - ).rejects.toThrow("spawn exploded"); + ).rejects.toBe(spawnError); await expect( runGitCommand(["status"], { diff --git a/packages/server/src/utils/run-git-command.ts b/packages/server/src/utils/run-git-command.ts index a476c79e1f..c66c587cd3 100644 --- a/packages/server/src/utils/run-git-command.ts +++ b/packages/server/src/utils/run-git-command.ts @@ -142,6 +142,7 @@ function waitForGitCommand( task: Promise, signal: AbortSignal | undefined, args: readonly string[], + hasStarted: () => boolean, ): Promise { if (!signal) return task; @@ -153,7 +154,13 @@ function waitForGitCommand( signal.removeEventListener("abort", onAbort); callback(); }; - const onAbort = () => finish(() => reject(createGitCancellationError(args))); + const onAbort = () => { + // A queued command has no process to join, so it can be canceled + // immediately. Once the limiter has started the task, its process owns + // settlement and will preserve the cancellation error until `close`. + if (hasStarted()) return; + finish(() => reject(createGitCancellationError(args))); + }; signal.addEventListener("abort", onAbort, { once: true }); void task.then( (value) => finish(() => resolve(value)), @@ -168,7 +175,9 @@ export function runGitCommand( options: GitCommandOptions, ): Promise { const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args)); + let started = false; const promise = gitLimit(() => { + started = true; if (options.signal?.aborted) { gitRuntimeMetrics.start(runtimeMetric); gitRuntimeMetrics.finish(runtimeMetric, { success: false, timedOut: false }); @@ -212,6 +221,9 @@ export function runGitCommand( let settled = false; let metricFinished = false; + let pendingError: Error | null = null; + let pendingErrorTimedOut = false; + let requestedSignal: NodeJS.Signals | null = null; let truncated = false; let stdoutBytes = 0; let stderrBytes = 0; @@ -233,36 +245,26 @@ export function runGitCommand( gitRuntimeMetrics.finish(runtimeMetric, { success: metric.success, timedOut }); }; + const rememberError = (error: Error, timedOut = false) => { + if (pendingError) return; + pendingError = error; + pendingErrorTimedOut = timedOut; + }; + + const requestTermination = (error: Error, timedOut = false) => { + rememberError(error, timedOut); + if (requestedSignal) return; + requestedSignal = "SIGKILL"; + child.kill(requestedSignal); + }; + const timer = setTimeout(() => { const error = new Error(`Git command timed out after ${timeout}ms: ${command}`); - child.kill("SIGKILL"); - finishMetricOnce( - { - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, - exitCode: null, - signal: "SIGKILL", - success: false, - }, - true, - ); - settle(() => reject(error)); + requestTermination(error, true); }, timeout); const onAbort = () => { - child.kill("SIGKILL"); - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, - exitCode: null, - signal: "SIGKILL", - success: false, - }); - settle(() => reject(createGitCancellationError(args))); + requestTermination(createGitCancellationError(args)); }; options.signal?.addEventListener("abort", onAbort, { once: true }); if (options.signal?.aborted) onAbort(); @@ -308,15 +310,7 @@ export function runGitCommand( }); child.on("error", (error) => { - finishMetricOnce({ - args, - cwd: options.cwd, - startedAtMs: startedAt, - durationMs: Date.now() - startedAt, - exitCode: null, - signal: null, - success: false, - }); + rememberError(error); if (logger && traceContext) { logger.trace( { @@ -327,7 +321,6 @@ export function runGitCommand( "Git command process error", ); } - settle(() => reject(error)); }); child.on("close", (exitCode, signal) => { @@ -353,6 +346,24 @@ export function runGitCommand( ); } + if (pendingError) { + const error = pendingError; + finishMetricOnce( + { + args, + cwd: options.cwd, + startedAtMs: startedAt, + durationMs: Date.now() - startedAt, + exitCode, + signal: signal ?? requestedSignal, + success: false, + }, + pendingErrorTimedOut, + ); + settle(() => reject(error)); + return; + } + if (!truncated && !acceptExitCodes.includes(exitCode ?? -1)) { finishMetricOnce({ args, @@ -390,7 +401,7 @@ export function runGitCommand( }); }); gitRuntimeMetrics.observeLimiter(gitLimit.activeCount, gitLimit.pendingCount); - return waitForGitCommand(promise, options.signal, args); + return waitForGitCommand(promise, options.signal, args, () => started); } function formatGitCommand(args: readonly string[]): string { From eb3975b45d20bc69336d8a69d67e0517b5c372b6 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 09:09:04 -0400 Subject: [PATCH 37/43] test(server): keep git discovery on real clock --- packages/server/src/server/session.workspaces.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index 6a1aea4db0..9a26d97e0c 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -8823,7 +8823,6 @@ test("workspace auto-name replaces the unchanged prompt title", async () => { }); test("workspace auto-name uses the backing root for a nested worktree", async () => { - vi.useFakeTimers(); const tempDir = realpathSync(mkdtempSync(path.join(tmpdir(), "workspace-auto-name-rejected-"))); const repoDir = path.join(tempDir, "repo"); mkdirSync(repoDir); @@ -8894,7 +8893,10 @@ test("workspace auto-name uses the backing root for a nested worktree", async () workspace, firstAgentContext: { prompt: "Fix checkout title" }, }); - await vi.runAllTimersAsync(); + // This integration path performs real Git discovery after the 0ms + // scheduling boundary. Keep it on the real clock so draining fake timers + // cannot advance the command's 30s timeout ahead of process closure. + await vi.waitFor(() => expect(emittedCwds).toEqual([workspaceCwd])); expect(generateCalls).toBe(1); expect(stored.get(workspace.workspaceId)).toMatchObject({ @@ -8916,7 +8918,6 @@ test("workspace auto-name uses the backing root for a nested worktree", async () expect(gitMutations).toEqual([]); expect(emittedCwds).toEqual([workspaceCwd]); } finally { - vi.useRealTimers(); rmSync(tempDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); From 64346de94a04d6eec171498dc354e984d3245f26 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 13:53:17 -0400 Subject: [PATCH 38/43] fix(server): close create and initialization races --- .../server/src/server/agent/agent-loading.ts | 39 ++++---- .../src/server/agent/agent-manager.test.ts | 54 ++++++++++ .../server/src/server/agent/agent-manager.ts | 11 +++ .../create-agent-lifecycle-dispatch.test.ts | 32 +++--- .../server/agent/create-agent/create.test.ts | 59 +++++++++++ .../src/server/agent/create-agent/create.ts | 11 ++- .../server/src/server/schedule/service.ts | 1 + packages/server/src/server/session.test.ts | 4 +- packages/server/src/server/session.ts | 51 +++++++--- .../src/server/session.workspaces.test.ts | 98 ++++++++++++++++++- .../src/server/test-utils/session-stubs.ts | 1 + 11 files changed, 312 insertions(+), 49 deletions(-) diff --git a/packages/server/src/server/agent/agent-loading.ts b/packages/server/src/server/agent/agent-loading.ts index e273236842..103f36ebe8 100644 --- a/packages/server/src/server/agent/agent-loading.ts +++ b/packages/server/src/server/agent/agent-loading.ts @@ -22,6 +22,7 @@ export type AgentLoaderManager = Pick< AgentManager, | "createAgent" | "getAgent" + | "getAgentInitializationState" | "getRegisteredProviderIds" | "hydrateTimelineFromProvider" | "resumeAgentFromPersistence" @@ -63,28 +64,26 @@ export async function ensureAgentLoaded( agentId: string, deps: EnsureAgentLoadedDeps, ): Promise { - await deps.agentManager.waitForAgentClose?.(agentId); + while (true) { + await deps.agentManager.waitForAgentClose?.(agentId); - const inflight = pendingAgentInitializations.get(agentId); - if (inflight) { - inflight.options.broadcastTimeline ||= deps.broadcastTimeline === true; - return inflight.promise; - } - - const existing = deps.agentManager.getAgent(agentId); - if (existing) { - return existing; - } - - // A close may have started after the first barrier observed no in-flight - // work. Once the live lookup is empty, this second barrier closes that gap - // before storage-backed resume begins. - await deps.agentManager.waitForAgentClose?.(agentId); + const inflight = pendingAgentInitializations.get(agentId); + if (inflight) { + inflight.options.broadcastTimeline ||= deps.broadcastTimeline === true; + return inflight.promise; + } - const laterInflight = pendingAgentInitializations.get(agentId); - if (laterInflight) { - laterInflight.options.broadcastTimeline ||= deps.broadcastTimeline === true; - return laterInflight.promise; + // The close barrier and the runtime lookup cannot be made atomic across an + // await. Re-read both pieces of state synchronously so a close that starts + // in that gap is joined on the next pass instead of returning its runtime. + const initializationState = deps.agentManager.getAgentInitializationState(agentId); + if (initializationState.closeInFlight) { + continue; + } + if (initializationState.agent) { + return initializationState.agent; + } + break; } const pendingOptions = { diff --git a/packages/server/src/server/agent/agent-manager.test.ts b/packages/server/src/server/agent/agent-manager.test.ts index 06fd21a59b..4b9c6575c2 100644 --- a/packages/server/src/server/agent/agent-manager.test.ts +++ b/packages/server/src/server/agent/agent-manager.test.ts @@ -248,6 +248,7 @@ class HeldReloadCloseClient extends TestAgentClient { private readonly closeAllowed = deferred(); originalSessionClosed = false; replacementSessionClosed = false; + resumeCount = 0; override async createSession(config: AgentSessionConfig): Promise { const signalCloseStarted = () => this.closeStarted.resolve(); @@ -268,6 +269,7 @@ class HeldReloadCloseClient extends TestAgentClient { _handle: AgentPersistenceHandle, config?: Partial, ): Promise { + this.resumeCount += 1; const recordReplacementClosed = () => { this.replacementSessionClosed = true; }; @@ -7916,6 +7918,58 @@ test("load waits for an in-flight explicit close and creates one resumed runtime } }); +test("load joins a close that starts after its barrier but before runtime lookup", async () => { + const workdir = mkdtempSync(join(tmpdir(), "agent-manager-close-start-gap-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const client = new HeldReloadCloseClient(); + const manager = new AgentManager({ clients: { codex: client }, registry: storage, logger }); + const agentId = "00000000-0000-4000-8000-000000000218"; + let closeTask: Promise | null = null; + + try { + await manager.createAgent({ provider: "codex", cwd: workdir }, agentId, { + workspaceId: undefined, + }); + const waitForAgentClose = manager.waitForAgentClose.bind(manager); + vi.spyOn(manager, "waitForAgentClose") + .mockImplementation(async (id) => waitForAgentClose(id)) + .mockImplementationOnce(async () => { + queueMicrotask(() => { + closeTask = manager.closeAgent(agentId); + }); + }); + + let loadSettled = false; + const load = ensureAgentLoaded(agentId, { + agentManager: manager, + agentStorage: storage, + logger, + }).then((agent) => { + loadSettled = true; + return agent; + }); + + await client.waitForCloseToStart(); + await Promise.resolve(); + expect(loadSettled).toBe(false); + expect(client.resumeCount).toBe(0); + + client.finishClosing(); + const resumed = await load; + await closeTask; + + expect(resumed).toMatchObject({ id: agentId, lifecycle: "idle" }); + expect(client.originalSessionClosed).toBe(true); + expect(client.resumeCount).toBe(1); + } finally { + client.finishClosing(); + await closeTask?.catch(() => undefined); + await manager.closeAgent(agentId).catch(() => undefined); + await storage.flush().catch(() => undefined); + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("concurrent explicit closes tear down the runtime once", async () => { const workdir = mkdtempSync(join(tmpdir(), "agent-manager-concurrent-close-")); const closeStarted = deferred(); diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index f66d36fe95..70a1577029 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -1001,6 +1001,17 @@ export class AgentManager { return agent ? { ...agent } : null; } + getAgentInitializationState(id: string): { + agent: ManagedAgent | null; + closeInFlight: boolean; + } { + const closeInFlight = this.inFlightAgentCloses.has(id); + return { + agent: closeInFlight ? null : this.getAgent(id), + closeInFlight, + }; + } + async waitForAgentClose(agentId: string): Promise { await this.inFlightAgentCloses?.get(agentId)?.catch(() => undefined); } diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index 8baf6298fe..0e54bf65d6 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -161,7 +161,9 @@ test("startup removes a side-effect-free pending creation reservation", async () expect(pendingCreations.size).toBe(0); }); -test("startup leaves persisted agent cleanup to its durable obligation", async () => { +test("startup adopts a published agent without cleaning its pending worktree", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-published-agent-worktree-")); + mkdirSync(join(worktreePath, ".git")); const agentId = "agent-pending-now-persisted"; const pendingCreations = new Map([ [ @@ -169,13 +171,7 @@ test("startup leaves persisted agent cleanup to its durable obligation", async ( { agentId, createdAt: "2025-01-01T00:00:00.000Z", - cleanupTarget: { - kind: "worktree", - targetPath: "/tmp/persisted-agent-worktree", - worktreeIncarnationId: PENDING_WORKTREE_INCARNATION_ID, - directoryIdentity: { device: "7", inode: "42" }, - metadataBaseRefName: "main", - }, + cleanupTarget: pendingWorktreeTarget(worktreePath), }, ], ]); @@ -184,7 +180,10 @@ test("startup leaves persisted agent cleanup to its durable obligation", async ( agentId, { id: agentId, - autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + autoArchiveObligation: { + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-published-agent" }, + }, } as StoredAgentRecord, ], ]); @@ -199,10 +198,19 @@ test("startup leaves persisted agent cleanup to its durable obligation", async ( "archiveWorktreePath", ); - await dispatch.recoverPendingAgentCreations(); + try { + await dispatch.recoverPendingAgentCreations(); - expect(archiveWorktreePath).not.toHaveBeenCalled(); - expect(pendingCreations.size).toBe(0); + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + expect(records.get(agentId)?.autoArchiveObligation).toEqual({ + phase: "armed", + target: { kind: "workspace", workspaceId: "ws-published-agent" }, + }); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } }); test("startup removes an exact pre-git worktree reservation without archiving", async () => { diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index c99f8f55e3..05ebdf9b26 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -136,6 +136,65 @@ test("session create persists and arms auto-archive before starting the initial expect(order).toEqual(["persisted", "armed", "prompt"]); }); +test("session create reports a durable agent when initial prompt start is not confirmed", async () => { + const agentId = "00000000-0000-4000-8000-000000000401"; + const promptStartTimeout = new Error("initial prompt start timed out"); + const snapshot = { + id: agentId, + provider: "codex", + cwd: "/tmp/paseo-create-test", + runtimeInfo: null, + } as ManagedAgent; + const liveSnapshot = { ...snapshot, lifecycle: "running" as const }; + const removePendingAgentCreation = vi.fn(async () => undefined); + const onCreated = vi.fn(); + + const result = await createAgentCommand( + { + agentManager: { + createAgent: vi.fn(async () => snapshot), + getAgent: vi.fn(() => liveSnapshot), + tryRunOutOfBand: vi.fn(() => false), + hasInFlightRun: vi.fn(() => false), + streamAgent: vi.fn(() => (async function* noop() {})()), + waitForAgentRunStart: vi.fn(async () => { + throw promptStartTimeout; + }), + } as unknown as AgentManager, + agentStorage: { + removePendingAgentCreation, + } as unknown as AgentStorage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + }, + { + kind: "session", + agentId, + config: { provider: "codex", cwd: "/tmp/paseo-create-test" }, + workspaceId: "ws-create-test", + initialPrompt: "start the work", + labels: {}, + provisionalTitle: null, + firstAgentContext: {}, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + onCreated, + buildSessionConfig: async (config) => ({ sessionConfig: config }), + }, + ); + + expect(result).toMatchObject({ + snapshot, + liveSnapshot, + initialPromptStarted: false, + initialPromptError: promptStartTimeout, + }); + expect(removePendingAgentCreation).toHaveBeenCalledWith(agentId); + expect(onCreated).toHaveBeenCalledWith({ + agentId, + autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, + }); +}); + test("legacy worktree create keeps its journal through durable agent registration", async () => { const workdir = mkdtempSync(join(tmpdir(), "create-agent-journal-test-")); const storage = new AgentStorage(join(workdir, "agents"), logger); diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index 1048415db5..4a45d3930e 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -355,7 +355,10 @@ async function resolveSessionCreateAgent( runOptions, setupContinuation, background: true, - promptFailure: "throw", + // Agent publication is already durable before the first prompt is + // dispatched. Keep a prompt-start failure separate from creation truth so + // callers do not retry and create a duplicate agent/worktree. + promptFailure: "return-error", promptLogger: dependencies.logger.child({ clientMessageId: resolveClientMessageId(input.clientMessageId), }), @@ -540,7 +543,11 @@ async function sendInitialPrompt( throw error; } if (resolved.promptFailure === "return-error") { - return { started: false, liveSnapshot: snapshot, error }; + return { + started: false, + liveSnapshot: dependencies.agentManager.getAgent(snapshot.id) ?? snapshot, + error, + }; } dependencies.logger.error({ err: error, agentId: snapshot.id }, "Failed to run initial prompt"); return { started: false, liveSnapshot: snapshot }; diff --git a/packages/server/src/server/schedule/service.ts b/packages/server/src/server/schedule/service.ts index 6a650ac92a..a25157df13 100644 --- a/packages/server/src/server/schedule/service.ts +++ b/packages/server/src/server/schedule/service.ts @@ -199,6 +199,7 @@ type ScheduleAgentManager = Pick< AgentManager, | "createAgent" | "getAgent" + | "getAgentInitializationState" | "getRegisteredProviderIds" | "hasInFlightRun" | "hydrateTimelineFromProvider" diff --git a/packages/server/src/server/session.test.ts b/packages/server/src/server/session.test.ts index 66f163c686..d7bcbf1209 100644 --- a/packages/server/src/server/session.test.ts +++ b/packages/server/src/server/session.test.ts @@ -4967,9 +4967,11 @@ describe("agent config setters", () => { function liveAgentManager(overrides: { [K in keyof SessionOptions["agentManager"]]?: unknown }): { [K in keyof SessionOptions["agentManager"]]?: unknown; } { + const agent = { id: "agent-1" }; return { waitForAgentClose: vi.fn().mockResolvedValue(undefined), - getAgent: vi.fn(() => ({ id: "agent-1" })), + getAgent: vi.fn(() => agent), + getAgentInitializationState: vi.fn(() => ({ agent, closeInFlight: false })), ...overrides, }; } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 2c14874fa1..b8012e2f8b 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -3109,7 +3109,7 @@ export class Session { : { kind: "agent" }, } : undefined; - const { snapshot, liveSnapshot } = await createAgentCommand( + const { snapshot, liveSnapshot, initialPromptError } = await createAgentCommand( { agentManager: this.agentManager, agentStorage: this.agentStorage, @@ -3167,18 +3167,8 @@ export class Session { { currentSelection: this.getFocusedAgentSelectionForCwd(resolvedIntent.config.cwd) }, ); } - if (requestId) { - const agentPayload = await this.buildAgentPayload(liveSnapshot); - this.emit({ - type: "status", - payload: { - status: "agent_created", - agentId: liveSnapshot.id, - requestId, - agent: agentPayload, - }, - }); - } + await this.emitAgentCreatedStatus(requestId, liveSnapshot); + this.reportInitialPromptStartFailure(snapshot.id, initialPromptError); this.sessionLogger.info( { agentId: snapshot.id, provider: snapshot.provider }, @@ -3215,6 +3205,41 @@ export class Session { } } + private async emitAgentCreatedStatus( + requestId: string | undefined, + liveSnapshot: ManagedAgent, + ): Promise { + if (!requestId) return; + const agentPayload = await this.buildAgentPayload(liveSnapshot); + this.emit({ + type: "status", + payload: { + status: "agent_created", + agentId: liveSnapshot.id, + requestId, + agent: agentPayload, + }, + }); + } + + private reportInitialPromptStartFailure(agentId: string, error: unknown | null): void { + if (!error) return; + const promptError = error instanceof Error ? error.message : String(error); + this.sessionLogger.warn( + { err: error, agentId }, + "Agent created but initial prompt start was not confirmed", + ); + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "error", + content: `Agent ${agentId} was created, but its initial prompt start was not confirmed: ${promptError}`, + }, + }); + } + private async reservePendingAgentCreation( autoArchive: boolean | undefined, ): Promise { diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index 9a26d97e0c..67c3037c08 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -552,6 +552,7 @@ function createSessionForWorkspaceTests( terminalManager?: TerminalManager | null; agentManager?: { [K in keyof SessionOptions["agentManager"]]?: unknown }; agentStorage?: { [K in keyof SessionOptions["agentStorage"]]?: unknown }; + createAgentLifecycleDispatch?: SessionOptions["createAgentLifecycleDispatch"]; projectRegistry?: SessionOptions["projectRegistry"]; workspaceRegistry?: SessionOptions["workspaceRegistry"]; github?: ForgeService; @@ -672,7 +673,8 @@ function createSessionForWorkspaceTests( upsert: async () => {}, ...options.agentStorage, }), - createAgentLifecycleDispatch: createAgentLifecycleDispatchStub(), + createAgentLifecycleDispatch: + options.createAgentLifecycleDispatch ?? createAgentLifecycleDispatchStub(), projectRegistry: options.projectRegistry ?? { initialize: async () => {}, existsOnDisk: async () => true, @@ -877,6 +879,100 @@ test("client heartbeat clears attention for the focused terminal", async () => { }); }); +test("create_agent_request returns the published agent when first-prompt start times out", async () => { + const agentId = "00000000-0000-4000-8000-000000000550"; + const updatedAt = "2026-08-01T12:00:00.000Z"; + const snapshot = makeManagedAgent({ + id: agentId, + cwd: REPO_CWD, + workspaceId: "ws-repo-running", + lifecycle: "idle", + updatedAt, + }) as unknown as ManagedAgent; + const stored = makeStoredAgent({ id: agentId, cwd: REPO_CWD, updatedAt }); + stored.workspaceId = "ws-repo-running"; + stored.lastStatus = "idle"; + stored.autoArchiveObligation = { phase: "armed", target: { kind: "agent" } }; + const promptStartTimeout = new Error("initial prompt start timed out"); + const emitted: SessionOutboundMessage[] = []; + const beginPendingAgentCreation = vi.fn(async () => ({ + agentId, + createdAt: updatedAt, + cleanupTarget: { kind: "agent" as const }, + })); + const removePendingAgentCreation = vi.fn(async () => undefined); + const registerAutoArchive = vi.fn(() => ({ + settled: Promise.resolve("cancelled" as const), + cancel: async () => undefined, + })); + const lifecycleDispatch = createAgentLifecycleDispatchStub(); + lifecycleDispatch.registerAutoArchive = registerAutoArchive; + const streamAgent = vi.fn(() => (async function* noEvents() {})()); + + const session = createSessionForWorkspaceTests({ + onMessage: (message) => emitted.push(message), + createAgentLifecycleDispatch: lifecycleDispatch, + agentManager: { + allocateAgentId: () => agentId, + createAgent: vi.fn(async (_config, requestedAgentId, options) => { + expect(requestedAgentId).toBe(agentId); + expect(options.autoArchiveObligation).toEqual({ + phase: "armed", + target: { kind: "agent" }, + }); + return snapshot; + }), + getAgent: () => snapshot, + tryRunOutOfBand: () => false, + hasInFlightRun: () => false, + streamAgent, + waitForAgentRunStart: vi.fn(async () => { + throw promptStartTimeout; + }), + }, + agentStorage: { + beginPendingAgentCreation, + removePendingAgentCreation, + get: async (requestedAgentId: string) => (requestedAgentId === agentId ? stored : null), + }, + }); + + await session.handleMessage({ + type: "create_agent_request", + requestId: "req-create-prompt-timeout", + workspaceId: "ws-repo-running", + config: { provider: "codex", cwd: REPO_CWD }, + initialPrompt: "start the assigned work", + autoArchive: true, + attachments: [], + }); + + const statuses = filterByType(emitted, "status").map((message) => message.payload); + expect(statuses).toEqual([ + expect.objectContaining({ + status: "agent_created", + requestId: "req-create-prompt-timeout", + agentId, + agent: expect.objectContaining({ id: agentId }), + }), + ]); + expect(filterByType(emitted, "activity_log")).toContainEqual( + expect.objectContaining({ + payload: expect.objectContaining({ + type: "error", + content: expect.stringContaining("initial prompt start was not confirmed"), + }), + }), + ); + expect(beginPendingAgentCreation).toHaveBeenCalledWith(agentId); + expect(removePendingAgentCreation).toHaveBeenCalledWith(agentId); + expect(registerAutoArchive).toHaveBeenCalledWith({ + agentId, + obligation: { phase: "armed", target: { kind: "agent" } }, + }); + expect(streamAgent).toHaveBeenCalledWith(agentId, "start the assigned work", undefined); +}); + test("create_agent_request keeps requested child cwd when grouped under an existing parent workspace", async () => { const workdir = mkdtempSync(path.join(tmpdir(), "paseo-create-agent-cwd-")); try { diff --git a/packages/server/src/server/test-utils/session-stubs.ts b/packages/server/src/server/test-utils/session-stubs.ts index ff76e04f1c..124104510a 100644 --- a/packages/server/src/server/test-utils/session-stubs.ts +++ b/packages/server/src/server/test-utils/session-stubs.ts @@ -53,6 +53,7 @@ export function createAgentLifecycleDispatchStub(): SessionOptions["createAgentL cancel: async () => {}, }), cleanupCreatedWorktreeAfterFailedAgentCreate: async () => {}, + recoverPendingAgentCreation: async () => {}, }); } From 976f1724c0a85417bbc5e6c11b247502f45501aa Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 14:31:27 -0400 Subject: [PATCH 39/43] fix(server): drain create ingress during shutdown --- .../server/agent/create-agent/create.test.ts | 111 ++++++++++- .../src/server/agent/create-agent/create.ts | 124 +++++++++--- .../server/src/server/bootstrap.smoke.test.ts | 187 +++++++++++++++++- packages/server/src/server/session.ts | 14 +- .../server/src/server/websocket-server.ts | 27 ++- 5 files changed, 427 insertions(+), 36 deletions(-) diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index 05ebdf9b26..bd3a5f6c27 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -1,4 +1,5 @@ -import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { expect, test, vi } from "vitest"; @@ -9,6 +10,7 @@ import { createProviderSnapshotManagerStub } from "../../test-utils/session-stub import { AgentManager } from "../agent-manager.js"; import { AgentStorage } from "../agent-storage.js"; import type { CreatePaseoWorktreeWorkflowResult } from "../../worktree-session.js"; +import type { CreatePaseoWorktreeInput } from "../../paseo-worktree-service.js"; import { createAgentCommand } from "./create.js"; import type { ManagedAgent } from "../agent-manager.js"; import { WorkspaceLifecycleCoordinator } from "../../workspace-lifecycle-coordinator.js"; @@ -405,6 +407,7 @@ test("session create releases setup reservation when agent creation fails", asyn test.each(["worktree callback", "provider resolution"] as const)( "MCP create releases a reserved setup continuation after %s fails", async (failurePoint) => { + const pendingAgentId = "00000000-0000-4000-8000-000000000560"; const releaseWithoutStarting = vi.fn(); const startAfterAgentCreate = vi.fn(); const createAgent = vi.fn(); @@ -429,12 +432,31 @@ test.each(["worktree callback", "provider resolution"] as const)( releaseWithoutStarting, }, } as unknown as CreatePaseoWorktreeWorkflowResult; + const beginPendingAgentCreation = vi.fn(async () => ({ + agentId: pendingAgentId, + createdAt: "2026-08-01T00:00:00.000Z", + cleanupTarget: { kind: "agent" as const }, + })); + const removePendingAgentCreation = vi.fn(async () => undefined); await expect( createAgentCommand( { - agentManager: { createAgent } as unknown as AgentManager, - agentStorage: {} as AgentStorage, + agentManager: { + allocateAgentId: () => pendingAgentId, + createAgent, + } as unknown as AgentManager, + agentStorage: { + beginPendingAgentCreation, + listPendingAgentCreations: async () => [ + { + agentId: pendingAgentId, + createdAt: "2026-08-01T00:00:00.000Z", + cleanupTarget: { kind: "agent" as const }, + }, + ], + removePendingAgentCreation, + } as unknown as AgentStorage, logger, providerSnapshotManager: providerSnapshotManager.manager, createPaseoWorktree: vi.fn(async () => createdWorktree), @@ -461,6 +483,8 @@ test.each(["worktree callback", "provider resolution"] as const)( expect(releaseWithoutStarting).toHaveBeenCalledOnce(); expect(startAfterAgentCreate).not.toHaveBeenCalled(); expect(createAgent).not.toHaveBeenCalled(); + expect(beginPendingAgentCreation).toHaveBeenCalledWith(pendingAgentId); + expect(removePendingAgentCreation).toHaveBeenCalledWith(pendingAgentId); }, ); @@ -819,6 +843,87 @@ test("mcp create exposes the created worktree before dispatching the initial pro } }); +test("mcp worktree create journals the exact path incarnation through durable agent publication", async () => { + const workdir = mkdtempSync(join(tmpdir(), "create-agent-mcp-journal-test-")); + const storage = new AgentStorage(join(workdir, "agents"), logger); + const agentManager = createRealAgentManager(storage); + const agentId = "00000000-0000-4000-8000-000000000561"; + const worktreePath = join(workdir, "worktree"); + const workspaceCwd = join(worktreePath, "packages", "app"); + const worktreeIncarnationId = randomUUID(); + let observedPending: Awaited> = []; + const createAgent = agentManager.createAgent.bind(agentManager); + vi.spyOn(agentManager, "allocateAgentId").mockReturnValue(agentId); + vi.spyOn(agentManager, "createAgent").mockImplementation(async (...args) => { + observedPending = await storage.listPendingAgentCreations(); + return createAgent(...args); + }); + + try { + await createAgentCommand( + { + agentManager, + agentStorage: storage, + logger, + providerSnapshotManager: createProviderSnapshotManagerStub().manager, + createPaseoWorktree: async (input: CreatePaseoWorktreeInput) => { + mkdirSync(workspaceCwd, { recursive: true }); + const directoryStat = statSync(worktreePath, { bigint: true }); + await input.onWorktreePathResolved?.(worktreePath, { + worktreeIncarnationId, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, + metadataBaseRefName: "main", + }); + return { + worktree: { worktreePath }, + intent: {}, + workspace: { workspaceId: "ws-mcp-journal", cwd: workspaceCwd }, + repoRoot: workdir, + created: true, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: () => {}, + releaseWithoutStarting: () => {}, + }, + } as unknown as CreatePaseoWorktreeWorkflowResult; + }, + }, + { + kind: "mcp", + provider: "codex", + cwd: workdir, + title: "journaled worktree", + background: true, + notifyOnFinish: false, + worktree: { worktreeName: "feature", baseBranch: "main" }, + }, + ); + + expect(observedPending).toEqual([ + expect.objectContaining({ + agentId, + cleanupTarget: { + kind: "worktree", + targetPath: worktreePath, + worktreeIncarnationId, + directoryIdentity: { + device: statSync(worktreePath, { bigint: true }).dev.toString(), + inode: statSync(worktreePath, { bigint: true }).ino.toString(), + }, + metadataBaseRefName: "main", + }, + }), + ]); + await expect(storage.listPendingAgentCreations()).resolves.toEqual([]); + await expect(storage.get(agentId)).resolves.toMatchObject({ id: agentId }); + } finally { + rmSync(workdir, { recursive: true, force: true }); + } +}); + test("session create keeps the prompt title after the initial prompt settles", async () => { const workdir = mkdtempSync(join(tmpdir(), "create-agent-title-test-")); const storage = new AgentStorage(join(workdir, "agents"), logger); diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index 4a45d3930e..c084fd6872 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -188,10 +188,8 @@ export async function createAgentCommand( dependencies: CreateAgentCommandDependencies, input: CreateAgentCommandInput, ): Promise { - const resolved = - input.kind === "session" - ? await resolveSessionCreateAgent(dependencies, input) - : await resolveMcpCreateAgent(dependencies, input); + const pendingAgentId = await reservePendingAgentId(dependencies, input); + const resolved = await resolveCreateAgent(dependencies, input, pendingAgentId); let snapshot: ManagedAgent; const lifecycleCoordinator = @@ -206,7 +204,7 @@ export async function createAgentCommand( () => dependencies.agentManager.createAgent( resolved.config, - input.kind === "session" ? input.agentId : undefined, + pendingAgentId, resolved.createOptions, ), ); @@ -222,22 +220,8 @@ export async function createAgentCommand( let liveSnapshot = snapshot; let initialPromptStarted = false; let initialPromptError: unknown | null = null; - if (input.kind === "session") { - if (input.agentId) { - await dependencies.agentStorage.removePendingAgentCreation(input.agentId).catch((error) => { - dependencies.logger.warn( - { err: error, agentId: input.agentId }, - "Failed to remove completed agent creation journal", - ); - }); - } - input.onCreated?.({ - agentId: snapshot.id, - autoArchiveObligation: resolved.createOptions.autoArchiveObligation, - }); - } else { - input.onCreated?.({ agentId: snapshot.id, createdWorktree: resolved.createdWorktree ?? null }); - } + await completePendingAgentCreation(dependencies, pendingAgentId); + notifyAgentCreated(input, resolved, snapshot); if (resolved.prompt !== undefined) { const sendResult = await sendInitialPrompt(dependencies, resolved, snapshot); initialPromptStarted = sendResult.started; @@ -266,6 +250,86 @@ export async function createAgentCommand( }; } +async function reservePendingAgentId( + dependencies: CreateAgentCommandDependencies, + input: CreateAgentCommandInput, +): Promise { + if (input.kind === "session") return input.agentId; + if (!shouldCreateMcpWorktree(input.worktree)) return undefined; + + const pendingAgentId = dependencies.agentManager.allocateAgentId(); + await dependencies.agentStorage.beginPendingAgentCreation(pendingAgentId); + return pendingAgentId; +} + +async function resolveCreateAgent( + dependencies: CreateAgentCommandDependencies, + input: CreateAgentCommandInput, + pendingAgentId: string | undefined, +): Promise { + try { + return input.kind === "session" + ? await resolveSessionCreateAgent(dependencies, input) + : await resolveMcpCreateAgent(dependencies, input, pendingAgentId); + } catch (error) { + await removeSideEffectFreePendingCreation(dependencies.agentStorage, pendingAgentId); + throw error; + } +} + +async function completePendingAgentCreation( + dependencies: CreateAgentCommandDependencies, + pendingAgentId: string | undefined, +): Promise { + if (!pendingAgentId) return; + await dependencies.agentStorage.removePendingAgentCreation(pendingAgentId).catch((error) => { + dependencies.logger.warn( + { err: error, agentId: pendingAgentId }, + "Failed to remove completed agent creation journal", + ); + }); +} + +function notifyAgentCreated( + input: CreateAgentCommandInput, + resolved: ResolvedCreateAgent, + snapshot: ManagedAgent, +): void { + if (input.kind === "session") { + input.onCreated?.({ + agentId: snapshot.id, + autoArchiveObligation: resolved.createOptions.autoArchiveObligation, + }); + return; + } + input.onCreated?.({ agentId: snapshot.id, createdWorktree: resolved.createdWorktree ?? null }); +} + +function shouldCreateMcpWorktree(worktree: CreateAgentFromMcpInput["worktree"]): boolean { + return Boolean( + worktree && + (worktree.worktreeName || worktree.refName || worktree.action || worktree.githubPrNumber), + ); +} + +async function removeSideEffectFreePendingCreation( + agentStorage: AgentStorage, + pendingAgentId: string | undefined, +): Promise { + if (!pendingAgentId) return; + try { + const pending = (await agentStorage.listPendingAgentCreations()).find( + (record) => record.agentId === pendingAgentId, + ); + if (pending?.cleanupTarget.kind === "agent") { + await agentStorage.removePendingAgentCreation(pendingAgentId); + } + } catch { + // Preserve the original create failure. Startup recovery owns any journal + // whose side-effect-free cleanup could not be confirmed here. + } +} + async function resolveSessionCreateAgent( dependencies: CreateAgentCommandDependencies, input: CreateAgentFromSessionInput, @@ -372,6 +436,7 @@ async function resolveSessionCreateAgent( async function resolveMcpCreateAgent( dependencies: CreateAgentCommandDependencies, input: CreateAgentFromMcpInput, + pendingAgentId?: string, ): Promise { const resolvedProviderModel = resolveProviderModel(input.provider); const provider = resolvedProviderModel.provider; @@ -385,6 +450,14 @@ async function resolveMcpCreateAgent( cwd, worktree: input.worktree, initialPrompt: input.initialPrompt ?? "", + onWorktreePathResolved: pendingAgentId + ? (worktreePath, reservation) => + dependencies.agentStorage.setPendingAgentCreationWorktree( + pendingAgentId, + worktreePath, + reservation, + ) + : undefined, }); try { if (createdWorktree) input.onWorktreeCreated?.(createdWorktree); @@ -586,6 +659,10 @@ async function resolveMcpCwd(params: { cwd: string; initialPrompt: string; worktree: CreateAgentFromMcpInput["worktree"]; + onWorktreePathResolved?: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; }): Promise<{ resolvedCwd: string; setupContinuation?: AgentWorktreeSetupContinuation; @@ -596,9 +673,7 @@ async function resolveMcpCwd(params: { if (!worktree) { return { resolvedCwd: params.cwd }; } - const shouldCreateWorktree = Boolean( - worktree.worktreeName || worktree.refName || worktree.action || worktree.githubPrNumber, - ); + const shouldCreateWorktree = shouldCreateMcpWorktree(worktree); if (!shouldCreateWorktree) { return { resolvedCwd: params.cwd }; } @@ -624,6 +699,7 @@ async function resolveMcpCwd(params: { runSetup: false, paseoHome: dependencies.paseoHome, worktreesRoot: dependencies.worktreesRoot, + onWorktreePathResolved: params.onWorktreePathResolved, }, createPaseoWorktree: dependencies.createPaseoWorktree, resolveDefaultBranch: baseBranch ? async () => baseBranch : undefined, diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 7b87c685fa..2a252a2763 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -1,7 +1,8 @@ import os from "node:os"; import http from "node:http"; import path from "node:path"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises"; import pino from "pino"; import { afterEach, describe, expect, test, vi } from "vitest"; import { WebSocket } from "ws"; @@ -16,6 +17,7 @@ import { DaemonClient } from "./test-utils/daemon-client.js"; import { isPlatform } from "../test-utils/platform.js"; import { findFreePort } from "./service-proxy.js"; import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; +import { readPaseoWorktreeIncarnationId } from "../utils/worktree-metadata.js"; import type { HubEnrollment, HubEnrollmentResult, @@ -43,6 +45,14 @@ type WebSocketProbeResult = | { status: "connected" } | { status: "rejected"; statusCode: number | null }; +function deferred(): { promise: Promise; resolve: (value: T) => void } { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + describe("paseo daemon bootstrap", () => { afterEach(() => { vi.restoreAllMocks(); @@ -343,6 +353,147 @@ describe("paseo daemon bootstrap", () => { } }); + test("shutdown joins a worktree create accepted on an established socket before ingress freezes", async () => { + const clients = createTestAgentClients(); + const createStarted = deferred(); + const allowCreate = deferred(); + const codexClient = clients.codex!; + const createSession = codexClient.createSession.bind(codexClient); + let createdSessionCwd: string | null = null; + vi.spyOn(codexClient, "createSession").mockImplementation(async (config, launchContext) => { + createdSessionCwd = config.cwd; + createStarted.resolve(); + await allowCreate.promise; + return createSession(config, launchContext); + }); + + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: clients, + }); + const { repoDir, tempRoot } = await createCommittedGitRepo("accepted-create"); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + + try { + await client.connect(); + const createPromise = client.createAgent({ + provider: "codex", + cwd: repoDir, + worktree: { mode: "branch-off", newBranch: "accepted-before-shutdown", base: "main" }, + autoArchive: false, + }); + await createStarted.promise; + + const [pendingCreation] = await daemonHandle.daemon.agentStorage.listPendingAgentCreations(); + expect(pendingCreation?.cleanupTarget).toMatchObject({ + kind: "worktree", + targetPath: createdSessionCwd, + }); + if (pendingCreation?.cleanupTarget.kind !== "worktree" || !createdSessionCwd) { + throw new Error("Expected an exact pending worktree creation journal"); + } + const directoryStat = await stat(createdSessionCwd, { bigint: true }); + expect(pendingCreation.cleanupTarget.directoryIdentity).toEqual({ + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }); + expect(readPaseoWorktreeIncarnationId(createdSessionCwd)).toBe( + pendingCreation.cleanupTarget.worktreeIncarnationId, + ); + + const stopPromise = daemonHandle.daemon.stop(); + const earlyStop = await Promise.race([ + stopPromise.then(() => "stopped" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(earlyStop).toBe("pending"); + + allowCreate.resolve(); + const created = await createPromise; + expect(created.cwd).toBe(createdSessionCwd); + await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + await stopPromise; + } finally { + allowCreate.resolve(); + await client.close().catch(() => undefined); + await daemonHandle.daemon.stop().catch(() => undefined); + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + + test("an established socket cannot start a worktree create after ingress freezes", async () => { + const heldAgentClose = holdAgentClose(); + const clients = createTestAgentClients({ closeSession: heldAgentClose.closeSession }); + const createSession = vi.spyOn(clients.codex!, "createSession"); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: clients, + }); + const { repoDir, tempRoot } = await createCommittedGitRepo("rejected-create"); + const initialAgentCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-shutdown-agent-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: initialAgentCwd }, + undefined, + { workspaceId: undefined }, + ); + expect(createSession).toHaveBeenCalledTimes(1); + + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + await heldAgentClose.started; + + const lateCreate = client + .createAgent({ + provider: "codex", + cwd: repoDir, + worktree: { mode: "branch-off", newBranch: "rejected-after-shutdown", base: "main" }, + }) + .then( + () => "created" as const, + () => "rejected" as const, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(createSession).toHaveBeenCalledTimes(1); + expect(listGitWorktreePaths(repoDir)).toHaveLength(1); + await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + + heldAgentClose.finish(); + await stopPromise; + await client.close(); + await expect(lateCreate).resolves.toBe("rejected"); + } finally { + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.daemon.stop().catch(() => undefined); + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(initialAgentCwd, { recursive: true, force: true }), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + test("continues teardown after a shutdown step fails", async () => { const daemonHandle = await createTestPaseoDaemon(); const failure = new Error("service proxy stop failed"); @@ -776,6 +927,40 @@ function holdAgentClose(): HeldAgentClose { }; } +async function createCommittedGitRepo(slug: string): Promise<{ + repoDir: string; + tempRoot: string; +}> { + const tempRoot = await mkdtemp(path.join(os.tmpdir(), `paseo-shutdown-${slug}-`)); + const repoDir = path.join(tempRoot, "repo"); + execFileSync("git", ["init", "-b", "main", repoDir], { stdio: "pipe" }); + execFileSync("git", ["config", "user.email", "test@getpaseo.local"], { + cwd: repoDir, + stdio: "pipe", + }); + execFileSync("git", ["config", "user.name", "Paseo Test"], { + cwd: repoDir, + stdio: "pipe", + }); + await writeFile(path.join(repoDir, "README.md"), "shutdown lifecycle\n"); + execFileSync("git", ["add", "README.md"], { cwd: repoDir, stdio: "pipe" }); + execFileSync("git", ["-c", "commit.gpgsign=false", "commit", "-m", "initial"], { + cwd: repoDir, + stdio: "pipe", + }); + return { repoDir, tempRoot }; +} + +function listGitWorktreePaths(repoDir: string): string[] { + return execFileSync("git", ["worktree", "list", "--porcelain"], { + cwd: repoDir, + stdio: "pipe", + }) + .toString() + .split("\n") + .flatMap((line) => (line.startsWith("worktree ") ? [line.slice("worktree ".length)] : [])); +} + async function beginDaemonShutdownWithAgentClosing(): Promise { const heldAgentClose = holdAgentClose(); const daemonHandle = await createTestPaseoDaemon({ diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index b8012e2f8b..774bb32a1a 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -257,6 +257,12 @@ function errorToFriendlyMessage(error: unknown): string { return "Unknown error"; } +function shouldJournalAgentCreation(msg: CreateAgentRequestMessage): boolean { + return ( + msg.autoArchive === true || Boolean(msg.worktree || msg.worktreeName || msg.git?.createWorktree) + ); +} + function resolveSubscriptionId( subscribe: unknown, requestedSubscriptionId: string | undefined, @@ -3080,7 +3086,9 @@ export class Session { ...(attachments && attachments.length > 0 ? { attachments } : {}), }; const workspacePromptTitle = resolveFirstAgentPromptTitle(firstAgentContext); - const pendingCreation = await this.reservePendingAgentCreation(autoArchive); + const pendingCreation = await this.reservePendingAgentCreation( + shouldJournalAgentCreation(msg), + ); pendingCreationAgentId = pendingCreation.agentId; const createdWorktree = await this.createWorktreeForCreateAgentRequest({ cwd: config.cwd, @@ -3241,9 +3249,9 @@ export class Session { } private async reservePendingAgentCreation( - autoArchive: boolean | undefined, + requiresDurableCreation: boolean, ): Promise { - if (autoArchive !== true) return {}; + if (!requiresDurableCreation) return {}; const agentId = this.agentManager.allocateAgentId(); await this.agentStorage.beginPendingAgentCreation(agentId); diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 63b2b7c6f2..5187ea235b 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -500,6 +500,7 @@ export class VoiceAssistantWebSocketServer { private readonly wss: WebSocketServer; private readonly pendingConnections: Map = new Map(); private readonly sessions: Map = new Map(); + private readonly inFlightCreateMessages = new Set>(); private readonly socketIdentities: Map = new Map(); private readonly externalSessionsByKey: Map = new Map(); private readonly serverId: string; @@ -951,12 +952,15 @@ export class VoiceAssistantWebSocketServer { connectionLogger.info("Hub session attached"); } - public prepareForShutdown(): void { + public async prepareForShutdown(): Promise { this.acceptingConnections = false; + while (this.inFlightCreateMessages.size > 0) { + await Promise.allSettled(Array.from(this.inFlightCreateMessages)); + } } public async close(): Promise { - this.prepareForShutdown(); + await this.prepareForShutdown(); this.unsubscribeSpeechReadiness?.(); this.unsubscribeSpeechReadiness = null; this.unsubscribeDaemonConfigChange?.(); @@ -2064,9 +2068,22 @@ export class VoiceAssistantWebSocketServer { } if (message.type === "session") { - void this.dispatchSessionMessage(ws, activeConnection, message).catch((error: unknown) => { - this.handleRawMessageError({ ws, data, error, log: activeConnection.connectionLogger }); - }); + const dispatch = this.dispatchSessionMessage(ws, activeConnection, message); + const trackedForShutdown = + message.message.type === "create_agent_request" || + message.message.type === "create_paseo_worktree_request"; + if (trackedForShutdown) { + this.inFlightCreateMessages.add(dispatch); + } + void dispatch + .catch((error: unknown) => { + this.handleRawMessageError({ ws, data, error, log: activeConnection.connectionLogger }); + }) + .finally(() => { + if (trackedForShutdown) { + this.inFlightCreateMessages.delete(dispatch); + } + }); } } catch (error) { this.handleRawMessageError({ ws, data, error, log }); From fe2948a3659e5172080d9d77f2b4604bac309d9e Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 15:34:02 -0400 Subject: [PATCH 40/43] fix(server): close lifecycle mutation gaps --- .../src/server/agent/agent-storage.test.ts | 21 +- .../server/src/server/agent/agent-storage.ts | 59 ++++- .../create-agent-lifecycle-dispatch.test.ts | 121 ++++++++- .../agent/create-agent-lifecycle-dispatch.ts | 176 ++++++++----- .../server/agent/create-agent/create.test.ts | 30 ++- .../src/server/agent/create-agent/create.ts | 43 ++-- .../src/server/agent/mcp-server.test.ts | 54 +++- .../server/src/server/bootstrap.smoke.test.ts | 142 ++++++++++- packages/server/src/server/bootstrap.ts | 74 +++++- .../src/server/schedule/service.test.ts | 51 ++++ .../server/src/server/schedule/service.ts | 44 +++- packages/server/src/server/session.ts | 52 ++-- .../server/workspace-archive-service.test.ts | 46 +++- .../src/server/workspace-archive-service.ts | 7 +- .../src/server/worktree-core.posix.test.ts | 8 + packages/server/src/server/worktree-core.ts | 3 + .../src/server/worktree-session.test.ts | 82 ++++++ .../server/src/server/worktree-session.ts | 238 +++++++++++------- packages/server/src/utils/worktree.ts | 87 ++++++- 19 files changed, 1098 insertions(+), 240 deletions(-) diff --git a/packages/server/src/server/agent/agent-storage.test.ts b/packages/server/src/server/agent/agent-storage.test.ts index c595577a95..9b3bf76e97 100644 --- a/packages/server/src/server/agent/agent-storage.test.ts +++ b/packages/server/src/server/agent/agent-storage.test.ts @@ -211,7 +211,25 @@ describe("AgentStorage", () => { test("persists pending creation intent separately from visible agents", async () => { await storage.beginPendingAgentCreation("agent-pending-create"); - await storage.setPendingAgentCreationWorktree( + await storage.planPendingAgentCreationWorktree( + "agent-pending-create", + "/tmp/paseo/worktrees/project/feature-2", + { + worktreeIncarnationId: "4d2ce498-4c27-4ea2-8ed3-46720de7194e", + metadataBaseRefName: "main", + }, + ); + await expect(storage.listPendingAgentCreations()).resolves.toMatchObject([ + { + cleanupTarget: { + kind: "worktree", + targetPath: "/tmp/paseo/worktrees/project/feature-2", + worktreeIncarnationId: "4d2ce498-4c27-4ea2-8ed3-46720de7194e", + directoryIdentity: null, + }, + }, + ]); + await storage.identifyPendingAgentCreationWorktree( "agent-pending-create", "/tmp/paseo/worktrees/project/feature-2", { @@ -227,6 +245,7 @@ describe("AgentStorage", () => { { agentId: "agent-pending-create", createdAt: expect.any(String), + ownerKind: "agent", cleanupTarget: { kind: "worktree", targetPath: "/tmp/paseo/worktrees/project/feature-2", diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 38c41f670d..4046399f2a 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -46,16 +46,21 @@ const AUTO_ARCHIVE_OBLIGATION_SCHEMA = z.object({ const PENDING_AGENT_CREATION_SCHEMA = z.object({ agentId: z.string(), createdAt: z.string(), + ownerKind: z.enum(["agent", "standalone-worktree"]).optional().default("agent"), cleanupTarget: z.discriminatedUnion("kind", [ z.object({ kind: z.literal("agent") }), z.object({ kind: z.literal("worktree"), targetPath: z.string(), worktreeIncarnationId: z.string().uuid(), - directoryIdentity: z.object({ - device: z.string(), - inode: z.string(), - }), + directoryIdentity: z + .object({ + device: z.string(), + inode: z.string(), + }) + .nullable() + .optional() + .default(null), metadataBaseRefName: z.string().min(1), }), ]), @@ -158,22 +163,25 @@ export class AgentStorage { await this.queueRecordWrite(record); } - async beginPendingAgentCreation(agentId: string): Promise { + async beginPendingAgentCreation( + agentId: string, + options?: { ownerKind?: PendingAgentCreation["ownerKind"] }, + ): Promise { const record: PendingAgentCreation = { agentId, createdAt: new Date().toISOString(), + ownerKind: options?.ownerKind ?? "agent", cleanupTarget: { kind: "agent" }, }; await writeJsonFileAtomic(this.pendingCreationPath(agentId), record); return record; } - async setPendingAgentCreationWorktree( + async planPendingAgentCreationWorktree( agentId: string, targetPath: string, - reservation: { + plan: { worktreeIncarnationId: string; - directoryIdentity: { device: string; inode: string }; metadataBaseRefName: string; }, ): Promise { @@ -181,6 +189,41 @@ export class AgentStorage { if (!record) { throw new Error(`Pending agent creation ${agentId} not found`); } + await writeJsonFileAtomic(this.pendingCreationPath(agentId), { + ...record, + cleanupTarget: { + kind: "worktree", + targetPath, + ...plan, + directoryIdentity: null, + }, + } satisfies PendingAgentCreation); + } + + async identifyPendingAgentCreationWorktree( + agentId: string, + targetPath: string, + reservation: { + worktreeIncarnationId: string; + directoryIdentity: { device: string; inode: string }; + metadataBaseRefName: string; + }, + ): Promise { + const record = await this.readPendingAgentCreation(agentId); + if ( + !record || + record.cleanupTarget.kind !== "worktree" || + record.cleanupTarget.targetPath !== targetPath || + record.cleanupTarget.worktreeIncarnationId !== reservation.worktreeIncarnationId + ) { + const recordedTarget = + record?.cleanupTarget.kind === "worktree" + ? `${record.cleanupTarget.targetPath} (${record.cleanupTarget.worktreeIncarnationId})` + : (record?.cleanupTarget.kind ?? "missing"); + throw new Error( + `Pending agent creation ${agentId} does not own ${targetPath} (${reservation.worktreeIncarnationId}); recorded ${recordedTarget}`, + ); + } await writeJsonFileAtomic(this.pendingCreationPath(agentId), { ...record, cleanupTarget: { kind: "worktree", targetPath, ...reservation }, diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index 0e54bf65d6..5d8a34c563 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve as resolvePath } from "node:path"; import { expect, test, vi } from "vitest"; @@ -247,6 +247,125 @@ test("startup removes an exact pre-git worktree reservation without archiving", } }); +test("startup retains the journal and directory after a crash between mkdir and identity", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-pre-identity-crash-")); + const agentId = "agent-pending-pre-identity-crash"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: { + kind: "worktree", + targetPath: worktreePath, + worktreeIncarnationId: PENDING_WORKTREE_INCARNATION_ID, + directoryIdentity: null, + metadataBaseRefName: "main", + }, + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.has(agentId)).toBe(true); + expect(existsSync(worktreePath)).toBe(true); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + +test("startup removes its marked directory after a crash before identity persistence", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-marked-pre-identity-crash-")); + writeFileSync( + join(worktreePath, ".paseo-worktree-creation"), + `${PENDING_WORKTREE_INCARNATION_ID}\n`, + ); + const agentId = "agent-pending-marked-pre-identity-crash"; + const pendingCreations = new Map([ + [ + agentId, + { + agentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: { + kind: "worktree", + targetPath: worktreePath, + worktreeIncarnationId: PENDING_WORKTREE_INCARNATION_ID, + directoryIdentity: null, + metadataBaseRefName: "main", + }, + }, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + }); + + await dispatch.recoverPendingAgentCreations(); + + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(false); +}); + +test("startup adopts a published standalone worktree instead of archiving it", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-published-standalone-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const creationId = "standalone-pending-now-persisted"; + const workspaceId = "ws-published-standalone"; + const pendingCreations = new Map([ + [ + creationId, + { + agentId: creationId, + createdAt: "2025-01-01T00:00:00.000Z", + ownerKind: "standalone-worktree", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const workspaces = [ + { + workspaceId, + cwd: worktreePath, + worktreeRoot: worktreePath, + archivedAt: null, + } as PersistedWorkspaceRecord, + ]; + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + workspaces, + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + test("startup retains and retries an orphan worktree pending creation", async () => { const worktreePath = mkdtempSync(join(tmpdir(), "paseo-orphan-worktree-")); mkdirSync(join(worktreePath, ".git")); diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index 37f7a45bc7..bbf4c585b4 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -6,7 +6,7 @@ import type pino from "pino"; import type { ForgeService } from "../../services/forge-service.js"; import { withTimeout } from "../../utils/promise-timeout.js"; -import { isPaseoOwnedWorktreeCwd } from "../../utils/worktree.js"; +import { isPaseoOwnedWorktreeCwd, readWorktreeCreationMarker } from "../../utils/worktree.js"; import { readPaseoWorktreeMetadata, writePaseoWorktreeMetadata, @@ -22,7 +22,7 @@ import type { WorkspaceGitService } from "../workspace-git-service.js"; import type { WorkspaceRegistry } from "../workspace-registry.js"; import type { SessionOutboundMessage } from "../messages.js"; import type { AgentManager, AgentSubscriber, SubscribeOptions } from "./agent-manager.js"; -import type { AgentStorage, AutoArchiveObligation } from "./agent-storage.js"; +import type { AgentStorage, AutoArchiveObligation, PendingAgentCreation } from "./agent-storage.js"; interface CreateAgentLifecycleDispatchDependencies { paseoHome: string; @@ -129,73 +129,28 @@ export class CreateAgentLifecycleDispatch { ); if (!pending) return; - if ( - (await this.dependencies.agentStorage.get(agentId)) || - pending.cleanupTarget.kind === "agent" - ) { + if (pending.cleanupTarget.kind === "agent") { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; + } + if (await this.pendingAgentWasPublished(pending)) { await this.dependencies.agentStorage.removePendingAgentCreation(agentId); return; } - const targetPath = resolvePath(pending.cleanupTarget.targetPath); + const pendingWorktree = { ...pending, cleanupTarget: pending.cleanupTarget }; + const targetPath = resolvePath(pendingWorktree.cleanupTarget.targetPath); try { - if (!existsSync(targetPath)) { - const ownsActiveWorkspace = (await this.dependencies.workspaceRegistry.list()).some( - (workspace) => - !workspace.archivedAt && - resolvePath(workspace.worktreeRoot ?? workspace.cwd) === targetPath, - ); - if (!ownsActiveWorkspace) { - await this.dependencies.agentStorage.removePendingAgentCreation(agentId); - return; - } + if (!existsSync(targetPath) && !(await this.activeWorkspaceOwnsPath(targetPath))) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return; } - if (existsSync(targetPath)) { - const targetStat = statSync(targetPath, { bigint: true }); - const currentDirectoryIdentity = { - device: targetStat.dev.toString(), - inode: targetStat.ino.toString(), - }; - if ( - currentDirectoryIdentity.device !== pending.cleanupTarget.directoryIdentity.device || - currentDirectoryIdentity.inode !== pending.cleanupTarget.directoryIdentity.inode - ) { - this.dependencies.logger.warn( - { agentId, worktreePath: targetPath }, - "Pending agent creation no longer owns its worktree path", - ); - await this.dependencies.agentStorage.removePendingAgentCreation(agentId); - return; - } - - const gitPath = resolvePath(targetPath, ".git"); - if (!existsSync(gitPath)) { - await rm(targetPath, { recursive: true, force: true }); - await this.dependencies.agentStorage.removePendingAgentCreation(agentId); - return; - } - const metadata = readPaseoWorktreeMetadata(targetPath); - if ( - metadata?.incarnationId && - metadata.incarnationId !== pending.cleanupTarget.worktreeIncarnationId - ) { - this.dependencies.logger.warn( - { agentId, worktreePath: targetPath }, - "Pending agent creation found a replacement worktree incarnation", - ); - await this.dependencies.agentStorage.removePendingAgentCreation(agentId); - return; - } - if (!metadata?.incarnationId) { - writePaseoWorktreeMetadata(targetPath, { - baseRefName: metadata?.baseRefName ?? pending.cleanupTarget.metadataBaseRefName, - incarnationId: pending.cleanupTarget.worktreeIncarnationId, - ...(metadata?.changeRequestLookupTarget - ? { changeRequestLookupTarget: metadata.changeRequestLookupTarget } - : {}), - }); - } + if ( + existsSync(targetPath) && + (await this.recoverExistingPendingDirectory(pendingWorktree, targetPath)) + ) { + return; } await this.archiveWorktreePath(targetPath); @@ -208,6 +163,103 @@ export class CreateAgentLifecycleDispatch { } } + private async pendingAgentWasPublished(pending: PendingAgentCreation): Promise { + if ((pending.ownerKind ?? "agent") !== "agent") return false; + return Boolean(await this.dependencies.agentStorage.get(pending.agentId)); + } + + private async activeWorkspaceOwnsPath(targetPath: string): Promise { + return (await this.dependencies.workspaceRegistry.list()).some( + (workspace) => + !workspace.archivedAt && + resolvePath(workspace.worktreeRoot ?? workspace.cwd) === targetPath, + ); + } + + private classifyPendingDirectoryOwnership( + pending: PendingAgentCreation & { + cleanupTarget: Extract; + }, + targetPath: string, + ): "owned" | "replaced" | "ambiguous" { + const expectedDirectoryIdentity = pending.cleanupTarget.directoryIdentity; + if (!expectedDirectoryIdentity) { + return readWorktreeCreationMarker(targetPath) === pending.cleanupTarget.worktreeIncarnationId + ? "owned" + : "ambiguous"; + } + const targetStat = statSync(targetPath, { bigint: true }); + return targetStat.dev.toString() === expectedDirectoryIdentity.device && + targetStat.ino.toString() === expectedDirectoryIdentity.inode + ? "owned" + : "replaced"; + } + + private async recoverExistingPendingDirectory( + pending: PendingAgentCreation & { + cleanupTarget: Extract; + }, + targetPath: string, + ): Promise { + const agentId = pending.agentId; + const directoryOwnership = this.classifyPendingDirectoryOwnership(pending, targetPath); + if (directoryOwnership === "replaced") { + this.dependencies.logger.warn( + { agentId, worktreePath: targetPath }, + "Pending agent creation no longer owns its worktree path", + ); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + if (directoryOwnership === "ambiguous") { + // The process may have stopped after mkdir and before persisting the + // directory identity. Without the matching in-directory marker we + // cannot distinguish our directory from a replacement, so retain the + // durable cleanup obligation and fail closed. + this.dependencies.logger.warn( + { agentId, worktreePath: targetPath }, + "Pending agent creation has ambiguous pre-identity worktree ownership", + ); + return true; + } + + if (!existsSync(resolvePath(targetPath, ".git"))) { + await rm(targetPath, { recursive: true, force: true }); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + const metadata = readPaseoWorktreeMetadata(targetPath); + if ( + metadata?.incarnationId && + metadata.incarnationId !== pending.cleanupTarget.worktreeIncarnationId + ) { + this.dependencies.logger.warn( + { agentId, worktreePath: targetPath }, + "Pending agent creation found a replacement worktree incarnation", + ); + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + if (!metadata?.incarnationId) { + writePaseoWorktreeMetadata(targetPath, { + baseRefName: metadata?.baseRefName ?? pending.cleanupTarget.metadataBaseRefName, + incarnationId: pending.cleanupTarget.worktreeIncarnationId, + ...(metadata?.changeRequestLookupTarget + ? { changeRequestLookupTarget: metadata.changeRequestLookupTarget } + : {}), + }); + } + + if ( + pending.ownerKind === "standalone-worktree" && + (await this.activeWorkspaceOwnsPath(targetPath)) + ) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } + return false; + } + async cleanupCreatedWorktreeAfterFailedAgentCreate(input: { createdWorktree: CreatePaseoWorktreeWorkflowResult | null; createdAgentId: string | null; diff --git a/packages/server/src/server/agent/create-agent/create.test.ts b/packages/server/src/server/agent/create-agent/create.test.ts index bd3a5f6c27..9937b9f727 100644 --- a/packages/server/src/server/agent/create-agent/create.test.ts +++ b/packages/server/src/server/agent/create-agent/create.test.ts @@ -237,11 +237,25 @@ test("legacy worktree create keeps its journal through durable agent registratio provisionalTitle: null, firstAgentContext: { attachments: [] }, autoArchiveObligation: { phase: "armed", target: { kind: "agent" } }, - buildSessionConfig: async (config, _git, _worktreeName, _context, onPath) => { - await onPath?.(resolvedWorktreePath, { + buildSessionConfig: async (config, _git, _worktreeName, _context, journal) => { + const plan = { worktreeIncarnationId: "30704df3-6339-4c9c-8277-1416544ed7cc", - directoryIdentity: { device: "7", inode: "42" }, metadataBaseRefName: "main", + }; + await journal?.onWorktreePathPlanned(resolvedWorktreePath, plan); + expect(await storage.listPendingAgentCreations()).toMatchObject([ + { cleanupTarget: { kind: "worktree", directoryIdentity: null } }, + ]); + mkdirSync(resolvedWorktreePath, { recursive: true }); + const directoryStat = statSync(resolvedWorktreePath, { + bigint: true, + }); + await journal?.onWorktreePathResolved(resolvedWorktreePath, { + ...plan, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, }); expect(await storage.listPendingAgentCreations()).toEqual([ expect.objectContaining({ @@ -250,13 +264,15 @@ test("legacy worktree create keeps its journal through durable agent registratio kind: "worktree", targetPath: resolvedWorktreePath, worktreeIncarnationId: "30704df3-6339-4c9c-8277-1416544ed7cc", - directoryIdentity: { device: "7", inode: "42" }, + directoryIdentity: { + device: directoryStat.dev.toString(), + inode: directoryStat.ino.toString(), + }, metadataBaseRefName: "main", }, }), ]); removalOrder.push("worktree-resolved"); - mkdirSync(resolvedWorktreePath, { recursive: true }); return { sessionConfig: { ...config, cwd: resolvedWorktreePath }, setupContinuation: { @@ -867,6 +883,10 @@ test("mcp worktree create journals the exact path incarnation through durable ag logger, providerSnapshotManager: createProviderSnapshotManagerStub().manager, createPaseoWorktree: async (input: CreatePaseoWorktreeInput) => { + await input.onWorktreePathPlanned?.(worktreePath, { + worktreeIncarnationId, + metadataBaseRefName: "main", + }); mkdirSync(workspaceCwd, { recursive: true }); const directoryStat = statSync(worktreePath, { bigint: true }); await input.onWorktreePathResolved?.(worktreePath, { diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index c084fd6872..52932546a7 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -1,7 +1,7 @@ import type { Logger } from "pino"; import type { TerminalManager } from "../../../terminal/terminal-manager.js"; -import type { WorktreeCreationReservation } from "../../../utils/worktree.js"; +import type { WorktreeCreationJournalCallbacks } from "../../../utils/worktree.js"; import type { CreatePaseoWorktreeInput } from "../../paseo-worktree-service.js"; import { expandUserPath, resolvePathFromBase } from "../../path-utils.js"; import { toWorktreeRequestError } from "../../worktree-errors.js"; @@ -83,10 +83,7 @@ export interface CreateAgentFromSessionInput { gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, - onWorktreePathResolved?: ( - worktreePath: string, - reservation: WorktreeCreationReservation, - ) => Promise, + worktreeCreationJournal?: WorktreeCreationJournalCallbacks, ) => Promise; } @@ -346,12 +343,7 @@ async function resolveSessionCreateAgent( input.worktreeName, input.firstAgentContext, pendingAgentId - ? (worktreePath, reservation) => - dependencies.agentStorage.setPendingAgentCreationWorktree( - pendingAgentId, - worktreePath, - reservation, - ) + ? pendingWorktreeCreationJournal(dependencies.agentStorage, pendingAgentId) : undefined, ); // Validate the requested mode against the provider's modes for the resolved @@ -450,13 +442,8 @@ async function resolveMcpCreateAgent( cwd, worktree: input.worktree, initialPrompt: input.initialPrompt ?? "", - onWorktreePathResolved: pendingAgentId - ? (worktreePath, reservation) => - dependencies.agentStorage.setPendingAgentCreationWorktree( - pendingAgentId, - worktreePath, - reservation, - ) + worktreeCreationJournal: pendingAgentId + ? pendingWorktreeCreationJournal(dependencies.agentStorage, pendingAgentId) : undefined, }); try { @@ -659,10 +646,7 @@ async function resolveMcpCwd(params: { cwd: string; initialPrompt: string; worktree: CreateAgentFromMcpInput["worktree"]; - onWorktreePathResolved?: ( - worktreePath: string, - reservation: WorktreeCreationReservation, - ) => Promise; + worktreeCreationJournal?: WorktreeCreationJournalCallbacks; }): Promise<{ resolvedCwd: string; setupContinuation?: AgentWorktreeSetupContinuation; @@ -699,7 +683,8 @@ async function resolveMcpCwd(params: { runSetup: false, paseoHome: dependencies.paseoHome, worktreesRoot: dependencies.worktreesRoot, - onWorktreePathResolved: params.onWorktreePathResolved, + onWorktreePathPlanned: params.worktreeCreationJournal?.onWorktreePathPlanned, + onWorktreePathResolved: params.worktreeCreationJournal?.onWorktreePathResolved, }, createPaseoWorktree: dependencies.createPaseoWorktree, resolveDefaultBranch: baseBranch ? async () => baseBranch : undefined, @@ -729,6 +714,18 @@ async function resolveMcpCwd(params: { }; } +function pendingWorktreeCreationJournal( + agentStorage: AgentStorage, + agentId: string, +): WorktreeCreationJournalCallbacks { + return { + onWorktreePathPlanned: (worktreePath, plan) => + agentStorage.planPendingAgentCreationWorktree(agentId, worktreePath, plan), + onWorktreePathResolved: (worktreePath, reservation) => + agentStorage.identifyPendingAgentCreationWorktree(agentId, worktreePath, reservation), + }; +} + interface CreateMcpWorktreeOptions { input: CreatePaseoWorktreeInput; createPaseoWorktree: CreatePaseoWorktreeWorkflowFn | undefined; diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 026f2e119c..9ce75c07df 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -1,4 +1,5 @@ import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { describe, expect, it, vi } from "vitest"; @@ -194,7 +195,9 @@ interface TestDeps { } function buildAgentManagerSpies() { + const getAgent = vi.fn(); return { + allocateAgentId: vi.fn(() => randomUUID()), createAgent: vi.fn(), waitForAgentEvent: vi.fn().mockResolvedValue({ status: "idle", @@ -210,7 +213,11 @@ function buildAgentManagerSpies() { updateAgentMetadata: vi.fn().mockResolvedValue(undefined), archiveAgent: vi.fn().mockResolvedValue({ archivedAt: new Date().toISOString() }), notifyAgentState: vi.fn(), - getAgent: vi.fn(), + getAgent, + getAgentInitializationState: vi.fn((agentId: string) => ({ + agent: getAgent(agentId), + closeInFlight: false, + })), listAgents: vi.fn().mockReturnValue([]), getTimeline: vi.fn().mockReturnValue([]), resumeAgentFromPersistence: vi.fn(), @@ -237,6 +244,16 @@ function buildAgentStorageSpies() { upsert: vi.fn().mockResolvedValue(undefined), applySnapshot: vi.fn(), list: vi.fn().mockResolvedValue([]), + beginPendingAgentCreation: vi.fn().mockImplementation(async (agentId: string) => ({ + agentId, + createdAt: new Date().toISOString(), + ownerKind: "agent" as const, + cleanupTarget: { kind: "agent" as const }, + })), + planPendingAgentCreationWorktree: vi.fn().mockResolvedValue(undefined), + identifyPendingAgentCreationWorktree: vi.fn().mockResolvedValue(undefined), + listPendingAgentCreations: vi.fn().mockResolvedValue([]), + removePendingAgentCreation: vi.fn().mockResolvedValue(undefined), remove: vi.fn(), }; } @@ -842,6 +859,10 @@ function createPaseoWorktreeForMcpTest(options: { logger: createTestLogger(), generateWorkspaceName: options.generateWorkspaceName ?? (async () => null), }); + const worktreeCreationStorage = new AgentStorage( + join(options.paseoHome, "test-worktree-creation-journal"), + createTestLogger(), + ); return async (input, serviceOptions) => { options.setupContinuations?.push(serviceOptions?.setupContinuation?.kind); @@ -860,9 +881,30 @@ function createPaseoWorktreeForMcpTest(options: { warmWorkspaceGitData: async () => {}, autoNameWorkspaceBranchForFirstAgent: (autoNameInput) => workspaceAutoName.scheduleForWorktree(autoNameInput), - emitWorkspaceUpdateForWorkspaceId: async (workspaceId) => { - options.broadcasts.push(workspaceId); + worktreeCreationJournal: { + beginPendingAgentCreation: (creationId, journalOptions) => + worktreeCreationStorage.beginPendingAgentCreation(creationId, journalOptions), + planPendingAgentCreationWorktree: (creationId, worktreePath, plan) => + worktreeCreationStorage.planPendingAgentCreationWorktree( + creationId, + worktreePath, + plan, + ), + identifyPendingAgentCreationWorktree: (creationId, worktreePath, reservation) => + worktreeCreationStorage.identifyPendingAgentCreationWorktree( + creationId, + worktreePath, + reservation, + ), + removePendingAgentCreation: (creationId) => + worktreeCreationStorage.removePendingAgentCreation(creationId), + recoverPendingCreation: (creationId) => + worktreeCreationStorage.removePendingAgentCreation(creationId), }, + // The helper records the initial publication synchronously below. Keep + // background setup refreshes out of that assertion stream; auto-name + // refreshes use the WorkspaceAutoName dependency above. + emitWorkspaceUpdateForWorkspaceId: async () => {}, cacheWorkspaceSetupSnapshot: () => {}, emit: () => {}, sessionLogger: createTestLogger(), @@ -1868,7 +1910,7 @@ describe("create_agent MCP tool", () => { expect.objectContaining({ cwd: expect.stringContaining("agent-worktree"), }), - undefined, + expect.any(String), { workspaceId: createdWorkspaceIds[0] }, ); } finally { @@ -2217,7 +2259,7 @@ describe("create_agent MCP tool", () => { expect.objectContaining({ title: "Explicit Agent Title", }), - undefined, + expect.any(String), { workspaceId }, ); expect(workspace).toMatchObject({ @@ -2541,7 +2583,7 @@ describe("create_agent MCP tool", () => { expect(startedAgentSetupIds).toEqual(["agent-pr-worktree"]); expect(spies.agentManager.createAgent).toHaveBeenCalledWith( expect.objectContaining({ cwd: "/tmp/worktrees/pr-123" }), - undefined, + expect.any(String), { workspaceId: "ws-pr-123" }, ); await waitForUnexpectedWorkspaceNamingSideEffects(); diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 2a252a2763..1d979740d6 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -6,6 +6,8 @@ import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises"; import pino from "pino"; import { afterEach, describe, expect, test, vi } from "vitest"; import { WebSocket } from "ws"; +import { experimental_createMCPClient } from "ai"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; import { createPaseoDaemon, parseListenString, type PaseoDaemonConfig } from "./bootstrap.js"; import { AgentManagerShuttingDownError } from "./agent/agent-manager.js"; @@ -18,6 +20,7 @@ import { isPlatform } from "../test-utils/platform.js"; import { findFreePort } from "./service-proxy.js"; import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { readPaseoWorktreeIncarnationId } from "../utils/worktree-metadata.js"; +import { getPaseoWorktreesRoot } from "../utils/worktree.js"; import type { HubEnrollment, HubEnrollmentResult, @@ -53,6 +56,13 @@ function deferred(): { promise: Promise; resolve: (value: T) => void } { return { promise, resolve }; } +async function createBootstrapAgentMcpClient(port: number) { + const transport = new StreamableHTTPClientTransport( + new URL(`http://127.0.0.1:${port}/mcp/agents`), + ); + return experimental_createMCPClient({ transport }); +} + describe("paseo daemon bootstrap", () => { afterEach(() => { vi.restoreAllMocks(); @@ -485,9 +495,136 @@ describe("paseo daemon bootstrap", () => { await client.close().catch(() => undefined); await daemonHandle.daemon.stop().catch(() => undefined); await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { + recursive: true, + force: true, + }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(initialAgentCwd, { recursive: true, force: true }), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins a standalone MCP worktree create accepted before ingress freezes", async () => { + const daemonHandle = await createTestPaseoDaemon({ cleanup: false }); + const { repoDir, tempRoot } = await createCommittedGitRepo("mcp-accepted-create"); + const mcpClient = await createBootstrapAgentMcpClient(daemonHandle.port); + const releaseMutation = deferred(); + const mutationStarted = deferred(); + const worktreesRoot = await getPaseoWorktreesRoot(repoDir, daemonHandle.paseoHome); + const heldMutation = defaultWorkspaceLifecycleCoordinator.runWorktreeMutationExclusive( + worktreesRoot, + async () => { + mutationStarted.resolve(); + await releaseMutation.promise; + }, + ); + await mutationStarted.promise; + + try { + const createPromise = mcpClient.callTool({ + name: "create_workspace", + args: { + isolation: "worktree", + path: repoDir, + worktreeSlug: "mcp-accepted-before-shutdown", + branchName: "feature/mcp-accepted-before-shutdown", + baseBranch: "main", + }, + }); + await vi.waitFor(async () => { + const pending = await daemonHandle.daemon.agentStorage.listPendingAgentCreations(); + expect(pending).toContainEqual( + expect.objectContaining({ ownerKind: "standalone-worktree" }), + ); + }); + + const stopPromise = daemonHandle.daemon.stop(); + const earlyStop = await Promise.race([ + stopPromise.then(() => "stopped" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(earlyStop).toBe("pending"); + + releaseMutation.resolve(); + const result = await createPromise; + expect(result.structuredContent).toMatchObject({ + isolation: "worktree", + cwd: expect.stringContaining("mcp-accepted-before-shutdown"), + }); + await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + await stopPromise; + } finally { + releaseMutation.resolve(); + await heldMutation.catch(() => undefined); + await mcpClient.close().catch(() => undefined); + await daemonHandle.daemon.stop().catch(() => undefined); + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); await Promise.all([ rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + + test("MCP cannot create a worktree after the shutdown closure snapshot", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ + closeSession: heldAgentClose.closeSession, + }), + }); + const { repoDir, tempRoot } = await createCommittedGitRepo("mcp-rejected-create"); + const initialAgentCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-mcp-shutdown-agent-")); + const mcpClient = await createBootstrapAgentMcpClient(daemonHandle.port); + + try { + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: initialAgentCwd }, + undefined, + { workspaceId: undefined }, + ); + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + await heldAgentClose.started; + const closureWorktrees = listGitWorktreePaths(repoDir); + + await expect( + mcpClient.callTool({ + name: "create_workspace", + args: { + isolation: "worktree", + path: repoDir, + worktreeSlug: "mcp-rejected-after-shutdown", + branchName: "feature/mcp-rejected-after-shutdown", + baseBranch: "main", + }, + }), + ).rejects.toThrow(); + expect(listGitWorktreePaths(repoDir)).toEqual(closureWorktrees); + await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + + heldAgentClose.finish(); + await stopPromise; + } finally { + heldAgentClose.finish(); + await mcpClient.close().catch(() => undefined); + await daemonHandle.daemon.stop().catch(() => undefined); + await daemonHandle.daemon.agentManager.flush().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { + recursive: true, + force: true, + }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), rm(initialAgentCwd, { recursive: true, force: true }), rm(tempRoot, { recursive: true, force: true }), ]); @@ -578,7 +715,10 @@ describe("paseo daemon bootstrap", () => { } finally { await daemonHandle.daemon.agentManager.flush().catch(() => undefined); await Promise.all([ - rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(path.dirname(daemonHandle.paseoHome), { + recursive: true, + force: true, + }), rm(daemonHandle.staticDir, { recursive: true, force: true }), ]); } diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 6b4579c271..864217b96a 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -482,6 +482,38 @@ class DaemonShutdownDeadlineError extends Error { } } +class LifecycleMutationIngressClosedError extends Error { + constructor() { + super("Lifecycle mutation ingress is closed"); + this.name = "LifecycleMutationIngressClosedError"; + } +} + +class LifecycleMutationIngress { + private accepting = true; + private readonly inFlight = new Set>(); + + run(operation: () => Promise): Promise { + if (!this.accepting) { + return Promise.reject(new LifecycleMutationIngressClosedError()); + } + const task = Promise.resolve().then(operation); + this.inFlight.add(task); + void task.then( + () => this.inFlight.delete(task), + () => this.inFlight.delete(task), + ); + return task; + } + + async closeAndDrain(): Promise { + this.accepting = false; + while (this.inFlight.size > 0) { + await Promise.allSettled(Array.from(this.inFlight)); + } + } +} + function waitForShutdownStep( task: Promise, signal: AbortSignal, @@ -1075,6 +1107,22 @@ export async function createPaseoDaemon( }, autoNameWorkspaceBranchForFirstAgent: (autoNameInput) => workspaceAutoName.scheduleForWorktree(autoNameInput), + worktreeCreationJournal: { + beginPendingAgentCreation: (creationId, options) => + agentStorage.beginPendingAgentCreation(creationId, options), + planPendingAgentCreationWorktree: (creationId, worktreePath, plan) => + agentStorage.planPendingAgentCreationWorktree(creationId, worktreePath, plan), + identifyPendingAgentCreationWorktree: (creationId, worktreePath, reservation) => + agentStorage.identifyPendingAgentCreationWorktree( + creationId, + worktreePath, + reservation, + ), + removePendingAgentCreation: (creationId) => + agentStorage.removePendingAgentCreation(creationId), + recoverPendingCreation: (creationId) => + hubAgentLifecycle.recoverPendingAgentCreation(creationId), + }, emitWorkspaceUpdateForWorkspaceId: async (workspaceId) => { await emitWorkspaceUpdatesExternal([workspaceId]); }, @@ -1410,6 +1458,7 @@ export async function createPaseoDaemon( agentManager.setPaseoToolsEnabled(config.mcpInjectIntoAgents !== false); const mcpEnabled = config.mcpEnabled ?? true; + const agentMcpMutationIngress = new LifecycleMutationIngress(); let agentMcpBaseUrl: string | null = null; if (mcpEnabled) { const agentMcpRoute = "/mcp/agents"; @@ -1519,7 +1568,18 @@ export async function createPaseoDaemon( }; const handleAgentMcpRequest: express.RequestHandler = (req, res) => { - void runAgentMcpRequest(req, res); + void agentMcpMutationIngress + .run(() => runAgentMcpRequest(req, res)) + .catch((error) => { + if (error instanceof LifecycleMutationIngressClosedError && !res.headersSent) { + res.status(503).json({ error: "Daemon is shutting down" }); + return; + } + logger.error({ err: error }, "Failed to dispatch Agent MCP request"); + if (!res.headersSent) { + res.status(500).json({ error: "Internal MCP server error" }); + } + }); }; app.post(agentMcpRoute, handleAgentMcpRequest); @@ -1751,8 +1811,15 @@ export async function createPaseoDaemon( await attempt("workspace cleanup retry service", () => workspaceCleanupRetryService.stop()); await attempt("workspace reconciliation", () => workspaceReconciliation.dispose()); await attempt("script health monitor", () => scriptHealthMonitor.stop()); - // Freeze both ingress and registration before taking the agent closure snapshot. - await attempt("WebSocket ingress", () => wsServer?.prepareForShutdown()); + // Freeze every worktree/agent mutation producer and join operations accepted + // before the boundary before taking the agent closure snapshot. + await attempt("lifecycle mutation ingress", async () => { + await Promise.all([ + wsServer?.prepareForShutdown(), + agentMcpMutationIngress.closeAndDrain(), + scheduleService.stop(), + ]); + }); await attempt("agent registration", () => agentManager.prepareForShutdown()); await attempt("create-agent lifecycle", async () => { const remainingMs = Math.max(0, shutdownDeadlineAt - Date.now()); @@ -1770,7 +1837,6 @@ export async function createPaseoDaemon( await attempt("provider snapshots", () => providerSnapshotManager.shutdown()); await attempt("terminals", () => terminalManager.killAll()); await attempt("speech", () => speechService.stop()); - await attempt("schedules", () => scheduleService.stop()); await attempt("relay", () => relayRuntime?.stop()); await attempt("WebSocket server", () => wsServer?.close()); await attempt("service proxy", () => serviceProxy.stopStandalone()); diff --git a/packages/server/src/server/schedule/service.test.ts b/packages/server/src/server/schedule/service.test.ts index a7fad87beb..a0bb625d50 100644 --- a/packages/server/src/server/schedule/service.test.ts +++ b/packages/server/src/server/schedule/service.test.ts @@ -2055,6 +2055,57 @@ describe("ScheduleService", () => { expect(inspected.runs[0]?.status).toBe("succeeded"); }); + test("stop freezes new runs and joins a run accepted before shutdown", async () => { + let markRunStarted: (() => void) | null = null; + const runStarted = new Promise((resolve) => { + markRunStarted = resolve; + }); + let finishRun: (() => void) | null = null; + const runBlocked = new Promise((resolve) => { + finishRun = resolve; + }); + const service = createScheduleService({ + paseoHome: tempDir, + logger: createTestLogger(), + agentManager: new AgentManager({ logger: createTestLogger() }), + agentStorage, + providerSnapshotManager: NO_UNATTENDED_SCHEDULE_POLICY, + now: () => now, + runner: async () => { + markRunStarted?.(); + await runBlocked; + return { agentId: null, output: "finished before shutdown" }; + }, + }); + const created = await service.create({ + prompt: "join on shutdown", + cadence: { type: "every", everyMs: 60_000 }, + target: { + type: "new-agent", + config: { provider: "claude", cwd: tempDir }, + }, + }); + now = new Date("2026-01-01T00:01:00.000Z"); + const tickPromise = service.tick(); + await runStarted; + + const stopPromise = service.stop(); + await expect( + Promise.race([ + stopPromise.then(() => "stopped" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 20)), + ]), + ).resolves.toBe("pending"); + await expect(service.runOnce(created.id)).rejects.toThrow("shutting down"); + + finishRun?.(); + await Promise.all([tickPromise, stopPromise]); + expect((await service.inspect(created.id)).runs[0]).toMatchObject({ + status: "succeeded", + output: "finished before shutdown", + }); + }); + test("rejects archived target agents before loading them", async () => { const manager = new AgentManager({ logger: createTestLogger() }); const service = createScheduleService({ diff --git a/packages/server/src/server/schedule/service.ts b/packages/server/src/server/schedule/service.ts index a25157df13..d5e71c2835 100644 --- a/packages/server/src/server/schedule/service.ts +++ b/packages/server/src/server/schedule/service.ts @@ -250,6 +250,9 @@ export class ScheduleService { runId: string, ) => Promise; private readonly runningScheduleIds = new Set(); + private readonly inFlightTicks = new Set>(); + private readonly inFlightRuns = new Set>(); + private acceptingRuns = true; private tickTimer: ReturnType | null = null; constructor(options: ScheduleServiceOptions) { @@ -266,6 +269,7 @@ export class ScheduleService { } async start(): Promise { + this.acceptingRuns = true; await this.recoverInterruptedRuns(); await this.sweepOrphanedSchedules(); if (this.tickTimer) { @@ -281,10 +285,14 @@ export class ScheduleService { } async stop(): Promise { + this.acceptingRuns = false; if (this.tickTimer) { clearInterval(this.tickTimer); this.tickTimer = null; } + while (this.inFlightTicks.size > 0 || this.inFlightRuns.size > 0) { + await Promise.allSettled([...this.inFlightTicks, ...this.inFlightRuns]); + } } async create(input: CreateScheduleInput): Promise { @@ -524,6 +532,9 @@ export class ScheduleService { } async runOnce(id: string): Promise { + if (!this.acceptingRuns) { + throw new Error("Schedule service is shutting down"); + } const schedule = await this.inspect(id); if (schedule.status === "completed") { throw new Error(`Schedule ${id} is already completed`); @@ -531,11 +542,22 @@ export class ScheduleService { if (this.runningScheduleIds.has(id)) { throw new Error(`Schedule ${id} is already running`); } - await this.runSchedule(schedule, this.now(), { manual: true }); + await this.runScheduleTracked(schedule, this.now(), { manual: true }); return this.inspect(id); } async tick(): Promise { + if (!this.acceptingRuns) return; + const task = this.tickInternal(); + this.inFlightTicks.add(task); + try { + await task; + } finally { + this.inFlightTicks.delete(task); + } + } + + private async tickInternal(): Promise { const now = this.now(); const schedules = await this.store.list(); for (const schedule of schedules) { @@ -552,7 +574,25 @@ export class ScheduleService { if (new Date(schedule.nextRunAt).getTime() > now.getTime()) { continue; } - await this.runSchedule(schedule, now); + if (!this.acceptingRuns) return; + await this.runScheduleTracked(schedule, now); + } + } + + private async runScheduleTracked( + schedule: StoredSchedule, + now: Date, + options?: { manual?: boolean }, + ): Promise { + if (!this.acceptingRuns) { + throw new Error("Schedule service is shutting down"); + } + const task = this.runSchedule(schedule, now, options); + this.inFlightRuns.add(task); + try { + await task; + } finally { + this.inFlightRuns.delete(task); } } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 774bb32a1a..16b67c141e 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -27,7 +27,7 @@ import type { TerminalManager, TerminalWorkspaceContributionChangedEvent, } from "../terminal/terminal-manager.js"; -import type { WorktreeCreationReservation } from "../utils/worktree.js"; +import type { WorktreeCreationJournalCallbacks } from "../utils/worktree.js"; import { TerminalSessionController } from "../terminal/terminal-session-controller.js"; import type { TerminalActivity } from "@getpaseo/protocol/terminal-activity"; import type { BinaryFrame } from "@getpaseo/protocol/binary-frames/index"; @@ -357,10 +357,7 @@ interface ResolvedSessionCreateAgentIntent { interface PendingAgentCreationReservation { agentId?: string; - onWorktreePathResolved?: ( - worktreePath: string, - reservation: WorktreeCreationReservation, - ) => Promise; + worktreeCreationJournal?: WorktreeCreationJournalCallbacks; } type FetchWorkspacesRequestMessage = Extract< @@ -3095,7 +3092,7 @@ export class Session { target: worktree, firstAgentContext, hasLegacyGitOptions: Boolean(git), - onWorktreePathResolved: pendingCreation.onWorktreePathResolved, + worktreeCreationJournal: pendingCreation.worktreeCreationJournal, }); createdWorktreeForCleanup = createdWorktree; const resolvedIntent = await this.resolveSessionCreateAgentIntent({ @@ -3257,8 +3254,16 @@ export class Session { await this.agentStorage.beginPendingAgentCreation(agentId); return { agentId, - onWorktreePathResolved: (worktreePath, reservation) => - this.agentStorage.setPendingAgentCreationWorktree(agentId, worktreePath, reservation), + worktreeCreationJournal: { + onWorktreePathPlanned: (worktreePath, plan) => + this.agentStorage.planPendingAgentCreationWorktree(agentId, worktreePath, plan), + onWorktreePathResolved: (worktreePath, reservation) => + this.agentStorage.identifyPendingAgentCreationWorktree( + agentId, + worktreePath, + reservation, + ), + }, }; } @@ -3625,10 +3630,7 @@ export class Session { gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, - onWorktreePathResolved?: ( - worktreePath: string, - reservation: WorktreeCreationReservation, - ) => Promise, + worktreeCreationJournal?: WorktreeCreationJournalCallbacks, ): Promise<{ sessionConfig: AgentSessionConfig; setupContinuation?: CreatePaseoWorktreeWorkflowResult["setupContinuation"]; @@ -3669,7 +3671,7 @@ export class Session { gitOptions, legacyWorktreeName, firstAgentContext, - onWorktreePathResolved, + worktreeCreationJournal, ); } @@ -5863,6 +5865,22 @@ export class Session { this.workspaceAutoName.scheduleForWorktree(autoNameInput, { currentSelection: this.getFocusedAgentSelectionForCwd(autoNameInput.workspace.cwd), }), + worktreeCreationJournal: { + beginPendingAgentCreation: (creationId, journalOptions) => + this.agentStorage.beginPendingAgentCreation(creationId, journalOptions), + planPendingAgentCreationWorktree: (creationId, worktreePath, plan) => + this.agentStorage.planPendingAgentCreationWorktree(creationId, worktreePath, plan), + identifyPendingAgentCreationWorktree: (creationId, worktreePath, reservation) => + this.agentStorage.identifyPendingAgentCreationWorktree( + creationId, + worktreePath, + reservation, + ), + removePendingAgentCreation: (creationId) => + this.agentStorage.removePendingAgentCreation(creationId), + recoverPendingCreation: (creationId) => + this.createAgentLifecycleDispatch.recoverPendingAgentCreation(creationId), + }, emitWorkspaceUpdateForWorkspaceId: (workspaceId) => this.emitWorkspaceUpdateForWorkspaceId(workspaceId), cacheWorkspaceSetupSnapshot: (workspaceId, snapshot) => { @@ -5891,10 +5909,7 @@ export class Session { target: CreateAgentWorktreeTarget | undefined; firstAgentContext: FirstAgentContext; hasLegacyGitOptions: boolean; - onWorktreePathResolved?: ( - worktreePath: string, - reservation: WorktreeCreationReservation, - ) => Promise; + worktreeCreationJournal?: WorktreeCreationJournalCallbacks; }): Promise { if (input.target && input.hasLegacyGitOptions) { throw new Error("create_agent_request worktree cannot be combined with git options"); @@ -5909,7 +5924,8 @@ export class Session { runSetup: false, paseoHome: this.paseoHome, worktreesRoot: this.worktreesRoot, - onWorktreePathResolved: input.onWorktreePathResolved, + onWorktreePathPlanned: input.worktreeCreationJournal?.onWorktreePathPlanned, + onWorktreePathResolved: input.worktreeCreationJournal?.onWorktreePathResolved, } as const; switch (input.target.mode) { case "branch-off": { diff --git a/packages/server/src/server/workspace-archive-service.test.ts b/packages/server/src/server/workspace-archive-service.test.ts index 1e12eb96a4..2ac415ead0 100644 --- a/packages/server/src/server/workspace-archive-service.test.ts +++ b/packages/server/src/server/workspace-archive-service.test.ts @@ -29,6 +29,7 @@ import { requireArchiveCleanupComplete, retryPendingWorkspaceCleanup, resolveWorkspaceIdAtPath, + WorkspaceArchiveTargetNotFoundError, } from "./workspace-archive-service.js"; import { WorkspaceCleanupRetryService } from "./workspace-cleanup-retry-service.js"; import { WorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; @@ -294,6 +295,34 @@ describe("archiveByScope", () => { }); }); + test("rejects truthfully when an explicit workspace vanishes during final refresh", async () => { + const { tempDir } = createGitRepo(); + const workspaceId = "ws-vanished-before-archive"; + const workspaceCwd = path.join(tempDir, "local-checkout"); + mkdirSync(workspaceCwd); + const deps = createArchiveDeps({ + paseoHome: path.join(tempDir, ".paseo"), + activeWorkspaces: [{ workspaceId, cwd: workspaceCwd, kind: "local_checkout" }], + }); + let activeReadCount = 0; + deps.listActiveWorkspaces = async () => { + activeReadCount += 1; + return activeReadCount === 1 + ? [{ workspaceId, cwd: workspaceCwd, kind: "local_checkout" }] + : []; + }; + deps.workspaceRegistry = { get: async () => null }; + deps.archiveWorkspaceRecord = vi.fn(deps.archiveWorkspaceRecord); + + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId }, + requestId: "req-vanished-before-archive", + }), + ).rejects.toBeInstanceOf(WorkspaceArchiveTargetNotFoundError); + expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); + }); + test("waits for ownership on a sibling discovered during final target refresh", async () => { const { tempDir } = createGitRepo(); const workspaceCwd = path.join(tempDir, "shared-checkout"); @@ -1481,7 +1510,7 @@ describe("archiveByScope", () => { expect(existsSync(worktree.worktreePath)).toBe(true); }); - test("workspace scope with unknown workspace id is a clean no-op", async () => { + test("workspace scope rejects an unknown explicit workspace id", async () => { const { tempDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -1494,15 +1523,12 @@ describe("archiveByScope", () => { return originalArchiveWorkspaceRecord(workspaceId); }); - const result = await archiveByScope(deps, { - scope: { kind: "workspace", workspaceId: "ws-does-not-exist" }, - requestId: "req-unknown-workspace", - }); - - assertArchiveResult(result, { - archivedWorkspaceIds: [], - removedDirectory: false, - }); + await expect( + archiveByScope(deps, { + scope: { kind: "workspace", workspaceId: "ws-does-not-exist" }, + requestId: "req-unknown-workspace", + }), + ).rejects.toBeInstanceOf(WorkspaceArchiveTargetNotFoundError); expect(deps.markWorkspaceArchiving).not.toHaveBeenCalled(); expect(deps.archiveWorkspaceRecord).not.toHaveBeenCalled(); expect(deps.emitWorkspaceUpdatesForWorkspaceIds).not.toHaveBeenCalled(); diff --git a/packages/server/src/server/workspace-archive-service.ts b/packages/server/src/server/workspace-archive-service.ts index 255fa018cc..ddc5abedbe 100644 --- a/packages/server/src/server/workspace-archive-service.ts +++ b/packages/server/src/server/workspace-archive-service.ts @@ -423,11 +423,8 @@ async function resolveArchiveTarget( if (persistedRecord?.cleanupPending) { return archiveTargetFromPendingCleanup(workspaceId, persistedRecord.cleanupPending); } - dependencies.sessionLogger?.warn( - { workspaceId }, - "Workspace not found for archive-by-scope; skipping", - ); - return { backing: null, teardownTargets: [], workspaceIds: [] }; + dependencies.sessionLogger?.warn({ workspaceId }, "Workspace not found for archive-by-scope"); + throw new WorkspaceArchiveTargetNotFoundError(workspaceId); } return { backing: await resolveWorkspaceBackingDirectory(record, dependencies), diff --git a/packages/server/src/server/worktree-core.posix.test.ts b/packages/server/src/server/worktree-core.posix.test.ts index 956d27854e..8a5c1867c0 100644 --- a/packages/server/src/server/worktree-core.posix.test.ts +++ b/packages/server/src/server/worktree-core.posix.test.ts @@ -436,6 +436,11 @@ describe.skipIf(isPlatform("win32"))("worktree-core POSIX-only", () => { worktreeSlug: "collision", paseoHome, runSetup: false, + onWorktreePathPlanned: async (worktreePath, plan) => { + expect(worktreePath).toBe(resolvedPath); + expect(existsSync(worktreePath)).toBe(false); + observedIncarnationId = plan.worktreeIncarnationId; + }, onWorktreePathResolved: async (worktreePath, reservation) => { observedPath = worktreePath; observedIncarnationId = reservation.worktreeIncarnationId; @@ -1493,6 +1498,9 @@ describe.skipIf(isPlatform("win32"))("worktree-core POSIX-only", () => { worktreeSlug: "reused-worktree", paseoHome, runSetup: false, + onWorktreePathPlanned: async (worktreePath) => { + resolvedPaths.push(worktreePath); + }, onWorktreePathResolved: async (worktreePath) => { resolvedPaths.push(worktreePath); }, diff --git a/packages/server/src/server/worktree-core.ts b/packages/server/src/server/worktree-core.ts index 1d7a0a05e7..90ae1e53b7 100644 --- a/packages/server/src/server/worktree-core.ts +++ b/packages/server/src/server/worktree-core.ts @@ -6,6 +6,7 @@ import { resolveExistingWorktreeForSlug, slugify, validateBranchSlug, + type WorktreeCreationPlan, type WorktreeCreationReservation, type WorktreeConfig, } from "../utils/worktree.js"; @@ -30,6 +31,7 @@ export interface CreateWorktreeCoreInput { paseoHome?: string; worktreesRoot?: string; runSetup?: boolean; + onWorktreePathPlanned?: (worktreePath: string, plan: WorktreeCreationPlan) => Promise; onWorktreePathResolved?: ( worktreePath: string, reservation: WorktreeCreationReservation, @@ -133,6 +135,7 @@ export async function createWorktreeCore( runSetup: input.runSetup ?? true, paseoHome: input.paseoHome, worktreesRoot: input.worktreesRoot, + onWorktreePathPlanned: input.onWorktreePathPlanned, onWorktreePathResolved: input.onWorktreePathResolved, }), intent, diff --git a/packages/server/src/server/worktree-session.test.ts b/packages/server/src/server/worktree-session.test.ts index 8fb3c7f355..c710ea7fca 100644 --- a/packages/server/src/server/worktree-session.test.ts +++ b/packages/server/src/server/worktree-session.test.ts @@ -94,6 +94,21 @@ function createLogger(): Logger { return logger; } +function createWorktreeCreationJournalStub() { + return { + beginPendingAgentCreation: vi.fn().mockImplementation(async (creationId: string) => ({ + agentId: creationId, + createdAt: new Date().toISOString(), + ownerKind: "standalone-worktree" as const, + cleanupTarget: { kind: "agent" as const }, + })), + planPendingAgentCreationWorktree: vi.fn().mockResolvedValue(undefined), + identifyPendingAgentCreationWorktree: vi.fn().mockResolvedValue(undefined), + removePendingAgentCreation: vi.fn().mockResolvedValue(undefined), + recoverPendingCreation: vi.fn().mockResolvedValue(undefined), + }; +} + function createWorkflowForRequestTest(options: { paseoHome: string; createPaseoWorktree?: CreatePaseoWorktreeFn; @@ -115,6 +130,7 @@ function createWorkflowForRequestTest(options: { createPaseoWorktree, warmWorkspaceGitData: options.warmWorkspaceGitData ?? (async () => {}), autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: createWorktreeCreationJournalStub(), emitWorkspaceUpdateForWorkspaceId: async () => {}, cacheWorkspaceSetupSnapshot: () => {}, emit: () => {}, @@ -458,6 +474,69 @@ describe("resolveGitCreateBaseBranch", () => { }); describe("create worktree setup boundary", () => { + test("journals standalone creation before mkdir and clears it after workspace publication", async () => { + const { tempDir, repoDir } = createGitRepo(); + const paseoHome = path.join(tempDir, ".paseo"); + const journal = createWorktreeCreationJournalStub(); + let plannedCreationId: string | null = null; + let plannedPath: string | null = null; + let plannedIncarnationId: string | null = null; + journal.planPendingAgentCreationWorktree.mockImplementation( + async (creationId, worktreePath, plan) => { + expect(existsSync(worktreePath)).toBe(false); + plannedCreationId = creationId; + plannedPath = worktreePath; + plannedIncarnationId = plan.worktreeIncarnationId; + }, + ); + journal.identifyPendingAgentCreationWorktree.mockImplementation( + async (creationId, worktreePath, reservation) => { + expect(creationId).toBe(plannedCreationId); + expect(worktreePath).toBe(plannedPath); + expect(reservation.worktreeIncarnationId).toBe(plannedIncarnationId); + expect(existsSync(worktreePath)).toBe(true); + }, + ); + + try { + const result = await createPaseoWorktreeWorkflow( + { + paseoHome, + createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), + warmWorkspaceGitData: async () => {}, + autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: journal, + emitWorkspaceUpdateForWorkspaceId: async () => {}, + cacheWorkspaceSetupSnapshot: () => {}, + emit: () => {}, + sessionLogger: createLogger(), + terminalManager: null, + archiveWorkspaceRecord: async () => {}, + serviceProxy: null, + scriptRuntimeStore: null, + getDaemonTcpPort: null, + getDaemonTcpHost: null, + onScriptsChanged: null, + }, + { + cwd: repoDir, + worktreeSlug: "standalone-journal-order", + runSetup: false, + paseoHome, + }, + ); + + expect(journal.beginPendingAgentCreation).toHaveBeenCalledWith(plannedCreationId, { + ownerKind: "standalone-worktree", + }); + expect(journal.identifyPendingAgentCreationWorktree).toHaveBeenCalledOnce(); + expect(journal.removePendingAgentCreation).toHaveBeenCalledWith(plannedCreationId); + expect(existsSync(result.worktree.worktreePath)).toBe(true); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + test("registers workspace setup before returning the created workspace", async () => { const { tempDir, repoDir } = createGitRepo(); const paseoHome = path.join(tempDir, ".paseo"); @@ -475,6 +554,7 @@ describe("create worktree setup boundary", () => { createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), warmWorkspaceGitData: async () => {}, autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: createWorktreeCreationJournalStub(), emitWorkspaceUpdateForWorkspaceId: async () => {}, cacheWorkspaceSetupSnapshot: () => {}, emit: () => {}, @@ -522,6 +602,7 @@ describe("create worktree setup boundary", () => { createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), warmWorkspaceGitData: async () => {}, autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: createWorktreeCreationJournalStub(), emitWorkspaceUpdateForWorkspaceId: async () => {}, cacheWorkspaceSetupSnapshot: () => {}, emit: (message) => workspaceSetupEvents.push(message), @@ -595,6 +676,7 @@ describe("create worktree setup boundary", () => { createPaseoWorktree: createPaseoWorktreeForTest({ paseoHome }), warmWorkspaceGitData: async () => {}, autoNameWorkspaceBranchForFirstAgent: () => {}, + worktreeCreationJournal: createWorktreeCreationJournalStub(), emitWorkspaceUpdateForWorkspaceId: async () => {}, cacheWorkspaceSetupSnapshot: () => {}, emit: () => {}, diff --git a/packages/server/src/server/worktree-session.ts b/packages/server/src/server/worktree-session.ts index 27dcb2520b..6a9c6bbf7f 100644 --- a/packages/server/src/server/worktree-session.ts +++ b/packages/server/src/server/worktree-session.ts @@ -1,7 +1,9 @@ import type { Logger } from "pino"; +import { randomUUID } from "node:crypto"; import { basename, resolve } from "node:path"; import type { AgentSessionConfig } from "./agent/agent-sdk-types.js"; +import type { AgentStorage } from "./agent/agent-storage.js"; import { type GitSetupOptions, type FirstAgentContext, @@ -33,7 +35,7 @@ import { runWorktreeSetupCommands, slugify, validateBranchSlug, - type WorktreeCreationReservation, + type WorktreeCreationJournalCallbacks, type WorktreeConfig, type WorktreeSetupCommandResult, WorktreeSetupError, @@ -131,6 +133,15 @@ interface CreatePaseoWorktreeWorkflowDependencies extends CreatePaseoWorktreeInB workspace: PersistedWorkspaceRecord; firstAgentContext: FirstAgentContext; }) => void; + worktreeCreationJournal: Pick< + AgentStorage, + | "beginPendingAgentCreation" + | "planPendingAgentCreationWorktree" + | "identifyPendingAgentCreationWorktree" + | "removePendingAgentCreation" + > & { + recoverPendingCreation: (creationId: string) => Promise; + }; } interface AgentWorktreeSetupContinuationInput { @@ -222,10 +233,7 @@ export async function buildAgentSessionConfig( gitOptions?: GitSetupOptions, legacyWorktreeName?: string, firstAgentContext?: FirstAgentContext, - onWorktreePathResolved?: ( - worktreePath: string, - reservation: WorktreeCreationReservation, - ) => Promise, + worktreeCreationJournal?: WorktreeCreationJournalCallbacks, ): Promise<{ sessionConfig: AgentSessionConfig; setupContinuation?: AgentWorktreeSetupContinuation; @@ -263,7 +271,8 @@ export async function buildAgentSessionConfig( runSetup: false, paseoHome: dependencies.paseoHome, worktreesRoot: dependencies.worktreesRoot, - onWorktreePathResolved, + onWorktreePathPlanned: worktreeCreationJournal?.onWorktreePathPlanned, + onWorktreePathResolved: worktreeCreationJournal?.onWorktreePathResolved, }, { resolveDefaultBranch: normalized.baseBranch @@ -626,97 +635,156 @@ export async function createPaseoWorktreeWorkflow( ): Promise { const lifecycleCoordinator = dependencies.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator; + if (Boolean(input.onWorktreePathPlanned) !== Boolean(input.onWorktreePathResolved)) { + throw new Error("Worktree creation journaling requires both planning and identity callbacks"); + } + const callerOwnsCreationJournal = Boolean( + input.onWorktreePathPlanned && input.onWorktreePathResolved, + ); + const standaloneCreationId = callerOwnsCreationJournal ? null : randomUUID(); + if (standaloneCreationId) { + await dependencies.worktreeCreationJournal.beginPendingAgentCreation(standaloneCreationId, { + ownerKind: "standalone-worktree", + }); + } + const journaledInput: CreatePaseoWorktreeInput = standaloneCreationId + ? { + ...input, + onWorktreePathPlanned: (worktreePath, plan) => + dependencies.worktreeCreationJournal.planPendingAgentCreationWorktree( + standaloneCreationId, + worktreePath, + plan, + ), + onWorktreePathResolved: (worktreePath, reservation) => + dependencies.worktreeCreationJournal.identifyPendingAgentCreationWorktree( + standaloneCreationId, + worktreePath, + reservation, + ), + } + : input; const worktreesRoot = await getPaseoWorktreesRoot( input.cwd, input.paseoHome ?? dependencies.paseoHome, input.worktreesRoot ?? dependencies.worktreesRoot, ); - return lifecycleCoordinator.runWorktreeMutationExclusive(resolve(worktreesRoot), async () => { - const createdWorktree = await dependencies.createPaseoWorktree( - { - ...input, - runSetup: false, - paseoHome: input.paseoHome ?? dependencies.paseoHome, - worktreesRoot: input.worktreesRoot ?? dependencies.worktreesRoot, - }, - options?.resolveDefaultBranch - ? { resolveDefaultBranch: options.resolveDefaultBranch } - : undefined, - ); - const slug = basename(createdWorktree.worktree.worktreePath); - const workspace = createdWorktree.workspace; - const setupContinuation = options?.setupContinuation ?? { kind: "workspace" }; - - setTimeout(() => { - if (input.firstAgentContext) { - dependencies.autoNameWorkspaceBranchForFirstAgent({ - workspace, - firstAgentContext: input.firstAgentContext, - }); - } - void dependencies.warmWorkspaceGitData(workspace).catch((error) => { - dependencies.sessionLogger.warn( - { err: error, workspaceId: workspace.workspaceId }, - "Failed to warm workspace git data after creating worktree", + const creation = + lifecycleCoordinator.runWorktreeMutationExclusive( + resolve(worktreesRoot), + async (): Promise => { + const createdWorktree = await dependencies.createPaseoWorktree( + { + ...journaledInput, + runSetup: false, + paseoHome: input.paseoHome ?? dependencies.paseoHome, + worktreesRoot: input.worktreesRoot ?? dependencies.worktreesRoot, + }, + options?.resolveDefaultBranch + ? { resolveDefaultBranch: options.resolveDefaultBranch } + : undefined, ); - }); - }, 0); + const slug = basename(createdWorktree.worktree.worktreePath); + const workspace = createdWorktree.workspace; + const setupContinuation = options?.setupContinuation ?? { kind: "workspace" }; - if (setupContinuation.kind === "workspace") { - const setupTask = new Promise((resolveSetup) => { setTimeout(() => { - void runWorktreeSetupInBackground(dependencies, { - requestCwd: input.cwd, - repoRoot: createdWorktree.repoRoot, - workspaceId: workspace.workspaceId, - worktree: createdWorktree.worktree, - shouldBootstrap: createdWorktree.created, - slug, - worktreePath: createdWorktree.worktree.worktreePath, - workspaceCwd: workspace.cwd, - }).then(resolveSetup, resolveSetup); + if (input.firstAgentContext) { + dependencies.autoNameWorkspaceBranchForFirstAgent({ + workspace, + firstAgentContext: input.firstAgentContext, + }); + } + void dependencies.warmWorkspaceGitData(workspace).catch((error) => { + dependencies.sessionLogger.warn( + { err: error, workspaceId: workspace.workspaceId }, + "Failed to warm workspace git data after creating worktree", + ); + }); }, 0); - }); - lifecycleCoordinator.trackWorkspaceSetup( - workspace.workspaceId, - setupTask, - createdWorktree.worktree.worktreePath, - ); - } - if (setupContinuation.kind === "agent") { - const reservation = lifecycleCoordinator.reserveWorkspaceSetup( - workspace.workspaceId, - createdWorktree.worktree.worktreePath, - ); - return { - ...createdWorktree, - setupContinuation: { - kind: "agent", - startAfterAgentCreate: ({ agentId }) => { - const setupTask = runAsyncWorktreeBootstrap({ - agentId, - workspaceId: workspace.workspaceId, - worktree: createdWorktree.worktree, - workspaceCwd: workspace.cwd, - shouldBootstrap: createdWorktree.created, - terminalManager: setupContinuation.terminalManager, - appendTimelineItem: (item) => setupContinuation.appendTimelineItem({ agentId, item }), - emitLiveTimelineItem: (item) => - setupContinuation.emitLiveTimelineItem({ agentId, item }), - logger: setupContinuation.logger, - }); - reservation.completeWith(setupTask); - void setupTask; - }, - releaseWithoutStarting: reservation.release, - }, - }; - } + if (setupContinuation.kind === "workspace") { + const setupTask = new Promise((resolveSetup) => { + setTimeout(() => { + void runWorktreeSetupInBackground(dependencies, { + requestCwd: input.cwd, + repoRoot: createdWorktree.repoRoot, + workspaceId: workspace.workspaceId, + worktree: createdWorktree.worktree, + shouldBootstrap: createdWorktree.created, + slug, + worktreePath: createdWorktree.worktree.worktreePath, + workspaceCwd: workspace.cwd, + }).then(resolveSetup, resolveSetup); + }, 0); + }); + lifecycleCoordinator.trackWorkspaceSetup( + workspace.workspaceId, + setupTask, + createdWorktree.worktree.worktreePath, + ); + } - return createdWorktree; - }); + if (setupContinuation.kind === "agent") { + const reservation = lifecycleCoordinator.reserveWorkspaceSetup( + workspace.workspaceId, + createdWorktree.worktree.worktreePath, + ); + return { + ...createdWorktree, + setupContinuation: { + kind: "agent", + startAfterAgentCreate: ({ agentId }: { agentId: string }) => { + const setupTask = runAsyncWorktreeBootstrap({ + agentId, + workspaceId: workspace.workspaceId, + worktree: createdWorktree.worktree, + workspaceCwd: workspace.cwd, + shouldBootstrap: createdWorktree.created, + terminalManager: setupContinuation.terminalManager, + appendTimelineItem: (item) => + setupContinuation.appendTimelineItem({ agentId, item }), + emitLiveTimelineItem: (item) => + setupContinuation.emitLiveTimelineItem({ agentId, item }), + logger: setupContinuation.logger, + }); + reservation.completeWith(setupTask); + void setupTask; + }, + releaseWithoutStarting: reservation.release, + }, + }; + } + + return createdWorktree; + }, + ); + if (!standaloneCreationId) return creation; + return creation.then( + async (result) => { + await dependencies.worktreeCreationJournal + .removePendingAgentCreation(standaloneCreationId) + .catch((error) => { + dependencies.sessionLogger.warn( + { err: error, creationId: standaloneCreationId }, + "Failed to clear a published standalone worktree creation journal", + ); + }); + return result; + }, + async (error) => { + await dependencies.worktreeCreationJournal + .recoverPendingCreation(standaloneCreationId) + .catch((recoveryError) => { + dependencies.sessionLogger.warn( + { err: recoveryError, creationId: standaloneCreationId }, + "Failed to recover a standalone worktree creation", + ); + }); + throw error; + }, + ); } export async function handleWorkspaceSetupStatusRequest( diff --git a/packages/server/src/utils/worktree.ts b/packages/server/src/utils/worktree.ts index 3606a25291..7aaa0c1078 100644 --- a/packages/server/src/utils/worktree.ts +++ b/packages/server/src/utils/worktree.ts @@ -1,4 +1,13 @@ -import { existsSync, mkdirSync, realpathSync, rmSync, statSync } from "fs"; +import { + existsSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + statSync, + unlinkSync, + writeFileSync, +} from "fs"; import { copyFile, rm, stat } from "fs/promises"; import { join, basename, dirname, isAbsolute, resolve, sep } from "path"; import net from "node:net"; @@ -208,12 +217,18 @@ export interface CreateWorktreeOptions { runSetup: boolean; paseoHome?: string; worktreesRoot?: string; + onWorktreePathPlanned?: (worktreePath: string, plan: WorktreeCreationPlan) => Promise; onWorktreePathResolved?: ( worktreePath: string, reservation: WorktreeCreationReservation, ) => Promise; } +export interface WorktreeCreationPlan { + worktreeIncarnationId: string; + metadataBaseRefName: string; +} + export interface WorktreeCreationReservation { worktreeIncarnationId: string; directoryIdentity: { @@ -223,6 +238,24 @@ export interface WorktreeCreationReservation { metadataBaseRefName: string; } +export interface WorktreeCreationJournalCallbacks { + onWorktreePathPlanned: (worktreePath: string, plan: WorktreeCreationPlan) => Promise; + onWorktreePathResolved: ( + worktreePath: string, + reservation: WorktreeCreationReservation, + ) => Promise; +} + +const WORKTREE_CREATION_MARKER_FILENAME = ".paseo-worktree-creation"; + +export function readWorktreeCreationMarker(worktreePath: string): string | null { + try { + return readFileSync(join(worktreePath, WORKTREE_CREATION_MARKER_FILENAME), "utf8").trim(); + } catch { + return null; + } +} + interface ResolveExistingWorktreeForSlugOptions { slug: string; repoRoot: string; @@ -918,6 +951,19 @@ function normalizePathForOwnership(input: string): string { } } +function normalizePlannedPathForOwnership(input: string): string { + const resolvedInput = resolve(input); + const missingSegments: string[] = []; + let existingAncestor = resolvedInput; + while (!existsSync(existingAncestor)) { + const parent = dirname(existingAncestor); + if (parent === existingAncestor) break; + missingSegments.unshift(basename(existingAncestor)); + existingAncestor = parent; + } + return join(normalizePathForOwnership(existingAncestor), ...missingSegments); +} + function resolveRepoRootFromGitCommonDir(commonDir: string): string { const normalizedCommonDir = normalizePathForOwnership(commonDir); return basename(normalizedCommonDir) === ".git" @@ -1281,20 +1327,36 @@ export const createWorktree = async ({ runSetup, paseoHome, worktreesRoot, + onWorktreePathPlanned, onWorktreePathResolved, }: CreateWorktreeOptions): Promise => { + if (Boolean(onWorktreePathPlanned) !== Boolean(onWorktreePathResolved)) { + throw new Error("Worktree creation journaling requires both planning and identity callbacks"); + } const sourcePlan = await resolveWorktreeSourcePlan({ cwd, source, desiredSlug: worktreeSlug }); let worktreePath = join(await getPaseoWorktreesRoot(cwd, paseoHome, worktreesRoot), worktreeSlug); - mkdirSync(dirname(worktreePath), { recursive: true }); - // Also handle worktree path collision. When creation is journaled, reserve - // the directory so the journal can identify this exact path incarnation. + // Also handle worktree path collision. Journaled creation first persists the + // exact candidate path and incarnation, then atomically claims the directory. let finalWorktreePath = worktreePath; let pathSuffix = 1; - if (onWorktreePathResolved) { + const worktreeIncarnationId = randomUUID(); + if (onWorktreePathPlanned && onWorktreePathResolved) { while (true) { + const normalizedCandidatePath = normalizePlannedPathForOwnership(finalWorktreePath); + if (existsSync(normalizedCandidatePath)) { + finalWorktreePath = `${worktreePath}-${pathSuffix}`; + pathSuffix++; + continue; + } + await onWorktreePathPlanned(normalizedCandidatePath, { + worktreeIncarnationId, + metadataBaseRefName: sourcePlan.metadataBaseRefName, + }); + mkdirSync(dirname(normalizedCandidatePath), { recursive: true }); try { - mkdirSync(finalWorktreePath); + mkdirSync(normalizedCandidatePath); + finalWorktreePath = normalizedCandidatePath; break; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; @@ -1303,6 +1365,7 @@ export const createWorktree = async ({ } } } else { + mkdirSync(dirname(finalWorktreePath), { recursive: true }); while (existsSync(finalWorktreePath)) { finalWorktreePath = `${worktreePath}-${pathSuffix}`; pathSuffix++; @@ -1310,10 +1373,15 @@ export const createWorktree = async ({ } const normalizedWorktreePath = normalizePathForOwnership(finalWorktreePath); - const worktreeIncarnationId = randomUUID(); - if (onWorktreePathResolved) { - const directoryStat = statSync(normalizedWorktreePath, { bigint: true }); + if (onWorktreePathPlanned && onWorktreePathResolved) { + const markerPath = join(normalizedWorktreePath, WORKTREE_CREATION_MARKER_FILENAME); try { + writeFileSync(markerPath, `${worktreeIncarnationId}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); + const directoryStat = statSync(normalizedWorktreePath, { bigint: true }); await onWorktreePathResolved(normalizedWorktreePath, { worktreeIncarnationId, directoryIdentity: { @@ -1322,6 +1390,7 @@ export const createWorktree = async ({ }, metadataBaseRefName: sourcePlan.metadataBaseRefName, }); + unlinkSync(markerPath); } catch (error) { rmSync(normalizedWorktreePath, { recursive: true, force: true }); throw error; From 0a2282ad166cc522226a61064ec5c41782a447a1 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 15:39:54 -0400 Subject: [PATCH 41/43] fix(server): gate native lifecycle tools on shutdown --- .../src/server/agent/mcp-server.test.ts | 3 ++ .../src/server/agent/tools/paseo-tools.ts | 32 ++++++++++++++++++- .../server/src/server/bootstrap.smoke.test.ts | 26 ++++++++------- packages/server/src/server/bootstrap.ts | 18 +++-------- 4 files changed, 52 insertions(+), 27 deletions(-) diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 9ce75c07df..889d22e5f8 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -2597,6 +2597,7 @@ describe("create_agent MCP tool", () => { const paseoHome = join(tempDir, ".paseo"); const broadcasts: string[] = []; const setupContinuations: Array<"workspace" | "agent" | undefined> = []; + const runLifecycleMutation = vi.fn(async (operation: () => Promise) => operation()); try { execFileSync("git", ["init", repoDir], { stdio: "pipe" }); @@ -2633,6 +2634,7 @@ describe("create_agent MCP tool", () => { WorkspaceGitService, "getSnapshot" | "listWorktrees" | "resolveRepoRoot" >, + runLifecycleMutation, logger, }); const tool = registeredTool(server, "create_workspace"); @@ -2649,6 +2651,7 @@ describe("create_agent MCP tool", () => { expect(response.structuredContent.workspaceId).toBe(broadcasts[0]); expect(workspaceGitService.getSnapshot).not.toHaveBeenCalled(); expect(setupContinuations).toEqual([undefined]); + expect(runLifecycleMutation).toHaveBeenCalledOnce(); expect(broadcasts).toHaveLength(1); expect(broadcasts[0]).toMatch(/^wks_[0-9a-f]{16}$/); } finally { diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index 4e4cd4466a..0d126f144b 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -132,6 +132,7 @@ export interface PaseoToolHostDependencies { paseoHome?: string; worktreesRoot?: string; lifecycleCoordinator?: WorkspaceLifecycleCoordinator; + runLifecycleMutation?: (operation: () => Promise) => Promise; /** * ID of the agent that is using this tool catalog. * Used for cwd/mode inheritance when agents spawn child agents. @@ -148,6 +149,32 @@ export interface PaseoToolHostDependencies { logger: Logger; } +const LIFECYCLE_MUTATION_TOOL_NAMES = new Set([ + "create_workspace", + "archive_workspace", + "create_agent", + "cancel_agent", + "archive_agent", + "kill_agent", + "update_agent", + "rename_workspace", + "start_workspace_script", + "stop_workspace_script", + "create_terminal", + "kill_terminal", + "send_terminal_keys", + "create_schedule", + "create_heartbeat", + "delete_heartbeat", + "pause_schedule", + "resume_schedule", + "delete_schedule", + "update_schedule", + "run_schedule_once", + "set_agent_mode", + "respond_to_permission", +]); + async function requireActiveWorkspaceForOwnership( options: Pick, workspaceId: string, @@ -615,7 +642,10 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase if (!tool) { throw new Error(`Paseo tool not found: ${name}`); } - return tool.handler(await parseToolInput(tool, input), context); + const operation = async () => tool.handler(await parseToolInput(tool, input), context); + return options.runLifecycleMutation && LIFECYCLE_MUTATION_TOOL_NAMES.has(name) + ? options.runLifecycleMutation(operation) + : operation(); }, }); diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 1d979740d6..b2e932079e 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -595,18 +595,20 @@ describe("paseo daemon bootstrap", () => { await heldAgentClose.started; const closureWorktrees = listGitWorktreePaths(repoDir); - await expect( - mcpClient.callTool({ - name: "create_workspace", - args: { - isolation: "worktree", - path: repoDir, - worktreeSlug: "mcp-rejected-after-shutdown", - branchName: "feature/mcp-rejected-after-shutdown", - baseBranch: "main", - }, - }), - ).rejects.toThrow(); + const lateCreate = await mcpClient.callTool({ + name: "create_workspace", + args: { + isolation: "worktree", + path: repoDir, + worktreeSlug: "mcp-rejected-after-shutdown", + branchName: "feature/mcp-rejected-after-shutdown", + baseBranch: "main", + }, + }); + expect(lateCreate).toMatchObject({ + isError: true, + content: [expect.objectContaining({ text: "Lifecycle mutation ingress is closed" })], + }); expect(listGitWorktreePaths(repoDir)).toEqual(closureWorktrees); await expect(daemonHandle.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( [], diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 864217b96a..6bb17ed951 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -1393,6 +1393,7 @@ export async function createPaseoDaemon( ); logger.info({ elapsed: elapsed() }, "Preparing voice and MCP runtime"); + const lifecycleMutationIngress = new LifecycleMutationIngress(); const createAgentToolHostDependencies = ( runtime: PaseoToolRuntimeContext, ): PaseoToolHostDependencies => ({ @@ -1445,6 +1446,7 @@ export async function createPaseoDaemon( browserToolsBroker, paseoHome: config.paseoHome, worktreesRoot: config.worktreesRoot, + runLifecycleMutation: (operation) => lifecycleMutationIngress.run(operation), callerAgentId: runtime.callerAgentId, enableVoiceTools: runtime.enableVoiceTools, voiceOnly: runtime.voiceOnly, @@ -1458,7 +1460,6 @@ export async function createPaseoDaemon( agentManager.setPaseoToolsEnabled(config.mcpInjectIntoAgents !== false); const mcpEnabled = config.mcpEnabled ?? true; - const agentMcpMutationIngress = new LifecycleMutationIngress(); let agentMcpBaseUrl: string | null = null; if (mcpEnabled) { const agentMcpRoute = "/mcp/agents"; @@ -1568,18 +1569,7 @@ export async function createPaseoDaemon( }; const handleAgentMcpRequest: express.RequestHandler = (req, res) => { - void agentMcpMutationIngress - .run(() => runAgentMcpRequest(req, res)) - .catch((error) => { - if (error instanceof LifecycleMutationIngressClosedError && !res.headersSent) { - res.status(503).json({ error: "Daemon is shutting down" }); - return; - } - logger.error({ err: error }, "Failed to dispatch Agent MCP request"); - if (!res.headersSent) { - res.status(500).json({ error: "Internal MCP server error" }); - } - }); + void runAgentMcpRequest(req, res); }; app.post(agentMcpRoute, handleAgentMcpRequest); @@ -1816,7 +1806,7 @@ export async function createPaseoDaemon( await attempt("lifecycle mutation ingress", async () => { await Promise.all([ wsServer?.prepareForShutdown(), - agentMcpMutationIngress.closeAndDrain(), + lifecycleMutationIngress.closeAndDrain(), scheduleService.stop(), ]); }); From f5539186e77b6baf72e1029165bf7c29809c5729 Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 16:29:59 -0400 Subject: [PATCH 42/43] fix: close remaining lifecycle shutdown races --- .../create-agent-lifecycle-dispatch.test.ts | 55 +++ .../agent/create-agent-lifecycle-dispatch.ts | 20 + .../src/server/agent/create-agent/create.ts | 4 + .../src/server/agent/mcp-server.test.ts | 98 ++++ .../src/server/agent/tools/paseo-tools.ts | 117 +++-- .../server/src/server/bootstrap.smoke.test.ts | 458 ++++++++++++++++++ packages/server/src/server/bootstrap.ts | 1 + .../server/src/server/websocket-server.ts | 66 ++- 8 files changed, 769 insertions(+), 50 deletions(-) diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts index 5d8a34c563..f3c95020c5 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.test.ts @@ -213,6 +213,61 @@ test("startup adopts a published agent without cleaning its pending worktree", a } }); +test("startup adopts a same-path successful retry instead of cleaning its stale pending worktree", async () => { + const worktreePath = mkdtempSync(join(tmpdir(), "paseo-retried-agent-worktree-")); + mkdirSync(join(worktreePath, ".git")); + const staleAgentId = "agent-failed-before-publish"; + const retryAgentId = "agent-retry-published"; + const workspaceId = "ws-retry-published"; + const pendingCreations = new Map([ + [ + staleAgentId, + { + agentId: staleAgentId, + createdAt: "2025-01-01T00:00:00.000Z", + cleanupTarget: pendingWorktreeTarget(worktreePath), + }, + ], + ]); + const records = new Map([ + [ + retryAgentId, + { + id: retryAgentId, + workspaceId, + } as StoredAgentRecord, + ], + ]); + const dispatch = createLifecycleDispatch(new AgentLifecycleEvents(), async () => undefined, { + pendingCreations, + records, + workspaces: [ + { + workspaceId, + cwd: worktreePath, + worktreeRoot: worktreePath, + archivedAt: null, + } as PersistedWorkspaceRecord, + ], + }); + const archiveWorktreePath = vi.spyOn( + dispatch as unknown as { + archiveWorktreePath(worktreePath: string, workspaceId?: string): Promise; + }, + "archiveWorktreePath", + ); + + try { + await dispatch.recoverPendingAgentCreations(); + + expect(archiveWorktreePath).not.toHaveBeenCalled(); + expect(pendingCreations.size).toBe(0); + expect(existsSync(worktreePath)).toBe(true); + } finally { + rmSync(worktreePath, { recursive: true, force: true }); + } +}); + test("startup removes an exact pre-git worktree reservation without archiving", async () => { const worktreePath = mkdtempSync(join(tmpdir(), "paseo-pre-git-reservation-")); const agentId = "agent-pending-pre-git-reservation"; diff --git a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts index bbf4c585b4..04d3b6ae62 100644 --- a/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts +++ b/packages/server/src/server/agent/create-agent-lifecycle-dispatch.ts @@ -176,6 +176,22 @@ export class CreateAgentLifecycleDispatch { ); } + private async activeWorkspaceWithPublishedAgentOwnsPath(targetPath: string): Promise { + const owningWorkspaceIds = new Set( + (await this.dependencies.workspaceRegistry.list()) + .filter( + (workspace) => + !workspace.archivedAt && + resolvePath(workspace.worktreeRoot ?? workspace.cwd) === targetPath, + ) + .map((workspace) => workspace.workspaceId), + ); + if (owningWorkspaceIds.size === 0) return false; + return (await this.dependencies.agentStorage.list()).some( + (record) => record.workspaceId && owningWorkspaceIds.has(record.workspaceId), + ); + } + private classifyPendingDirectoryOwnership( pending: PendingAgentCreation & { cleanupTarget: Extract; @@ -202,6 +218,10 @@ export class CreateAgentLifecycleDispatch { targetPath: string, ): Promise { const agentId = pending.agentId; + if (await this.activeWorkspaceWithPublishedAgentOwnsPath(targetPath)) { + await this.dependencies.agentStorage.removePendingAgentCreation(agentId); + return true; + } const directoryOwnership = this.classifyPendingDirectoryOwnership(pending, targetPath); if (directoryOwnership === "replaced") { this.dependencies.logger.warn( diff --git a/packages/server/src/server/agent/create-agent/create.ts b/packages/server/src/server/agent/create-agent/create.ts index 52932546a7..d18b0acd22 100644 --- a/packages/server/src/server/agent/create-agent/create.ts +++ b/packages/server/src/server/agent/create-agent/create.ts @@ -111,6 +111,7 @@ export interface CreateAgentFromMcpInput { agentId: string; createdWorktree: CreatePaseoWorktreeWorkflowResult | null; }) => void; + onPendingAgentCreation?: (agentId: string) => void; onWorktreeCreated?: (createdWorktree: CreatePaseoWorktreeWorkflowResult) => void; callerAgentId?: string; callerContext?: { @@ -186,6 +187,9 @@ export async function createAgentCommand( input: CreateAgentCommandInput, ): Promise { const pendingAgentId = await reservePendingAgentId(dependencies, input); + if (input.kind === "mcp" && pendingAgentId) { + input.onPendingAgentCreation?.(pendingAgentId); + } const resolved = await resolveCreateAgent(dependencies, input, pendingAgentId); let snapshot: ManagedAgent; diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 889d22e5f8..86cee3993e 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -1918,6 +1918,76 @@ describe("create_agent MCP tool", () => { } }); + it("cleans and recovers a native MCP worktree journal when create_agent fails", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + const createdWorktree = { + worktree: { + branchName: "failed-agent-worktree", + worktreePath: "/tmp/worktrees/failed-agent-worktree", + }, + intent: { + kind: "branch-off" as const, + branchName: "failed-agent-worktree", + baseBranch: "main", + }, + workspace: createPersistedWorkspaceRecord({ + workspaceId: "ws-failed-agent-worktree", + projectId: REPO_CWD, + cwd: "/tmp/worktrees/failed-agent-worktree", + kind: "worktree", + displayName: "failed-agent-worktree", + createdAt: "2026-08-01T00:00:00.000Z", + updatedAt: "2026-08-01T00:00:00.000Z", + }), + repoRoot: REPO_CWD, + created: true, + setupContinuation: { + kind: "agent" as const, + startAfterAgentCreate: vi.fn(), + releaseWithoutStarting: vi.fn(), + }, + }; + const cleanupCreatedWorktreeAfterFailedAgentCreate = vi.fn().mockResolvedValue(undefined); + const recoverPendingAgentCreation = vi.fn().mockResolvedValue(undefined); + spies.agentManager.createAgent.mockRejectedValue(new Error("provider creation failed")); + + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + providerSnapshotManager: createOpenCodeManager().manager, + createPaseoWorktree: vi.fn().mockResolvedValue(createdWorktree), + createAgentLifecycleDispatch: { + cleanupCreatedWorktreeAfterFailedAgentCreate, + recoverPendingAgentCreation, + }, + logger, + }); + + await expect( + registeredTool(server, "create_agent").handler({ + ...detachedWorktreeWorkspace(REPO_CWD, { + kind: "branch-off", + worktreeSlug: "failed-agent-worktree", + branchName: "failed-agent-worktree", + baseBranch: "main", + }), + title: "Failed worktree agent", + provider: "codex/gpt-5.4", + initialPrompt: "Fail after creating the worktree", + background: true, + }), + ).rejects.toThrow("provider creation failed"); + + const pendingAgentId = z + .string() + .parse(spies.agentStorage.beginPendingAgentCreation.mock.calls[0]?.[0]); + expect(cleanupCreatedWorktreeAfterFailedAgentCreate).toHaveBeenCalledWith({ + createdWorktree, + createdAgentId: null, + }); + expect(recoverPendingAgentCreation).toHaveBeenCalledWith(pendingAgentId); + }); + it("creates a create_agent branch-off worktree without invoking the legacy metadata branch rename", async () => { const { agentManager, agentStorage, spies } = createTestDeps(); const tempDir = await mkdtemp(join(tmpdir(), "paseo-mcp-agent-worktree-name-context-")); @@ -3712,6 +3782,34 @@ describe("send_agent_prompt MCP tool", () => { ); }); + it("runs prompts inside lifecycle shutdown admission", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + const runLifecycleMutation = vi.fn(async (operation: () => Promise) => operation()); + spies.agentManager.getAgent.mockReturnValue({ + id: "child-agent", + cwd: existingCwd, + lifecycle: "running", + currentModeId: null, + availableModes: [], + config: { title: "Child" }, + } as ManagedAgent); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + providerSnapshotManager: createOpenCodeManager().manager, + runLifecycleMutation, + logger, + }); + + await invokeToolWithParsedInput(registeredTool(server, "send_agent_prompt"), { + agentId: "child-agent", + prompt: "Follow up", + background: true, + }); + + expect(runLifecycleMutation).toHaveBeenCalledOnce(); + }); + it("keeps top-level prompts blocking by default", async () => { const { agentManager, agentStorage, spies } = createTestDeps(); spies.agentManager.getAgent.mockReturnValue({ diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index 0d126f144b..86f8eda452 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -21,6 +21,7 @@ import { import { curateAgentActivity } from "../activity-curator.js"; import { selectItemsByProjectedLimit } from "../timeline-projection.js"; import type { AgentStorage } from "../agent-storage.js"; +import type { CreateAgentLifecycleDispatch } from "../create-agent-lifecycle-dispatch.js"; import { ensureAgentLoaded } from "../agent-loading.js"; import { isStoredAgentProviderAvailable } from "../../persistence-hooks.js"; import { @@ -35,7 +36,10 @@ import type { FirstAgentContext } from "../../messages.js"; import { everyMsToFiveFieldCron } from "@getpaseo/protocol/schedule/cadence"; import { expandUserPath, isSameOrDescendantPath, resolvePathFromBase } from "../../path-utils.js"; import type { TerminalManager } from "../../../terminal/terminal-manager.js"; -import type { CreatePaseoWorktreeWorkflowFn } from "../../worktree-session.js"; +import type { + CreatePaseoWorktreeWorkflowFn, + CreatePaseoWorktreeWorkflowResult, +} from "../../worktree-session.js"; import type { ScheduleService } from "../../schedule/service.js"; import { ScheduleRunSchema, @@ -133,6 +137,10 @@ export interface PaseoToolHostDependencies { worktreesRoot?: string; lifecycleCoordinator?: WorkspaceLifecycleCoordinator; runLifecycleMutation?: (operation: () => Promise) => Promise; + createAgentLifecycleDispatch?: Pick< + CreateAgentLifecycleDispatch, + "cleanupCreatedWorktreeAfterFailedAgentCreate" | "recoverPendingAgentCreation" + >; /** * ID of the agent that is using this tool catalog. * Used for cwd/mode inheritance when agents spawn child agents. @@ -153,6 +161,7 @@ const LIFECYCLE_MUTATION_TOOL_NAMES = new Set([ "create_workspace", "archive_workspace", "create_agent", + "send_agent_prompt", "cancel_agent", "archive_agent", "kill_agent", @@ -1477,47 +1486,71 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase requestedBackground = resolvedArgs.parsedArgs.background; notifyOnFinish = resolvedArgs.parsedArgs.notifyOnFinish ?? false; } - const { - snapshot, - background: createdInBackground, - initialPromptStarted, - } = await createAgentCommand( - { - agentManager, - agentStorage, - logger: childLogger, - paseoHome: options.paseoHome, - worktreesRoot: options.worktreesRoot, - terminalManager, - providerSnapshotManager, - lifecycleCoordinator: - options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator, - requireActiveWorkspaceForOwnership: (workspaceId) => - requireActiveWorkspaceForOwnership(options, workspaceId), - createPaseoWorktree: options.createPaseoWorktree, - ...(options.ensureWorkspaceForCreate - ? { ensureWorkspaceForCreate: options.ensureWorkspaceForCreate } - : {}), - }, - { - kind: "mcp", - provider: parsedArgs.provider, - title: parsedArgs.title, - initialPrompt: parsedArgs.initialPrompt, - cwd: resolvedArgs.cwd, - workspaceId: resolvedArgs.workspaceId, - thinking: parsedArgs.settings?.thinkingOptionId, - features: parsedArgs.settings?.features, - labels: parsedArgs.labels, - mode: parsedArgs.settings?.modeId, - background: requestedBackground, - notifyOnFinish, - detached: resolvedArgs.detached, - callerAgentId, - callerContext, - worktree, - }, - ); + let createdWorktreeForCleanup: CreatePaseoWorktreeWorkflowResult | null = null; + let createdAgentId: string | null = null; + let pendingCreationAgentId: string | undefined; + const createResult = await (async () => { + try { + return await createAgentCommand( + { + agentManager, + agentStorage, + logger: childLogger, + paseoHome: options.paseoHome, + worktreesRoot: options.worktreesRoot, + terminalManager, + providerSnapshotManager, + lifecycleCoordinator: + options.lifecycleCoordinator ?? defaultWorkspaceLifecycleCoordinator, + requireActiveWorkspaceForOwnership: (workspaceId) => + requireActiveWorkspaceForOwnership(options, workspaceId), + createPaseoWorktree: options.createPaseoWorktree, + ...(options.ensureWorkspaceForCreate + ? { ensureWorkspaceForCreate: options.ensureWorkspaceForCreate } + : {}), + }, + { + kind: "mcp", + provider: parsedArgs.provider, + title: parsedArgs.title, + initialPrompt: parsedArgs.initialPrompt, + cwd: resolvedArgs.cwd, + workspaceId: resolvedArgs.workspaceId, + thinking: parsedArgs.settings?.thinkingOptionId, + features: parsedArgs.settings?.features, + labels: parsedArgs.labels, + mode: parsedArgs.settings?.modeId, + background: requestedBackground, + notifyOnFinish, + detached: resolvedArgs.detached, + callerAgentId, + callerContext, + worktree, + onPendingAgentCreation: (agentId) => { + pendingCreationAgentId = agentId; + }, + onWorktreeCreated: (createdWorktree) => { + createdWorktreeForCleanup = createdWorktree; + }, + onCreated: ({ agentId }) => { + createdAgentId = agentId; + }, + }, + ); + } catch (error) { + await options.createAgentLifecycleDispatch?.cleanupCreatedWorktreeAfterFailedAgentCreate({ + createdWorktree: createdWorktreeForCleanup, + createdAgentId, + }); + if (pendingCreationAgentId) { + await options.createAgentLifecycleDispatch?.recoverPendingAgentCreation( + pendingCreationAgentId, + ); + } + throw error; + } + })(); + const { snapshot, background: createdInBackground, initialPromptStarted } = createResult; try { if (!createdInBackground && initialPromptStarted) { diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index b2e932079e..03955ffada 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -21,6 +21,8 @@ import { findFreePort } from "./service-proxy.js"; import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { readPaseoWorktreeIncarnationId } from "../utils/worktree-metadata.js"; import { getPaseoWorktreesRoot } from "../utils/worktree.js"; +import { Session } from "./session.js"; +import { FileBackedProjectRegistry, FileBackedWorkspaceRegistry } from "./workspace-registry.js"; import type { HubEnrollment, HubEnrollmentResult, @@ -507,6 +509,279 @@ describe("paseo daemon bootstrap", () => { } }); + test("an established socket cannot start a workspace mutation after shutdown admission closes", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ closeSession: heldAgentClose.closeSession }), + }); + const blockerCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-late-workspace-blocker-")); + const workspaceCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-late-workspace-target-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: blockerCwd }, + undefined, + { workspaceId: undefined }, + ); + const originalHandleMessage = Session.prototype.handleMessage; + const handledWorkspaceCreates: string[] = []; + vi.spyOn(Session.prototype, "handleMessage").mockImplementation( + async function (message, source) { + if (message.type === "workspace.create.request") { + handledWorkspaceCreates.push(message.source.path); + } + return originalHandleMessage.call(this, message, source); + }, + ); + + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + await heldAgentClose.started; + + const lateCreate = client + .createWorkspace({ source: { kind: "directory", path: workspaceCwd } }) + .then( + () => "created" as const, + () => "rejected" as const, + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(handledWorkspaceCreates).toEqual([]); + + heldAgentClose.finish(); + await stopPromise; + await expect(lateCreate).resolves.toBe("rejected"); + } finally { + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(blockerCwd, { recursive: true, force: true }), + rm(workspaceCwd, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins workspace archive paused before lifecycle coordinator admission", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ closeSession: heldAgentClose.closeSession }), + }); + const blockerCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-archive-blocker-")); + const workspaceCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-archive-target-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + const registryStarted = deferred(); + const releaseRegistry = deferred(); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: blockerCwd }, + undefined, + { workspaceId: undefined }, + ); + const created = await client.createWorkspace({ + source: { kind: "directory", path: workspaceCwd }, + }); + if (!created.workspace) throw new Error(created.error ?? "Failed to create target workspace"); + + const originalList = FileBackedWorkspaceRegistry.prototype.list; + let pauseNextList = true; + vi.spyOn(FileBackedWorkspaceRegistry.prototype, "list").mockImplementation(async function () { + if (pauseNextList) { + pauseNextList = false; + registryStarted.resolve(); + await releaseRegistry.promise; + } + return originalList.call(this); + }); + + const archivePromise = client.archiveWorkspace(created.workspace.id); + await registryStarted.promise; + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + + const closureBeforeRelease = await Promise.race([ + heldAgentClose.started.then(() => "closing" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(closureBeforeRelease).toBe("pending"); + + releaseRegistry.resolve(); + await expect(archivePromise).resolves.toMatchObject({ + workspaceId: created.workspace.id, + error: null, + }); + await heldAgentClose.started; + heldAgentClose.finish(); + await stopPromise; + } finally { + releaseRegistry.resolve(); + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(blockerCwd, { recursive: true, force: true }), + rm(workspaceCwd, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins project removal paused before lifecycle coordinator admission", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ closeSession: heldAgentClose.closeSession }), + }); + const blockerCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-project-remove-blocker-")); + const workspaceCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-project-remove-target-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + const registryStarted = deferred(); + const releaseRegistry = deferred(); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: blockerCwd }, + undefined, + { workspaceId: undefined }, + ); + const created = await client.createWorkspace({ + source: { kind: "directory", path: workspaceCwd }, + }); + if (!created.workspace) throw new Error(created.error ?? "Failed to create target project"); + + const originalGet = FileBackedProjectRegistry.prototype.get; + let pauseTargetGet = true; + vi.spyOn(FileBackedProjectRegistry.prototype, "get").mockImplementation( + async function (projectId) { + if (pauseTargetGet && projectId === created.workspace?.projectId) { + pauseTargetGet = false; + registryStarted.resolve(); + await releaseRegistry.promise; + } + return originalGet.call(this, projectId); + }, + ); + + const removePromise = client.removeProject(created.workspace.projectId); + await registryStarted.promise; + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + + const closureBeforeRelease = await Promise.race([ + heldAgentClose.started.then(() => "closing" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(closureBeforeRelease).toBe("pending"); + + releaseRegistry.resolve(); + await expect(removePromise).resolves.toEqual({ + removedWorkspaceIds: [created.workspace.id], + }); + await heldAgentClose.started; + heldAgentClose.finish(); + await stopPromise; + } finally { + releaseRegistry.resolve(); + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(blockerCwd, { recursive: true, force: true }), + rm(workspaceCwd, { recursive: true, force: true }), + ]); + } + }); + + test("shutdown joins workspace create paused before session mutation dispatch", async () => { + const heldAgentClose = holdAgentClose(); + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ closeSession: heldAgentClose.closeSession }), + }); + const blockerCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-workspace-create-blocker-")); + const workspaceCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-workspace-create-target-")); + const client = new DaemonClient({ + url: `ws://127.0.0.1:${daemonHandle.port}/ws`, + appVersion: "0.1.82", + }); + const dispatchStarted = deferred(); + const releaseDispatch = deferred(); + + try { + await client.connect(); + await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: blockerCwd }, + undefined, + { workspaceId: undefined }, + ); + + const originalHandleMessage = Session.prototype.handleMessage; + let pauseWorkspaceCreate = true; + vi.spyOn(Session.prototype, "handleMessage").mockImplementation( + async function (message, source) { + if (pauseWorkspaceCreate && message.type === "workspace.create.request") { + pauseWorkspaceCreate = false; + dispatchStarted.resolve(); + await releaseDispatch.promise; + } + return originalHandleMessage.call(this, message, source); + }, + ); + + const createPromise = client.createWorkspace({ + source: { kind: "directory", path: workspaceCwd }, + }); + await dispatchStarted.promise; + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + + const closureBeforeRelease = await Promise.race([ + heldAgentClose.started.then(() => "closing" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(closureBeforeRelease).toBe("pending"); + + releaseDispatch.resolve(); + const created = await createPromise; + expect(created.error).toBeNull(); + expect(created.workspace?.id).toBeTruthy(); + await heldAgentClose.started; + heldAgentClose.finish(); + await stopPromise; + } finally { + releaseDispatch.resolve(); + heldAgentClose.finish(); + await client.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(blockerCwd, { recursive: true, force: true }), + rm(workspaceCwd, { recursive: true, force: true }), + ]); + } + }); + test("shutdown joins a standalone MCP worktree create accepted before ingress freezes", async () => { const daemonHandle = await createTestPaseoDaemon({ cleanup: false }); const { repoDir, tempRoot } = await createCommittedGitRepo("mcp-accepted-create"); @@ -572,6 +847,101 @@ describe("paseo daemon bootstrap", () => { } }); + test("failed native MCP create can retry the same worktree slug and survive startup recovery", async () => { + const paseoHomeRoot = await mkdtemp(path.join(os.tmpdir(), "paseo-mcp-create-retry-home-")); + const { repoDir, tempRoot } = await createCommittedGitRepo("mcp-create-retry"); + let firstDaemon: Awaited> | null = null; + let secondDaemon: Awaited> | null = null; + let firstMcpClient: Awaited> | null = null; + const staticDirs: string[] = []; + + try { + firstDaemon = await createTestPaseoDaemon({ paseoHomeRoot, cleanup: false }); + staticDirs.push(firstDaemon.staticDir); + firstMcpClient = await createBootstrapAgentMcpClient(firstDaemon.port); + + const failed = await firstMcpClient.callTool({ + name: "create_agent", + args: { + cwd: repoDir, + worktreeName: "same-slug-retry", + branchName: "same-slug-retry", + baseBranch: "main", + title: "Failed same-slug create", + provider: "codex/gpt-5.4", + mode: "invalid-test-mode", + initialPrompt: "This create should fail after making its worktree", + background: true, + }, + }); + expect(failed.isError).toBe(true); + await expect(firstDaemon.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + expect(listGitWorktreePaths(repoDir)).toHaveLength(1); + + const retried = await firstMcpClient.callTool({ + name: "create_agent", + args: { + cwd: repoDir, + worktreeName: "same-slug-retry", + refName: "same-slug-retry", + title: "Successful same-slug retry", + provider: "codex/gpt-5.4", + mode: "full-access", + initialPrompt: "Complete the retry", + background: true, + }, + }); + expect(retried.isError).not.toBe(true); + const structured = retried.structuredContent as + | { agentId?: unknown; cwd?: unknown; workspaceId?: unknown } + | undefined; + if ( + typeof structured?.agentId !== "string" || + typeof structured.cwd !== "string" || + typeof structured.workspaceId !== "string" + ) { + throw new Error("Expected successful MCP retry agent/workspace identifiers"); + } + const { agentId, cwd, workspaceId } = structured as { + agentId: string; + cwd: string; + workspaceId: string; + }; + expect(path.basename(cwd)).toBe("same-slug-retry"); + await expect(firstDaemon.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + + await firstMcpClient.close(); + firstMcpClient = null; + await firstDaemon.daemon.stop(); + await firstDaemon.daemon.agentManager.flush(); + + secondDaemon = await createTestPaseoDaemon({ paseoHomeRoot, cleanup: false }); + staticDirs.push(secondDaemon.staticDir); + + const recoveredAgent = await secondDaemon.daemon.agentStorage.get(agentId); + expect(recoveredAgent).toMatchObject({ id: agentId, workspaceId }); + expect(recoveredAgent?.archivedAt).toBeFalsy(); + await expect(secondDaemon.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( + [], + ); + await expect(stat(cwd)).resolves.toMatchObject({}); + expect(listGitWorktreePaths(repoDir)).toContain(cwd); + } finally { + await firstMcpClient?.close().catch(() => undefined); + await secondDaemon?.close().catch(() => undefined); + await firstDaemon?.close().catch(() => undefined); + await Promise.all([ + rm(paseoHomeRoot, { recursive: true, force: true }), + ...staticDirs.map((staticDir) => rm(staticDir, { recursive: true, force: true })), + rm(tempRoot, { recursive: true, force: true }), + ]); + } + }); + test("MCP cannot create a worktree after the shutdown closure snapshot", async () => { const heldAgentClose = holdAgentClose(); const daemonHandle = await createTestPaseoDaemon({ @@ -633,6 +1003,94 @@ describe("paseo daemon bootstrap", () => { } }); + test("shutdown joins an admitted native MCP prompt and rejects prompts after ingress closes", async () => { + const heldAgentClose = holdAgentClose(); + const startedPrompts: unknown[] = []; + const daemonHandle = await createTestPaseoDaemon({ + cleanup: false, + agentClients: createTestAgentClients({ + closeSession: heldAgentClose.closeSession, + onStartTurn: (prompt) => startedPrompts.push(prompt), + }), + }); + const agentCwd = await mkdtemp(path.join(os.tmpdir(), "paseo-mcp-prompt-shutdown-agent-")); + const mcpClient = await createBootstrapAgentMcpClient(daemonHandle.port); + const storageReadStarted = deferred(); + const releaseStorageRead = deferred(); + + try { + const agent = await daemonHandle.daemon.agentManager.createAgent( + { provider: "codex", cwd: agentCwd }, + undefined, + { workspaceId: undefined }, + ); + const originalGet = daemonHandle.daemon.agentStorage.get.bind( + daemonHandle.daemon.agentStorage, + ); + let pauseTargetRead = true; + vi.spyOn(daemonHandle.daemon.agentStorage, "get").mockImplementation(async (agentId) => { + if (pauseTargetRead && agentId === agent.id) { + pauseTargetRead = false; + storageReadStarted.resolve(); + await releaseStorageRead.promise; + } + return originalGet(agentId); + }); + + const admittedPrompt = mcpClient.callTool({ + name: "send_agent_prompt", + args: { + agentId: agent.id, + prompt: "Admitted before shutdown", + background: true, + }, + }); + await storageReadStarted.promise; + heldAgentClose.arm(); + const stopPromise = daemonHandle.daemon.stop(); + + const closureBeforeRelease = await Promise.race([ + heldAgentClose.started.then(() => "closing" as const), + new Promise<"pending">((resolve) => setTimeout(() => resolve("pending"), 50)), + ]); + expect(closureBeforeRelease).toBe("pending"); + + releaseStorageRead.resolve(); + await expect(admittedPrompt).resolves.toMatchObject({ + structuredContent: { success: true }, + }); + expect(startedPrompts).toEqual(["Admitted before shutdown"]); + await heldAgentClose.started; + + const latePrompt = await mcpClient.callTool({ + name: "send_agent_prompt", + args: { + agentId: agent.id, + prompt: "Rejected after shutdown", + background: true, + }, + }); + expect(latePrompt).toMatchObject({ + isError: true, + content: [expect.objectContaining({ text: "Lifecycle mutation ingress is closed" })], + }); + expect(startedPrompts).toEqual(["Admitted before shutdown"]); + + heldAgentClose.finish(); + await stopPromise; + } finally { + releaseStorageRead.resolve(); + heldAgentClose.finish(); + await mcpClient.close().catch(() => undefined); + await daemonHandle.close().catch(() => undefined); + await Promise.all([ + rm(path.dirname(daemonHandle.paseoHome), { recursive: true, force: true }), + rm(daemonHandle.staticDir, { recursive: true, force: true }), + rm(agentCwd, { recursive: true, force: true }), + ]); + } + }); + test("continues teardown after a shutdown step fails", async () => { const daemonHandle = await createTestPaseoDaemon(); const failure = new Error("service proxy stop failed"); diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index 6bb17ed951..509717b528 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -1447,6 +1447,7 @@ export async function createPaseoDaemon( paseoHome: config.paseoHome, worktreesRoot: config.worktreesRoot, runLifecycleMutation: (operation) => lifecycleMutationIngress.run(operation), + createAgentLifecycleDispatch: hubAgentLifecycle, callerAgentId: runtime.callerAgentId, enableVoiceTools: runtime.enableVoiceTools, voiceOnly: runtime.voiceOnly, diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 5187ea235b..9cba086222 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -19,6 +19,7 @@ import type { CheckoutDiffManager, CheckoutDiffMetrics } from "./checkout-diff-m import type { DaemonConfigStore, MutableDaemonConfig } from "./daemon-config-store.js"; import { type ServerInfoStatusPayload, + type SessionInboundMessage, type SessionOutboundMessage, type WorkspaceSetupSnapshot, type WSHelloMessage, @@ -456,6 +457,51 @@ const WS_CLOSE_SERVER_SHUTDOWN = 1001; const WS_PROTOCOL_VERSION = 1; const WS_RUNTIME_METRICS_FLUSH_MS = 30_000; +const SHUTDOWN_TRACKED_SESSION_MUTATIONS = new Set([ + "agent.detach.request", + "agent.fork_context.request", + "agent.rewind.request", + "agent_permission_response", + "archive_agent_request", + "archive_workspace_request", + "cancel_agent_request", + "close_items_request", + "create_agent_request", + "create_paseo_worktree_request", + "create_terminal_request", + "delete_agent_request", + "hub.execution.agent.create.request", + "hub.execution.control.request", + "import_agent_request", + "kill_terminal_request", + "loop/run", + "loop/stop", + "open_project_request", + "paseo_worktree_archive_request", + "project.add.request", + "project.create_directory.request", + "project.github.clone.request", + "project.remove.request", + "refresh_agent_request", + "resume_agent_request", + "schedule/create", + "schedule/delete", + "schedule/pause", + "schedule/resume", + "schedule/run-once", + "schedule/update", + "send_agent_message_request", + "set_agent_feature_request", + "set_agent_mode_request", + "set_agent_model_request", + "set_agent_thinking_request", + "start_workspace_script_request", + "workspace.create.request", + "workspace.recovery.restore.request", + "workspace.script.start.request", + "workspace.script.stop.request", +]); + export class MissingDaemonVersionError extends Error { constructor() { super("VoiceAssistantWebSocketServer requires a non-empty daemonVersion."); @@ -500,7 +546,8 @@ export class VoiceAssistantWebSocketServer { private readonly wss: WebSocketServer; private readonly pendingConnections: Map = new Map(); private readonly sessions: Map = new Map(); - private readonly inFlightCreateMessages = new Set>(); + private readonly inFlightLifecycleMutations = new Set>(); + private acceptingLifecycleMutations = true; private readonly socketIdentities: Map = new Map(); private readonly externalSessionsByKey: Map = new Map(); private readonly serverId: string; @@ -954,8 +1001,9 @@ export class VoiceAssistantWebSocketServer { public async prepareForShutdown(): Promise { this.acceptingConnections = false; - while (this.inFlightCreateMessages.size > 0) { - await Promise.allSettled(Array.from(this.inFlightCreateMessages)); + this.acceptingLifecycleMutations = false; + while (this.inFlightLifecycleMutations.size > 0) { + await Promise.allSettled(Array.from(this.inFlightLifecycleMutations)); } } @@ -2068,12 +2116,14 @@ export class VoiceAssistantWebSocketServer { } if (message.type === "session") { + const trackedForShutdown = SHUTDOWN_TRACKED_SESSION_MUTATIONS.has(message.message.type); + if (trackedForShutdown && !this.acceptingLifecycleMutations) { + ws.close(WS_CLOSE_SERVER_SHUTDOWN, "Server shutting down"); + return; + } const dispatch = this.dispatchSessionMessage(ws, activeConnection, message); - const trackedForShutdown = - message.message.type === "create_agent_request" || - message.message.type === "create_paseo_worktree_request"; if (trackedForShutdown) { - this.inFlightCreateMessages.add(dispatch); + this.inFlightLifecycleMutations.add(dispatch); } void dispatch .catch((error: unknown) => { @@ -2081,7 +2131,7 @@ export class VoiceAssistantWebSocketServer { }) .finally(() => { if (trackedForShutdown) { - this.inFlightCreateMessages.delete(dispatch); + this.inFlightLifecycleMutations.delete(dispatch); } }); } From f7cfc6ca5bf011091d76f784c242c669017ae47f Mon Sep 17 00:00:00 2001 From: Timi Ajiboye Date: Sat, 1 Aug 2026 16:59:35 -0400 Subject: [PATCH 43/43] fix: preserve canonical worktree workspace ownership --- .../server/src/server/bootstrap.smoke.test.ts | 83 +++++++++++++++++-- .../src/server/paseo-worktree-service.test.ts | 21 +++-- .../workspace-provisioning-service.test.ts | 38 +++++++++ .../workspace-provisioning-service.ts | 63 ++++++++------ 4 files changed, 169 insertions(+), 36 deletions(-) diff --git a/packages/server/src/server/bootstrap.smoke.test.ts b/packages/server/src/server/bootstrap.smoke.test.ts index 03955ffada..9489e3d5f7 100644 --- a/packages/server/src/server/bootstrap.smoke.test.ts +++ b/packages/server/src/server/bootstrap.smoke.test.ts @@ -2,7 +2,7 @@ import os from "node:os"; import http from "node:http"; import path from "node:path"; import { execFileSync } from "node:child_process"; -import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import pino from "pino"; import { afterEach, describe, expect, test, vi } from "vitest"; import { WebSocket } from "ws"; @@ -21,6 +21,7 @@ import { findFreePort } from "./service-proxy.js"; import { defaultWorkspaceLifecycleCoordinator } from "./workspace-lifecycle-coordinator.js"; import { readPaseoWorktreeIncarnationId } from "../utils/worktree-metadata.js"; import { getPaseoWorktreesRoot } from "../utils/worktree.js"; +import { createRealpathAwarePathMatcher } from "../utils/path.js"; import { Session } from "./session.js"; import { FileBackedProjectRegistry, FileBackedWorkspaceRegistry } from "./workspace-registry.js"; import type { @@ -853,24 +854,46 @@ describe("paseo daemon bootstrap", () => { let firstDaemon: Awaited> | null = null; let secondDaemon: Awaited> | null = null; let firstMcpClient: Awaited> | null = null; + let secondDaemonClient: DaemonClient | null = null; const staticDirs: string[] = []; try { firstDaemon = await createTestPaseoDaemon({ paseoHomeRoot, cleanup: false }); staticDirs.push(firstDaemon.staticDir); firstMcpClient = await createBootstrapAgentMcpClient(firstDaemon.port); + const originalWorkspaceResult = await firstMcpClient.callTool({ + name: "create_workspace", + args: { + isolation: "worktree", + path: repoDir, + worktreeSlug: "same-slug-retry", + branchName: "same-slug-retry", + baseBranch: "main", + }, + }); + const originalWorkspace = originalWorkspaceResult.structuredContent as + | { cwd?: unknown; workspaceId?: unknown } + | undefined; + if ( + typeof originalWorkspace?.cwd !== "string" || + typeof originalWorkspace.workspaceId !== "string" + ) { + throw new Error("Expected original MCP worktree workspace identifiers"); + } + const worktreeCwd = originalWorkspace.cwd; + const originalWorkspaceId = originalWorkspace.workspaceId; + expect(listGitWorktreePaths(repoDir)).toContain(worktreeCwd); const failed = await firstMcpClient.callTool({ name: "create_agent", args: { cwd: repoDir, worktreeName: "same-slug-retry", - branchName: "same-slug-retry", - baseBranch: "main", + refName: "same-slug-retry", title: "Failed same-slug create", provider: "codex/gpt-5.4", mode: "invalid-test-mode", - initialPrompt: "This create should fail after making its worktree", + initialPrompt: "This create should fail after reusing its worktree", background: true, }, }); @@ -878,7 +901,10 @@ describe("paseo daemon bootstrap", () => { await expect(firstDaemon.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( [], ); - expect(listGitWorktreePaths(repoDir)).toHaveLength(1); + expect(await activeWorkspaceIdsAtWorktreeRoot(firstDaemon.paseoHome, worktreeCwd)).toEqual([ + originalWorkspaceId, + ]); + expect(listGitWorktreePaths(repoDir)).toContain(worktreeCwd); const retried = await firstMcpClient.callTool({ name: "create_agent", @@ -909,10 +935,14 @@ describe("paseo daemon bootstrap", () => { cwd: string; workspaceId: string; }; - expect(path.basename(cwd)).toBe("same-slug-retry"); + expect(cwd).toBe(worktreeCwd); + expect(workspaceId).toBe(originalWorkspaceId); await expect(firstDaemon.daemon.agentStorage.listPendingAgentCreations()).resolves.toEqual( [], ); + expect(await activeWorkspaceIdsAtWorktreeRoot(firstDaemon.paseoHome, worktreeCwd)).toEqual([ + originalWorkspaceId, + ]); await firstMcpClient.close(); firstMcpClient = null; @@ -921,6 +951,11 @@ describe("paseo daemon bootstrap", () => { secondDaemon = await createTestPaseoDaemon({ paseoHomeRoot, cleanup: false }); staticDirs.push(secondDaemon.staticDir); + secondDaemonClient = new DaemonClient({ + url: `ws://127.0.0.1:${secondDaemon.port}/ws`, + appVersion: "0.1.82", + }); + await secondDaemonClient.connect(); const recoveredAgent = await secondDaemon.daemon.agentStorage.get(agentId); expect(recoveredAgent).toMatchObject({ id: agentId, workspaceId }); @@ -930,7 +965,21 @@ describe("paseo daemon bootstrap", () => { ); await expect(stat(cwd)).resolves.toMatchObject({}); expect(listGitWorktreePaths(repoDir)).toContain(cwd); + expect(await activeWorkspaceIdsAtWorktreeRoot(secondDaemon.paseoHome, worktreeCwd)).toEqual([ + workspaceId, + ]); + + await expect(secondDaemonClient.archiveWorkspace(workspaceId)).resolves.toMatchObject({ + workspaceId, + error: null, + }); + await expect(stat(worktreeCwd)).rejects.toMatchObject({ code: "ENOENT" }); + expect(listGitWorktreePaths(repoDir)).not.toContain(worktreeCwd); + expect(await activeWorkspaceIdsAtWorktreeRoot(secondDaemon.paseoHome, worktreeCwd)).toEqual( + [], + ); } finally { + await secondDaemonClient?.close().catch(() => undefined); await firstMcpClient?.close().catch(() => undefined); await secondDaemon?.close().catch(() => undefined); await firstDaemon?.close().catch(() => undefined); @@ -1561,6 +1610,28 @@ function listGitWorktreePaths(repoDir: string): string[] { .flatMap((line) => (line.startsWith("worktree ") ? [line.slice("worktree ".length)] : [])); } +async function activeWorkspaceIdsAtWorktreeRoot( + paseoHome: string, + worktreeRoot: string, +): Promise { + const records = JSON.parse( + await readFile(path.join(paseoHome, "projects", "workspaces.json"), "utf8"), + ) as Array<{ + workspaceId: string; + cwd: string; + worktreeRoot?: string | null; + archivedAt?: string | null; + }>; + const matchesWorktreeRoot = createRealpathAwarePathMatcher(worktreeRoot); + return records + .filter( + (workspace) => + !workspace.archivedAt && matchesWorktreeRoot(workspace.worktreeRoot ?? workspace.cwd), + ) + .map((workspace) => workspace.workspaceId) + .sort(); +} + async function beginDaemonShutdownWithAgentClosing(): Promise { const heldAgentClose = holdAgentClose(); const daemonHandle = await createTestPaseoDaemon({ diff --git a/packages/server/src/server/paseo-worktree-service.test.ts b/packages/server/src/server/paseo-worktree-service.test.ts index 5fec6bad05..f24a932487 100644 --- a/packages/server/src/server/paseo-worktree-service.test.ts +++ b/packages/server/src/server/paseo-worktree-service.test.ts @@ -460,7 +460,7 @@ test("an explicit project FK remains unchanged when its worktree comes from anot // POSIX-only: Windows git worktree paths need separate canonicalization coverage. test.skipIf(isPlatform("win32"))( - "reuses an existing worktree and still upserts the workspace", + "reuses an existing worktree with its canonical workspace identity", async () => { const { repoDir, tempDir } = createGitRepo(); cleanupPaths.push(tempDir); @@ -493,11 +493,20 @@ test.skipIf(isPlatform("win32"))( ); expect(second.created).toBe(false); - expect(second.worktree.worktreePath).toBe(first.worktree.worktreePath); - expect(events).toContain(`workspace:${second.workspace.workspaceId}`); - // Creation never dedupes by directory: the same worktree path yields a - // distinct workspace record on the second call. - expect(second.workspace.workspaceId).not.toBe(first.workspace.workspaceId); + expect( + createRealpathAwarePathMatcher(first.worktree.worktreePath)(second.worktree.worktreePath), + ).toBe(true); + expect(second.workspace.workspaceId).toBe(first.workspace.workspaceId); + expect(events).toEqual([]); + expect( + Array.from(deps.workspaces.values()).filter( + (workspace) => + !workspace.archivedAt && + createRealpathAwarePathMatcher(first.worktree.worktreePath)( + workspace.worktreeRoot ?? workspace.cwd, + ), + ), + ).toEqual([first.workspace]); }, ); diff --git a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts index 810f0b7219..b9e98d96fd 100644 --- a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts +++ b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.test.ts @@ -145,6 +145,44 @@ test("re-opening an active workspace by exact path returns the same record witho expect(await workspaceRegistry.list()).toHaveLength(1); }); +test("concurrent managed worktree provisioning preserves one canonical active owner", async () => { + const repoRoot = path.join(tmpDir, "repo"); + const worktreeRoot = path.join(tmpDir, "worktrees", "same-slug"); + gitRoots.add(repoRoot); + gitRoots.add(worktreeRoot); + gitBranches.set(worktreeRoot, "same-slug"); + const project = await projectRegistry.getOrCreateActiveByRoot({ + rootPath: repoRoot, + kind: "git", + displayName: "repo", + timestamp: "2026-08-01T00:00:00.000Z", + }); + const input = { + sourceCwd: repoRoot, + projectId: project.projectId, + repoRoot, + cwd: worktreeRoot, + worktreeRoot, + branch: "same-slug", + baseBranch: "main", + title: "Canonical owner", + expectsInitialAgent: false, + }; + + const [created, reused] = await Promise.all([ + provisioning.createWorkspaceForWorktree(input), + provisioning.createWorkspaceForWorktree({ ...input, title: "Duplicate attempt" }), + ]); + + expect(reused.workspaceId).toBe(created.workspaceId); + expect(reused.title).toBe("Canonical owner"); + expect( + (await workspaceRegistry.list()).filter( + (workspace) => !workspace.archivedAt && workspace.worktreeRoot === worktreeRoot, + ), + ).toEqual([created]); +}); + test("re-opening Windows-equivalent workspace cwd spellings reuses the active and archived record", async () => { const cwd = path.join(tmpDir, "workspace"); const created = await provisioning.findOrCreateWorkspaceForDirectory(cwd); diff --git a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts index c652eae0f0..37ffae1803 100644 --- a/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts +++ b/packages/server/src/server/session/workspace-provisioning/workspace-provisioning-service.ts @@ -249,31 +249,46 @@ export function createWorkspaceProvisioningService(deps: { const repoRoot = resolve(input.repoRoot); const cwd = resolve(input.cwd); const worktreeRoot = resolve(input.worktreeRoot); - const project = await resolveSourceProjectForWorktree({ - sourceCwd, - projectId: input.projectId, - repoRoot, - }); - const timestamp = new Date().toISOString(); - const workspace = createPersistedWorkspaceRecord({ - workspaceId: generateWorkspaceId(), - projectId: project.projectId, - ...initialWorkspacePlacement({ - source: "created_worktree", - cwd, - worktreeRoot, - branch: input.branch, - baseBranch: input.baseBranch, - mainRepoRoot: repoRoot, - }), - title: input.title, - createdAt: timestamp, - updatedAt: timestamp, - }); - await workspaceRegistry.upsert(workspace, { - expectsInitialAgent: input.expectsInitialAgent, + return lifecycleCoordinator.runDirectoryExclusive(worktreeRoot, async () => { + const matchesWorktreeRoot = createRealpathAwarePathMatcher(worktreeRoot); + const activeOwner = (await workspaceRegistry.list()) + .filter( + (workspace) => + !workspace.archivedAt && matchesWorktreeRoot(workspace.worktreeRoot ?? workspace.cwd), + ) + .sort( + (left, right) => + Date.parse(left.createdAt) - Date.parse(right.createdAt) || + left.workspaceId.localeCompare(right.workspaceId), + )[0]; + if (activeOwner) return activeOwner; + + const project = await resolveSourceProjectForWorktree({ + sourceCwd, + projectId: input.projectId, + repoRoot, + }); + const timestamp = new Date().toISOString(); + const workspace = createPersistedWorkspaceRecord({ + workspaceId: generateWorkspaceId(), + projectId: project.projectId, + ...initialWorkspacePlacement({ + source: "created_worktree", + cwd, + worktreeRoot, + branch: input.branch, + baseBranch: input.baseBranch, + mainRepoRoot: repoRoot, + }), + title: input.title, + createdAt: timestamp, + updatedAt: timestamp, + }); + await workspaceRegistry.upsert(workspace, { + expectsInitialAgent: input.expectsInitialAgent, + }); + return workspace; }); - return workspace; } async function resolveSourceProjectForWorktree(input: {