From b3eeafeb29b4005847d0a40e9cbfe3837e8d02cc Mon Sep 17 00:00:00 2001 From: Wibus Wu <62133302+wibus-wee@users.noreply.github.com> Date: Sun, 6 Sep 2026 10:09:36 +0800 Subject: [PATCH 1/3] fix(cli): prevent operation completion replay Fence delivery execution by Worker generation and exclusive claims, bound recovery, preserve uncertain outcomes, and retry terminal settlement without relaunching ACP. Model: gpt-5 --- apps/cli/src/orchestration/AGENTS.md | 41 +- .../operation-coordinator.test.ts | 1344 ++++++++++++++++- .../orchestration/operation-coordinator.ts | 520 ++++++- .../src/orchestration/operation-model.test.ts | 208 ++- apps/cli/src/orchestration/operation-model.ts | 193 ++- .../src/orchestration/operation-store.test.ts | 570 ++++++- apps/cli/src/orchestration/operation-store.ts | 559 ++++++- .../src/session/session-execution-service.ts | 181 ++- .../tests/session-execution-service.test.ts | 293 +++- locales/en.json | 1 + locales/zh_CN.json | 1 + .../components/src/components/ai-gui/view.tsx | 16 +- packages/shared/src/message-schemas.ts | 8 +- packages/shared/src/session-orchestration.ts | 11 +- 14 files changed, 3731 insertions(+), 215 deletions(-) diff --git a/apps/cli/src/orchestration/AGENTS.md b/apps/cli/src/orchestration/AGENTS.md index 7850b8887..59c2baa57 100644 --- a/apps/cli/src/orchestration/AGENTS.md +++ b/apps/cli/src/orchestration/AGENTS.md @@ -45,7 +45,9 @@ Root and `apps/cli/AGENTS.md` apply. Normative behavior lives in store work per raw fs event. - The MCP server process also holds ONE store connection (lazy singleton), opened with `maintenance: false` so non-owner opens are not themselves write - transactions; the daemon coordinator owns open-time repair/cleanup. Do not + transactions; current-schema detection is read-only and migration takes the + writer lock only when that probe finds work. The daemon coordinator owns + open-time repair/cleanup. Do not reintroduce per-call open/close: each close checkpoints against the shared WAL and each default open writes, which is the "database is locked" source. - WAL allows one writer machine-wide. Every writing store transaction runs @@ -59,6 +61,43 @@ Root and `apps/cli/AGENTS.md` apply. Normative behavior lives in - Delivery never writes user dispatch pointers. Pending user input wins every idle boundary; completion uses a stable `role: system` `operation_completion` Turn and then the existing Session execution mutex. + Its Assistant Turn id is `assistant:` even though it has no user + dispatch ownership. Assistant `finished`/`endedAt` is never Delivery completion + evidence: teardown writes the same terminal footprint. Delivery execution has three + fencing layers: the Host lease excludes other Hosts; each CLI Worker process owns one boot + id and starts only after the supervisor/Host-lease lifecycle barrier; and each attempt owns + a fresh token. + Execution fields live in `delivery_execution_state`, not `deliveries`: stable binaries parse + `SELECT * FROM deliveries` strictly, so adding columns there makes a downgraded binary unable + to read the shared local database. + Normal claims require no active token and never take over another owner. Paths that write + a terminal continuation failure or consume without execution must acquire the same + exclusive token first; the history write and token-matched consume happen while it is + held. Failed finalization retains the token and unfinished steps for later wakes: retry + history before consume, never ACP or a cleanup write. Recheck ownership after history + awaits; do not rewrite durable history. Stop drops this memory; replacement Workers use + durable state. Once per Worker startup, + the coordinator clears tokens owned by older boot ids + without resetting the attempt count. A claim records `claimed`, becomes `prepared` only + after the completion Turn is durable (which spends one bounded preparation attempt), and + becomes `started` immediately before calling ACP. Release and consume must match both the + boot id and claim token. Claim contention exits before history or ACP side effects and + records no failure. Only a confirmed pre-provider interruption releases a prepared claim; + a rejected start-fence write settles as not started and follows that same release path instead + of becoming a handled turn. User cancellation consumes it. A missing settlement after ACP started becomes `uncertain` + and is never automatically replayed: reconciliation writes + `DELIVERY_EXECUTION_UNCERTAIN` under a terminal claim, preserves existing output, and tells + the user to continue manually if needed. Provider-accepted steer settles the original + Delivery immediately, so cancellation of a later user-owned turn cannot reopen it. + Settlement write failure retains the claim-bound outcome in the live coordinator and retries + it on later wakes without ACP; replacement-Worker recovery converts any still-fenced started + claim to `uncertain`, never to runnable. A + coordinated workspace stop abandons only that coordinator's claims before closing its store: + `claimed`/`prepared` become runnable and `started` becomes `uncertain`. At most + one confirmed pre-provider recovery is allowed; after two prepared attempts, + `DELIVERY_ATTEMPTS_EXHAUSTED` is written and consumed without invoking ACP. A pending + Delivery from the pre-claim schema migrates as `uncertain`; its prior execution count is + unknowable and must not be fabricated. - Missing Session metadata, a recoverable tombstone, or an unsynchronized Machine Flock document is uncertainty, not permanent deletion/configuration absence. Keep the item/Delivery pending until positive evidence or deadline. diff --git a/apps/cli/src/orchestration/operation-coordinator.test.ts b/apps/cli/src/orchestration/operation-coordinator.test.ts index 6e733657c..3bb0035a2 100644 --- a/apps/cli/src/orchestration/operation-coordinator.test.ts +++ b/apps/cli/src/orchestration/operation-coordinator.test.ts @@ -25,6 +25,14 @@ import { LodyOperationStore } from './operation-store'; const roots = new Set(); const TEST_NOW_MS = Date.parse('2026-07-20T00:00:00Z'); +type DeliveryDispatchOptions = { + onTurnClaimed?: () => Promise; + onTurnStarted?: () => Promise; + onTurnSettled?: ( + settlement: 'handled' | 'cancelled' | 'not_started' | 'uncertain' + ) => Promise; +}; + const makeHarness = async (options?: { deadlineAt?: string; requesterArchived?: boolean; @@ -34,6 +42,7 @@ const makeHarness = async (options?: { agentConfigId?: string; configurationSyncSucceeds?: boolean; configurationSync?: () => Promise; + beforeTurnClaim?: () => Promise; beforeTargetMetaRead?: () => Promise; now?: () => number; machineAgentConfig?: AgentConfigMeta; @@ -44,11 +53,14 @@ const makeHarness = async (options?: { materializationFailuresBeforeSuccess?: number; materializationWritesBeforeFailure?: boolean; materializationWritesDocBeforeFailure?: boolean; + historyFailuresBeforeSuccess?: number; + beforeRequesterHistoryWrite?: () => Promise; materializeTargetOverride?: () => Promise; targetDocSync?: () => Promise<{ history?: SessionHistoryInput[]; meta?: SessionMeta; } | void>; + workerBootId?: string; }) => { const root = await mkdtemp(path.join(os.tmpdir(), 'lody-operation-coordinator-')); roots.add(root); @@ -101,6 +113,7 @@ const makeHarness = async (options?: { ...(targetInputDurable ? ([[targetSessionId, targetMeta]] as const) : []), ]); const subscribers = new Map void>>(); + let historyUpdateAttempt = 0; const sessionDoc = (sessionId: SessionId) => ({ mirror: { subscribe: (callback: () => void) => { @@ -112,6 +125,13 @@ const makeHarness = async (options?: { }, getHistory: async () => histories.get(sessionId) ?? [], updateHistory: async (update: (history: SessionHistoryInput[]) => SessionHistoryInput[]) => { + if (sessionId === requesterSessionId) { + await options?.beforeRequesterHistoryWrite?.(); + historyUpdateAttempt += 1; + if (historyUpdateAttempt <= (options?.historyFailuresBeforeSuccess ?? 0)) { + throw new Error('transient history write failure'); + } + } histories.set(sessionId, update(histories.get(sessionId) ?? [])); }, }); @@ -155,20 +175,24 @@ const makeHarness = async (options?: { let pendingUser = options?.pendingUser ?? false; let busy = options?.busy ?? false; const continueSession = vi.fn(async (message: unknown, dispatchOptions: unknown) => { - const typedMessage = message as { sessionId: SessionId }; - const typedOptions = dispatchOptions as { onTurnClaimed?: () => Promise }; - await typedOptions.onTurnClaimed?.(); + const typedMessage = message as { sessionId: SessionId; userTurnId: string }; + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + await options?.beforeTurnClaim?.(); + if ((await typedOptions.onTurnClaimed?.()) === false) return; + const assistantTurnId = `assistant:${typedMessage.userTurnId}`; histories.set(typedMessage.sessionId, [ ...(histories.get(typedMessage.sessionId) ?? []), { - id: `assistant:${histories.get(typedMessage.sessionId)?.length ?? 0}`, + id: assistantTurnId, role: 'assistant', + userTurnId: typedMessage.userTurnId, timestamp: '2026-07-20T00:00:01.000Z', items: [{ type: 'text', text: 'continued' }], fileDiff: [], finished: true, }, ]); + await typedOptions.onTurnSettled?.('handled'); }); const logger = { warn: vi.fn(), debug: vi.fn() }; const syncMachineFlockDoc = vi.fn( @@ -236,7 +260,7 @@ const makeHarness = async (options?: { } if (shouldFail) throw new Error('transient Streams failure'); }); - const coordinator = new LodyOperationCoordinator({ + const coordinatorOptions = { workspaceId, machineId, userId: 'user-1', @@ -259,12 +283,14 @@ const makeHarness = async (options?: { storeFactory, storePath, now: options?.now ?? (() => TEST_NOW_MS), + ...(options?.workerBootId ? { workerBootId: options.workerBootId } : {}), operationStoreWatchFactory: (_directory, onChange) => { operationStoreWake = onChange; return { close: vi.fn() }; }, materializeTarget, - }); + } satisfies ConstructorParameters[0]; + const coordinator = new LodyOperationCoordinator(coordinatorOptions); const store = new LodyOperationStore(storePath, () => TEST_NOW_MS); store.accept({ workspaceId, @@ -292,6 +318,7 @@ const makeHarness = async (options?: { store.close(); return { coordinator, + coordinatorOptions, continueSession, resolveUser, histories, @@ -816,7 +843,32 @@ describe('LodyOperationCoordinator', () => { it('expires a Delivery 8h past its Operation deadline instead of waking the requester', async () => { // deadline + 8h grace lands exactly on TEST_NOW: stranded completions from // a long-dead store or downtime must not restart old conversations. - const harness = await makeHarness({ deadlineAt: '2026-07-19T16:00:00.000Z' }); + const harness = await makeHarness({ + deadlineAt: '2026-07-19T16:00:00.000Z', + workerBootId: 'worker-new', + }); + const oldStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + oldStore.finish(harness.requesterSessionId, 'review-round-1', { type: 'cancelled' }); + oldStore.claimDeliveryExecution(harness.requesterSessionId, 'review-round-1', { + claimId: 'stale-attempt', + workerBootId: 'worker-old', + }); + oldStore.prepareClaimedDeliveryExecution( + harness.requesterSessionId, + 'review-round-1', + 'worker-old', + 'stale-attempt' + ); + oldStore.markClaimedDeliveryExecutionStarted( + harness.requesterSessionId, + 'review-round-1', + 'worker-old', + 'stale-attempt' + ); + } finally { + oldStore.close(); + } harness.coordinator.start(); await harness.coordinator.idle(); harness.coordinator.stop(); @@ -869,7 +921,7 @@ describe('LodyOperationCoordinator', () => { }); harness.continueSession.mockImplementation(async (message, dispatchOptions) => { const typedMessage = message as { sessionId: SessionId }; - const typedOptions = dispatchOptions as { onTurnClaimed?: () => Promise }; + const typedOptions = dispatchOptions as DeliveryDispatchOptions; await typedOptions.onTurnClaimed?.(); markDeliveryClaimed(); await deliveryReleased; @@ -884,6 +936,7 @@ describe('LodyOperationCoordinator', () => { finished: true, }, ]); + await typedOptions.onTurnSettled?.('handled'); }); harness.coordinator.start(); @@ -1028,7 +1081,7 @@ describe('LodyOperationCoordinator', () => { }); harness.continueSession.mockImplementation(async (message, dispatchOptions) => { const typedMessage = message as { sessionId: SessionId }; - const typedOptions = dispatchOptions as { onTurnClaimed?: () => Promise }; + const typedOptions = dispatchOptions as DeliveryDispatchOptions; await typedOptions.onTurnClaimed?.(); harness.histories.set(typedMessage.sessionId, [ ...(harness.histories.get(typedMessage.sessionId) ?? []), @@ -1047,6 +1100,7 @@ describe('LodyOperationCoordinator', () => { finished: true, }, ]); + await typedOptions.onTurnSettled?.('handled'); }); harness.coordinator.start(); @@ -1188,19 +1242,415 @@ describe('LodyOperationCoordinator', () => { ]); }); - it('consumes existing continuation evidence before configuration lookup or sync', async () => { + it('retries terminal settlement after its consume write fails', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + agentConfigId: 'removed-agent-config', + configurationSyncSucceeds: true, + }); + const consume = vi.spyOn(LodyOperationStore.prototype, 'consumeClaimedDelivery'); + const originalConsume = consume.getMockImplementation(); + consume.mockImplementationOnce(() => { + throw new Error('terminal settlement write failed'); + }); + if (originalConsume) consume.mockImplementation(originalConsume); + + try { + harness.coordinator.start(); + await harness.coordinator.idle(); + + const afterFailure = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + const delivery = afterFailure.getDelivery(harness.requesterSessionId, 'review-round-1'); + expect(delivery).toMatchObject({ state: 'pending', attemptCount: 0 }); + expect(delivery.activeClaimId).toEqual(expect.any(String)); + expect(delivery.activeClaimWorkerBootId).toEqual(expect.any(String)); + } finally { + afterFailure.close(); + } + expect(harness.histories.get(harness.requesterSessionId)).toEqual([ + expect.objectContaining({ + items: [ + expect.objectContaining({ + type: 'operation_completion', + continuation: { + status: 'not_started', + reason: expect.objectContaining({ code: 'CONFIGURATION_UNAVAILABLE' }), + }, + }), + ], + }), + ]); + + await harness.coordinator.wake('retry-terminal-settlement-1'); + await harness.coordinator.idle(); + await harness.coordinator.wake('retry-terminal-settlement-2'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).not.toHaveBeenCalled(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 0, + }); + } finally { + finalStore.close(); + } + expect(harness.histories.get(harness.requesterSessionId)).toHaveLength(1); + } finally { + harness.coordinator.stop(); + consume.mockRestore(); + } + }); + + it.each([0, 2])( + 'recovers terminal settlement during a write outage with %i failed history writes', + async (historyFailuresBeforeSuccess) => { + let rejectFurtherHistoryWrites = false; + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + agentConfigId: 'removed-agent-config', + configurationSyncSucceeds: true, + historyFailuresBeforeSuccess, + workerBootId: 'worker-a', + beforeRequesterHistoryWrite: async () => { + if (rejectFurtherHistoryWrites) throw new Error('history is unavailable again'); + }, + }); + let writeOutage = true; + const originalConsume = LodyOperationStore.prototype.consumeClaimedDelivery; + const originalRelease = LodyOperationStore.prototype.releaseDeliveryClaim; + const consume = vi.spyOn(LodyOperationStore.prototype, 'consumeClaimedDelivery'); + const release = vi.spyOn(LodyOperationStore.prototype, 'releaseDeliveryClaim'); + consume.mockImplementation(function (this: LodyOperationStore, ...args) { + if (writeOutage) throw new Error('terminal database write outage'); + return originalConsume.apply(this, args); + }); + release.mockImplementation(function (this: LodyOperationStore, ...args) { + if (writeOutage) throw new Error('terminal database write outage'); + return originalRelease.apply(this, args); + }); + const inspect = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + harness.coordinator.start(); + await harness.coordinator.idle(); + const claimed = inspect.getDelivery(harness.requesterSessionId, 'review-round-1'); + expect(claimed.activeClaimId).toEqual(expect.any(String)); + if (historyFailuresBeforeSuccess > 0) { + expect(harness.histories.get(harness.requesterSessionId)).toEqual([]); + await harness.coordinator.wake('history-still-unavailable'); + await harness.coordinator.idle(); + expect(harness.histories.get(harness.requesterSessionId)).toEqual([]); + expect(inspect.getDelivery(harness.requesterSessionId, 'review-round-1').state).toBe( + 'pending' + ); + } + for (let wake = 0; wake < 3; wake += 1) { + await harness.coordinator.wake('terminal-database-still-unavailable'); + await harness.coordinator.idle(); + } + expect(inspect.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'pending', + attemptCount: 0, + activeClaimId: claimed.activeClaimId, + }); + expect(harness.histories.get(harness.requesterSessionId)).toHaveLength(1); + + rejectFurtherHistoryWrites = true; + writeOutage = false; + await harness.coordinator.wake('terminal-database-recovered'); + await harness.coordinator.idle(); + expect(inspect.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 0, + }); + await harness.coordinator.wake('already-finalized'); + await harness.coordinator.idle(); + expect(harness.continueSession).not.toHaveBeenCalled(); + expect(harness.histories.get(harness.requesterSessionId)).toHaveLength(1); + } finally { + consume.mockRestore(); + release.mockRestore(); + harness.coordinator.stop(); + inspect.close(); + } + } + ); + + it.each(['worker-a', 'worker-b'])( + 'does not consume a replacement terminal claim owned by %s after history yields', + async (replacementBootId) => { + let historyStarted!: () => void; + let finishHistory!: () => void; + const started = new Promise((resolve) => { + historyStarted = resolve; + }); + const finish = new Promise((resolve) => { + finishHistory = resolve; + }); + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + agentConfigId: 'removed-agent-config', + configurationSyncSucceeds: true, + workerBootId: 'worker-a', + beforeRequesterHistoryWrite: async () => { + historyStarted(); + await finish; + }, + }); + const store = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + harness.coordinator.start(); + await started; + store.abandonDeliveryClaimsOwnedBy('workspace-1' as WorkspaceId, 'worker-a'); + expect( + store.claimDeliveryFinalization(harness.requesterSessionId, 'review-round-1', { + claimId: 'replacement-terminal-claim', + workerBootId: replacementBootId, + }).status + ).toBe('claimed'); + finishHistory(); + await harness.coordinator.idle(); + await harness.coordinator.wake('stale-finalization'); + await harness.coordinator.idle(); + expect(store.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'pending', + activeClaimId: 'replacement-terminal-claim', + activeClaimWorkerBootId: replacementBootId, + attemptCount: 0, + }); + expect(harness.continueSession).not.toHaveBeenCalled(); + } finally { + finishHistory(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + store.close(); + } + } + ); + + it('does not write configuration failure when another Worker claims during config sync', async () => { + let markSyncStarted!: () => void; + let resolveSync!: (value: boolean) => void; + const syncStarted = new Promise((resolve) => { + markSyncStarted = resolve; + }); + const syncResult = new Promise((resolve) => { + resolveSync = resolve; + }); + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + agentConfigId: 'removed-agent-config', + workerBootId: 'worker-a', + configurationSync: async () => { + markSyncStarted(); + return await syncResult; + }, + }); + + harness.coordinator.start(); + await syncStarted; + const competitorStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect( + competitorStore.claimDeliveryExecution(harness.requesterSessionId, 'review-round-1', { + claimId: 'attempt-b', + workerBootId: 'worker-b', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 0 } }); + } finally { + competitorStore.close(); + } + resolveSync(true); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).not.toHaveBeenCalled(); + expect(harness.histories.get(harness.requesterSessionId)).toEqual([]); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'pending', + activeClaimId: 'attempt-b', + activeClaimWorkerBootId: 'worker-b', + }); + } finally { + finalStore.close(); + } + }); + + it('does not finalize a Delivery after stopping during configuration sync', async () => { + let markSyncStarted!: () => void; + let resolveSync!: (value: boolean) => void; + const syncStarted = new Promise((resolve) => { + markSyncStarted = resolve; + }); + const syncResult = new Promise((resolve) => { + resolveSync = resolve; + }); const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z', agentConfigId: 'removed-agent-config', + configurationSync: async () => { + markSyncStarted(); + return await syncResult; + }, + }); + + harness.coordinator.start(); + await syncStarted; + const oldWork = harness.coordinator.idle(); + harness.coordinator.stop(); + resolveSync(true); + await oldWork; + + expect(harness.continueSession).not.toHaveBeenCalled(); + expect(harness.histories.get(harness.requesterSessionId)).toEqual([]); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + const delivery = finalStore.getDelivery(harness.requesterSessionId, 'review-round-1'); + expect(delivery).toMatchObject({ + state: 'pending', + executionPhase: 'ready', + }); + expect(delivery.activeClaimId).toBeUndefined(); + } finally { + finalStore.close(); + } + }); + + it('does not claim a Delivery after stopping before the execution claim', async () => { + let markClaimPending!: () => void; + let releaseClaim!: () => void; + const claimPending = new Promise((resolve) => { + markClaimPending = resolve; + }); + const claimReleased = new Promise((resolve) => { + releaseClaim = resolve; + }); + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + beforeTurnClaim: async () => { + markClaimPending(); + await claimReleased; + }, }); + + harness.coordinator.start(); + await claimPending; + const oldWork = harness.coordinator.idle(); + harness.coordinator.stop(); + releaseClaim(); + await oldWork; + + expect(harness.continueSession).toHaveBeenCalledOnce(); + expect(harness.histories.get(harness.requesterSessionId)).toEqual([]); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + const delivery = finalStore.getDelivery(harness.requesterSessionId, 'review-round-1'); + expect(delivery).toMatchObject({ + state: 'pending', + executionPhase: 'ready', + attemptCount: 0, + }); + expect(delivery.activeClaimId).toBeUndefined(); + } finally { + finalStore.close(); + } + }); + + it('clears a stale non-started marker when recovered execution begins', async () => { + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); harness.histories.set(harness.requesterSessionId, [ { id: 'operation-completion:requester-1:review-round-1', role: 'system', timestamp: '2026-07-20T00:00:00.000Z', - items: [], + items: [ + { + type: 'operation_completion', + deliveryId: 'operation:requester-1:review-round-1:completion', + operationId: 'review-round-1', + operationKind: 'session_chat', + completion: { type: 'cancelled' }, + continuation: { + status: 'not_started', + reason: { + code: 'CONFIGURATION_UNAVAILABLE', + message: 'The frozen continuation agent configuration is no longer available.', + }, + }, + }, + ], + fileDiff: [], + finished: true, + }, + ]); + + harness.coordinator.start(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const completion = harness.histories + .get(harness.requesterSessionId) + ?.find((entry) => entry.role === 'system') + ?.items?.find((item) => item.type === 'operation_completion'); + expect(completion).not.toHaveProperty('continuation'); + }); + + it('does not replay after graceful teardown terminalizes a started Delivery turn', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + workerBootId: 'daemon-after-restart', + }); + const shutdownStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + shutdownStore.finish(harness.requesterSessionId, 'review-round-1', { type: 'cancelled' }); + expect( + shutdownStore.claimDeliveryExecution(harness.requesterSessionId, 'review-round-1', { + claimId: 'attempt-before-graceful-shutdown', + workerBootId: 'daemon-before-restart', + }) + ).toMatchObject({ status: 'claimed' }); + expect( + shutdownStore.prepareClaimedDeliveryExecution( + harness.requesterSessionId, + 'review-round-1', + 'daemon-before-restart', + 'attempt-before-graceful-shutdown' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 1 } }); + expect( + shutdownStore.markClaimedDeliveryExecutionStarted( + harness.requesterSessionId, + 'review-round-1', + 'daemon-before-restart', + 'attempt-before-graceful-shutdown' + ) + ).toBe(true); + } finally { + shutdownStore.close(); + } + harness.histories.set(harness.requesterSessionId, [ + { + id: 'operation-completion:requester-1:review-round-1', + role: 'system', + timestamp: '2026-07-20T00:00:00.000Z', + items: [ + { + type: 'operation_completion', + deliveryId: 'operation:requester-1:review-round-1:completion', + operationId: 'review-round-1', + operationKind: 'session_chat', + completion: { type: 'cancelled' }, + }, + ], fileDiff: [], finished: true, + endedAt: TEST_NOW_MS - 1, }, { id: 'assistant:operation-completion:requester-1:review-round-1', @@ -1216,55 +1666,263 @@ describe('LodyOperationCoordinator', () => { await harness.coordinator.idle(); harness.coordinator.stop(); - expect(harness.openFlockDoc).not.toHaveBeenCalled(); - expect(harness.syncMachineFlockDoc).not.toHaveBeenCalled(); expect(harness.continueSession).not.toHaveBeenCalled(); const store = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); try { expect(store.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([]); + expect(store.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + executionPhase: 'uncertain', + attemptCount: 1, + }); } finally { store.close(); } + expect(harness.histories.get(harness.requesterSessionId)?.[0]?.items).toEqual([ + expect.objectContaining({ + type: 'operation_completion', + continuation: { + status: 'uncertain', + reason: expect.objectContaining({ code: 'DELIVERY_EXECUTION_UNCERTAIN' }), + }, + }), + ]); }); - it('keeps an active continuation pending until durable completion evidence exists', async () => { - const systemTurnId = 'operation-completion:requester-1:review-round-1'; - const harness = await makeHarness({ - deadlineAt: '2026-07-19T23:59:59.000Z', - busy: true, - activeTurnId: `assistant:${systemTurnId}`, + it('recovers a started claim after the workspace coordinator restarts in one Worker', async () => { + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + let markStarted!: () => void; + const started = new Promise((resolve) => { + markStarted = resolve; + }); + let releaseOldExecution!: () => void; + const oldExecutionReleased = new Promise((resolve) => { + releaseOldExecution = resolve; + }); + let markOldExecutionReturned!: () => void; + const oldExecutionReturned = new Promise((resolve) => { + markOldExecutionReturned = resolve; + }); + harness.continueSession.mockImplementationOnce(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + expect(await typedOptions.onTurnClaimed?.()).toBe(true); + expect(await typedOptions.onTurnStarted?.()).toBe(true); + markStarted(); + await oldExecutionReleased; + markOldExecutionReturned(); }); - harness.histories.set(harness.requesterSessionId, [ - { - id: systemTurnId, - role: 'system', - timestamp: '2026-07-20T00:00:00.000Z', - items: [], - fileDiff: [], - finished: true, - }, - ]); harness.coordinator.start(); - await harness.coordinator.idle(); + await started; + const oldStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(oldStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'pending', + executionPhase: 'started', + activeClaimId: expect.any(String), + activeClaimWorkerBootId: expect.any(String), + }); + } finally { + oldStore.close(); + } + harness.coordinator.stop(); - expect(harness.continueSession).not.toHaveBeenCalled(); - const storeWhileActive = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + const replacement = new LodyOperationCoordinator(harness.coordinatorOptions); try { - expect(storeWhileActive.listPendingDeliveries('workspace-1' as WorkspaceId)).toHaveLength(1); + replacement.start(); + await replacement.idle(); } finally { - storeWhileActive.close(); + replacement.stop(); + releaseOldExecution(); + await oldExecutionReturned; } - // Model a hard-crashed turn: the active marker disappears without an - // assistant or chat_failed history entry. The pending Delivery must retry. - harness.setBusy(false); - await harness.coordinator.wake('active-turn-disappeared'); + expect(harness.continueSession).toHaveBeenCalledOnce(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + executionPhase: 'uncertain', + attemptCount: 1, + }); + } finally { + finalStore.close(); + } + }); + + it('consumes the Delivery when a user turn lands before its completed assistant turn', async () => { + const systemTurnId = 'operation-completion:requester-1:review-round-1'; + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + harness.continueSession.mockImplementation(async (message, dispatchOptions) => { + const typedMessage = message as { sessionId: SessionId }; + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + await typedOptions.onTurnClaimed?.(); + harness.histories.set(typedMessage.sessionId, [ + ...(harness.histories.get(typedMessage.sessionId) ?? []), + { + id: 'ordinary-user-turn', + role: 'user', + timestamp: '2026-07-20T00:00:00.500Z', + items: [{ type: 'text', text: 'new work' }], + fileDiff: [], + status: 'pending', + }, + { + id: `assistant:${systemTurnId}`, + role: 'assistant', + userTurnId: systemTurnId, + timestamp: '2026-07-20T00:00:01.000Z', + items: [{ type: 'text', text: 'completion handled' }], + fileDiff: [], + finished: true, + }, + ]); + await typedOptions.onTurnSettled?.('handled'); + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + await harness.coordinator.wake('duplicate-history-event'); await harness.coordinator.idle(); harness.coordinator.stop(); expect(harness.continueSession).toHaveBeenCalledOnce(); - const storeAfterRetry = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + const store = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(store.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([]); + } finally { + store.close(); + } + }); + + it('retries a Delivery when only its eager nonterminal assistant entry survived a crash', async () => { + const systemTurnId = 'operation-completion:requester-1:review-round-1'; + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + harness.histories.set(harness.requesterSessionId, [ + { + id: systemTurnId, + role: 'system', + timestamp: '2026-07-20T00:00:00.000Z', + items: [], + fileDiff: [], + finished: true, + }, + { + id: `assistant:${systemTurnId}`, + role: 'assistant', + userTurnId: systemTurnId, + timestamp: '2026-07-20T00:00:00.500Z', + items: [{ type: 'text', text: 'partial output before crash' }], + fileDiff: [], + finished: false, + }, + ]); + + harness.coordinator.start(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const store = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(store.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([]); + } finally { + store.close(); + } + }); + + it('does not consume a Delivery from an unrelated assistant turn', async () => { + const systemTurnId = 'operation-completion:requester-1:review-round-1'; + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + harness.histories.set(harness.requesterSessionId, [ + { + id: systemTurnId, + role: 'system', + timestamp: '2026-07-20T00:00:00.000Z', + items: [], + fileDiff: [], + finished: true, + }, + { + id: 'assistant:unrelated-user-turn', + role: 'assistant', + userTurnId: 'unrelated-user-turn', + timestamp: '2026-07-20T00:00:00.500Z', + items: [], + fileDiff: [], + finished: true, + }, + ]); + harness.continueSession.mockImplementation(async (message, dispatchOptions) => { + const typedMessage = message as { sessionId: SessionId }; + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + await typedOptions.onTurnClaimed?.(); + harness.histories.set(typedMessage.sessionId, [ + ...(harness.histories.get(typedMessage.sessionId) ?? []), + { + id: `assistant:${systemTurnId}`, + role: 'assistant', + userTurnId: systemTurnId, + timestamp: '2026-07-20T00:00:01.000Z', + items: [], + fileDiff: [], + finished: true, + }, + ]); + await typedOptions.onTurnSettled?.('handled'); + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const store = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(store.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([]); + } finally { + store.close(); + } + }); + + it('keeps an active continuation pending until durable completion evidence exists', async () => { + const systemTurnId = 'operation-completion:requester-1:review-round-1'; + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + busy: true, + activeTurnId: `assistant:${systemTurnId}`, + }); + harness.histories.set(harness.requesterSessionId, [ + { + id: systemTurnId, + role: 'system', + timestamp: '2026-07-20T00:00:00.000Z', + items: [], + fileDiff: [], + finished: true, + }, + ]); + + harness.coordinator.start(); + await harness.coordinator.idle(); + + expect(harness.continueSession).not.toHaveBeenCalled(); + const storeWhileActive = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(storeWhileActive.listPendingDeliveries('workspace-1' as WorkspaceId)).toHaveLength(1); + } finally { + storeWhileActive.close(); + } + + // Model a hard-crashed turn: the active marker disappears without an + // assistant or chat_failed history entry. The pending Delivery must retry. + harness.setBusy(false); + await harness.coordinator.wake('active-turn-disappeared'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const storeAfterRetry = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); try { expect(storeAfterRetry.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([]); } finally { @@ -1272,6 +1930,614 @@ describe('LodyOperationCoordinator', () => { } }); + it('recovers one prepared pre-provider Delivery after a replacement Worker starts', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + workerBootId: 'daemon-new', + }); + const oldStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + oldStore.finish(harness.requesterSessionId, 'review-round-1', { type: 'cancelled' }); + expect( + oldStore.claimDeliveryExecution(harness.requesterSessionId, 'review-round-1', { + claimId: 'attempt-before-crash', + workerBootId: 'daemon-old', + }) + ).toMatchObject({ status: 'claimed' }); + expect( + oldStore.prepareClaimedDeliveryExecution( + harness.requesterSessionId, + 'review-round-1', + 'daemon-old', + 'attempt-before-crash' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 1 } }); + } finally { + oldStore.close(); + } + + harness.coordinator.start(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 2, + }); + } finally { + finalStore.close(); + } + }); + + it('does not replay provider execution left uncertain by an exited Worker', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + workerBootId: 'daemon-new', + }); + const oldStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + oldStore.finish(harness.requesterSessionId, 'review-round-1', { type: 'cancelled' }); + oldStore.claimDeliveryExecution(harness.requesterSessionId, 'review-round-1', { + claimId: 'attempt-before-crash', + workerBootId: 'daemon-old', + }); + oldStore.prepareClaimedDeliveryExecution( + harness.requesterSessionId, + 'review-round-1', + 'daemon-old', + 'attempt-before-crash' + ); + expect( + oldStore.markClaimedDeliveryExecutionStarted( + harness.requesterSessionId, + 'review-round-1', + 'daemon-old', + 'attempt-before-crash' + ) + ).toBe(true); + } finally { + oldStore.close(); + } + + harness.coordinator.start(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).not.toHaveBeenCalled(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + executionPhase: 'uncertain', + attemptCount: 1, + }); + } finally { + finalStore.close(); + } + expect(harness.histories.get(harness.requesterSessionId)).toEqual([ + expect.objectContaining({ + role: 'system', + items: [ + expect.objectContaining({ + type: 'operation_completion', + continuation: { + status: 'uncertain', + reason: expect.objectContaining({ code: 'DELIVERY_EXECUTION_UNCERTAIN' }), + }, + }), + ], + }), + ]); + }); + + it('exhausts two orphaned pre-provider attempts without starting ACP a third time', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + workerBootId: 'worker-c', + }); + const crashedStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + crashedStore.finish(harness.requesterSessionId, 'review-round-1', { type: 'cancelled' }); + expect( + crashedStore.claimDeliveryExecution(harness.requesterSessionId, 'review-round-1', { + claimId: 'attempt-a', + workerBootId: 'worker-a', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 0 } }); + expect( + crashedStore.prepareClaimedDeliveryExecution( + harness.requesterSessionId, + 'review-round-1', + 'worker-a', + 'attempt-a' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 1 } }); + expect( + crashedStore.recoverOrphanedDeliveryClaims('workspace-1' as WorkspaceId, 'worker-b') + ).toBe(1); + expect( + crashedStore.claimDeliveryExecution(harness.requesterSessionId, 'review-round-1', { + claimId: 'attempt-b', + workerBootId: 'worker-b', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 1 } }); + expect( + crashedStore.prepareClaimedDeliveryExecution( + harness.requesterSessionId, + 'review-round-1', + 'worker-b', + 'attempt-b' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 2 } }); + } finally { + crashedStore.close(); + } + + harness.coordinator.start(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).not.toHaveBeenCalled(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 2, + }); + } finally { + finalStore.close(); + } + expect(harness.histories.get(harness.requesterSessionId)).toEqual([ + expect.objectContaining({ + role: 'system', + items: [ + expect.objectContaining({ + type: 'operation_completion', + continuation: { + status: 'not_started', + reason: expect.objectContaining({ code: 'DELIVERY_ATTEMPTS_EXHAUSTED' }), + }, + }), + ], + }), + ]); + }); + + it('silently loses a Delivery claim when another Worker wins after the idle check', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + workerBootId: 'worker-a', + }); + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const competitorStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect( + competitorStore.claimDeliveryExecution(harness.requesterSessionId, 'review-round-1', { + claimId: 'attempt-b', + workerBootId: 'worker-b', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 0 } }); + } finally { + competitorStore.close(); + } + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + expect(await typedOptions.onTurnClaimed?.()).toBe(false); + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + expect(harness.histories.get(harness.requesterSessionId)).toEqual([]); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'pending', + attemptCount: 0, + activeClaimId: 'attempt-b', + activeClaimWorkerBootId: 'worker-b', + }); + } finally { + finalStore.close(); + } + }); + + it('does not replay after claimed consume commits even if the coordinator tail fails', async () => { + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + await typedOptions.onTurnClaimed?.(); + await typedOptions.onTurnSettled?.('handled'); + throw new Error('coordinator crashed after durable consume'); + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + await harness.coordinator.wake('restart-after-ack'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 1, + }); + } finally { + finalStore.close(); + } + }); + + it('retries settlement persistence without replaying handled provider execution', async () => { + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + const consume = vi.spyOn(LodyOperationStore.prototype, 'consumeClaimedDelivery'); + const originalConsume = consume.getMockImplementation(); + consume.mockImplementationOnce(() => { + throw new Error('settlement write failed'); + }); + if (originalConsume) consume.mockImplementation(originalConsume); + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + expect(await typedOptions.onTurnClaimed?.()).toBe(true); + expect(await typedOptions.onTurnStarted?.()).toBe(true); + try { + await typedOptions.onTurnSettled?.('handled'); + } catch { + // SessionExecutionService logs settlement persistence failures and returns. + } + }); + + try { + harness.coordinator.start(); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 1, + }); + } finally { + finalStore.close(); + } + const completion = harness.histories + .get(harness.requesterSessionId) + ?.find((entry) => entry.role === 'system') + ?.items?.find((item) => item.type === 'operation_completion'); + expect(completion).not.toHaveProperty('continuation'); + } finally { + consume.mockRestore(); + } + }); + + it('retries an observed settlement on a later wake without replaying provider execution', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + workerBootId: 'worker-a', + }); + const originalConsume = LodyOperationStore.prototype.consumeClaimedDelivery; + const consume = vi.spyOn(LodyOperationStore.prototype, 'consumeClaimedDelivery'); + consume + .mockImplementationOnce(() => { + throw new Error('settlement callback write failed'); + }) + .mockImplementationOnce(() => { + throw new Error('settlement fallback write failed'); + }) + .mockImplementationOnce(() => { + throw new Error('first wake settlement write failed'); + }) + .mockImplementation(originalConsume); + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + expect(await typedOptions.onTurnClaimed?.()).toBe(true); + expect(await typedOptions.onTurnStarted?.()).toBe(true); + try { + await typedOptions.onTurnSettled?.('handled'); + } catch { + // SessionExecutionService logs settlement persistence failures and returns. + } + }); + + try { + harness.coordinator.start(); + await harness.coordinator.idle(); + + const strandedStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect( + strandedStore.getDelivery(harness.requesterSessionId, 'review-round-1') + ).toMatchObject({ + state: 'pending', + executionPhase: 'started', + activeClaimWorkerBootId: 'worker-a', + }); + } finally { + strandedStore.close(); + } + + await harness.coordinator.wake('retry-observed-settlement-1'); + await harness.coordinator.idle(); + const retryStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(retryStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'pending', + executionPhase: 'started', + activeClaimWorkerBootId: 'worker-a', + }); + } finally { + retryStore.close(); + } + + await harness.coordinator.wake('retry-observed-settlement-2'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 1, + }); + } finally { + finalStore.close(); + } + } finally { + consume.mockRestore(); + } + }); + + it('does not spend execution attempts when completion history is not durable', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + historyFailuresBeforeSuccess: 2, + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + await harness.coordinator.wake('retry-first-history-failure'); + await harness.coordinator.idle(); + + const beforeDurableHistory = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect( + beforeDurableHistory.getDelivery(harness.requesterSessionId, 'review-round-1') + ).toMatchObject({ + state: 'pending', + attemptCount: 0, + }); + } finally { + beforeDurableHistory.close(); + } + + await harness.coordinator.wake('retry-after-history-recovers'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledTimes(3); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 1, + }); + } finally { + finalStore.close(); + } + const completion = harness.histories + .get(harness.requesterSessionId) + ?.find((entry) => entry.role === 'system') + ?.items?.find((item) => item.type === 'operation_completion'); + expect(completion).not.toHaveProperty('continuation'); + }); + + it('releases a prepared claim when the pre-provider start fence write fails', async () => { + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + const markStarted = vi + .spyOn(LodyOperationStore.prototype, 'markClaimedDeliveryExecutionStarted') + .mockImplementationOnce(() => { + throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); + }); + let providerStarts = 0; + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + if ((await typedOptions.onTurnClaimed?.()) === false) return; + try { + if ((await typedOptions.onTurnStarted?.()) === false) return; + } catch { + await typedOptions.onTurnSettled?.('not_started'); + return; + } + providerStarts += 1; + await typedOptions.onTurnSettled?.('handled'); + }); + + try { + harness.coordinator.start(); + await harness.coordinator.idle(); + + const afterFailure = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + const deliveryAfterFailure = afterFailure.getDelivery( + harness.requesterSessionId, + 'review-round-1' + ); + expect(deliveryAfterFailure).toMatchObject({ + state: 'pending', + executionPhase: 'ready', + attemptCount: 1, + }); + expect(deliveryAfterFailure).not.toHaveProperty('activeClaimId'); + expect(deliveryAfterFailure).not.toHaveProperty('activeClaimWorkerBootId'); + } finally { + afterFailure.close(); + } + expect(providerStarts).toBe(0); + + await harness.coordinator.wake('retry-after-start-fence-write-failure'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(providerStarts).toBe(1); + expect(harness.continueSession).toHaveBeenCalledTimes(2); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 2, + }); + } finally { + finalStore.close(); + } + } finally { + harness.coordinator.stop(); + markStarted.mockRestore(); + } + }); + + it('recovers once when execution exits after claim without reporting a settlement', async () => { + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + let executionCount = 0; + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + await typedOptions.onTurnClaimed?.(); + executionCount += 1; + if (executionCount === 1) { + throw new Error('execution exited without a settlement'); + } + await typedOptions.onTurnSettled?.('handled'); + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + await harness.coordinator.wake('recover-missing-settlement'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledTimes(2); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + attemptCount: 2, + }); + } finally { + finalStore.close(); + } + }); + + it('records uncertainty when provider execution returns without settlement', async () => { + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + expect(await typedOptions.onTurnClaimed?.()).toBe(true); + expect(await typedOptions.onTurnStarted?.()).toBe(true); + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + await harness.coordinator.wake('duplicate-wake-after-missing-settlement'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledOnce(); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + expect(finalStore.getDelivery(harness.requesterSessionId, 'review-round-1')).toMatchObject({ + state: 'consumed', + executionPhase: 'uncertain', + attemptCount: 1, + }); + } finally { + finalStore.close(); + } + expect(harness.histories.get(harness.requesterSessionId)).toEqual([ + expect.objectContaining({ + items: [ + expect.objectContaining({ + continuation: { + status: 'uncertain', + reason: expect.objectContaining({ code: 'DELIVERY_EXECUTION_UNCERTAIN' }), + }, + }), + ], + }), + ]); + }); + + it('records a static failure after two executions exit without a settlement', async () => { + const harness = await makeHarness({ deadlineAt: '2026-07-19T23:59:59.000Z' }); + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + await typedOptions.onTurnClaimed?.(); + throw new Error('execution exited without a settlement'); + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + await harness.coordinator.wake('recover-missing-settlement'); + await harness.coordinator.idle(); + await harness.coordinator.wake('duplicate-wake-after-exhaustion'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledTimes(2); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + const delivery = finalStore.getDelivery(harness.requesterSessionId, 'review-round-1'); + expect(delivery).toMatchObject({ state: 'consumed', attemptCount: 2 }); + } finally { + finalStore.close(); + } + }); + + it('consumes a user-cancelled Delivery without starting ACP again', async () => { + const harness = await makeHarness({ + deadlineAt: '2026-07-19T23:59:59.000Z', + workerBootId: 'daemon-1', + }); + harness.continueSession.mockImplementation(async (_message, dispatchOptions) => { + const typedOptions = dispatchOptions as DeliveryDispatchOptions; + await typedOptions.onTurnClaimed?.(); + await typedOptions.onTurnStarted?.(); + await typedOptions.onTurnSettled?.('cancelled'); + }); + + harness.coordinator.start(); + await harness.coordinator.idle(); + await harness.coordinator.wake('recover-interrupted-delivery'); + await harness.coordinator.idle(); + await harness.coordinator.wake('duplicate-history-event'); + await harness.coordinator.idle(); + harness.coordinator.stop(); + + expect(harness.continueSession).toHaveBeenCalledTimes(1); + const finalStore = new LodyOperationStore(harness.storePath, () => TEST_NOW_MS); + try { + const delivery = finalStore.getDelivery(harness.requesterSessionId, 'review-round-1'); + expect(delivery).toMatchObject({ + state: 'consumed', + attemptCount: 1, + }); + } finally { + finalStore.close(); + } + expect(harness.histories.get(harness.requesterSessionId)).toEqual([ + expect.objectContaining({ + id: 'operation-completion:requester-1:review-round-1', + items: [expect.objectContaining({ type: 'operation_completion' })], + }), + ]); + }); + it('coalesces repeated wakes into one serial follow-up Delivery attempt', async () => { let resolveSync!: (value: boolean) => void; let markSyncStarted!: () => void; diff --git a/apps/cli/src/orchestration/operation-coordinator.ts b/apps/cli/src/orchestration/operation-coordinator.ts index 200acc715..712873ff4 100644 --- a/apps/cli/src/orchestration/operation-coordinator.ts +++ b/apps/cli/src/orchestration/operation-coordinator.ts @@ -34,12 +34,25 @@ import type { SessionDispatchWatcher } from '@/session/session-dispatch-watcher' import type { SessionExecutionService } from '@/session/session-execution-service'; import type { SessionUserResolver } from '@/session/session-user-resolver'; -import { getLodyOperationStorePath, LodyOperationStore } from './operation-store'; +import { + DELIVERY_MAX_ATTEMPTS, + getLodyOperationStorePath, + LodyOperationStore, +} from './operation-store'; type TargetSubscription = { unsubscribe: () => void; }; +type DeliveryTurnSettlement = 'handled' | 'cancelled' | 'not_started' | 'uncertain'; + +type ObservedDeliverySettlement = { + claimId: string; +} & ( + | { outcome: DeliveryTurnSettlement } + | { outcome: 'finalize'; pendingHistoryWrite?: () => Promise } +); + const TARGET_OUTPUT_PREVIEW_MAX_BYTES = 8 * 1024; const MATERIALIZATION_RETRY_MIN_MS = 1_000; const MATERIALIZATION_RETRY_MAX_MS = 30_000; @@ -50,6 +63,11 @@ const MATERIALIZATION_RETRY_MAX_MS = 30_000; // default Operation deadline is 24h, so a legitimately finished completion // always has at least this window to reach the requester's idle boundary. const DELIVERY_EXPIRY_GRACE_MS = 8 * 60 * 60 * 1_000; +// This module is loaded once by each CLI Worker process. All workspace +// coordinators in that Worker share one boot identity, while a replacement +// Worker necessarily receives a fresh identity after the supervisor's child +// exit barrier (or the foreground Host-lease acquisition barrier). +const WORKER_BOOT_ID = randomUUID(); const truncateTargetOutput = ( text: string @@ -120,6 +138,7 @@ export type LodyOperationCoordinatorOptions = { directory: string, onChange: (filename: string | Buffer | null) => void ) => Pick; + workerBootId?: string; materializeTarget: ( operation: StoredLodyOperation, item: Extract, @@ -128,16 +147,14 @@ export type LodyOperationCoordinatorOptions = { ) => Promise; }; +const isTerminalAssistantEntry = (entry: SessionHistoryInput): boolean => + entry.role === 'assistant' && (entry.finished === true || typeof entry.endedAt === 'number'); + const terminalAssistantFor = ( history: SessionHistoryInput[], userTurnId: string ): SessionHistoryInput | undefined => - history.find( - (entry) => - entry.role === 'assistant' && - entry.userTurnId === userTurnId && - (entry.finished === true || typeof entry.endedAt === 'number') - ); + history.find((entry) => entry.userTurnId === userTurnId && isTerminalAssistantEntry(entry)); const completionText = (operation: StoredLodyOperation): string => [ @@ -158,6 +175,7 @@ export class LodyOperationCoordinator { private readonly deliveryChains = new Map>(); private readonly queuedDeliveryIds = new Set(); private readonly dirtyDeliveryReasons = new Map(); + private readonly observedDeliverySettlements = new Map(); private readonly operationAbortControllers = new Map(); private metaWatch: RepoWatchHandle | null = null; private store: LodyOperationStore | null = null; @@ -165,11 +183,13 @@ export class LodyOperationCoordinator { private storeWakeTimer: ReturnType | null = null; private started = false; private readonly materializationClaimToken = randomUUID(); + private readonly workerBootId: string; constructor(private readonly options: LodyOperationCoordinatorOptions) { const storePath = options.storePath ?? getLodyOperationStorePath(options.machineId); this.storeFactory = options.storeFactory ?? (() => new LodyOperationStore(storePath)); this.now = options.now ?? getServerNow; + this.workerBootId = options.workerBootId ?? WORKER_BOOT_ID; } start(): void { @@ -193,6 +213,15 @@ export class LodyOperationCoordinator { // event loop (multiple workspace coordinators watching the shared // machine-level store amplify it). this.store = this.storeFactory(); + const recoveredClaims = this.store.recoverOrphanedDeliveryClaims( + this.options.workspaceId, + this.workerBootId + ); + if (recoveredClaims > 0) { + this.options.logger.warn( + `[orchestration] Recovered ${recoveredClaims} orphaned Delivery claim(s) from an exited Worker` + ); + } const storePath = this.options.storePath ?? getLodyOperationStorePath(this.options.machineId); const storeBasename = path.basename(storePath); const watchOperationStore = @@ -224,6 +253,25 @@ export class LodyOperationCoordinator { this.storeWatch = null; if (this.storeWakeTimer) clearTimeout(this.storeWakeTimer); this.storeWakeTimer = null; + if (this.store) { + try { + const abandonedClaims = this.store.abandonDeliveryClaimsOwnedBy( + this.options.workspaceId, + this.workerBootId + ); + if (abandonedClaims > 0) { + this.options.logger.warn( + `[orchestration] Abandoned ${abandonedClaims} Delivery claim(s) while stopping the workspace coordinator` + ); + } + } catch (error) { + this.options.logger.warn( + `[orchestration] Could not abandon Delivery claims during coordinator stop: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } this.store?.close(); this.store = null; for (const subscription of this.targetSubscriptions.values()) { @@ -243,6 +291,7 @@ export class LodyOperationCoordinator { this.deliveryChains.clear(); this.queuedDeliveryIds.clear(); this.dirtyDeliveryReasons.clear(); + this.observedDeliverySettlements.clear(); for (const controller of this.operationAbortControllers.values()) controller.abort(); this.operationAbortControllers.clear(); } @@ -780,8 +829,105 @@ export class LodyOperationCoordinator { ); } + private async settleObservedDeliveryClaim( + delivery: StoredLodyDelivery, + settlement: ObservedDeliverySettlement + ): Promise { + const previous = this.observedDeliverySettlements.get(delivery.deliveryId); + if (!previous || previous.claimId === settlement.claimId) { + this.observedDeliverySettlements.set(delivery.deliveryId, settlement); + } + if (settlement.outcome === 'finalize' && settlement.pendingHistoryWrite) { + const current = this.withStore((store) => + store.getDelivery(delivery.requesterSessionId, delivery.operationId) + ); + if ( + current.activeClaimWorkerBootId !== this.workerBootId || + current.activeClaimId !== settlement.claimId + ) { + if (this.observedDeliverySettlements.get(delivery.deliveryId) === settlement) { + this.observedDeliverySettlements.delete(delivery.deliveryId); + } + return current; + } + await settlement.pendingHistoryWrite(); + delete settlement.pendingHistoryWrite; + } + // History writes can yield across workspace stop or claim replacement. + const current = this.withStore((store) => { + let latest = store.getDelivery(delivery.requesterSessionId, delivery.operationId); + if ( + latest.activeClaimWorkerBootId !== this.workerBootId || + latest.activeClaimId !== settlement.claimId + ) { + return latest; + } + if ( + settlement.outcome === 'handled' || + settlement.outcome === 'cancelled' || + settlement.outcome === 'finalize' + ) { + return store.consumeClaimedDelivery( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + settlement.claimId + ).delivery; + } + if (settlement.outcome === 'uncertain') { + store.markClaimedDeliveryExecutionUncertain( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + settlement.claimId + ); + } else { + store.releaseDeliveryClaim( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + settlement.claimId + ); + } + latest = store.getDelivery(delivery.requesterSessionId, delivery.operationId); + return latest; + }); + if ( + current.activeClaimWorkerBootId !== this.workerBootId || + current.activeClaimId !== settlement.claimId + ) { + if (this.observedDeliverySettlements.get(delivery.deliveryId) === settlement) { + this.observedDeliverySettlements.delete(delivery.deliveryId); + } + } + return current; + } + private async deliverIfRunnable(delivery: StoredLodyDelivery, reason: string): Promise { if (!this.started) return; + delivery = this.withStore((store) => + store.getDelivery(delivery.requesterSessionId, delivery.operationId) + ); + if (delivery.state === 'consumed') { + this.observedDeliverySettlements.delete(delivery.deliveryId); + return; + } + // A claim is exclusive regardless of owner. Foreign-boot claims + // are removed only once during Worker startup, after the lifecycle barrier. + if (delivery.activeClaimId || delivery.activeClaimWorkerBootId) { + const settlement = this.observedDeliverySettlements.get(delivery.deliveryId); + if ( + settlement && + delivery.activeClaimWorkerBootId === this.workerBootId && + delivery.activeClaimId === settlement.claimId + ) { + await this.settleObservedDeliveryClaim(delivery, settlement); + } else if (settlement) { + this.observedDeliverySettlements.delete(delivery.deliveryId); + } + return; + } + this.observedDeliverySettlements.delete(delivery.deliveryId); const metaRecord = await this.options.workspaceDocument.repo.getDocMeta( getSessionRoomId(delivery.requesterSessionId) ); @@ -793,33 +939,35 @@ export class LodyOperationCoordinator { ); this.subscribeTarget(delivery.requesterSessionId, sessionDoc); - // Durable recovery evidence is deliberately checked before configuration - // lookup or remote sync. A completed continuation (or an authoritative - // unavailable completion) must only consume the Delivery, regardless of - // repo-meta size or later configuration changes. An active turn alone is - // not durable evidence: leave the Delivery pending until history records an - // assistant response or chat_failed notice. const execution = this.options.executionService.getExecutionSnapshot( delivery.requesterSessionId ); - const historyBeforeDispatch = await sessionDoc.getHistory(); - const continuationEvidence = this.getContinuationEvidence( - historyBeforeDispatch, - delivery.systemTurnId - ); - if (continuationEvidence) { - this.consumeDelivery(delivery, reason, continuationEvidence); - return; - } const operation = this.withStore((store) => store.get(delivery.requesterSessionId, delivery.operationId) ); if (this.now() >= Date.parse(operation.deadlineAt) + DELIVERY_EXPIRY_GRACE_MS) { - this.consumeDelivery(delivery, reason, 'expired_stale'); + await this.finalizeDeliveryWithoutExecution( + sessionDoc, + operation, + delivery, + reason, + 'expired_stale', + undefined, + false, + delivery.executionPhase === 'uncertain' ? 'uncertain' : 'ready' + ); + return; + } + if (delivery.executionPhase === 'uncertain') { + await this.failUncertainDelivery(sessionDoc, operation, delivery, reason); return; } if (execution.hasActiveTurn) return; if (this.options.dispatchWatcher.hasPendingDispatch(delivery.requesterSessionId)) return; + if (delivery.attemptCount >= DELIVERY_MAX_ATTEMPTS) { + await this.failExhaustedDelivery(sessionDoc, operation, delivery, reason); + return; + } const configuration = await this.resolveFrozenConfiguration(operation, delivery, reason); if (configuration === 'unknown') { @@ -827,14 +975,57 @@ export class LodyOperationCoordinator { return; } if (configuration === 'unavailable') { - await this.writeCompletionTurn(sessionDoc, operation, delivery, false); - this.consumeDelivery(delivery, reason, 'configuration_unavailable'); + await this.finalizeDeliveryWithoutExecution( + sessionDoc, + operation, + delivery, + reason, + 'configuration_unavailable', + { + code: 'CONFIGURATION_UNAVAILABLE', + message: 'The frozen continuation agent configuration is no longer available.', + } + ); return; } const frozen = operation.frozenContinuationConfig.inputConfig; const requester = await this.resolveRequesterIdentity(operation.requesterUserId); - await this.options.executionService.continueSession( + const attemptId = randomUUID(); + let observedSettlement: DeliveryTurnSettlement | undefined; + const settleResidualClaim = () => + observedSettlement + ? this.settleObservedDeliveryClaim(delivery, { + claimId: attemptId, + outcome: observedSettlement, + }) + : this.withStore((store) => { + let current = store.getDelivery(delivery.requesterSessionId, delivery.operationId); + if ( + current.activeClaimWorkerBootId !== this.workerBootId || + current.activeClaimId !== attemptId + ) { + return current; + } + if (current.executionPhase === 'started') { + store.markClaimedDeliveryExecutionUncertain( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + attemptId + ); + } else { + store.releaseDeliveryClaim( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + attemptId + ); + } + current = store.getDelivery(delivery.requesterSessionId, delivery.operationId); + return current; + }); + const continuation = this.options.executionService.continueSession( { type: 'session/chat', sessionId: delivery.requesterSessionId, @@ -860,29 +1051,197 @@ export class LodyOperationCoordinator { { dispatchSource: 'delivery', onTurnClaimed: async () => { - await this.writeCompletionTurn(sessionDoc, operation, delivery, true); + if (!this.started) return false; + const claim = this.withStore((store) => + store.claimDeliveryExecution(delivery.requesterSessionId, delivery.operationId, { + claimId: attemptId, + workerBootId: this.workerBootId, + }) + ); + if (claim.status !== 'claimed') { + this.options.logger.debug( + `[orchestration] Delivery ${delivery.deliveryId} claim lost status=${claim.status}` + ); + return false; + } + try { + await this.writeCompletionTurn(sessionDoc, operation, delivery); + const prepared = this.withStore((store) => + store.prepareClaimedDeliveryExecution( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + attemptId + ) + ); + if (!prepared.prepared) { + this.withStore((store) => + store.releaseDeliveryClaim( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + attemptId + ) + ); + this.options.logger.debug( + `[orchestration] Delivery ${delivery.deliveryId} execution start lost its claim` + ); + return false; + } + } catch (error) { + this.withStore((store) => + store.releaseDeliveryClaim( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + attemptId + ) + ); + throw error; + } + return true; + }, + onTurnStarted: async () => + this.withStore((store) => + store.markClaimedDeliveryExecutionStarted( + delivery.requesterSessionId, + delivery.operationId, + this.workerBootId, + attemptId + ) + ), + onTurnSettled: async (outcome) => { + observedSettlement = outcome; + const settled = await this.settleObservedDeliveryClaim(delivery, { + claimId: attemptId, + outcome, + }); + if (outcome === 'handled' || outcome === 'cancelled') { + this.options.logger.debug( + `[orchestration] Delivery ${delivery.deliveryId} consumed=${String(settled.state === 'consumed')}` + ); + } }, } ); - const historyAfterExecution = await sessionDoc.getHistory(); - const evidenceAfterExecution = this.getContinuationEvidence( - historyAfterExecution, - delivery.systemTurnId - ); - if (evidenceAfterExecution) { - this.consumeDelivery(delivery, reason, evidenceAfterExecution); + try { + await continuation; + } catch (error) { + const afterInterruption = await settleResidualClaim(); + if (afterInterruption.executionPhase === 'uncertain') { + await this.failUncertainDelivery(sessionDoc, operation, afterInterruption, reason); + } + if ( + afterInterruption.state === 'pending' && + !afterInterruption.activeClaimId && + afterInterruption.attemptCount >= DELIVERY_MAX_ATTEMPTS + ) { + await this.failExhaustedDelivery(sessionDoc, operation, afterInterruption, reason); + } + throw error; + } + const afterExecution = await settleResidualClaim(); + if (afterExecution.executionPhase === 'uncertain') { + await this.failUncertainDelivery(sessionDoc, operation, afterExecution, reason); + return; + } + if ( + afterExecution.state === 'pending' && + !afterExecution.activeClaimId && + afterExecution.attemptCount >= DELIVERY_MAX_ATTEMPTS + ) { + await this.failExhaustedDelivery(sessionDoc, operation, afterExecution, reason); } } - private consumeDelivery( + private async failExhaustedDelivery( + sessionDoc: SessionDocument, + operation: StoredLodyOperation, + delivery: StoredLodyDelivery, + wakeReason: string + ): Promise { + await this.finalizeDeliveryWithoutExecution( + sessionDoc, + operation, + delivery, + wakeReason, + 'attempts_exhausted', + { + code: 'DELIVERY_ATTEMPTS_EXHAUSTED', + message: 'The completion continuation did not settle after two delivery attempts.', + }, + true + ); + } + + private async failUncertainDelivery( + sessionDoc: SessionDocument, + operation: StoredLodyOperation, + delivery: StoredLodyDelivery, + wakeReason: string + ): Promise { + await this.finalizeDeliveryWithoutExecution( + sessionDoc, + operation, + delivery, + wakeReason, + 'execution_uncertain', + { + status: 'uncertain', + code: 'DELIVERY_EXECUTION_UNCERTAIN', + message: + 'The completion continuation may have started before execution was interrupted. It was not replayed; review the session output and continue manually if needed.', + }, + false, + 'uncertain' + ); + } + + private async finalizeDeliveryWithoutExecution( + sessionDoc: SessionDocument, + operation: StoredLodyOperation, delivery: StoredLodyDelivery, wakeReason: string, - evidence: string - ): void { + evidence: string, + continuationFailure?: { + status?: 'not_started' | 'uncertain'; + code: + | 'CONFIGURATION_UNAVAILABLE' + | 'DELIVERY_ATTEMPTS_EXHAUSTED' + | 'DELIVERY_EXECUTION_UNCERTAIN'; + message: string; + }, + requireAttemptsExhausted = false, + requiredExecutionPhase: 'ready' | 'uncertain' = 'ready' + ): Promise { + if (!this.started) return false; const startedAt = performance.now(); - this.withStore((store) => - store.consumeDelivery(delivery.requesterSessionId, delivery.operationId) + const claimId = randomUUID(); + const claim = this.withStore((store) => + store.claimDeliveryFinalization(delivery.requesterSessionId, delivery.operationId, { + claimId, + workerBootId: this.workerBootId, + requireAttemptsExhausted, + requiredExecutionPhase, + }) ); + if (claim.status !== 'claimed') { + this.options.logger.debug( + `[orchestration] Delivery ${delivery.deliveryId} finalization skipped status=${claim.status} reason=${evidence} wake=${wakeReason}` + ); + return false; + } + const settled = await this.settleObservedDeliveryClaim(delivery, { + claimId, + outcome: 'finalize', + ...(continuationFailure + ? { + pendingHistoryWrite: () => + this.writeCompletionTurn(sessionDoc, operation, delivery, continuationFailure), + } + : {}), + }); + if (settled.state !== 'consumed') return false; const timer = this.configurationTimers.get(delivery.requesterSessionId); if (timer) clearTimeout(timer); this.configurationTimers.delete(delivery.requesterSessionId); @@ -891,6 +1250,7 @@ export class LodyOperationCoordinator { performance.now() - startedAt ).toFixed(2)}` ); + return true; } /** @@ -980,7 +1340,14 @@ export class LodyOperationCoordinator { sessionDoc: SessionDocument, operation: StoredLodyOperation, delivery: StoredLodyDelivery, - configAvailable: boolean + continuationFailure?: { + status?: 'not_started' | 'uncertain'; + code: + | 'CONFIGURATION_UNAVAILABLE' + | 'DELIVERY_ATTEMPTS_EXHAUSTED' + | 'DELIVERY_EXECUTION_UNCERTAIN'; + message: string; + } ): Promise { if (!operation.completion) { throw new Error(`Finished Operation ${operation.operationId} has no completion.`); @@ -991,13 +1358,13 @@ export class LodyOperationCoordinator { operationId: operation.operationId, operationKind: operation.kind, completion: operation.completion, - ...(!configAvailable + ...(continuationFailure ? { continuation: { - status: 'not_started' as const, + status: continuationFailure.status ?? ('not_started' as const), reason: { - code: 'CONFIGURATION_UNAVAILABLE' as const, - message: 'The frozen continuation agent configuration is no longer available.', + code: continuationFailure.code, + message: continuationFailure.message, }, }, } @@ -1017,36 +1384,39 @@ export class LodyOperationCoordinator { chainDepth: operation.initiatorChainDepth + 1, }, }; - await sessionDoc.updateHistory((history) => - history.some((entry) => entry.id === delivery.systemTurnId) ? history : [...history, turn] - ); - } - - private getContinuationEvidence( - history: SessionHistoryInput[], - systemTurnId: string - ): string | null { - const index = history.findIndex( - (entry) => entry.id === systemTurnId && entry.role === 'system' - ); - if (index < 0) return null; - const completionWasUnavailable = history[index]?.items?.some( - (item) => - item.type === 'operation_completion' && - item.continuation?.status === 'not_started' && - item.continuation.reason.code === 'CONFIGURATION_UNAVAILABLE' - ); - if (completionWasUnavailable) return 'configuration_unavailable'; - for (const entry of history.slice(index + 1)) { - if (entry.role === 'assistant') return 'assistant_history'; - if (entry.role === 'user') return null; - if ( - entry.role === 'system' && - entry.items?.some((item) => item.type === 'system_notice' && item.name === 'chat_failed') - ) { - return 'chat_failed'; - } - } - return null; + await sessionDoc.updateHistory((history) => { + const existing = history.find((entry) => entry.id === delivery.systemTurnId); + if (!existing) return [...history, turn]; + if (existing.role !== 'system') return history; + return history.map((entry) => + entry.id !== delivery.systemTurnId + ? entry + : { + ...entry, + items: entry.items?.map((existingItem) => { + if ( + existingItem.type !== 'operation_completion' || + existingItem.deliveryId !== delivery.deliveryId + ) { + return existingItem; + } + if (continuationFailure) { + return { + ...existingItem, + continuation: { + status: continuationFailure.status ?? ('not_started' as const), + reason: { + code: continuationFailure.code, + message: continuationFailure.message, + }, + }, + }; + } + const { continuation: _continuation, ...withoutContinuation } = existingItem; + return withoutContinuation; + }), + } + ); + }); } } diff --git a/apps/cli/src/orchestration/operation-model.test.ts b/apps/cli/src/orchestration/operation-model.test.ts index 40c39b4bd..25dbef202 100644 --- a/apps/cli/src/orchestration/operation-model.test.ts +++ b/apps/cli/src/orchestration/operation-model.test.ts @@ -24,31 +24,76 @@ describe('Operation delivery executable model', () => { ); expect(afterDelivery).toMatchObject({ activeTurn: 'delivery', - delivery: 'consumed', - completionTurnWrites: 1, + delivery: 'claimed', + completionTurnWrites: 0, }); + const prepared = stepOrchestrationModel(afterDelivery, 'prepare_turn'); + const started = stepOrchestrationModel(prepared, 'start_turn'); + expect(started).toMatchObject({ delivery: 'started', completionTurnWrites: 1 }); + expect(stepOrchestrationModel(started, 'complete_turn').delivery).toBe('consumed'); }); it('keeps archived delivery pending until restore', () => { const archived = trace('accept', 'materialize_success', 'archive', 'finish', 'schedule'); expect(archived).toMatchObject({ delivery: 'pending', completionTurnWrites: 0 }); - const restored = stepOrchestrationModel( - stepOrchestrationModel(archived, 'restore'), - 'schedule' - ); - expect(restored.delivery).toBe('consumed'); + const restored = trace('accept', 'materialize_success', 'archive', 'finish', 'restore'); + const claimed = stepOrchestrationModel(restored, 'schedule'); + expect(claimed.delivery).toBe('claimed'); + const prepared = stepOrchestrationModel(claimed, 'prepare_turn'); + const started = stepOrchestrationModel(prepared, 'start_turn'); + expect(started.delivery).toBe('started'); }); it('writes the result once without starting an assistant when configuration is gone', () => { - expect( - trace('accept', 'materialize_success', 'delete_configuration', 'finish', 'schedule') - ).toMatchObject({ + const finalizing = trace( + 'accept', + 'materialize_success', + 'delete_configuration', + 'finish', + 'schedule' + ); + expect(finalizing).toMatchObject({ + delivery: 'finalizing', + deliveryClaimOwner: 'current', + activeTurn: 'none', + completionTurnWrites: 0, + }); + expect(stepOrchestrationModel(finalizing, 'complete_finalization')).toMatchObject({ delivery: 'consumed', activeTurn: 'none', completionTurnWrites: 1, }); }); + it('retains terminal ownership through repeated history and consume failures', () => { + const failed = trace( + 'accept', + 'materialize_success', + 'delete_configuration', + 'finish', + 'schedule', + 'history_write_fail', + 'schedule', + 'history_write_fail', + 'finalization_consume_fail', + 'schedule', + 'finalization_consume_fail' + ); + expect(failed).toMatchObject({ + delivery: 'finalizing', + deliveryClaimOwner: 'current', + completionTurnWrites: 1, + deliveryAttempts: 0, + activeTurn: 'none', + }); + expect(stepOrchestrationModel(failed, 'complete_finalization')).toMatchObject({ + delivery: 'consumed', + deliveryClaimOwner: 'none', + completionTurnWrites: 1, + deliveryAttempts: 0, + }); + }); + it('uses deadline as a terminal backstop', () => { expect(trace('accept', 'deadline')).toMatchObject({ operation: 'finished', @@ -56,6 +101,137 @@ describe('Operation delivery executable model', () => { }); }); + it('permits one confirmed pre-provider recovery and consumes without a third attempt', () => { + const firstAttempt = trace( + 'accept', + 'materialize_success', + 'finish', + 'schedule', + 'prepare_turn' + ); + const secondAttempt = stepOrchestrationModel( + stepOrchestrationModel(stepOrchestrationModel(firstAttempt, 'interrupt_turn'), 'schedule'), + 'prepare_turn' + ); + expect(secondAttempt).toMatchObject({ delivery: 'prepared', deliveryAttempts: 2 }); + const exhausted = stepOrchestrationModel( + stepOrchestrationModel(secondAttempt, 'interrupt_turn'), + 'schedule' + ); + expect(exhausted).toMatchObject({ + delivery: 'finalizing', + deliveryClaimOwner: 'current', + deliveryAttempts: 2, + activeTurn: 'none', + }); + expect(stepOrchestrationModel(exhausted, 'complete_finalization').delivery).toBe('consumed'); + }); + + it('consumes a user cancellation without reopening the Delivery', () => { + const started = trace( + 'accept', + 'materialize_success', + 'finish', + 'schedule', + 'prepare_turn', + 'start_turn' + ); + const cancelled = stepOrchestrationModel(started, 'cancel_turn'); + expect(cancelled).toMatchObject({ + delivery: 'consumed', + deliveryClaimOwner: 'none', + activeTurn: 'none', + deliveryAttempts: 1, + }); + expect(stepOrchestrationModel(cancelled, 'schedule')).toEqual(cancelled); + }); + + it('keeps a crashed attempt fenced until a replacement Worker recovers the old boot', () => { + const firstAttempt = trace( + 'accept', + 'materialize_success', + 'finish', + 'schedule', + 'prepare_turn', + 'start_turn' + ); + const afterExit = stepOrchestrationModel(firstAttempt, 'restart'); + expect(afterExit).toMatchObject({ + delivery: 'started', + deliveryClaimOwner: 'previous', + activeTurn: 'none', + deliveryAttempts: 1, + }); + expect(stepOrchestrationModel(afterExit, 'schedule')).toEqual(afterExit); + + const recovered = stepOrchestrationModel(afterExit, 'recover_orphans'); + expect(recovered).toMatchObject({ + delivery: 'uncertain', + deliveryClaimOwner: 'none', + activeTurn: 'none', + deliveryAttempts: 1, + }); + const finalizing = stepOrchestrationModel(recovered, 'schedule'); + expect(finalizing).toMatchObject({ + delivery: 'uncertain_finalizing', + deliveryClaimOwner: 'current', + activeTurn: 'none', + }); + expect(stepOrchestrationModel(finalizing, 'complete_finalization').delivery).toBe('consumed'); + }); + + it('releases pre-start claims without spending the execution budget', () => { + const firstClaim = trace('accept', 'materialize_success', 'finish', 'schedule'); + expect(firstClaim).toMatchObject({ + delivery: 'claimed', + deliveryAttempts: 0, + completionTurnWrites: 0, + }); + const firstFailure = stepOrchestrationModel(firstClaim, 'history_write_fail'); + const secondClaim = stepOrchestrationModel(firstFailure, 'schedule'); + const secondFailure = stepOrchestrationModel(secondClaim, 'history_write_fail'); + expect(secondFailure).toMatchObject({ + delivery: 'pending', + deliveryAttempts: 0, + deliveryClaimOwner: 'none', + }); + const recovered = stepOrchestrationModel( + stepOrchestrationModel(stepOrchestrationModel(secondFailure, 'schedule'), 'prepare_turn'), + 'start_turn' + ); + expect(recovered).toMatchObject({ delivery: 'started', deliveryAttempts: 1 }); + }); + + it('keeps terminal history finalization fenced across Worker replacement', () => { + const finalizing = trace( + 'accept', + 'materialize_success', + 'delete_configuration', + 'finish', + 'schedule' + ); + const afterExit = stepOrchestrationModel(finalizing, 'restart'); + expect(afterExit).toMatchObject({ + delivery: 'finalizing', + deliveryClaimOwner: 'previous', + completionTurnWrites: 0, + }); + expect(stepOrchestrationModel(afterExit, 'schedule')).toEqual(afterExit); + + const recovered = stepOrchestrationModel(afterExit, 'recover_orphans'); + const reclaimed = stepOrchestrationModel(recovered, 'schedule'); + expect(reclaimed).toMatchObject({ + delivery: 'finalizing', + deliveryClaimOwner: 'current', + completionTurnWrites: 0, + }); + expect(stepOrchestrationModel(reclaimed, 'complete_finalization')).toMatchObject({ + delivery: 'consumed', + deliveryClaimOwner: 'none', + completionTurnWrites: 1, + }); + }); + it('keeps a failed materialization pending until its owned retry fires', () => { const failed = trace('accept', 'materialize_fail', 'finish'); expect(failed).toMatchObject({ @@ -97,5 +273,17 @@ describe('Operation delivery executable model', () => { chainDepth: 6, }) ).toThrow(/exceeded the fixed depth cap/); + expect(() => + assertOrchestrationModelSafety({ + ...initialOrchestrationModelState(), + deliveryAttempts: 3, + }) + ).toThrow(/bounded attempt count/); + expect(() => + assertOrchestrationModelSafety({ + ...initialOrchestrationModelState(), + deliveryClaimOwner: 'current', + }) + ).toThrow(/claim state/); }); }); diff --git a/apps/cli/src/orchestration/operation-model.ts b/apps/cli/src/orchestration/operation-model.ts index 02b1bda76..49aac9495 100644 --- a/apps/cli/src/orchestration/operation-model.ts +++ b/apps/cli/src/orchestration/operation-model.ts @@ -6,7 +6,18 @@ export type OrchestrationModelState = { operation: 'absent' | 'active' | 'finished'; targetInput: 'absent' | 'missing' | 'retry_scheduled' | 'durable'; - delivery: 'absent' | 'pending' | 'consumed'; + delivery: + | 'absent' + | 'pending' + | 'claimed' + | 'prepared' + | 'started' + | 'uncertain' + | 'finalizing' + | 'uncertain_finalizing' + | 'consumed'; + deliveryClaimOwner: 'none' | 'current' | 'previous'; + deliveryAttempts: number; activeTurn: 'none' | 'user' | 'delivery'; queuedUsers: number; archived: boolean; @@ -24,7 +35,17 @@ export type OrchestrationModelAction = | 'deadline' | 'enqueue_user' | 'schedule' + | 'prepare_turn' + | 'start_turn' + | 'history_write_fail' | 'complete_turn' + | 'fail_turn' + | 'interrupt_turn' + | 'cancel_turn' + | 'complete_finalization' + | 'finalization_consume_fail' + | 'restart' + | 'recover_orphans' | 'archive' | 'restore' | 'delete_configuration'; @@ -33,6 +54,8 @@ export const initialOrchestrationModelState = (): OrchestrationModelState => ({ operation: 'absent', targetInput: 'absent', delivery: 'absent', + deliveryClaimOwner: 'none', + deliveryAttempts: 0, activeTurn: 'none', queuedUsers: 0, archived: false, @@ -92,18 +115,145 @@ export const stepOrchestrationModel = ( if (next.queuedUsers > 0) { next.queuedUsers -= 1; next.activeTurn = 'user'; + } else if (next.delivery === 'uncertain') { + next.delivery = 'uncertain_finalizing'; + next.deliveryClaimOwner = 'current'; } else if (next.delivery === 'pending') { - next.completionTurnWrites += 1; - next.delivery = 'consumed'; - if (next.configurationAvailable) { + if (!next.configurationAvailable || next.deliveryAttempts >= 2) { + next.delivery = 'finalizing'; + next.deliveryClaimOwner = 'current'; + } else { + next.delivery = 'claimed'; + next.deliveryClaimOwner = 'current'; next.activeTurn = 'delivery'; + } + } + break; + case 'prepare_turn': + if ( + next.activeTurn === 'delivery' && + next.delivery === 'claimed' && + next.deliveryClaimOwner === 'current' && + next.deliveryAttempts < 2 + ) { + next.completionTurnWrites = Math.max(1, next.completionTurnWrites); + next.delivery = 'prepared'; + if (next.deliveryAttempts === 0) { next.chainDepth += 1; } + next.deliveryAttempts += 1; + } + break; + case 'start_turn': + if ( + next.activeTurn === 'delivery' && + next.delivery === 'prepared' && + next.deliveryClaimOwner === 'current' + ) { + next.delivery = 'started'; + } + break; + case 'history_write_fail': + if ( + next.activeTurn === 'delivery' && + next.delivery === 'claimed' && + next.deliveryClaimOwner === 'current' + ) { + next.delivery = 'pending'; + next.deliveryClaimOwner = 'none'; + next.activeTurn = 'none'; } break; case 'complete_turn': + if ( + next.activeTurn === 'delivery' && + (next.delivery === 'prepared' || next.delivery === 'started') + ) { + next.delivery = 'consumed'; + next.deliveryClaimOwner = 'none'; + } + next.activeTurn = 'none'; + break; + case 'fail_turn': + if ( + next.activeTurn === 'delivery' && + (next.delivery === 'prepared' || next.delivery === 'started') + ) { + next.delivery = 'consumed'; + next.deliveryClaimOwner = 'none'; + } next.activeTurn = 'none'; break; + case 'interrupt_turn': + if (next.activeTurn === 'delivery' && next.delivery === 'started') { + next.delivery = 'uncertain'; + next.deliveryClaimOwner = 'none'; + } else if ( + next.activeTurn === 'delivery' && + (next.delivery === 'claimed' || next.delivery === 'prepared') + ) { + next.delivery = 'pending'; + next.deliveryClaimOwner = 'none'; + } + next.activeTurn = 'none'; + break; + case 'cancel_turn': + if ( + next.activeTurn === 'delivery' && + (next.delivery === 'claimed' || next.delivery === 'prepared' || next.delivery === 'started') + ) { + next.delivery = 'consumed'; + next.deliveryClaimOwner = 'none'; + } + next.activeTurn = 'none'; + break; + case 'finalization_consume_fail': + if ( + (next.delivery === 'finalizing' || next.delivery === 'uncertain_finalizing') && + next.deliveryClaimOwner === 'current' + ) { + next.completionTurnWrites = Math.max(1, next.completionTurnWrites); + } + break; + case 'complete_finalization': + if ( + (next.delivery === 'finalizing' || next.delivery === 'uncertain_finalizing') && + next.deliveryClaimOwner === 'current' + ) { + next.completionTurnWrites = Math.max(1, next.completionTurnWrites); + next.delivery = 'consumed'; + next.deliveryClaimOwner = 'none'; + } + break; + case 'restart': + if ( + (next.delivery === 'claimed' || + next.delivery === 'prepared' || + next.delivery === 'started' || + next.delivery === 'finalizing' || + next.delivery === 'uncertain_finalizing') && + next.deliveryClaimOwner === 'current' + ) { + next.deliveryClaimOwner = 'previous'; + } + next.activeTurn = 'none'; + break; + case 'recover_orphans': + if ( + (next.delivery === 'claimed' || + next.delivery === 'prepared' || + next.delivery === 'started' || + next.delivery === 'finalizing' || + next.delivery === 'uncertain_finalizing') && + next.deliveryClaimOwner === 'previous' + ) { + next.delivery = + next.delivery === 'started' || next.delivery === 'uncertain_finalizing' + ? 'uncertain' + : 'pending'; + next.deliveryClaimOwner = 'none'; + } + break; case 'archive': next.archived = true; break; @@ -130,8 +280,29 @@ export const assertOrchestrationModelSafety = (state: OrchestrationModelState): if (state.operation === 'finished' && state.targetInput === 'retry_scheduled') { throw new Error('a terminal Operation retained a materialization retry'); } - if (state.activeTurn === 'delivery' && state.delivery !== 'consumed') { - throw new Error('a Delivery continuation started before the Delivery was claimed'); + if ( + state.activeTurn === 'delivery' && + state.delivery !== 'claimed' && + state.delivery !== 'prepared' && + state.delivery !== 'started' + ) { + throw new Error('a Delivery continuation is active without a durable attempt claim'); + } + if ( + (state.delivery === 'claimed' || + state.delivery === 'prepared' || + state.delivery === 'started' || + state.delivery === 'finalizing' || + state.delivery === 'uncertain_finalizing') !== + (state.deliveryClaimOwner !== 'none') + ) { + throw new Error('Delivery claim state and its fenced owner disagree'); + } + if (state.activeTurn === 'delivery' && state.deliveryClaimOwner !== 'current') { + throw new Error('a Delivery turn is active under a non-current Worker boot'); + } + if (state.deliveryAttempts > 2) { + throw new Error('a Delivery exceeded its bounded attempt count'); } if (state.chainDepth > 5) { throw new Error('machine-originated chain exceeded the fixed depth cap'); @@ -148,7 +319,17 @@ export const enumerateOrchestrationModel = (maxDepth: number): OrchestrationMode 'deadline', 'enqueue_user', 'schedule', + 'prepare_turn', + 'start_turn', + 'history_write_fail', 'complete_turn', + 'fail_turn', + 'interrupt_turn', + 'cancel_turn', + 'complete_finalization', + 'finalization_consume_fail', + 'restart', + 'recover_orphans', 'archive', 'restore', 'delete_configuration', diff --git a/apps/cli/src/orchestration/operation-store.test.ts b/apps/cli/src/orchestration/operation-store.test.ts index 4755933fb..c80ead4b7 100644 --- a/apps/cli/src/orchestration/operation-store.test.ts +++ b/apps/cli/src/orchestration/operation-store.test.ts @@ -73,6 +73,130 @@ describe('LodyOperationStore', () => { } }); + it('marks an existing legacy pending Delivery as execution-uncertain', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'lody-operation-store-migration-')); + roots.add(root); + const dbPath = path.join(root, 'operations.sqlite3'); + const legacy = new Database(dbPath); + try { + legacy.exec(` + CREATE TABLE operations ( + workspace_id TEXT NOT NULL, + owner_machine_id TEXT NOT NULL, + requester_session_id TEXT NOT NULL, + requester_user_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + kind TEXT NOT NULL, + fingerprint TEXT NOT NULL, + canonical_command_json TEXT NOT NULL, + frozen_config_json TEXT NOT NULL, + initiator_chain_depth INTEGER NOT NULL, + created_at TEXT NOT NULL, + deadline_at TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('active', 'finished')), + items_json TEXT NOT NULL, + completion_json TEXT, + finished_at TEXT, + PRIMARY KEY (requester_session_id, operation_id) + ); + CREATE TABLE deliveries ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + workspace_id TEXT NOT NULL, + requester_session_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + delivery_id TEXT NOT NULL UNIQUE, + system_turn_id TEXT NOT NULL UNIQUE, + state TEXT NOT NULL CHECK (state IN ('pending', 'consumed')), + initiator_chain_depth INTEGER NOT NULL, + completion_json TEXT NOT NULL, + consumed_at TEXT, + FOREIGN KEY (requester_session_id, operation_id) + REFERENCES operations (requester_session_id, operation_id) + ON DELETE CASCADE + ); + INSERT INTO operations ( + workspace_id, owner_machine_id, requester_session_id, requester_user_id, + operation_id, kind, fingerprint, canonical_command_json, frozen_config_json, + initiator_chain_depth, created_at, deadline_at, state, items_json, + completion_json, finished_at + ) VALUES ( + 'workspace-1', 'machine-1', 'legacy-requester', 'user-1', + 'legacy-operation', 'session_chat', 'legacy-fingerprint', '{}', + '{"inputConfig":{}}', 0, '2026-07-20T00:00:00.000Z', + '2026-07-20T01:00:00.000Z', 'finished', '[]', + '{"type":"cancelled"}', '2026-07-20T00:01:00.000Z' + ); + INSERT INTO deliveries ( + workspace_id, requester_session_id, operation_id, delivery_id, + system_turn_id, state, initiator_chain_depth, completion_json + ) VALUES ( + 'workspace-1', 'legacy-requester', 'legacy-operation', + 'operation:legacy-requester:legacy-operation:completion', + 'operation-completion:legacy-requester:legacy-operation', + 'pending', 0, '{"type":"cancelled"}' + ); + `); + } finally { + legacy.close(); + } + + const migrated = new LodyOperationStore(dbPath); + try { + migrated.accept(baseInput()); + migrated.finish('requester-1' as SessionId, 'review-round-1', { type: 'cancelled' }); + expect(migrated.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([ + expect.objectContaining({ + operationId: 'legacy-operation', + attemptCount: 0, + executionPhase: 'uncertain', + }), + expect.objectContaining({ attemptCount: 0, executionPhase: 'ready', state: 'pending' }), + ]); + } finally { + migrated.close(); + } + + const legacyReader = new Database(dbPath, { readonly: true }); + try { + const columns = legacyReader.pragma('table_info(deliveries)') as Array<{ name: string }>; + expect(columns.map((column) => column.name)).toEqual([ + 'sequence', + 'workspace_id', + 'requester_session_id', + 'operation_id', + 'delivery_id', + 'system_turn_id', + 'state', + 'initiator_chain_depth', + 'completion_json', + 'consumed_at', + ]); + expect( + legacyReader + .prepare( + 'SELECT execution_phase, attempt_count, active_claim_id, active_claim_worker_boot_id FROM delivery_execution_state WHERE requester_session_id = ? AND operation_id = ?' + ) + .get('legacy-requester', 'legacy-operation') + ).toEqual({ + execution_phase: 'uncertain', + attempt_count: 0, + active_claim_id: null, + active_claim_worker_boot_id: null, + }); + } finally { + legacyReader.close(); + } + + const reopened = new LodyOperationStore(dbPath); + try { + expect( + reopened.getDelivery('legacy-requester' as SessionId, 'legacy-operation') + ).toMatchObject({ state: 'pending', attemptCount: 0, executionPhase: 'uncertain' }); + } finally { + reopened.close(); + } + }); + it('accepts once and returns the same Operation for canonical-equivalent retries', async () => { const store = await makeStore(); try { @@ -233,6 +357,415 @@ describe('LodyOperationStore', () => { } }); + it('consumes only the active Delivery claim atomically', async () => { + const store = await makeStore(); + try { + store.accept(baseInput()); + store.finish('requester-1' as SessionId, 'review-round-1', { type: 'cancelled' }); + + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-1', + workerBootId: 'daemon-1', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 0 } }); + expect( + store.prepareClaimedDeliveryExecution( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'stale-attempt' + ) + ).toMatchObject({ prepared: false, delivery: { attemptCount: 0 } }); + expect( + store.prepareClaimedDeliveryExecution( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'attempt-1' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 1 } }); + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'stale-attempt' + ) + ).toMatchObject({ consumed: false, delivery: { state: 'pending' } }); + + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'attempt-1' + ) + ).toMatchObject({ + consumed: true, + delivery: { state: 'consumed' }, + }); + expect(store.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([]); + } finally { + store.close(); + } + }); + + it('fences terminal finalization without spending an execution attempt', async () => { + const store = await makeStore(); + try { + store.accept(baseInput()); + store.finish('requester-1' as SessionId, 'review-round-1', { type: 'cancelled' }); + + expect( + store.claimDeliveryFinalization('requester-1' as SessionId, 'review-round-1', { + claimId: 'premature-exhaustion', + workerBootId: 'worker-a', + requireAttemptsExhausted: true, + }) + ).toMatchObject({ status: 'not_ready', delivery: { attemptCount: 0 } }); + expect( + store.claimDeliveryFinalization('requester-1' as SessionId, 'review-round-1', { + claimId: 'finalization-a', + workerBootId: 'worker-a', + }) + ).toMatchObject({ + status: 'claimed', + delivery: { attemptCount: 0, activeClaimId: 'finalization-a' }, + }); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-b', + workerBootId: 'worker-b', + }) + ).toMatchObject({ + status: 'in_flight', + delivery: { attemptCount: 0, activeClaimId: 'finalization-a' }, + }); + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-b', + 'finalization-a' + ) + ).toMatchObject({ consumed: false, delivery: { state: 'pending' } }); + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'finalization-a' + ) + ).toMatchObject({ consumed: true, delivery: { state: 'consumed', attemptCount: 0 } }); + } finally { + store.close(); + } + }); + + it('recovers an orphaned terminal claim without changing the execution count', async () => { + const store = await makeStore(); + try { + store.accept(baseInput()); + store.finish('requester-1' as SessionId, 'review-round-1', { type: 'cancelled' }); + expect( + store.claimDeliveryFinalization('requester-1' as SessionId, 'review-round-1', { + claimId: 'finalization-a', + workerBootId: 'worker-a', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 0 } }); + + expect(store.recoverOrphanedDeliveryClaims('workspace-1' as WorkspaceId, 'worker-b')).toBe(1); + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'finalization-a' + ) + ).toMatchObject({ consumed: false, delivery: { state: 'pending', attemptCount: 0 } }); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-b', + workerBootId: 'worker-b', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 0 } }); + } finally { + store.close(); + } + }); + + it('abandons only the stopped coordinator owner and fences its prepared callback', async () => { + const store = await makeStore(); + try { + store.accept(baseInput()); + store.finish('requester-1' as SessionId, 'review-round-1', { type: 'cancelled' }); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-a', + workerBootId: 'worker-a', + }) + ).toMatchObject({ status: 'claimed' }); + expect( + store.prepareClaimedDeliveryExecution( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'attempt-a' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 1 } }); + + expect(store.abandonDeliveryClaimsOwnedBy('workspace-1' as WorkspaceId, 'worker-b')).toBe(0); + expect(store.getDelivery('requester-1' as SessionId, 'review-round-1')).toMatchObject({ + executionPhase: 'prepared', + activeClaimId: 'attempt-a', + activeClaimWorkerBootId: 'worker-a', + }); + expect(store.abandonDeliveryClaimsOwnedBy('workspace-1' as WorkspaceId, 'worker-a')).toBe(1); + expect(store.getDelivery('requester-1' as SessionId, 'review-round-1')).toMatchObject({ + executionPhase: 'ready', + attemptCount: 1, + }); + expect( + store.markClaimedDeliveryExecutionStarted( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'attempt-a' + ) + ).toBe(false); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-b', + workerBootId: 'worker-a', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 1 } }); + } finally { + store.close(); + } + }); + + it('quarantines an orphaned claim after provider execution starts', async () => { + const store = await makeStore(); + try { + store.accept(baseInput()); + store.finish('requester-1' as SessionId, 'review-round-1', { type: 'cancelled' }); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-a', + workerBootId: 'worker-a', + }) + ).toMatchObject({ status: 'claimed', delivery: { executionPhase: 'claimed' } }); + expect( + store.prepareClaimedDeliveryExecution( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'attempt-a' + ) + ).toMatchObject({ + prepared: true, + delivery: { attemptCount: 1, executionPhase: 'prepared' }, + }); + expect( + store.markClaimedDeliveryExecutionStarted( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'attempt-a' + ) + ).toBe(true); + + expect(store.recoverOrphanedDeliveryClaims('workspace-1' as WorkspaceId, 'worker-b')).toBe(1); + const recovered = store.getDelivery('requester-1' as SessionId, 'review-round-1'); + expect(recovered).toMatchObject({ + state: 'pending', + executionPhase: 'uncertain', + attemptCount: 1, + }); + expect(recovered).not.toHaveProperty('activeClaimId'); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-b', + workerBootId: 'worker-b', + }) + ).toMatchObject({ status: 'in_flight', delivery: { executionPhase: 'uncertain' } }); + } finally { + store.close(); + } + }); + + it('permits one prepared Delivery recovery and then exhausts attempts', async () => { + const store = await makeStore(); + try { + store.accept(baseInput()); + store.finish('requester-1' as SessionId, 'review-round-1', { type: 'cancelled' }); + + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-1', + workerBootId: 'daemon-1', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 0 } }); + expect( + store.prepareClaimedDeliveryExecution( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'attempt-1' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 1 } }); + expect( + store.releaseDeliveryClaim( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'attempt-1' + ) + ).toBe(true); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-2', + workerBootId: 'daemon-1', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 1 } }); + expect( + store.prepareClaimedDeliveryExecution( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'attempt-2' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 2 } }); + expect( + store.releaseDeliveryClaim( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'attempt-2' + ) + ).toBe(true); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-3', + workerBootId: 'daemon-1', + }) + ).toMatchObject({ status: 'exhausted', delivery: { attemptCount: 2, state: 'pending' } }); + expect( + store.claimDeliveryFinalization('requester-1' as SessionId, 'review-round-1', { + claimId: 'finalization-1', + workerBootId: 'daemon-1', + requireAttemptsExhausted: true, + }) + ).toMatchObject({ + status: 'claimed', + delivery: { attemptCount: 2, activeClaimId: 'finalization-1' }, + }); + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'finalization-1' + ) + ).toMatchObject({ consumed: true, delivery: { state: 'consumed', attemptCount: 2 } }); + } finally { + store.close(); + } + }); + + it('fences concurrent Workers and recovers only attempts from an older boot', async () => { + const store = await makeStore(); + try { + store.accept(baseInput()); + store.finish('requester-1' as SessionId, 'review-round-1', { type: 'cancelled' }); + + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-a', + workerBootId: 'worker-a', + }) + ).toMatchObject({ status: 'claimed', delivery: { attemptCount: 0 } }); + expect( + store.prepareClaimedDeliveryExecution( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'attempt-a' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 1 } }); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-b-before-barrier', + workerBootId: 'worker-b', + }) + ).toMatchObject({ + status: 'in_flight', + delivery: { + attemptCount: 1, + activeClaimId: 'attempt-a', + activeClaimWorkerBootId: 'worker-a', + }, + }); + expect(store.recoverOrphanedDeliveryClaims('workspace-1' as WorkspaceId, 'worker-a')).toBe(0); + expect(store.recoverOrphanedDeliveryClaims('workspace-1' as WorkspaceId, 'worker-b')).toBe(1); + expect( + store.claimDeliveryExecution('requester-1' as SessionId, 'review-round-1', { + claimId: 'attempt-b', + workerBootId: 'worker-b', + }) + ).toMatchObject({ + status: 'claimed', + delivery: { attemptCount: 1, activeClaimWorkerBootId: 'worker-b' }, + }); + expect( + store.prepareClaimedDeliveryExecution( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-b', + 'attempt-b' + ) + ).toMatchObject({ prepared: true, delivery: { attemptCount: 2 } }); + + expect( + store.releaseDeliveryClaim( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'attempt-a' + ) + ).toBe(false); + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'attempt-a' + ) + ).toMatchObject({ + consumed: false, + delivery: { state: 'pending', activeClaimId: 'attempt-b' }, + }); + expect( + store.claimDeliveryFinalization('requester-1' as SessionId, 'review-round-1', { + claimId: 'finalization-a', + workerBootId: 'worker-a', + requireAttemptsExhausted: true, + }) + ).toMatchObject({ status: 'in_flight', delivery: { activeClaimId: 'attempt-b' } }); + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'worker-a', + 'finalization-a' + ) + ).toMatchObject({ consumed: false, delivery: { state: 'pending' } }); + } finally { + store.close(); + } + }); + it('delivers the same caller-chosen operation id independently for two Sessions', async () => { const store = await makeStore(); try { @@ -309,7 +842,20 @@ describe('LodyOperationStore', () => { nowMs += 8 * 24 * 60 * 60 * 1_000; expect(store.get('requester-1' as SessionId, 'review-round-1').state).toBe('finished'); - store.consumeDelivery('requester-1' as SessionId, 'review-round-1'); + expect( + store.claimDeliveryFinalization('requester-1' as SessionId, 'review-round-1', { + claimId: 'retention-finalization', + workerBootId: 'daemon-1', + }) + ).toMatchObject({ status: 'claimed' }); + expect( + store.consumeClaimedDelivery( + 'requester-1' as SessionId, + 'review-round-1', + 'daemon-1', + 'retention-finalization' + ) + ).toMatchObject({ consumed: true }); nowMs += 8 * 24 * 60 * 60 * 1_000; expect(() => store.get('requester-1' as SessionId, 'review-round-1')).toThrowError( @@ -467,6 +1013,28 @@ const readLastCleanupAtMs = (dbPath: string): string | undefined => { }; describe('maintenance-free open', () => { + it('does not acquire the SQLite writer lock when the schema is current', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'lody-operation-store-readonly-open-')); + roots.add(root); + const dbPath = path.join(root, 'operations.sqlite3'); + const owner = new LodyOperationStore(dbPath); + owner.close(); + + const writer = new Database(dbPath); + writer.exec('BEGIN IMMEDIATE'); + try { + const nonOwner = new LodyOperationStore(dbPath, undefined, { maintenance: false }); + try { + expect(nonOwner.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([]); + } finally { + nonOwner.close(); + } + } finally { + writer.exec('ROLLBACK'); + writer.close(); + } + }); + it('skips open-time repair/cleanup writes but still serves reads and writes', async () => { const root = await mkdtemp(path.join(os.tmpdir(), 'lody-operation-store-maint-')); roots.add(root); diff --git a/apps/cli/src/orchestration/operation-store.ts b/apps/cli/src/orchestration/operation-store.ts index e5c4651c0..1fec873cc 100644 --- a/apps/cli/src/orchestration/operation-store.ts +++ b/apps/cli/src/orchestration/operation-store.ts @@ -28,6 +28,19 @@ import { getLodyDataDir } from '@lody/shared/node/installation-profile'; const DAY_MS = 24 * 60 * 60 * 1_000; const TERMINAL_RETENTION_MS = 7 * DAY_MS; export const MATERIALIZATION_CLAIM_MS = 60_000; +export const DELIVERY_MAX_ATTEMPTS = 2; + +type DeliveryExecutionClaim = + | { status: 'claimed'; delivery: StoredLodyDelivery } + | { status: 'in_flight'; delivery: StoredLodyDelivery } + | { status: 'exhausted'; delivery: StoredLodyDelivery } + | { status: 'consumed'; delivery: StoredLodyDelivery }; + +type DeliveryFinalizationClaim = + | { status: 'claimed'; delivery: StoredLodyDelivery } + | { status: 'in_flight'; delivery: StoredLodyDelivery } + | { status: 'not_ready'; delivery: StoredLodyDelivery } + | { status: 'consumed'; delivery: StoredLodyDelivery }; const OperationKindSchema = z.enum([ 'session_create', @@ -166,6 +179,29 @@ const DeliveryRowSchema = z }) .strict(); +const DeliveryReadRowSchema = DeliveryRowSchema.extend({ + execution_phase: z.enum(['ready', 'claimed', 'prepared', 'started', 'uncertain']), + attempt_count: z.number().int().nonnegative(), + active_claim_id: z.string().nullable(), + active_claim_worker_boot_id: z.string().nullable(), +}).strict(); + +const DELIVERY_READ_COLUMNS = ` + deliveries.sequence, + deliveries.workspace_id, + deliveries.requester_session_id, + deliveries.operation_id, + deliveries.delivery_id, + deliveries.system_turn_id, + deliveries.state, + deliveries.initiator_chain_depth, + deliveries.completion_json, + deliveries.consumed_at, + delivery_execution_state.execution_phase, + delivery_execution_state.attempt_count, + delivery_execution_state.active_claim_id, + delivery_execution_state.active_claim_worker_boot_id`; + export class LodyOperationStoreError extends Error { constructor( readonly code: string, @@ -694,6 +730,13 @@ export class LodyOperationStore { current.initiatorChainDepth, JSON.stringify(durableCompletion) ); + this.db + .prepare( + `INSERT OR IGNORE INTO delivery_execution_state ( + requester_session_id, operation_id, attempt_count + ) VALUES (?, ?, 0)` + ) + .run(current.requesterSessionId, current.operationId); const updated = this.getStored(requesterSessionId, operationId); if (!updated) { throw new Error('Finished Operation disappeared during transaction.'); @@ -731,32 +774,403 @@ export class LodyOperationStore { const rows = requesterSessionId ? this.db .prepare( - `SELECT * FROM deliveries - WHERE workspace_id = ? AND requester_session_id = ? AND state = 'pending' - ORDER BY sequence ASC` + `SELECT ${DELIVERY_READ_COLUMNS} + FROM deliveries + JOIN delivery_execution_state USING (requester_session_id, operation_id) + WHERE deliveries.workspace_id = ? + AND deliveries.requester_session_id = ? + AND deliveries.state = 'pending' + ORDER BY deliveries.sequence ASC` ) .all(workspaceId, requesterSessionId) : this.db .prepare( - `SELECT * FROM deliveries - WHERE workspace_id = ? AND state = 'pending' - ORDER BY sequence ASC` + `SELECT ${DELIVERY_READ_COLUMNS} + FROM deliveries + JOIN delivery_execution_state USING (requester_session_id, operation_id) + WHERE deliveries.workspace_id = ? AND deliveries.state = 'pending' + ORDER BY deliveries.sequence ASC` ) .all(workspaceId); return rows.map((row) => this.decodeDelivery(row)); } - consumeDelivery( + getDelivery(requesterSessionId: SessionId, operationId: string): StoredLodyDelivery { + const row = this.db + .prepare( + `SELECT ${DELIVERY_READ_COLUMNS} + FROM deliveries + JOIN delivery_execution_state USING (requester_session_id, operation_id) + WHERE deliveries.requester_session_id = ? AND deliveries.operation_id = ?` + ) + .get(requesterSessionId, operationId); + if (row === undefined) { + throw new LodyOperationStoreError( + 'DELIVERY_NOT_FOUND', + `Delivery not found for Operation: ${operationId}`, + false + ); + } + return this.decodeDelivery(row); + } + + /** Run once after the Worker lifecycle barrier, never during ordinary reconciliation. */ + recoverOrphanedDeliveryClaims(workspaceId: WorkspaceId, workerBootId: string): number { + const result = this.db + .prepare( + `UPDATE delivery_execution_state + SET execution_phase = CASE + WHEN execution_phase = 'started' THEN 'uncertain' + WHEN execution_phase IN ('claimed', 'prepared') THEN 'ready' + ELSE execution_phase + END, + active_claim_id = NULL, + active_claim_worker_boot_id = NULL + WHERE (active_claim_id IS NOT NULL OR active_claim_worker_boot_id IS NOT NULL) + AND ( + active_claim_id IS NULL + OR active_claim_worker_boot_id IS NULL + OR active_claim_worker_boot_id <> ? + ) + AND EXISTS ( + SELECT 1 FROM deliveries + WHERE deliveries.requester_session_id = delivery_execution_state.requester_session_id + AND deliveries.operation_id = delivery_execution_state.operation_id + AND deliveries.workspace_id = ? AND deliveries.state = 'pending' + )` + ) + .run(workerBootId, workspaceId); + return result.changes; + } + + /** Run only after this coordinator has stopped scheduling new Delivery work. */ + abandonDeliveryClaimsOwnedBy(workspaceId: WorkspaceId, workerBootId: string): number { + const result = this.db + .prepare( + `UPDATE delivery_execution_state + SET execution_phase = CASE + WHEN execution_phase = 'started' THEN 'uncertain' + WHEN execution_phase IN ('claimed', 'prepared') THEN 'ready' + ELSE execution_phase + END, + active_claim_id = NULL, + active_claim_worker_boot_id = NULL + WHERE active_claim_worker_boot_id = ? + AND EXISTS ( + SELECT 1 FROM deliveries + WHERE deliveries.requester_session_id = delivery_execution_state.requester_session_id + AND deliveries.operation_id = delivery_execution_state.operation_id + AND deliveries.workspace_id = ? AND deliveries.state = 'pending' + )` + ) + .run(workerBootId, workspaceId); + return result.changes; + } + + claimDeliveryExecution( requesterSessionId: SessionId, operationId: string, - consumedAt = new Date(this.now()).toISOString() - ): void { - this.db + claim: { claimId: string; workerBootId: string } + ): DeliveryExecutionClaim { + const transaction = this.db.transaction((): DeliveryExecutionClaim => { + const current = this.getDelivery(requesterSessionId, operationId); + if (current.state === 'consumed') return { status: 'consumed', delivery: current }; + if ( + current.activeClaimId === claim.claimId && + current.activeClaimWorkerBootId === claim.workerBootId + ) { + return { status: 'claimed', delivery: current }; + } + if (current.activeClaimId !== undefined || current.activeClaimWorkerBootId !== undefined) { + return { status: 'in_flight', delivery: current }; + } + if (current.executionPhase !== 'ready') { + return { status: 'in_flight', delivery: current }; + } + if (current.attemptCount >= DELIVERY_MAX_ATTEMPTS) { + return { status: 'exhausted', delivery: current }; + } + const result = this.db + .prepare( + `UPDATE delivery_execution_state + SET execution_phase = 'claimed', active_claim_id = ?, active_claim_worker_boot_id = ? + WHERE requester_session_id = ? AND operation_id = ? + AND attempt_count < ? + AND execution_phase = 'ready' + AND active_claim_id IS NULL AND active_claim_worker_boot_id IS NULL + AND EXISTS ( + SELECT 1 FROM deliveries + WHERE deliveries.requester_session_id = delivery_execution_state.requester_session_id + AND deliveries.operation_id = delivery_execution_state.operation_id + AND deliveries.state = 'pending' + )` + ) + .run( + claim.claimId, + claim.workerBootId, + requesterSessionId, + operationId, + DELIVERY_MAX_ATTEMPTS + ); + if (result.changes !== 1) { + const latest = this.getDelivery(requesterSessionId, operationId); + if (latest.state === 'consumed') return { status: 'consumed', delivery: latest }; + if (latest.activeClaimId !== undefined || latest.activeClaimWorkerBootId !== undefined) { + return { status: 'in_flight', delivery: latest }; + } + return { status: 'exhausted', delivery: latest }; + } + return { + status: 'claimed', + delivery: this.getDelivery(requesterSessionId, operationId), + }; + }); + return transaction.immediate(); + } + + prepareClaimedDeliveryExecution( + requesterSessionId: SessionId, + operationId: string, + workerBootId: string, + claimId: string + ): { prepared: boolean; delivery: StoredLodyDelivery } { + const transaction = this.db.transaction(() => { + const current = this.getDelivery(requesterSessionId, operationId); + if ( + current.activeClaimWorkerBootId === workerBootId && + current.activeClaimId === claimId && + (current.executionPhase === 'prepared' || current.executionPhase === 'started') + ) { + return { prepared: true, delivery: current }; + } + const result = this.db + .prepare( + `UPDATE delivery_execution_state + SET execution_phase = 'prepared', attempt_count = attempt_count + 1 + WHERE requester_session_id = ? AND operation_id = ? + AND attempt_count < ? + AND execution_phase = 'claimed' + AND active_claim_worker_boot_id = ? AND active_claim_id = ? + AND EXISTS ( + SELECT 1 FROM deliveries + WHERE deliveries.requester_session_id = delivery_execution_state.requester_session_id + AND deliveries.operation_id = delivery_execution_state.operation_id + AND deliveries.state = 'pending' + )` + ) + .run(requesterSessionId, operationId, DELIVERY_MAX_ATTEMPTS, workerBootId, claimId); + return { + prepared: result.changes === 1, + delivery: this.getDelivery(requesterSessionId, operationId), + }; + }); + return transaction.immediate(); + } + + markClaimedDeliveryExecutionStarted( + requesterSessionId: SessionId, + operationId: string, + workerBootId: string, + claimId: string + ): boolean { + const result = this.db + .prepare( + `UPDATE delivery_execution_state + SET execution_phase = 'started' + WHERE requester_session_id = ? AND operation_id = ? + AND execution_phase = 'prepared' + AND active_claim_worker_boot_id = ? AND active_claim_id = ? + AND EXISTS ( + SELECT 1 FROM deliveries + WHERE deliveries.requester_session_id = delivery_execution_state.requester_session_id + AND deliveries.operation_id = delivery_execution_state.operation_id + AND deliveries.state = 'pending' + )` + ) + .run(requesterSessionId, operationId, workerBootId, claimId); + if (result.changes === 1) return true; + const current = this.getDelivery(requesterSessionId, operationId); + return ( + current.executionPhase === 'started' && + current.activeClaimWorkerBootId === workerBootId && + current.activeClaimId === claimId + ); + } + + markClaimedDeliveryExecutionUncertain( + requesterSessionId: SessionId, + operationId: string, + workerBootId: string, + claimId: string + ): boolean { + const result = this.db + .prepare( + `UPDATE delivery_execution_state + SET execution_phase = 'uncertain', + active_claim_id = NULL, + active_claim_worker_boot_id = NULL + WHERE requester_session_id = ? AND operation_id = ? + AND execution_phase = 'started' + AND active_claim_worker_boot_id = ? AND active_claim_id = ? + AND EXISTS ( + SELECT 1 FROM deliveries + WHERE deliveries.requester_session_id = delivery_execution_state.requester_session_id + AND deliveries.operation_id = delivery_execution_state.operation_id + AND deliveries.state = 'pending' + )` + ) + .run(requesterSessionId, operationId, workerBootId, claimId); + return result.changes === 1; + } + + claimDeliveryFinalization( + requesterSessionId: SessionId, + operationId: string, + claim: { + claimId: string; + workerBootId: string; + requireAttemptsExhausted?: boolean; + requiredExecutionPhase?: 'ready' | 'uncertain'; + } + ): DeliveryFinalizationClaim { + const transaction = this.db.transaction((): DeliveryFinalizationClaim => { + const current = this.getDelivery(requesterSessionId, operationId); + if (current.state === 'consumed') return { status: 'consumed', delivery: current }; + if ( + current.activeClaimId === claim.claimId && + current.activeClaimWorkerBootId === claim.workerBootId + ) { + return { status: 'claimed', delivery: current }; + } + if (current.activeClaimId !== undefined || current.activeClaimWorkerBootId !== undefined) { + return { status: 'in_flight', delivery: current }; + } + const minimumAttemptCount = claim.requireAttemptsExhausted ? DELIVERY_MAX_ATTEMPTS : 0; + const requiredExecutionPhase = claim.requiredExecutionPhase ?? 'ready'; + if (current.attemptCount < minimumAttemptCount) { + return { status: 'not_ready', delivery: current }; + } + if (current.executionPhase !== requiredExecutionPhase) { + return { status: 'not_ready', delivery: current }; + } + const result = this.db + .prepare( + `UPDATE delivery_execution_state + SET active_claim_id = ?, active_claim_worker_boot_id = ? + WHERE requester_session_id = ? AND operation_id = ? + AND attempt_count >= ? + AND execution_phase = ? + AND active_claim_id IS NULL AND active_claim_worker_boot_id IS NULL + AND EXISTS ( + SELECT 1 FROM deliveries + WHERE deliveries.requester_session_id = delivery_execution_state.requester_session_id + AND deliveries.operation_id = delivery_execution_state.operation_id + AND deliveries.state = 'pending' + )` + ) + .run( + claim.claimId, + claim.workerBootId, + requesterSessionId, + operationId, + minimumAttemptCount, + requiredExecutionPhase + ); + if (result.changes !== 1) { + const latest = this.getDelivery(requesterSessionId, operationId); + if (latest.state === 'consumed') return { status: 'consumed', delivery: latest }; + if (latest.activeClaimId !== undefined || latest.activeClaimWorkerBootId !== undefined) { + return { status: 'in_flight', delivery: latest }; + } + return { status: 'not_ready', delivery: latest }; + } + return { + status: 'claimed', + delivery: this.getDelivery(requesterSessionId, operationId), + }; + }); + return transaction.immediate(); + } + + releaseDeliveryClaim( + requesterSessionId: SessionId, + operationId: string, + workerBootId: string, + claimId: string + ): boolean { + const result = this.db .prepare( - `UPDATE deliveries SET state = 'consumed', consumed_at = ? - WHERE requester_session_id = ? AND operation_id = ? AND state = 'pending'` + `UPDATE delivery_execution_state + SET execution_phase = CASE + WHEN execution_phase IN ('claimed', 'prepared') THEN 'ready' + ELSE execution_phase + END, + active_claim_id = NULL, + active_claim_worker_boot_id = NULL + WHERE requester_session_id = ? AND operation_id = ? + AND active_claim_worker_boot_id = ? AND active_claim_id = ? + AND execution_phase <> 'started' + AND EXISTS ( + SELECT 1 FROM deliveries + WHERE deliveries.requester_session_id = delivery_execution_state.requester_session_id + AND deliveries.operation_id = delivery_execution_state.operation_id + AND deliveries.state = 'pending' + )` ) - .run(consumedAt, requesterSessionId, operationId); + .run(requesterSessionId, operationId, workerBootId, claimId); + return result.changes === 1; + } + + consumeClaimedDelivery( + requesterSessionId: SessionId, + operationId: string, + workerBootId: string, + claimId: string, + consumedAt = new Date(this.now()).toISOString() + ): { consumed: boolean; delivery: StoredLodyDelivery } { + const transaction = this.db.transaction(() => { + const current = this.getDelivery(requesterSessionId, operationId); + if ( + current.state === 'consumed' || + current.activeClaimWorkerBootId !== workerBootId || + current.activeClaimId !== claimId + ) { + return { consumed: false, delivery: current }; + } + const result = this.db + .prepare( + `UPDATE deliveries + SET state = 'consumed', consumed_at = ? + WHERE requester_session_id = ? AND operation_id = ? AND state = 'pending' + AND EXISTS ( + SELECT 1 FROM delivery_execution_state + WHERE delivery_execution_state.requester_session_id = deliveries.requester_session_id + AND delivery_execution_state.operation_id = deliveries.operation_id + AND delivery_execution_state.active_claim_worker_boot_id = ? + AND delivery_execution_state.active_claim_id = ? + )` + ) + .run(consumedAt, requesterSessionId, operationId, workerBootId, claimId); + if (result.changes === 1) { + const released = this.db + .prepare( + `UPDATE delivery_execution_state + SET active_claim_id = NULL, active_claim_worker_boot_id = NULL + WHERE requester_session_id = ? AND operation_id = ? + AND active_claim_worker_boot_id = ? AND active_claim_id = ?` + ) + .run(requesterSessionId, operationId, workerBootId, claimId); + if (released.changes !== 1) { + throw new Error('Consumed Delivery lost its active claim during transaction.'); + } + } + const latest = this.getDelivery(requesterSessionId, operationId); + return { + consumed: result.changes === 1, + delivery: latest, + }; + }); + return transaction.immediate(); } deleteRequesterSession(requesterSessionId: SessionId): void { @@ -866,7 +1280,7 @@ export class LodyOperationStore { } private decodeDelivery(row: unknown): StoredLodyDelivery { - const parsed = DeliveryRowSchema.parse(row); + const parsed = DeliveryReadRowSchema.parse(row); return { sequence: parsed.sequence, workspaceId: parsed.workspace_id as WorkspaceId, @@ -875,6 +1289,12 @@ export class LodyOperationStore { deliveryId: parsed.delivery_id, systemTurnId: parsed.system_turn_id, state: parsed.state, + executionPhase: parsed.execution_phase, + attemptCount: parsed.attempt_count, + ...(parsed.active_claim_id ? { activeClaimId: parsed.active_claim_id } : {}), + ...(parsed.active_claim_worker_boot_id + ? { activeClaimWorkerBootId: parsed.active_claim_worker_boot_id } + : {}), initiatorChainDepth: parsed.initiator_chain_depth, completion: parseJson( parsed.completion_json, @@ -947,7 +1367,24 @@ export class LodyOperationStore { } private migrate(): void { - this.db.exec(` + if (!this.isMigrationRequired()) return; + + this.db + .transaction(() => { + const hadDeliveryExecutionState = + this.db + .prepare( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'delivery_execution_state'" + ) + .get() !== undefined; + const hadDeliveryExecutionPhase = + hadDeliveryExecutionState && + ( + this.db.prepare(`PRAGMA table_info(delivery_execution_state)`).all() as Array<{ + name: string; + }> + ).some((column) => column.name === 'execution_phase'); + this.db.exec(` CREATE TABLE IF NOT EXISTS operations ( workspace_id TEXT NOT NULL, owner_machine_id TEXT NOT NULL, @@ -990,6 +1427,20 @@ export class LodyOperationStore { CREATE INDEX IF NOT EXISTS deliveries_pending_session ON deliveries (workspace_id, requester_session_id, state, sequence); + CREATE TABLE IF NOT EXISTS delivery_execution_state ( + requester_session_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + execution_phase TEXT NOT NULL DEFAULT 'ready' + CHECK (execution_phase IN ('ready', 'claimed', 'prepared', 'started', 'uncertain')), + attempt_count INTEGER NOT NULL DEFAULT 0, + active_claim_id TEXT, + active_claim_worker_boot_id TEXT, + PRIMARY KEY (requester_session_id, operation_id), + FOREIGN KEY (requester_session_id, operation_id) + REFERENCES operations (requester_session_id, operation_id) + ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS operation_item_materializations ( requester_session_id TEXT NOT NULL, operation_id TEXT NOT NULL, @@ -1007,6 +1458,76 @@ export class LodyOperationStore { value TEXT NOT NULL ); `); + if (hadDeliveryExecutionState && !hadDeliveryExecutionPhase) { + this.db.exec( + `ALTER TABLE delivery_execution_state + ADD COLUMN execution_phase TEXT NOT NULL DEFAULT 'ready'` + ); + this.db.exec( + `UPDATE delivery_execution_state + SET execution_phase = CASE + WHEN attempt_count > 0 + OR active_claim_id IS NOT NULL + OR active_claim_worker_boot_id IS NOT NULL + THEN 'uncertain' + ELSE 'ready' + END` + ); + } + this.db + .prepare( + `INSERT OR IGNORE INTO delivery_execution_state ( + requester_session_id, operation_id, execution_phase, attempt_count + ) + SELECT requester_session_id, operation_id, + CASE WHEN state = 'pending' THEN ? ELSE 'ready' END, + 0 + FROM deliveries` + ) + .run(hadDeliveryExecutionState ? 'ready' : 'uncertain'); + }) + .immediate(); + } + + private isMigrationRequired(): boolean { + const requiredObjects = new Set([ + 'table:operations', + 'index:operations_active_owner', + 'table:deliveries', + 'index:deliveries_pending_session', + 'table:delivery_execution_state', + 'table:operation_item_materializations', + 'table:orchestration_meta', + ]); + const existingObjects = this.db + .prepare( + `SELECT type, name FROM sqlite_master + WHERE type IN ('table', 'index')` + ) + .all() as Array<{ type: string; name: string }>; + for (const { type, name } of existingObjects) { + requiredObjects.delete(`${type}:${name}`); + } + if (requiredObjects.size > 0) return true; + + const executionStateColumns = this.db + .prepare(`PRAGMA table_info(delivery_execution_state)`) + .all() as Array<{ name: string }>; + if (!executionStateColumns.some((column) => column.name === 'execution_phase')) return true; + + return ( + this.db + .prepare( + `SELECT 1 + FROM deliveries + LEFT JOIN delivery_execution_state + ON delivery_execution_state.requester_session_id = deliveries.requester_session_id + AND delivery_execution_state.operation_id = deliveries.operation_id + WHERE delivery_execution_state.requester_session_id IS NULL + LIMIT 1` + ) + .get() !== undefined + ); } private repairTerminalDeliveries(): void { @@ -1038,6 +1559,14 @@ export class LodyOperationStore { )` ) .run(); + this.db + .prepare( + `INSERT OR IGNORE INTO delivery_execution_state ( + requester_session_id, operation_id, execution_phase, attempt_count + ) + SELECT requester_session_id, operation_id, 'ready', 0 FROM deliveries` + ) + .run(); }) .immediate(); } diff --git a/apps/cli/src/session/session-execution-service.ts b/apps/cli/src/session/session-execution-service.ts index 9c1f840c6..a384d01fe 100644 --- a/apps/cli/src/session/session-execution-service.ts +++ b/apps/cli/src/session/session-execution-service.ts @@ -240,6 +240,11 @@ type TurnRuntimeState = { cancelFinalized: boolean; interruptRequested: boolean; terminateSessionOnCancel: boolean; + settlement?: { + callback: (settlement: SessionTurnSettlement) => Promise; + forcedOutcome?: SessionTurnSettlement; + completed: boolean; + }; /** Logical prompt tail currently owned by the one session-owner fiber. */ activePromptRun?: PromptHandoffRun; /** Serialized ancillary finalization for yielded logical turns. */ @@ -317,6 +322,14 @@ type VisibleSessionTurnOptions = { session?: ISession; userTurnId?: string; invocation?: TurnInvocation; + /** + * Turn that deterministically owns the assistant history entry. Delivery + * uses its system Turn here while leaving userTurnId absent so it cannot + * mutate user dispatch status or pointers. + */ + assistantEntryParentTurnId?: string; + onTurnStarted?: () => Promise; + onTurnSettled?: (settlement: SessionTurnSettlement) => Promise; /** * How the turn payload reached this machine. 'rpc' turns can start before the * user's history entry syncs locally, so their turn-scoped history writes go @@ -341,11 +354,19 @@ type SessionDispatchOptions = { /** * Runs only after this process has synchronously claimed the per-Session * visible-turn owner. Delivery uses this to append its system cause without - * racing a user dispatch between the idle check and the history write. + * racing a user dispatch between the idle check and the history write. False + * means an external durable claim lost contention; the turn is released + * without history, ACP, failure, or settlement side effects. */ - onTurnClaimed?: () => Promise; + onTurnClaimed?: () => Promise; + /** Runs immediately before the request can cross into the ACP provider. */ + onTurnStarted?: () => Promise; + /** Reports whether the provider was handled, cancelled, never started, or left uncertain. */ + onTurnSettled?: (settlement: SessionTurnSettlement) => Promise; }; +type SessionTurnSettlement = 'handled' | 'cancelled' | 'not_started' | 'uncertain'; + export type PreparedSessionDispatchRequest = | { mode: 'create'; request: SessionCreateRequestValidated } | { mode: 'continue'; request: SessionChatRequestValidated }; @@ -377,6 +398,17 @@ class SessionTurnHalted extends Data.TaggedError('SessionTurnHalted')<{ reason: ChatFailedReason; }> {} +class SessionTurnClaimContended extends Data.TaggedError('SessionTurnClaimContended')<{ + sessionId: SessionId; + turnId: string; +}> {} + +class SessionTurnStartFenceFailed extends Data.TaggedError('SessionTurnStartFenceFailed')<{ + sessionId: SessionId; + turnId: string; + cause: unknown; +}> {} + const isSessionTurnCancelled = (error: unknown): error is SessionTurnCancelled => { return ( typeof error === 'object' && @@ -395,6 +427,24 @@ const isSessionTurnHalted = (error: unknown): error is SessionTurnHalted => { ); }; +const isSessionTurnClaimContended = (error: unknown): error is SessionTurnClaimContended => { + return ( + typeof error === 'object' && + error !== null && + '_tag' in error && + error._tag === 'SessionTurnClaimContended' + ); +}; + +const isSessionTurnStartFenceFailed = (error: unknown): error is SessionTurnStartFenceFailed => { + return ( + typeof error === 'object' && + error !== null && + '_tag' in error && + error._tag === 'SessionTurnStartFenceFailed' + ); +}; + function truncateAnalyticsString(value: string, maxLength = 1_000): string { return value.length > maxLength ? `${value.slice(0, maxLength)}...` : value; } @@ -1289,6 +1339,9 @@ export class SessionExecutionService { requesterUserId: options.userId, inputConfig: options.inputConfig, }; + // Provider acceptance hands the original dispatch forward. A later + // user-owned steer turn must not cancel or reopen that responsibility. + await this.settleVisibleTurn(runtime, 'handled', { force: true }); try { await this.finalizeYieldedTurnOutput(runtime, options.sessionId, previousTurnId); @@ -1548,7 +1601,7 @@ export class SessionExecutionService { private createTurnRuntime( options: Pick< VisibleSessionTurnOptions, - 'sessionId' | 'session' | 'userTurnId' | 'invocation' + 'sessionId' | 'session' | 'userTurnId' | 'invocation' | 'onTurnSettled' > & { turnId: string } ): TurnRuntimeState { return { @@ -1568,10 +1621,32 @@ export class SessionExecutionService { cancelFinalized: false, interruptRequested: false, terminateSessionOnCancel: false, + ...(options.onTurnSettled + ? { settlement: { callback: options.onTurnSettled, completed: false } } + : {}), yieldedFinalization: Promise.resolve(), }; } + private async settleVisibleTurn( + runtime: TurnRuntimeState, + outcome: SessionTurnSettlement, + options: { force?: boolean } = {} + ): Promise { + const settlement = runtime.settlement; + if (!settlement || settlement.completed) return; + if (options.force) settlement.forcedOutcome = outcome; + const effectiveOutcome = settlement.forcedOutcome ?? outcome; + try { + await settlement.callback(effectiveOutcome); + settlement.completed = true; + } catch (error) { + this.deps.logger.error( + `[${runtime.sessionId}] Failed to persist ${effectiveOutcome} turn settlement: ${formatErrorMessage(error)}` + ); + } + } + private getTurnRuntime(sessionId: SessionId, turnId: string): TurnRuntimeState | undefined { const runtime = this.turnRuntimeBySession.get(sessionId); return runtime?.turnId === turnId ? runtime : undefined; @@ -2548,6 +2623,7 @@ export class SessionExecutionService { body: (ctx: VisibleSessionTurnContext) => Effect.Effect ): Promise { const { sessionId, sessionDoc, userTurnId } = options; + const assistantEntryParentTurnId = options.assistantEntryParentTurnId ?? userTurnId; const span = startTraceSpan(this.deps.logger, 'execution.visible_turn', { sessionId, ...(userTurnId ? { userTurnId } : {}), @@ -2576,7 +2652,7 @@ export class SessionExecutionService { let turnId!: string; let runtime!: TurnRuntimeState; try { - turnId = this.deps.beginConversationTurn(sessionId, userTurnId, { + turnId = this.deps.beginConversationTurn(sessionId, assistantEntryParentTurnId, { ...(options.dispatchSource ? { dispatchSource: options.dispatchSource } : {}), sessionDoc, deferACPUpdateTarget: true, @@ -2715,7 +2791,7 @@ export class SessionExecutionService { sessionDoc, runtime.turnId, runtime.session?.agentClient?.currentModel, - userTurnId + assistantEntryParentTurnId ) ) ); @@ -2761,6 +2837,27 @@ export class SessionExecutionService { yield* Effect.fail(new Error('Agent session was not ready')); return undefined; } + if (options.onTurnStarted) { + const started = yield* self.tryPromise(options.onTurnStarted).pipe( + Effect.mapError( + (cause) => + new SessionTurnStartFenceFailed({ + sessionId, + turnId: runtime.turnId, + cause, + }) + ) + ); + if (!started) { + yield* Effect.fail( + new SessionTurnClaimContended({ + sessionId, + turnId: runtime.turnId, + }) + ); + return undefined; + } + } runtime.terminateSessionOnCancel = false; self.deps.activateConversationTurnForACPUpdates(sessionId, runtime.turnId); runtime.promptStarted = true; @@ -2827,13 +2924,23 @@ export class SessionExecutionService { const fiber = Effect.runFork(program); runtime.fiber = fiber; + let settlement: SessionTurnSettlement | undefined; try { await this.awaitTurnFiber(fiber, sessionId, turnId); if (outcome === 'unknown') { outcome = 'completed'; } + settlement = 'handled'; } catch (error) { - if (isSessionTurnHalted(error)) { + if (isSessionTurnClaimContended(error)) { + outcome = 'claim-contended'; + } else if (isSessionTurnStartFenceFailed(error)) { + outcome = 'start-fence-failed'; + this.deps.logger.warn( + `[${sessionId}] Delivery start fence failed before provider execution: ${formatErrorMessage(error.cause)}` + ); + settlement = 'not_started'; + } else if (isSessionTurnHalted(error)) { outcome = `halted-${error.reason}`; await this.finalizeHaltedTurn({ sessionId, @@ -2841,10 +2948,8 @@ export class SessionExecutionService { turnId: runtime.turnId, reason: error.reason, }); - return outcome; - } - - if ( + settlement = 'handled'; + } else if ( isSessionTurnCancelled(error) || runtime.cancelFinalized || runtime.cancelRequested || @@ -2852,26 +2957,38 @@ export class SessionExecutionService { (await this.isUserTurnCancelled(sessionDoc, runtime.userTurnId)) ) { outcome = 'cancelled'; - return outcome; + const explicitlyCancelled = + runtime.cancelRequested || + this.isTurnCancelled(sessionId, runtime.turnId) || + (await this.isUserTurnCancelled(sessionDoc, runtime.userTurnId)); + settlement = explicitlyCancelled + ? 'cancelled' + : runtime.promptStarted + ? 'uncertain' + : 'not_started'; + } else { + await this.handleVisibleTurnUnhandledError({ + sessionId, + sessionDoc, + userTurnId: runtime.userTurnId, + runtime, + error, + code: effectiveErrorContext.code, + describe: effectiveErrorContext.describe, + onUnhandledError: effectiveErrorContext.onUnhandledError, + }); + outcome = 'unhandled-error-recorded'; + settlement = 'handled'; } - - await this.handleVisibleTurnUnhandledError({ - sessionId, - sessionDoc, - userTurnId: runtime.userTurnId, - runtime, - error, - code: effectiveErrorContext.code, - describe: effectiveErrorContext.describe, - onUnhandledError: effectiveErrorContext.onUnhandledError, - }); - outcome = 'unhandled-error-recorded'; } finally { if (!runtime.promptStarted) { this.deps.clearConversationTurn(sessionId, runtime.turnId); } span.end({ outcome, turnId }); } + if (settlement) { + await this.settleVisibleTurn(runtime, settlement); + } return outcome; } @@ -3317,7 +3434,18 @@ export class SessionExecutionService { } const body = dispatchOptions?.onTurnClaimed ? (ctx: VisibleSessionTurnContext) => - Effect.promise(dispatchOptions.onTurnClaimed!).pipe(Effect.flatMap(() => turn.body(ctx))) + Effect.promise(dispatchOptions.onTurnClaimed!).pipe( + Effect.flatMap((claimed) => + claimed + ? turn.body(ctx) + : Effect.fail( + new SessionTurnClaimContended({ + sessionId: message.sessionId, + turnId: ctx.turnId, + }) + ) + ) + ) : turn.body; await this.runVisibleSessionTurn(turn.options, body); } @@ -4046,6 +4174,11 @@ export class SessionExecutionService { requesterUserId: userId, inputConfig: acpSessionConfig, }, + ...(dispatchOptions?.dispatchSource === 'delivery' + ? { assistantEntryParentTurnId: userTurnId } + : {}), + ...(dispatchOptions?.onTurnStarted ? { onTurnStarted: dispatchOptions.onTurnStarted } : {}), + ...(dispatchOptions?.onTurnSettled ? { onTurnSettled: dispatchOptions.onTurnSettled } : {}), ...(dispatchOptions?.dispatchSource ? { dispatchSource: dispatchOptions.dispatchSource } : {}), diff --git a/apps/cli/tests/session-execution-service.test.ts b/apps/cli/tests/session-execution-service.test.ts index 20170acd4..bcaa9562b 100644 --- a/apps/cli/tests/session-execution-service.test.ts +++ b/apps/cli/tests/session-execution-service.test.ts @@ -32,6 +32,7 @@ import { } from '../src/agent/agent-client'; import { AcpAuthenticationManager } from '../src/agent/acp-authentication'; import { GitExecutableNotFoundError } from '../src/session/worktree/git-process-error'; +import { LodyOperationStore } from '../src/orchestration/operation-store'; const capabilityConfigId = 'config-1' as AgentConfigId; @@ -264,6 +265,7 @@ describe('SessionExecutionService', () => { turnId: 'assistant:user-1', promptPromise: new Promise(() => {}), }); + const onTurnSettled = vi.fn(async () => {}); const runtime = { sessionId, turnId: 'assistant:user-1', @@ -277,6 +279,7 @@ describe('SessionExecutionService', () => { }, activePromptRun: initialPromptRun, yieldedFinalization: Promise.resolve(), + settlement: { callback: onTurnSettled, completed: false }, }; ( service as unknown as { @@ -294,6 +297,8 @@ describe('SessionExecutionService', () => { inputConfig: { prompt: 'change direction' }, }) ).resolves.toMatchObject({ applied: true, disposition: 'applied' }); + expect(onTurnSettled).toHaveBeenCalledOnce(); + expect(onTurnSettled).toHaveBeenCalledWith('handled'); expect(deps.turnFinalization.finalizeACPState).toHaveBeenCalledWith( sessionId, @@ -401,6 +406,7 @@ describe('SessionExecutionService', () => { }) ).resolves.toEqual({ success: true }); await vi.waitFor(() => expect(cancel).toHaveBeenCalledWith('acp-steer')); + expect(onTurnSettled).toHaveBeenCalledOnce(); }); it('completes A to B to C when yielded prompts never settle', async () => { @@ -499,18 +505,22 @@ describe('SessionExecutionService', () => { ), }); const service = new SessionExecutionService(deps); - const lifecycle = service.continueSession({ - type: 'session/chat', - sessionId, - machineId: 'machine-1', - workspaceId: 'workspace-1' as WorkspaceId, - project: undefined, - acpSessionConfig: { prompt: 'A', cliType: 'builtin', agentType: 'claude' }, - userTurnId: 'user-a', - userId: 'user-1', - userName: 'User', - userEmail: 'user@example.com', - }); + const onTurnSettled = vi.fn(async () => {}); + const lifecycle = service.continueSession( + { + type: 'session/chat', + sessionId, + machineId: 'machine-1', + workspaceId: 'workspace-1' as WorkspaceId, + project: undefined, + acpSessionConfig: { prompt: 'A', cliType: 'builtin', agentType: 'claude' }, + userTurnId: 'user-a', + userId: 'user-1', + userName: 'User', + userEmail: 'user@example.com', + }, + { onTurnSettled } + ); await vi.waitFor(() => expect(prompt).toHaveBeenCalledTimes(1)); const steerB = service.steerSession({ @@ -531,6 +541,8 @@ describe('SessionExecutionService', () => { const releaseB = vi.fn(); applicationB.resolve({ steerId: 'steer-b', release: releaseB }); await expect(steerB).resolves.toMatchObject({ applied: true, disposition: 'applied' }); + expect(onTurnSettled).toHaveBeenCalledOnce(); + expect(onTurnSettled).toHaveBeenCalledWith('handled'); expect(releaseB).toHaveBeenCalledOnce(); expect(deps.activateConversationTurnForACPUpdates).toHaveBeenCalledTimes(2); expect( @@ -611,6 +623,7 @@ describe('SessionExecutionService', () => { expect(deps.turnFinalization.notifySessionCompleted).toHaveBeenCalledTimes(1); expect(deps.processMessageQueue).toHaveBeenCalledTimes(1); expect(service.getExecutionSnapshot(sessionId)).toMatchObject({ hasActiveTurn: false }); + expect(onTurnSettled).toHaveBeenCalledOnce(); }); it('rejects steer without mutating the active turn when the agent lacks support', async () => { @@ -1135,6 +1148,12 @@ describe('SessionExecutionService', () => { const runSilentPromptTurn = async (options: { sessionId: string; hasPromptOutputForTurn: boolean; + dispatchSource?: 'delivery'; + onTurnClaimed?: () => Promise; + onTurnStarted?: () => Promise; + onTurnSettled?: ( + settlement: 'handled' | 'cancelled' | 'not_started' | 'uncertain' + ) => Promise; }) => { let history: Array> = [ { @@ -1174,8 +1193,12 @@ describe('SessionExecutionService', () => { }), }; const notifySessionCompleted = vi.fn(async () => {}); + const onTurnSettled = options.onTurnSettled ?? vi.fn(async () => {}); const upsertDocMeta = vi.fn(async () => {}); const deps = createBaseDeps({ + beginConversationTurn: vi.fn( + (_sessionId: SessionId, parentTurnId?: string) => `assistant:${parentTurnId ?? 'unbound'}` + ), sessionManager: { getSession: vi.fn(() => activeSession), getPendingSession: vi.fn(() => null), @@ -1200,18 +1223,28 @@ describe('SessionExecutionService', () => { }); const service = new SessionExecutionService(deps); - await service.continueSession({ - type: 'session/chat', - sessionId: options.sessionId as SessionId, - machineId: 'machine-1', - workspaceId: 'workspace-1' as WorkspaceId, - project: undefined, - acpSessionConfig: { prompt: 'hi', cliType: 'builtin', agentType: 'codex' }, - userTurnId: 'turn-user-1', - userId: 'user-1', - userName: 'User', - userEmail: 'user@example.com', - }); + await service.continueSession( + { + type: 'session/chat', + sessionId: options.sessionId as SessionId, + machineId: 'machine-1', + workspaceId: 'workspace-1' as WorkspaceId, + project: undefined, + acpSessionConfig: { prompt: 'hi', cliType: 'builtin', agentType: 'codex' }, + userTurnId: 'turn-user-1', + userId: 'user-1', + userName: 'User', + userEmail: 'user@example.com', + }, + options.dispatchSource + ? { + dispatchSource: options.dispatchSource, + onTurnSettled, + ...(options.onTurnClaimed ? { onTurnClaimed: options.onTurnClaimed } : {}), + ...(options.onTurnStarted ? { onTurnStarted: options.onTurnStarted } : {}), + } + : undefined + ); return { deps, @@ -1219,6 +1252,7 @@ describe('SessionExecutionService', () => { notifySessionCompleted, upsertDocMeta, agentClient, + onTurnSettled, getHistory: () => history, }; }; @@ -1261,6 +1295,188 @@ describe('SessionExecutionService', () => { expect(notifySessionCompleted).toHaveBeenCalledTimes(1); }); + it('binds a Delivery assistant to its system turn without claiming user dispatch state', async () => { + const { deps, upsertDocMeta, onTurnSettled, getHistory } = await runSilentPromptTurn({ + sessionId: 'session-delivery-turn', + hasPromptOutputForTurn: true, + dispatchSource: 'delivery', + }); + const sessionDoc = await deps.workspaceDocument.getOrCreateSessionDoc( + 'session-delivery-turn' as SessionId + ); + + expect(deps.beginConversationTurn).toHaveBeenCalledWith( + 'session-delivery-turn', + 'turn-user-1', + { + dispatchSource: 'delivery', + sessionDoc, + deferACPUpdateTarget: true, + } + ); + expect(deps.createAssistantEntryForTurn).toHaveBeenCalledWith( + 'session-delivery-turn', + sessionDoc, + 'assistant:turn-user-1', + undefined, + 'turn-user-1' + ); + expect(getHistory()[0]?.status).toBe('pending'); + expect(onTurnSettled).toHaveBeenCalledWith('handled'); + expect( + upsertDocMeta.mock.calls.some(([, patch]) => { + const fields = patch as Record; + return 'processingUserMsgId' in fields || 'lastHandledUserMsgId' in fields; + }) + ).toBe(false); + }); + + it('settles a silent Delivery turn as durably handled', async () => { + const { onTurnSettled } = await runSilentPromptTurn({ + sessionId: 'session-silent-delivery-turn', + hasPromptOutputForTurn: false, + dispatchSource: 'delivery', + }); + + expect(onTurnSettled).toHaveBeenCalledWith('handled'); + }); + + it('crosses the durable Delivery start fence before calling the provider', async () => { + const onTurnStarted = vi.fn(async () => true); + const { agentClient } = await runSilentPromptTurn({ + sessionId: 'session-delivery-start-fence', + hasPromptOutputForTurn: true, + dispatchSource: 'delivery', + onTurnStarted, + }); + + expect(onTurnStarted).toHaveBeenCalledOnce(); + expect(onTurnStarted.mock.invocationCallOrder[0]).toBeLessThan( + agentClient.prompt.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY + ); + }); + + it('settles a failed Delivery start fence as not started without calling the provider', async () => { + const onTurnSettled = vi.fn(async () => {}); + const { deps, agentClient, getHistory } = await runSilentPromptTurn({ + sessionId: 'session-delivery-start-fence-failure', + hasPromptOutputForTurn: false, + dispatchSource: 'delivery', + onTurnStarted: async () => { + throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); + }, + onTurnSettled, + }); + + expect(agentClient.prompt).not.toHaveBeenCalled(); + expect(deps.recordChatFailure).not.toHaveBeenCalled(); + expect(onTurnSettled).toHaveBeenCalledOnce(); + expect(onTurnSettled).toHaveBeenCalledWith('not_started'); + expect(getHistory()[0]?.status).toBe('pending'); + }); + + it('does not replay a real stored Delivery when settlement persistence fails', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'lody-delivery-execution-integration-')); + const store = new LodyOperationStore(path.join(root, 'operations.sqlite3')); + const requesterSessionId = 'session-delivery-store' as SessionId; + const operationId = 'operation-delivery-store'; + const workerBootId = 'worker-integration'; + const claimId = 'claim-integration'; + try { + store.accept({ + workspaceId: 'workspace-1' as WorkspaceId, + ownerMachineId: 'machine-1' as MachineId, + requesterSessionId, + requesterUserId: 'user-1', + operationId, + kind: 'session_chat', + canonicalCommand: { prompt: 'integration' }, + frozenContinuationConfig: { + inputConfig: { cliType: 'builtin', agentType: 'codex', chainDepth: 0 }, + }, + initiatorChainDepth: 0, + createdAt: '2026-09-05T00:00:00.000Z', + deadlineAt: '2026-09-05T01:00:00.000Z', + items: [], + }); + store.finish(requesterSessionId, operationId, { type: 'cancelled' }); + + const { agentClient } = await runSilentPromptTurn({ + sessionId: requesterSessionId, + hasPromptOutputForTurn: true, + dispatchSource: 'delivery', + onTurnClaimed: async () => { + const claim = store.claimDeliveryExecution(requesterSessionId, operationId, { + claimId, + workerBootId, + }); + if (claim.status !== 'claimed') return false; + return store.prepareClaimedDeliveryExecution( + requesterSessionId, + operationId, + workerBootId, + claimId + ).prepared; + }, + onTurnStarted: async () => + store.markClaimedDeliveryExecutionStarted( + requesterSessionId, + operationId, + workerBootId, + claimId + ), + onTurnSettled: async (settlement) => { + expect(settlement).toBe('handled'); + throw new Error('settlement write failed'); + }, + }); + + expect(agentClient.prompt).toHaveBeenCalledOnce(); + expect(store.getDelivery(requesterSessionId, operationId)).toMatchObject({ + state: 'pending', + executionPhase: 'started', + attemptCount: 1, + activeClaimId: claimId, + }); + expect( + store.recoverOrphanedDeliveryClaims('workspace-1' as WorkspaceId, 'worker-replacement') + ).toBe(1); + expect( + store.claimDeliveryExecution(requesterSessionId, operationId, { + claimId: 'replacement-claim', + workerBootId: 'worker-replacement', + }) + ).toMatchObject({ + status: 'in_flight', + delivery: { executionPhase: 'uncertain', attemptCount: 1 }, + }); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it('silently releases a Delivery turn when its durable attempt claim loses contention', async () => { + const onTurnClaimed = vi.fn(async () => false); + const { deps, agentClient, onTurnSettled, getHistory } = await runSilentPromptTurn({ + sessionId: 'session-contended-delivery-turn', + hasPromptOutputForTurn: true, + dispatchSource: 'delivery', + onTurnClaimed, + }); + + expect(onTurnClaimed).toHaveBeenCalledOnce(); + expect(agentClient.prompt).not.toHaveBeenCalled(); + expect(deps.createAssistantEntryForTurn).not.toHaveBeenCalled(); + expect(deps.recordChatFailure).not.toHaveBeenCalled(); + expect(onTurnSettled).not.toHaveBeenCalled(); + expect(deps.clearConversationTurn).toHaveBeenCalledWith( + 'session-contended-delivery-turn', + 'assistant:turn-user-1' + ); + expect(getHistory()[0]?.status).toBe('pending'); + }); + it('rejects a chat turn before prompt when memory pressure persists', async () => { let history: Array> = [ { @@ -4665,19 +4881,23 @@ describe('SessionExecutionService', () => { expect(history[0]).toMatchObject({ id: 'turn-prompt-cancel', status: 'canceled' }); }); + const onTurnSettled = vi.fn(async () => {}); service = new SessionExecutionService(deps); - await service.continueSession({ - type: 'session/chat', - sessionId: 'session-prompt-cancel' as SessionId, - machineId: 'machine-1', - workspaceId: 'workspace-1' as WorkspaceId, - project: { kind: 'github', repoFullName: 'owner/repo', branch: 'main' }, - acpSessionConfig: { prompt: 'hello', cliType: 'builtin', agentType: 'codex' }, - userTurnId: 'turn-prompt-cancel', - userId: 'user-1', - userName: 'User', - userEmail: 'user@example.com', - }); + await service.continueSession( + { + type: 'session/chat', + sessionId: 'session-prompt-cancel' as SessionId, + machineId: 'machine-1', + workspaceId: 'workspace-1' as WorkspaceId, + project: { kind: 'github', repoFullName: 'owner/repo', branch: 'main' }, + acpSessionConfig: { prompt: 'hello', cliType: 'builtin', agentType: 'codex' }, + userTurnId: 'turn-prompt-cancel', + userId: 'user-1', + userName: 'User', + userEmail: 'user@example.com', + }, + { onTurnSettled } + ); expect(agentClient.cancel).toHaveBeenCalledWith('acp-prompt-cancel'); expect(deps.turnFinalization.finalizeACPState).toHaveBeenCalledTimes(1); @@ -4688,6 +4908,7 @@ describe('SessionExecutionService', () => { lastHandledUserMsgId: 'turn-prompt-cancel', processingUserMsgId: undefined, }); + expect(onTurnSettled).toHaveBeenCalledWith('cancelled'); }); it('does not wait for ACP cancel before interrupting an in-flight prompt', async () => { diff --git a/locales/en.json b/locales/en.json index 91870f24f..594432d65 100644 --- a/locales/en.json +++ b/locales/en.json @@ -3480,6 +3480,7 @@ "orchestration.operationCancelled": "Operation {{id}} cancelled", "orchestration.operationItemSummary": "{{total}} items · {{succeeded}} succeeded · {{failed}} failed · {{cancelled}} cancelled", "orchestration.continuationNotStarted": "The saved agent configuration is unavailable, so no continuation was started.", + "orchestration.continuationUncertain": "The continuation may have started before it was interrupted. It was not replayed; review the output and continue manually if needed.", "commands.tasks.quickAdd": "New Task", "commands.tasks.open": "Open Tasks", "tasks.title": "Tasks", diff --git a/locales/zh_CN.json b/locales/zh_CN.json index c5e5e9cca..095d2a7ee 100644 --- a/locales/zh_CN.json +++ b/locales/zh_CN.json @@ -3480,6 +3480,7 @@ "orchestration.operationCancelled": "操作 {{id}} 已取消", "orchestration.operationItemSummary": "共 {{total}} 项 · {{succeeded}} 项成功 · {{failed}} 项失败 · {{cancelled}} 项取消", "orchestration.continuationNotStarted": "保存的 Agent 配置不可用，因此没有启动后续执行。", + "orchestration.continuationUncertain": "后续执行可能在中断前已经启动，因此没有自动重放。请检查现有输出，并在需要时手动继续。", "commands.tasks.quickAdd": "新建任务", "commands.tasks.open": "打开任务", "tasks.title": "任务", diff --git a/packages/components/src/components/ai-gui/view.tsx b/packages/components/src/components/ai-gui/view.tsx index ed7a8ca76..46c163e26 100644 --- a/packages/components/src/components/ai-gui/view.tsx +++ b/packages/components/src/components/ai-gui/view.tsx @@ -2018,9 +2018,13 @@ const OperationCompletionView = ({ })} ) : null} - {completion.continuation?.status === 'not_started' ? ( + {completion.continuation ? (
- {t('orchestration.continuationNotStarted')} + {t( + completion.continuation.status === 'uncertain' + ? 'orchestration.continuationUncertain' + : 'orchestration.continuationNotStarted' + )}
) : null} @@ -2057,9 +2061,13 @@ const OperationCompletionView = ({ })} ) : null} - {completion.continuation?.status === 'not_started' ? ( + {completion.continuation ? (
- {t('orchestration.continuationNotStarted')} + {t( + completion.continuation.status === 'uncertain' + ? 'orchestration.continuationUncertain' + : 'orchestration.continuationNotStarted' + )}
) : null} diff --git a/packages/shared/src/message-schemas.ts b/packages/shared/src/message-schemas.ts index 05d228b70..3e961379b 100644 --- a/packages/shared/src/message-schemas.ts +++ b/packages/shared/src/message-schemas.ts @@ -3226,10 +3226,14 @@ export const NonSystemNoticeMessageContentSchema = z.discriminatedUnion('type', completion: z.unknown(), continuation: z .object({ - status: z.literal('not_started'), + status: z.enum(['not_started', 'uncertain']), reason: z .object({ - code: z.literal('CONFIGURATION_UNAVAILABLE'), + code: z.enum([ + 'CONFIGURATION_UNAVAILABLE', + 'DELIVERY_ATTEMPTS_EXHAUSTED', + 'DELIVERY_EXECUTION_UNCERTAIN', + ]), message: z.string(), }) .strict(), diff --git a/packages/shared/src/session-orchestration.ts b/packages/shared/src/session-orchestration.ts index 9ae2c6343..ae9e4d207 100644 --- a/packages/shared/src/session-orchestration.ts +++ b/packages/shared/src/session-orchestration.ts @@ -166,6 +166,10 @@ export type StoredLodyDelivery = { deliveryId: string; systemTurnId: string; state: 'pending' | 'consumed'; + executionPhase: 'ready' | 'claimed' | 'prepared' | 'started' | 'uncertain'; + attemptCount: number; + activeClaimId?: string; + activeClaimWorkerBootId?: string; initiatorChainDepth: number; completion: LodyOperationCompletion; consumedAt?: string; @@ -178,9 +182,12 @@ export type OperationCompletionContent = { operationKind: LodyOperationKind; completion: LodyOperationCompletion; continuation?: { - status: 'not_started'; + status: 'not_started' | 'uncertain'; reason: { - code: 'CONFIGURATION_UNAVAILABLE'; + code: + | 'CONFIGURATION_UNAVAILABLE' + | 'DELIVERY_ATTEMPTS_EXHAUSTED' + | 'DELIVERY_EXECUTION_UNCERTAIN'; message: string; }; }; From 61b75b34a50e094e8abf7c010b768846eb9e7570 Mon Sep 17 00:00:00 2001 From: Wibus <62133302+wibus-wee@users.noreply.github.com> Date: Sun, 6 Sep 2026 23:00:48 +0800 Subject: [PATCH 2/3] fix(cli): fence delivery starts once and finalize rejected attempts Skip the start fence during stale ACP prompt recovery. Finalize rejected start attempts while the runtime still owns the session, restoring idle without consuming user dispatch state. Cover repeated fence failures and stale-session recovery in regression tests. Model: gpt-5 --- apps/cli/src/orchestration/AGENTS.md | 68 ++-- .../src/session/session-execution-service.ts | 10 +- .../tests/session-execution-service.test.ts | 341 +++++++++++------- 3 files changed, 240 insertions(+), 179 deletions(-) diff --git a/apps/cli/src/orchestration/AGENTS.md b/apps/cli/src/orchestration/AGENTS.md index 59c2baa57..94c41a456 100644 --- a/apps/cli/src/orchestration/AGENTS.md +++ b/apps/cli/src/orchestration/AGENTS.md @@ -58,46 +58,34 @@ Root and `apps/cli/AGENTS.md` apply. Normative behavior lives in paths must not add blocking waits on top of the driver's `busy_timeout`. - `operation-model.ts` is the reduced executable race model. Update its bounded exploration and concrete traces whenever scheduling semantics change. -- Delivery never writes user dispatch pointers. Pending user input wins every - idle boundary; completion uses a stable `role: system` - `operation_completion` Turn and then the existing Session execution mutex. - Its Assistant Turn id is `assistant:` even though it has no user - dispatch ownership. Assistant `finished`/`endedAt` is never Delivery completion - evidence: teardown writes the same terminal footprint. Delivery execution has three - fencing layers: the Host lease excludes other Hosts; each CLI Worker process owns one boot - id and starts only after the supervisor/Host-lease lifecycle barrier; and each attempt owns - a fresh token. - Execution fields live in `delivery_execution_state`, not `deliveries`: stable binaries parse - `SELECT * FROM deliveries` strictly, so adding columns there makes a downgraded binary unable - to read the shared local database. - Normal claims require no active token and never take over another owner. Paths that write - a terminal continuation failure or consume without execution must acquire the same - exclusive token first; the history write and token-matched consume happen while it is - held. Failed finalization retains the token and unfinished steps for later wakes: retry - history before consume, never ACP or a cleanup write. Recheck ownership after history - awaits; do not rewrite durable history. Stop drops this memory; replacement Workers use - durable state. Once per Worker startup, - the coordinator clears tokens owned by older boot ids - without resetting the attempt count. A claim records `claimed`, becomes `prepared` only - after the completion Turn is durable (which spends one bounded preparation attempt), and - becomes `started` immediately before calling ACP. Release and consume must match both the - boot id and claim token. Claim contention exits before history or ACP side effects and - records no failure. Only a confirmed pre-provider interruption releases a prepared claim; - a rejected start-fence write settles as not started and follows that same release path instead - of becoming a handled turn. User cancellation consumes it. A missing settlement after ACP started becomes `uncertain` - and is never automatically replayed: reconciliation writes - `DELIVERY_EXECUTION_UNCERTAIN` under a terminal claim, preserves existing output, and tells - the user to continue manually if needed. Provider-accepted steer settles the original - Delivery immediately, so cancellation of a later user-owned turn cannot reopen it. - Settlement write failure retains the claim-bound outcome in the live coordinator and retries - it on later wakes without ACP; replacement-Worker recovery converts any still-fenced started - claim to `uncertain`, never to runnable. A - coordinated workspace stop abandons only that coordinator's claims before closing its store: - `claimed`/`prepared` become runnable and `started` becomes `uncertain`. At most - one confirmed pre-provider recovery is allowed; after two prepared attempts, - `DELIVERY_ATTEMPTS_EXHAUSTED` is written and consumed without invoking ACP. A pending - Delivery from the pre-claim schema migrates as `uncertain`; its prior execution count is - unknowable and must not be fabricated. +- Delivery never writes user dispatch pointers. Pending users win idle boundaries; + completion uses a stable system `operation_completion` Turn, the Session execution mutex, + and Assistant id `assistant:`. Assistant `finished`/`endedAt` is not completion + evidence: teardown writes it too. + Fencing has three layers: exclusive Host lease, per-process Worker boot id after the + supervisor/Host-lease lifecycle barrier, and fresh per-attempt token. Execution fields stay + in `delivery_execution_state`: stable binaries strictly parse `SELECT * FROM deliveries`. + Claims never take over active tokens. Contention exits before history/ACP side effects + without recording failure. Release and consume match both boot id and token. + Terminal history and consume-without-execution acquire the same token. Failed + finalization retains it and unfinished steps for later wakes: history before consume, + never ACP or a cleanup write. Recheck ownership after history awaits; do not rewrite durable + history. Stop drops this memory; replacement Workers use durable state. + `claimed` becomes `prepared` after durable completion history, spending one attempt, then + `started` before the first ACP prompt. Stale-ACP recovery never repeats the fence. + A failed fence write finalizes the Assistant and restores idle under Session ownership, + then settles `not_started`, not handled. Only confirmed + pre-provider interruption releases a prepared claim. User cancellation consumes it; + accepted steer settles the original Delivery immediately so later turns cannot reopen it. + Missing post-start settlement becomes `uncertain`, never automatic replay. Reconciliation + writes `DELIVERY_EXECUTION_UNCERTAIN` under a terminal claim, preserving output and directing + manual continuation. Observed outcomes survive write failures in the live coordinator, + bound to the claim; later wakes retry settlement without ACP. + Startup clears older-boot tokens once without resetting attempts; started becomes uncertain. + Workspace stop abandons only its own claims before closing the store: + `claimed`/`prepared` become runnable, `started` becomes uncertain. After one pre-provider + recovery (two prepared attempts), write and consume `DELIVERY_ATTEMPTS_EXHAUSTED` without ACP. + Pre-claim pending Deliveries migrate as uncertain; execution counts stay unknown. - Missing Session metadata, a recoverable tombstone, or an unsynchronized Machine Flock document is uncertainty, not permanent deletion/configuration absence. Keep the item/Delivery pending until positive evidence or deadline. diff --git a/apps/cli/src/session/session-execution-service.ts b/apps/cli/src/session/session-execution-service.ts index a384d01fe..2c06333fa 100644 --- a/apps/cli/src/session/session-execution-service.ts +++ b/apps/cli/src/session/session-execution-service.ts @@ -2837,7 +2837,7 @@ export class SessionExecutionService { yield* Effect.fail(new Error('Agent session was not ready')); return undefined; } - if (options.onTurnStarted) { + if (!runtime.promptStarted && options.onTurnStarted) { const started = yield* self.tryPromise(options.onTurnStarted).pipe( Effect.mapError( (cause) => @@ -2846,6 +2846,14 @@ export class SessionExecutionService { turnId: runtime.turnId, cause, }) + ), + // Finalize while this runtime still owns the session; settlement is not success. + Effect.tapError(() => + self.ignoreWithWarning( + sessionId, + 'Failed to finalize a rejected Delivery start fence', + self.tryPromise(() => self.handleTurnError(sessionId, sessionDoc)) + ) ) ); if (!started) { diff --git a/apps/cli/tests/session-execution-service.test.ts b/apps/cli/tests/session-execution-service.test.ts index bcaa9562b..ff4d6a754 100644 --- a/apps/cli/tests/session-execution-service.test.ts +++ b/apps/cli/tests/session-execution-service.test.ts @@ -21,6 +21,7 @@ import { type SessionGoalMessage, type SessionHistoryInput, type SessionId, + type SessionMeta, type SessionInputBlock, type WorkspaceId, } from '@lody/shared'; @@ -33,6 +34,8 @@ import { import { AcpAuthenticationManager } from '../src/agent/acp-authentication'; import { GitExecutableNotFoundError } from '../src/session/worktree/git-process-error'; import { LodyOperationStore } from '../src/orchestration/operation-store'; +import { markAssistantTurnFinished } from '../src/lib/assistant-turn-finalize'; +import { shouldWatchSession } from '../src/session/session-dispatch-logic'; const capabilityConfigId = 'config-1' as AgentConfigId; @@ -1147,6 +1150,7 @@ describe('SessionExecutionService', () => { // left the user with an unanswered message and no error anywhere. const runSilentPromptTurn = async (options: { sessionId: string; + attempts?: number; hasPromptOutputForTurn: boolean; dispatchSource?: 'delivery'; onTurnClaimed?: () => Promise; @@ -1183,9 +1187,12 @@ describe('SessionExecutionService', () => { createAgent: vi.fn(async () => 'acp-silent'), applyExecutionPlaneLimits: vi.fn(async () => {}), }; + let status = SessionStatusFactory.idle(); const sessionDoc = { getMetaState: vi.fn(async () => ({ isArchived: false })), - setStatus: vi.fn(async () => {}), + setStatus: vi.fn(async (next: typeof status) => { + status = next; + }), setLastMessageAt: vi.fn(async () => {}), getHistory: vi.fn(async () => history), updateHistory: vi.fn(async (updater: (prev: typeof history) => typeof history) => { @@ -1194,11 +1201,18 @@ describe('SessionExecutionService', () => { }; const notifySessionCompleted = vi.fn(async () => {}); const onTurnSettled = options.onTurnSettled ?? vi.fn(async () => {}); + const finalizationOwners: boolean[] = []; const upsertDocMeta = vi.fn(async () => {}); const deps = createBaseDeps({ beginConversationTurn: vi.fn( (_sessionId: SessionId, parentTurnId?: string) => `assistant:${parentTurnId ?? 'unbound'}` ), + createAssistantEntryForTurn: vi.fn(async (_sessionId, _sessionDoc, turnId) => { + const entry = { id: turnId, role: 'assistant', finished: false, endedAt: undefined }; + history = history.some((item) => item.id === turnId) + ? history.map((item) => (item.id === turnId ? entry : item)) + : [...history, entry]; + }), sessionManager: { getSession: vi.fn(() => activeSession), getPendingSession: vi.fn(() => null), @@ -1218,33 +1232,44 @@ describe('SessionExecutionService', () => { turnFinalization: { ...createBaseDeps({}).turnFinalization, notifySessionCompleted, + finalizeACPState: vi.fn(async (_sessionId, turnId) => { + finalizationOwners.push( + service.getExecutionSnapshot(options.sessionId as SessionId).hasActiveTurn + ); + history = markAssistantTurnFinished(history as SessionHistoryInput[], { + turnId, + endedAt: 42, + }); + }), }, observePromptOutputForTurn: vi.fn(() => options.hasPromptOutputForTurn), }); const service = new SessionExecutionService(deps); - await service.continueSession( - { - type: 'session/chat', - sessionId: options.sessionId as SessionId, - machineId: 'machine-1', - workspaceId: 'workspace-1' as WorkspaceId, - project: undefined, - acpSessionConfig: { prompt: 'hi', cliType: 'builtin', agentType: 'codex' }, - userTurnId: 'turn-user-1', - userId: 'user-1', - userName: 'User', - userEmail: 'user@example.com', - }, - options.dispatchSource - ? { - dispatchSource: options.dispatchSource, - onTurnSettled, - ...(options.onTurnClaimed ? { onTurnClaimed: options.onTurnClaimed } : {}), - ...(options.onTurnStarted ? { onTurnStarted: options.onTurnStarted } : {}), - } - : undefined - ); + for (let attempt = 0; attempt < (options.attempts ?? 1); attempt += 1) { + await service.continueSession( + { + type: 'session/chat', + sessionId: options.sessionId as SessionId, + machineId: 'machine-1', + workspaceId: 'workspace-1' as WorkspaceId, + project: undefined, + acpSessionConfig: { prompt: 'hi', cliType: 'builtin', agentType: 'codex' }, + userTurnId: 'turn-user-1', + userId: 'user-1', + userName: 'User', + userEmail: 'user@example.com', + }, + options.dispatchSource + ? { + dispatchSource: options.dispatchSource, + onTurnSettled, + ...(options.onTurnClaimed ? { onTurnClaimed: options.onTurnClaimed } : {}), + ...(options.onTurnStarted ? { onTurnStarted: options.onTurnStarted } : {}), + } + : undefined + ); + } return { deps, @@ -1254,6 +1279,9 @@ describe('SessionExecutionService', () => { agentClient, onTurnSettled, getHistory: () => history, + getStatus: () => status, + finalizationOwners, + service, }; }; @@ -1356,23 +1384,44 @@ describe('SessionExecutionService', () => { ); }); - it('settles a failed Delivery start fence as not started without calling the provider', async () => { + it('settles failed Delivery start fences as not started and restores idle state', async () => { const onTurnSettled = vi.fn(async () => {}); - const { deps, agentClient, getHistory } = await runSilentPromptTurn({ - sessionId: 'session-delivery-start-fence-failure', - hasPromptOutputForTurn: false, - dispatchSource: 'delivery', - onTurnStarted: async () => { - throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); - }, - onTurnSettled, - }); + const { deps, agentClient, getHistory, getStatus, service, finalizationOwners } = + await runSilentPromptTurn({ + sessionId: 'session-delivery-start-fence-failure', + attempts: 2, + hasPromptOutputForTurn: false, + dispatchSource: 'delivery', + onTurnStarted: async () => { + throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); + }, + onTurnSettled, + }); expect(agentClient.prompt).not.toHaveBeenCalled(); expect(deps.recordChatFailure).not.toHaveBeenCalled(); - expect(onTurnSettled).toHaveBeenCalledOnce(); + expect(onTurnSettled).toHaveBeenCalledTimes(2); expect(onTurnSettled).toHaveBeenCalledWith('not_started'); expect(getHistory()[0]?.status).toBe('pending'); + expect(getHistory().find((entry) => entry.role === 'assistant')).toMatchObject({ + id: 'assistant:turn-user-1', + finished: true, + endedAt: 42, + }); + expect(getStatus()).toEqual(SessionStatusFactory.idle()); + expect(finalizationOwners).toEqual([true, true]); + expect(deps.turnFinalization.notifySessionCompleted).not.toHaveBeenCalled(); + expect( + service.getExecutionSnapshot('session-delivery-start-fence-failure' as SessionId) + ).toMatchObject({ hasActiveTurn: false }); + expect( + shouldWatchSession({ + meta: { status: getStatus() } as SessionMeta, + hasUnprocessedCancelRequest: false, + hasRpcTurnOffer: false, + hasAccessRetry: false, + }) + ).toBe(false); }); it('does not replay a real stored Delivery when settlement persistence fails', async () => { @@ -1952,113 +2001,129 @@ describe('SessionExecutionService', () => { expect(activeClearedAt).toBeGreaterThan(finalizeStartedAt); }); - it('restores and retries a stale in-memory ACP session when the connection is closed', async () => { - const sessionId = 'session-stale-acp' as SessionId; - const acpSessionId = 'acp-stale' as ACPSessionId; - const restoredAcpSessionId = 'acp-restored' as ACPSessionId; - let history: Array> = [ - { - id: 'turn-user-1', - role: 'user', - status: 'pending', - read: false, - }, - ]; - const agentClient = { - isCreated: vi.fn(() => true), - cancel: vi.fn(async () => {}), - prompt: vi.fn(async () => { - throw new Error('ACP connection closed'); - }), - currentModel: undefined, - }; - const restoredAgentClient = { - isCreated: vi.fn(() => true), - cancel: vi.fn(async () => {}), - prompt: vi.fn(async () => ({})), - currentModel: undefined, - }; - const exec = vi.fn(async (command: string, args: string[]) => { - const key = `${command} ${args.join(' ')}`; - if (key === 'git rev-parse --is-inside-work-tree') return 'true\n'; - if (key === 'git rev-parse HEAD') return 'abc123\n'; - return ''; - }); - const activeSession = { - sessionId, - acpSessionId, - agentClient, - terminalManager: {} as unknown, - getWorkdir: () => '/tmp', - getHostWorkdir: () => '/tmp', - getParentSessionId: () => undefined, - exec, - terminate: vi.fn(async () => {}), - updateGitIdentity: vi.fn(), - createAgent: vi.fn(async () => acpSessionId), - applyExecutionPlaneLimits: vi.fn(async () => {}), - }; - const restoredSession = { - ...activeSession, - acpSessionId: restoredAcpSessionId, - agentClient: restoredAgentClient, - createAgent: vi.fn(async () => restoredAcpSessionId), - }; - const sessionDoc = { - getMetaState: vi.fn(async () => ({ isArchived: false })), - setStatus: vi.fn(async () => {}), - waitUntilSynced: vi.fn(async () => {}), - getHistory: vi.fn(async () => history), - updateHistory: vi.fn(async (updater: (prev: typeof history) => typeof history) => { - history = updater(history); - }), - }; - const deps = createBaseDeps({}); - const sessionManager = deps.sessionManager as unknown as { - getSession: ReturnType; - terminateSession: ReturnType; - createSession: ReturnType; - }; - const workspaceDocument = deps.workspaceDocument as unknown as { - getOrCreateSessionDoc: ReturnType; - }; - sessionManager.getSession.mockReturnValue(activeSession); - sessionManager.createSession.mockResolvedValue(restoredSession); - workspaceDocument.getOrCreateSessionDoc.mockResolvedValue(sessionDoc); + it.each([undefined, 'delivery'] as const)( + 'restores a stale ACP session without repeating the start fence for %s dispatch', + async (dispatchSource) => { + const sessionId = 'session-stale-acp' as SessionId; + const acpSessionId = 'acp-stale' as ACPSessionId; + const restoredAcpSessionId = 'acp-restored' as ACPSessionId; + let history: Array> = [ + { + id: 'turn-user-1', + role: 'user', + status: 'pending', + read: false, + }, + ]; + const agentClient = { + isCreated: vi.fn(() => true), + cancel: vi.fn(async () => {}), + prompt: vi.fn(async () => { + throw new Error('ACP connection closed'); + }), + currentModel: undefined, + }; + const restoredAgentClient = { + isCreated: vi.fn(() => true), + cancel: vi.fn(async () => {}), + prompt: vi.fn(async () => ({})), + currentModel: undefined, + }; + const exec = vi.fn(async (command: string, args: string[]) => { + const key = `${command} ${args.join(' ')}`; + if (key === 'git rev-parse --is-inside-work-tree') return 'true\n'; + if (key === 'git rev-parse HEAD') return 'abc123\n'; + return ''; + }); + const activeSession = { + sessionId, + acpSessionId, + agentClient, + terminalManager: {} as unknown, + getWorkdir: () => '/tmp', + getHostWorkdir: () => '/tmp', + getParentSessionId: () => undefined, + exec, + terminate: vi.fn(async () => {}), + updateGitIdentity: vi.fn(), + createAgent: vi.fn(async () => acpSessionId), + applyExecutionPlaneLimits: vi.fn(async () => {}), + }; + const restoredSession = { + ...activeSession, + acpSessionId: restoredAcpSessionId, + agentClient: restoredAgentClient, + createAgent: vi.fn(async () => restoredAcpSessionId), + }; + const sessionDoc = { + getMetaState: vi.fn(async () => ({ isArchived: false })), + setStatus: vi.fn(async () => {}), + waitUntilSynced: vi.fn(async () => {}), + getHistory: vi.fn(async () => history), + updateHistory: vi.fn(async (updater: (prev: typeof history) => typeof history) => { + history = updater(history); + }), + }; + const deps = createBaseDeps({}); + const sessionManager = deps.sessionManager as unknown as { + getSession: ReturnType; + terminateSession: ReturnType; + createSession: ReturnType; + }; + const workspaceDocument = deps.workspaceDocument as unknown as { + getOrCreateSessionDoc: ReturnType; + }; + sessionManager.getSession.mockReturnValue(activeSession); + sessionManager.createSession.mockResolvedValue(restoredSession); + workspaceDocument.getOrCreateSessionDoc.mockResolvedValue(sessionDoc); - const service = new SessionExecutionService(deps); - await service.continueSession({ - type: 'session/chat', - sessionId, - machineId: 'machine-1', - workspaceId: 'workspace-1' as WorkspaceId, - project: undefined, - acpSessionConfig: { prompt: 'hi', cliType: 'builtin', agentType: 'codex' }, - userTurnId: 'turn-user-1', - userId: 'user-1', - userName: 'User', - userEmail: 'user@example.com', - }); + const onTurnStarted = vi + .fn(async () => { + throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' }); + }) + .mockResolvedValueOnce(true); + const onTurnSettled = vi.fn(async () => {}); + const service = new SessionExecutionService(deps); + await service.continueSession( + { + type: 'session/chat', + sessionId, + machineId: 'machine-1', + workspaceId: 'workspace-1' as WorkspaceId, + project: undefined, + acpSessionConfig: { prompt: 'hi', cliType: 'builtin', agentType: 'codex' }, + userTurnId: 'turn-user-1', + userId: 'user-1', + userName: 'User', + userEmail: 'user@example.com', + }, + dispatchSource ? { dispatchSource, onTurnStarted, onTurnSettled } : undefined + ); - expect(agentClient.prompt).toHaveBeenCalledWith( - 'acp-stale', - [{ type: 'text', text: 'hello' }], - { - signal: expect.any(AbortSignal), - } - ); - expect(sessionManager.terminateSession).toHaveBeenCalledWith(sessionId, true); - expect(sessionManager.createSession).toHaveBeenCalled(); - expect(restoredAgentClient.prompt).toHaveBeenCalledWith( - 'acp-restored', - [{ type: 'text', text: 'hello' }], - { - signal: expect.any(AbortSignal), + expect(agentClient.prompt).toHaveBeenCalledWith( + 'acp-stale', + [{ type: 'text', text: 'hello' }], + { + signal: expect.any(AbortSignal), + } + ); + expect(sessionManager.terminateSession).toHaveBeenCalledWith(sessionId, true); + expect(sessionManager.createSession).toHaveBeenCalled(); + expect(restoredAgentClient.prompt).toHaveBeenCalledWith( + 'acp-restored', + [{ type: 'text', text: 'hello' }], + { + signal: expect.any(AbortSignal), + } + ); + expect(deps.recordChatFailure).not.toHaveBeenCalled(); + expect(history[0]?.status).toBe(dispatchSource === 'delivery' ? 'pending' : 'handled'); + if (dispatchSource === 'delivery') { + expect(onTurnSettled).toHaveBeenCalledWith('handled'); + expect(onTurnStarted).toHaveBeenCalledOnce(); } - ); - expect(deps.recordChatFailure).not.toHaveBeenCalled(); - expect(history[0]?.status).toBe('handled'); - }); + } + ); it('does not write legacy code-session tags when Code Collab is enabled for new turns', async () => { let history: Array> = [ From 3be488eba76312f7804bfe90022a4e4daf2da5b1 Mon Sep 17 00:00:00 2001 From: Wibus <62133302+wibus-wee@users.noreply.github.com> Date: Mon, 7 Sep 2026 13:01:32 +0800 Subject: [PATCH 3/3] fix(cli): backfill legacy delivery execution state Maintain delivery execution rows with a SQLite insert trigger so legacy writers remain visible to a running coordinator. Remove redundant application dual-writes and simplify delivery scheduling and model state after ablation checks. Model: gpt-5 --- apps/cli/src/orchestration/AGENTS.md | 40 ++++++++--------- .../orchestration/operation-coordinator.ts | 30 ++++++------- apps/cli/src/orchestration/operation-model.ts | 12 ------ .../src/orchestration/operation-store.test.ts | 43 +++++++++++++++++++ apps/cli/src/orchestration/operation-store.ts | 42 +++++------------- 5 files changed, 87 insertions(+), 80 deletions(-) diff --git a/apps/cli/src/orchestration/AGENTS.md b/apps/cli/src/orchestration/AGENTS.md index b5ff67b84..528c1d015 100644 --- a/apps/cli/src/orchestration/AGENTS.md +++ b/apps/cli/src/orchestration/AGENTS.md @@ -43,9 +43,8 @@ Root and `apps/cli/AGENTS.md` apply; `specs/session-orchestration.md` owns behav opened with `maintenance: false` so non-owner opens are not themselves write transactions; current-schema detection is read-only and migration takes the writer lock only when that probe finds work. The daemon coordinator owns - open-time repair/cleanup. Do not - reintroduce per-call open/close: each close checkpoints against the shared - WAL and each default open writes, which is the "database is locked" source. + open-time repair/cleanup. Do not reintroduce per-call open/close: each close + checkpoints WAL and each default open writes, causing "database is locked". - WAL allows one writer machine-wide. Every writing store transaction runs `BEGIN IMMEDIATE` (deferred read→write upgrades fail with `SQLITE_BUSY_SNAPSHOT`, which `busy_timeout` cannot wait out). Subprocess @@ -54,24 +53,23 @@ Root and `apps/cli/AGENTS.md` apply; `specs/session-orchestration.md` owns behav paths must not add blocking waits on top of the driver's `busy_timeout`. - `operation-model.ts` is the reduced executable race model. Update its bounded exploration and concrete traces whenever scheduling semantics change. -- Delivery never writes user dispatch pointers; pending users win idle boundaries. Completion uses - one stable system Turn, the Session mutex, and Assistant id `assistant:`; - Assistant `finished`/`endedAt` is not evidence because teardown also writes it. - Fences are the Host lease, Worker boot id, and attempt token. Execution fields stay - in `delivery_execution_state` because stable binaries parse `SELECT * FROM deliveries`. Active - claims cannot be taken over; contention has no history/ACP effects, and release/consume match both - ids. Terminal and no-execution paths claim too, write history before consume, retain failed - finalization for settlement-only retries, recheck ownership after awaits, and never rewrite - durable history. `claimed` becomes `prepared` after history and spends an attempt; `started` - precedes provider prompt, and stale-ACP recovery skips that fence. A failed start fence - finalizes the Assistant, restores idle under Session ownership, and settles `not_started`; only - confirmed pre-provider interruption releases prepared work. Cancellation or accepted steer - consumes the Delivery. Missing post-start settlement becomes `uncertain`, never replay, and emits - `DELIVERY_EXECUTION_UNCERTAIN` while preserving output. Claim-bound live outcomes survive store - failures for settlement-only retry. Startup recovers older boots without resetting attempts; - started work becomes uncertain. Stop abandons only its Worker: claimed/prepared work becomes - runnable and started work uncertain. After two prepared attempts, consume with - `DELIVERY_ATTEMPTS_EXHAUSTED` without ACP. Pre-claim pending migration is uncertain. +- Delivery never writes user dispatch pointers; pending users win idle boundaries. Completion owns + one stable system Turn under the Session mutex and Assistant `assistant:`; + `finished`/`endedAt` is not evidence because teardown writes it too. Host lease, Worker boot id, + and attempt token fence execution. Fields remain in `delivery_execution_state` because stable + binaries parse `SELECT * FROM deliveries`; its insert trigger atomically covers current and legacy + writers, migration backfills older rows, and app writers do not dual-write. Claims are exclusive; + contention has no history/ACP effects, and release/consume match both ids. Terminal/no-execution + paths claim, write history before consume, retain failed finalization for settlement-only retry, + recheck after awaits, and never rewrite durable history. `claimed` becomes `prepared` after history + and spends one attempt; `started` precedes the provider, while stale-ACP recovery skips that fence. + Failed start fencing finalizes the Assistant, restores idle under Session ownership, and settles + `not_started`; only confirmed pre-provider interruption releases prepared work. Cancellation or + accepted steer consumes. Missing post-start settlement becomes `uncertain`, never replays, emits + `DELIVERY_EXECUTION_UNCERTAIN`, and preserves output. Claim-bound outcomes survive store failures. + Startup recovers older boots without resetting attempts; stop abandons only its Worker. Both make + claimed/prepared work runnable and started work uncertain. After two prepared attempts, consume + with `DELIVERY_ATTEMPTS_EXHAUSTED` without ACP. Pre-claim migration is uncertain. - Create Operations may also maintain one stable `role: system` `operation_progress` Turn in the requester Session, written only by the Host-lease Worker, never by MCP replicas. It is durable UI state, never agent input/dispatch. Repair duplicate ids before keyed diff --git a/apps/cli/src/orchestration/operation-coordinator.ts b/apps/cli/src/orchestration/operation-coordinator.ts index fb305387d..9a3f56144 100644 --- a/apps/cli/src/orchestration/operation-coordinator.ts +++ b/apps/cli/src/orchestration/operation-coordinator.ts @@ -181,7 +181,7 @@ export class LodyOperationCoordinator { private readonly reconcileChains = new Map>(); private readonly deliveryChains = new Map>(); private readonly queuedDeliveryIds = new Set(); - private readonly dirtyDeliveryReasons = new Map(); + private readonly dirtyDeliveryIds = new Set(); private readonly observedDeliverySettlements = new Map(); private readonly operationAbortControllers = new Map(); private metaWatch: RepoWatchHandle | null = null; @@ -300,7 +300,7 @@ export class LodyOperationCoordinator { this.reconcileChains.clear(); this.deliveryChains.clear(); this.queuedDeliveryIds.clear(); - this.dirtyDeliveryReasons.clear(); + this.dirtyDeliveryIds.clear(); this.observedDeliverySettlements.clear(); for (const controller of this.operationAbortControllers.values()) controller.abort(); this.operationAbortControllers.clear(); @@ -927,7 +927,7 @@ export class LodyOperationCoordinator { // A wake can carry the state transition that makes an earlier transient // return runnable. Coalesce duplicates, but remember to make one serial // follow-up attempt after the current attempt finishes. - this.dirtyDeliveryReasons.set(delivery.deliveryId, reason); + this.dirtyDeliveryIds.add(delivery.deliveryId); return; } this.queuedDeliveryIds.add(delivery.deliveryId); @@ -938,7 +938,7 @@ export class LodyOperationCoordinator { .then(async () => { let attemptReason = reason; for (;;) { - this.dirtyDeliveryReasons.delete(delivery.deliveryId); + this.dirtyDeliveryIds.delete(delivery.deliveryId); try { await this.deliverIfRunnable(delivery, attemptReason); } catch (error: unknown) { @@ -948,11 +948,11 @@ export class LodyOperationCoordinator { }` ); } - const coalescedReason = this.dirtyDeliveryReasons.get(delivery.deliveryId); - if (!this.started || !coalescedReason || !this.isDeliveryPending(delivery)) { + const retry = this.dirtyDeliveryIds.delete(delivery.deliveryId); + if (!this.started || !retry || !this.isDeliveryPending(delivery)) { return; } - attemptReason = coalescedReason; + attemptReason = 'coalesced'; } }) .catch((error: unknown) => { @@ -963,17 +963,16 @@ export class LodyOperationCoordinator { ); }) .finally(() => { - const lateCoalescedReason = this.dirtyDeliveryReasons.get(delivery.deliveryId); + const lateRetry = this.dirtyDeliveryIds.delete(delivery.deliveryId); this.queuedDeliveryIds.delete(delivery.deliveryId); - this.dirtyDeliveryReasons.delete(delivery.deliveryId); if (this.deliveryChains.get(sessionId) === next) { this.deliveryChains.delete(sessionId); } // A wake can land after the worker loop decides it is clean but before // this chain's Promise settles. Clear this chain's ownership first so // the follow-up is enqueued behind any newer session work. - if (this.started && lateCoalescedReason && this.isDeliveryPending(delivery)) { - this.enqueueDelivery(delivery, lateCoalescedReason); + if (this.started && lateRetry && this.isDeliveryPending(delivery)) { + this.enqueueDelivery(delivery, 'coalesced'); } }); this.deliveryChains.set(sessionId, next); @@ -1371,8 +1370,8 @@ export class LodyOperationCoordinator { }, requireAttemptsExhausted = false, requiredExecutionPhase: 'ready' | 'uncertain' = 'ready' - ): Promise { - if (!this.started) return false; + ): Promise { + if (!this.started) return; const startedAt = performance.now(); const claimId = randomUUID(); const claim = this.withStore((store) => @@ -1387,7 +1386,7 @@ export class LodyOperationCoordinator { this.options.logger.debug( `[orchestration] Delivery ${delivery.deliveryId} finalization skipped status=${claim.status} reason=${evidence} wake=${wakeReason}` ); - return false; + return; } const settled = await this.settleObservedDeliveryClaim(delivery, { claimId, @@ -1399,7 +1398,7 @@ export class LodyOperationCoordinator { } : {}), }); - if (settled.state !== 'consumed') return false; + if (settled.state !== 'consumed') return; const timer = this.configurationTimers.get(delivery.requesterSessionId); if (timer) clearTimeout(timer); this.configurationTimers.delete(delivery.requesterSessionId); @@ -1408,7 +1407,6 @@ export class LodyOperationCoordinator { performance.now() - startedAt ).toFixed(2)}` ); - return true; } /** diff --git a/apps/cli/src/orchestration/operation-model.ts b/apps/cli/src/orchestration/operation-model.ts index 608ef38c2..bd87198ce 100644 --- a/apps/cli/src/orchestration/operation-model.ts +++ b/apps/cli/src/orchestration/operation-model.ts @@ -43,7 +43,6 @@ export type OrchestrationModelAction = | 'start_turn' | 'history_write_fail' | 'complete_turn' - | 'fail_turn' | 'interrupt_turn' | 'cancel_turn' | 'complete_finalization' @@ -189,16 +188,6 @@ export const stepOrchestrationModel = ( } next.activeTurn = 'none'; break; - case 'fail_turn': - if ( - next.activeTurn === 'delivery' && - (next.delivery === 'prepared' || next.delivery === 'started') - ) { - next.delivery = 'consumed'; - next.deliveryClaimOwner = 'none'; - } - next.activeTurn = 'none'; - break; case 'interrupt_turn': if (next.activeTurn === 'delivery' && next.delivery === 'started') { next.delivery = 'uncertain'; @@ -343,7 +332,6 @@ export const enumerateOrchestrationModel = (maxDepth: number): OrchestrationMode 'start_turn', 'history_write_fail', 'complete_turn', - 'fail_turn', 'interrupt_turn', 'cancel_turn', 'complete_finalization', diff --git a/apps/cli/src/orchestration/operation-store.test.ts b/apps/cli/src/orchestration/operation-store.test.ts index bf82ea1d1..e1f6195c8 100644 --- a/apps/cli/src/orchestration/operation-store.test.ts +++ b/apps/cli/src/orchestration/operation-store.test.ts @@ -197,6 +197,49 @@ describe('LodyOperationStore', () => { } }); + it('exposes a Delivery inserted by a legacy writer after the store opens', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'lody-operation-store-live-legacy-')); + roots.add(root); + const dbPath = path.join(root, 'operations.sqlite3'); + const preTriggerStore = new LodyOperationStore(dbPath); + preTriggerStore.accept(baseInput()); + preTriggerStore.close(); + + const legacyWriter = new Database(dbPath); + legacyWriter.exec('DROP TRIGGER deliveries_insert_execution_state'); + const store = new LodyOperationStore(dbPath, undefined, { maintenance: false }); + try { + legacyWriter.exec(` + BEGIN IMMEDIATE; + UPDATE operations + SET state = 'finished', completion_json = '{"type":"cancelled"}', + finished_at = '2026-07-20T00:01:00.000Z' + WHERE requester_session_id = 'requester-1' AND operation_id = 'review-round-1'; + INSERT INTO deliveries ( + workspace_id, requester_session_id, operation_id, delivery_id, + system_turn_id, state, initiator_chain_depth, completion_json + ) VALUES ( + 'workspace-1', 'requester-1', 'review-round-1', + 'operation:requester-1:review-round-1:completion', + 'operation-completion:requester-1:review-round-1', 'pending', 0, + '{"type":"cancelled"}' + ); + COMMIT; + `); + + expect(store.listPendingDeliveries('workspace-1' as WorkspaceId)).toEqual([ + expect.objectContaining({ + operationId: 'review-round-1', + executionPhase: 'ready', + attemptCount: 0, + }), + ]); + } finally { + legacyWriter.close(); + store.close(); + } + }); + it('accepts once and returns the same Operation for canonical-equivalent retries', async () => { const store = await makeStore(); try { diff --git a/apps/cli/src/orchestration/operation-store.ts b/apps/cli/src/orchestration/operation-store.ts index 100bdede7..04ffe9c7a 100644 --- a/apps/cli/src/orchestration/operation-store.ts +++ b/apps/cli/src/orchestration/operation-store.ts @@ -763,13 +763,6 @@ export class LodyOperationStore { current.initiatorChainDepth, JSON.stringify(durableCompletion) ); - this.db - .prepare( - `INSERT OR IGNORE INTO delivery_execution_state ( - requester_session_id, operation_id, attempt_count - ) VALUES (?, ?, 0)` - ) - .run(current.requesterSessionId, current.operationId); const updated = this.getStored(requesterSessionId, operationId); if (!updated) { throw new Error('Finished Operation disappeared during transaction.'); @@ -1479,6 +1472,14 @@ export class LodyOperationStore { ON DELETE CASCADE ); + CREATE TRIGGER IF NOT EXISTS deliveries_insert_execution_state + AFTER INSERT ON deliveries + BEGIN + INSERT OR IGNORE INTO delivery_execution_state ( + requester_session_id, operation_id, execution_phase, attempt_count + ) VALUES (NEW.requester_session_id, NEW.operation_id, 'ready', 0); + END; + CREATE TABLE IF NOT EXISTS operation_item_materializations ( requester_session_id TEXT NOT NULL, operation_id TEXT NOT NULL, @@ -1542,6 +1543,7 @@ export class LodyOperationStore { 'table:deliveries', 'index:deliveries_pending_session', 'table:delivery_execution_state', + 'trigger:deliveries_insert_execution_state', 'table:operation_item_materializations', 'table:operation_progress_settlements', 'table:orchestration_meta', @@ -1549,7 +1551,7 @@ export class LodyOperationStore { const existingObjects = this.db .prepare( `SELECT type, name FROM sqlite_master - WHERE type IN ('table', 'index')` + WHERE type IN ('table', 'index', 'trigger')` ) .all() as Array<{ type: string; name: string }>; for (const { type, name } of existingObjects) { @@ -1560,21 +1562,7 @@ export class LodyOperationStore { const executionStateColumns = this.db .prepare(`PRAGMA table_info(delivery_execution_state)`) .all() as Array<{ name: string }>; - if (!executionStateColumns.some((column) => column.name === 'execution_phase')) return true; - - return ( - this.db - .prepare( - `SELECT 1 - FROM deliveries - LEFT JOIN delivery_execution_state - ON delivery_execution_state.requester_session_id = deliveries.requester_session_id - AND delivery_execution_state.operation_id = deliveries.operation_id - WHERE delivery_execution_state.requester_session_id IS NULL - LIMIT 1` - ) - .get() !== undefined - ); + return !executionStateColumns.some((column) => column.name === 'execution_phase'); } private repairTerminalDeliveries(): void { @@ -1606,14 +1594,6 @@ export class LodyOperationStore { )` ) .run(); - this.db - .prepare( - `INSERT OR IGNORE INTO delivery_execution_state ( - requester_session_id, operation_id, execution_phase, attempt_count - ) - SELECT requester_session_id, operation_id, 'ready', 0 FROM deliveries` - ) - .run(); }) .immediate(); }