diff --git a/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md b/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md new file mode 100644 index 00000000..7386e6fb --- /dev/null +++ b/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md @@ -0,0 +1,32 @@ +--- +date: 2026-11-16 +title: "Re-establish a model claim whose directory vanished" +--- + +# 2026-11-16 — Re-establish a model claim whose directory vanished + +- **Context:** `acquireModelClaim` in `src/lock/model-claim.ts` creates the per-model claim directory + before the engine starts and writes the record again as `ready` only after `runtime.load` resolved + (`src/core/sessions.ts:112`). `writeClaim` wrapped every failure of that second write as + `IO_ERROR`, so a directory removed during the multi-minute start answered a load that had actually + succeeded as failed — observed in production as `IO_ERROR ... runtime load` with an `ENOENT` cause + on Windows. Worse, `acquireNow` keeps the handle when `runtime.findSession(modelId)` is true, so + every later load or attach of the same resident model repeated the write on the same dead path and + failed identically until the model was unloaded. Nothing in the core removes a live claim + directory: only `ModelClaimHandle.release()` does, serialized against the load or awaited by + `releaseAll()` after the runtimes stop. +- **Decision:** Stage updates under a unique name and check the existing record's claim ID before + replacing it. If the directory is removed after that check, the staged file disappears with it. + On `ENOENT`, use the same exclusive claim-directory creation as initial acquisition and retry the + write once. An incumbent or a process that wins creation makes the update return + `CORE_ALREADY_RUNNING`; other filesystem failures retain their coded `IO_ERROR` and errno cause. +- **Consequences:** A load whose engine is up keeps its claim recorded and returns its session + instead of failing, and later attaches no longer inherit a broken handle. The retry is bounded to + one attempt, so a path that keeps disappearing surfaces as the same `IO_ERROR` instead of looping. + Telemetry is unchanged: the event keeps its `IO_ERROR` code and its `ENOENT` cause when the retry + also fails. Watch for claim directories being deleted by other tools on a host — the record now + heals, but the deletion itself remains outside the claim protocol. +- **Owner:** team. +- **Links:** `src/lock/model-claim.ts`, `src/lock/model-claim-recovery.test.ts`, + `docs/decisions/2026-11-12-wrap-model-claim-filesystem-failures-as-coded-io-errors.md`, + Sentry ATOMIC-CHAT-CORE-CC. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 2c72bb5e..af5a7074 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -258,6 +258,7 @@ add a new one that says which record it supersedes. - [Preserve error cause on staged engine-build install failures](2026-11-06-preserve-cause-on-staged-engine-install-errors.md) - [Tag MLX install probe failures with finite diagnostic codes](2026-11-12-extend-bounded-diagnostics-to-mlx-install-probe.md) - [Wrap model-claim filesystem failures as coded IO errors](2026-11-12-wrap-model-claim-filesystem-failures-as-coded-io-errors.md) +- [Re-establish a model claim whose directory vanished](2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md) - [Accept the app's client keys next to the server key](2026-11-12-accept-app-client-keys-next-to-the-server-key.md) — `PUT /atomic/v1/server/client-keys`: in-memory extra keys next to a non-empty `api_key`; keyring wins, keyless stays open - [Downloads report the disk reason they already tag](2026-11-11-downloads-report-their-own-disk-reason.md) - [Keep the stream failure that ended a download retry ladder](2026-11-16-downloads-keep-the-stream-failure-that-ended-a-retry-ladder.md) diff --git a/src/lock/model-claim-recovery.test.ts b/src/lock/model-claim-recovery.test.ts new file mode 100644 index 00000000..c72acfbf --- /dev/null +++ b/src/lock/model-claim-recovery.test.ts @@ -0,0 +1,76 @@ +import { readFile, rm, writeFile } from 'node:fs/promises' +import type * as Fs from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { makeTmpDataFolder } from '../../test/helpers/tmp-data-folder.js' +import type { TmpDataFolder } from '../../test/helpers/tmp-data-folder.js' +import { acquireModelClaim } from './model-claim.js' + +vi.mock('node:fs/promises', async (importOriginal) => { + const fs = await importOriginal() + return { ...fs, writeFile: vi.fn(fs.writeFile) } +}) + +let data: TmpDataFolder + +beforeEach(async () => { + data = await makeTmpDataFolder('atomic-core-claim-recovery-') +}) + +afterEach(async () => { + await data.cleanup() +}) + +describe('a model claim whose directory vanished', () => { + it('writes the record again instead of failing the update that follows a finished load', async () => { + const claim = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') + // The directory is created before a multi-minute engine start and only written again "ready" + // after it: anything that removes it in between leaves an owned claim with no record on disk. + await rm(claim.path, { recursive: true, force: true }) + await expect(claim.update('ready')).resolves.toBeUndefined() + const record = JSON.parse(await readFile(join(claim.path, 'claim.json'), 'utf8')) as { + claim_id: string + state: string + } + expect(record).toMatchObject({ claim_id: claim.claim.claim_id, state: 'ready' }) + }) + + it('refuses to update a claim already acquired by another owner', async () => { + const first = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') + await rm(first.path, { recursive: true, force: true }) + const replacement = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-b') + const record = join(first.path, 'claim.json') + const before = await readFile(record, 'utf8') + + await expect(first.update('ready')).rejects.toMatchObject({ code: 'CORE_ALREADY_RUNNING' }) + expect(first.claim.state).toBe('loading') + await first.release() + expect(await readFile(record, 'utf8')).toBe(before) + await expect(replacement.update('ready')).resolves.toBeUndefined() + }) + + it('refuses recovery when a contender creates the directory after the failed write', async () => { + const first = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') + await rm(first.path, { recursive: true, force: true }) + const record = join(first.path, 'claim.json') + let replacementRecord: string | undefined + vi.mocked(writeFile).mockImplementationOnce(async () => { + await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-b') + replacementRecord = await readFile(record, 'utf8') + throw Object.assign(new Error('Claim directory disappeared'), { code: 'ENOENT' }) + }) + + await expect(first.update('ready')).rejects.toMatchObject({ code: 'CORE_ALREADY_RUNNING' }) + expect(replacementRecord).toBeDefined() + expect(await readFile(record, 'utf8')).toBe(replacementRecord) + }) + + it('still reports an unreachable claims folder as a coded IO_ERROR', async () => { + const claim = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') + await rm(claim.path, { recursive: true, force: true }) + await rm(data.layout.core.modelClaims, { recursive: true, force: true }) + await writeFile(data.layout.core.modelClaims, 'not a directory') + const error: unknown = await claim.update('ready').catch((e: unknown) => e) + expect(error).toMatchObject({ name: 'AtomicCoreError', code: 'IO_ERROR' }) + }) +}) diff --git a/src/lock/model-claim.ts b/src/lock/model-claim.ts index 38e24b9d..eac9a70d 100644 --- a/src/lock/model-claim.ts +++ b/src/lock/model-claim.ts @@ -1,7 +1,7 @@ /** Atomic cross-process ownership claim for a model shared by the app and the core. */ import { createHash, randomUUID } from 'node:crypto' -import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' -import { join } from 'node:path' +import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' import { AtomicCoreError } from '../contracts/index.js' import type { DataLayout } from '../config/index.js' import { isProcessAlive, processStartEpoch } from './process-identity.js' @@ -26,6 +26,7 @@ export interface ModelClaimHandle { } const isEexist = (e: unknown) => (e as NodeJS.ErrnoException).code === 'EEXIST' +const isEnoent = (e: unknown) => (e as NodeJS.ErrnoException).code === 'ENOENT' /** Every filesystem failure leaves the core as a coded error, keeping the errno as its cause. */ function claimIoError(action: string, path: string, cause: unknown): AtomicCoreError { @@ -37,6 +38,14 @@ function claimIoError(action: string, path: string, cause: unknown): AtomicCoreE ) } +function alreadyClaimed(modelId: string, path: string, owner?: ModelClaim): AtomicCoreError { + return new AtomicCoreError( + 'CORE_ALREADY_RUNNING', + `Another runtime already owns "${modelId}".`, + owner ? `${owner.owner_kind} pid ${owner.owner_pid} (${owner.state})` : path + ) +} + export function modelClaimKey(provider: string, modelId: string): string { return createHash('sha256').update(provider).update('\0').update(modelId).digest('hex') } @@ -70,7 +79,9 @@ export async function acquireModelClaim( if (isEexist(cause)) throw cause throw claimIoError('create', path, cause) }) - await writeClaim(path, claim) + await writeClaim(path, claim).catch((cause: unknown) => { + throw claimIoError('write', join(path, 'claim.json'), cause) + }) return handle(path, claim) } catch (e) { if (!isEexist(e)) throw e @@ -78,11 +89,7 @@ export async function acquireModelClaim( if (existing?.owner_kind === 'core' && existing.owner_instance_id === ownerInstanceId) return handle(path, existing) if (!existing || !(await claimOwnerIsStale(existing))) { - throw new AtomicCoreError( - 'CORE_ALREADY_RUNNING', - `Another runtime already owns "${modelId}".`, - existing ? `${existing.owner_kind} pid ${existing.owner_pid} (${existing.state})` : path - ) + throw alreadyClaimed(modelId, path, existing) } const stale = `${path}.stale-${randomUUID()}` await rename(path, stale).catch(() => undefined) @@ -107,15 +114,47 @@ async function readClaim(path: string): Promise { } } -async function writeClaim(path: string, claim: ModelClaim): Promise { +async function writeClaim(path: string, claim: ModelClaim, updating = false): Promise { const target = join(path, 'claim.json') const temp = join(path, `claim.${claim.claim_id}.${randomUUID()}.tmp`) try { await writeFile(temp, `${JSON.stringify(claim, null, 2)}\n`) + if (updating) { + // Check after staging. If the directory is replaced after this check, the unique temp file + // disappears with it and rename cannot overwrite the replacement owner's record. + const owner = await readClaim(path) + if (owner?.claim_id !== claim.claim_id) { + if (!owner) await stat(path) + throw alreadyClaimed(claim.model_id, path, owner) + } + } await rename(temp, target) } catch (cause) { - throw claimIoError('write', target, cause) + await rm(temp, { force: true }).catch(() => {}) + throw cause + } +} + +async function updateClaim(path: string, claim: ModelClaim): Promise { + const target = join(path, 'claim.json') + try { + await writeClaim(path, claim, true) + return + } catch (cause) { + if (cause instanceof AtomicCoreError) throw cause + if (!isEnoent(cause)) throw claimIoError('write', target, cause) } + // Only the winner of the same exclusive mkdir used by acquire can restore a missing claim. + await mkdir(dirname(path), { recursive: true }).catch((cause: unknown) => { + throw claimIoError('create the claims folder for', path, cause) + }) + await mkdir(path).catch(async (cause: unknown) => { + if (isEexist(cause)) throw alreadyClaimed(claim.model_id, path, await readClaim(path)) + throw claimIoError('create', path, cause) + }) + await writeClaim(path, claim).catch((cause: unknown) => { + throw claimIoError('write', target, cause) + }) } function handle(path: string, claim: ModelClaim): ModelClaimHandle { @@ -123,9 +162,9 @@ function handle(path: string, claim: ModelClaim): ModelClaimHandle { claim, path, async update(state) { - claim.state = state - claim.updated_at = new Date().toISOString() - await writeClaim(path, claim) + const next = { ...claim, state, updated_at: new Date().toISOString() } + await updateClaim(path, next) + Object.assign(claim, next) }, async release() { const current = await readClaim(path)