From dc7424c3bc7d592ea5470924a079f13d8879d080 Mon Sep 17 00:00:00 2001 From: atomic-sentry-bot Date: Mon, 16 Nov 2026 01:35:13 +0000 Subject: [PATCH 1/3] fix: address Sentry issue 7849148048 --- ...-a-model-claim-whose-directory-vanished.md | 32 +++++++++++++++ docs/decisions/INDEX.md | 1 + src/lock/model-claim-recovery.test.ts | 40 +++++++++++++++++++ src/lock/model-claim.ts | 24 +++++++++-- 4 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md create mode 100644 src/lock/model-claim-recovery.test.ts diff --git a/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md b/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md new file mode 100644 index 00000000..22cd92c4 --- /dev/null +++ b/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md @@ -0,0 +1,32 @@ +--- +date: 2026-11-16 +title: "Re-establish a model claim whose directory vanished" +--- + +# 2026-11-16 — Re-establish a model claim whose directory vanished + +- **Context:** `acquireModelClaim` in `src/lock/model-claim.ts` creates the per-model claim directory + before the engine starts and writes the record again as `ready` only after `runtime.load` resolved + (`src/core/sessions.ts:112`). `writeClaim` wrapped every failure of that second write as + `IO_ERROR`, so a directory removed during the multi-minute start answered a load that had actually + succeeded as failed — observed in production as `IO_ERROR ... runtime load` with an `ENOENT` cause + on Windows. Worse, `acquireNow` keeps the handle when `runtime.findSession(modelId)` is true, so + every later load or attach of the same resident model repeated the write on the same dead path and + failed identically until the model was unloaded. Nothing in the core removes a live claim + directory: only `ModelClaimHandle.release()` does, serialized against the load or awaited by + `releaseAll()` after the runtimes stop. +- **Decision:** In `writeClaim`, treat `ENOENT` from the record write as a vanished directory: + re-create `path` with `mkdir(path, { recursive: true })` and write the record once more. A second + failure, and every other errno, keep the existing `claimIoError('write', …)` wrapping, so a full + volume, a file where the folder belongs, or an unwritable folder stay coded `IO_ERROR`s and still + refuse the load. +- **Consequences:** A load whose engine is up keeps its claim recorded and returns its session + instead of failing, and later attaches no longer inherit a broken handle. The retry is bounded to + one attempt, so a path that keeps disappearing surfaces as the same `IO_ERROR` instead of looping. + Telemetry is unchanged: the event keeps its `IO_ERROR` code and its `ENOENT` cause when the retry + also fails. Watch for claim directories being deleted by other tools on a host — the record now + heals, but the deletion itself remains outside the claim protocol. +- **Owner:** team. +- **Links:** `src/lock/model-claim.ts`, `src/lock/model-claim-recovery.test.ts`, + `docs/decisions/2026-11-12-wrap-model-claim-filesystem-failures-as-coded-io-errors.md`, + Sentry ATOMIC-CHAT-CORE-CC. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index a8af0842..2cfa0602 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -258,3 +258,4 @@ add a new one that says which record it supersedes. - [An audio shutdown refusal is not an engine crash](2026-11-12-audio-shutdown-refusal-is-not-a-crash.md) - [Check MLX builds with the engine candidate workflow](2026-11-12-check-mlx-builds-with-the-engine-candidate.md) — `engine=mlx`: the fork's `vX.Y.Z`, conf's `mlxvlm-macos-arm64-` mirror; everything on macos-15, watched folders, folder live models - [Spell the sd.cpp argv for the installed build](2026-11-13-spell-the-sd-cpp-argv-for-the-installed-build.md) — `--backend te=cpu,vae=cpu` and `--llm` everywhere; `--flash-attn`/`--conv-direct` from build 982, the deprecated pair before; no `-M` +- [Re-establish a model claim whose directory vanished](2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md) diff --git a/src/lock/model-claim-recovery.test.ts b/src/lock/model-claim-recovery.test.ts new file mode 100644 index 00000000..78a62ec8 --- /dev/null +++ b/src/lock/model-claim-recovery.test.ts @@ -0,0 +1,40 @@ +import { readFile, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { makeTmpDataFolder } from '../../test/helpers/tmp-data-folder.js' +import type { TmpDataFolder } from '../../test/helpers/tmp-data-folder.js' +import { acquireModelClaim } from './model-claim.js' + +let data: TmpDataFolder + +beforeEach(async () => { + data = await makeTmpDataFolder('atomic-core-claim-recovery-') +}) + +afterEach(async () => { + await data.cleanup() +}) + +describe('a model claim whose directory vanished', () => { + it('writes the record again instead of failing the update that follows a finished load', async () => { + const claim = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') + // The directory is created before a multi-minute engine start and only written again "ready" + // after it: anything that removes it in between leaves an owned claim with no record on disk. + await rm(claim.path, { recursive: true, force: true }) + await expect(claim.update('ready')).resolves.toBeUndefined() + const record = JSON.parse(await readFile(join(claim.path, 'claim.json'), 'utf8')) as { + claim_id: string + state: string + } + expect(record).toMatchObject({ claim_id: claim.claim.claim_id, state: 'ready' }) + }) + + it('still reports an unreachable claims folder as a coded IO_ERROR', async () => { + const claim = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') + await rm(claim.path, { recursive: true, force: true }) + await rm(data.layout.core.modelClaims, { recursive: true, force: true }) + await writeFile(data.layout.core.modelClaims, 'not a directory') + const error: unknown = await claim.update('ready').catch((e: unknown) => e) + expect(error).toMatchObject({ name: 'AtomicCoreError', code: 'IO_ERROR' }) + }) +}) diff --git a/src/lock/model-claim.ts b/src/lock/model-claim.ts index 38e24b9d..8c87d991 100644 --- a/src/lock/model-claim.ts +++ b/src/lock/model-claim.ts @@ -26,6 +26,7 @@ export interface ModelClaimHandle { } const isEexist = (e: unknown) => (e as NodeJS.ErrnoException).code === 'EEXIST' +const isEnoent = (e: unknown) => (e as NodeJS.ErrnoException).code === 'ENOENT' /** Every filesystem failure leaves the core as a coded error, keeping the errno as its cause. */ function claimIoError(action: string, path: string, cause: unknown): AtomicCoreError { @@ -107,14 +108,31 @@ async function readClaim(path: string): Promise { } } +/** + * Write the record through a temp file. The claim directory is created before a load starts and the + * record is only written again when the engine reports itself ready, so a directory removed in the + * meantime leaves an owned claim with no record. The load behind it is already up and this process + * still holds the claim, so a missing directory is re-created once and the record written again + * instead of answering a finished load as failed. Nothing inside the core removes a live claim + * directory; a second ENOENT, or any other cause, stays the coded `IO_ERROR` it was. + */ async function writeClaim(path: string, claim: ModelClaim): Promise { const target = join(path, 'claim.json') - const temp = join(path, `claim.${claim.claim_id}.${randomUUID()}.tmp`) - try { + const write = async (): Promise => { + const temp = join(path, `claim.${claim.claim_id}.${randomUUID()}.tmp`) await writeFile(temp, `${JSON.stringify(claim, null, 2)}\n`) await rename(temp, target) + } + try { + await write() } catch (cause) { - throw claimIoError('write', target, cause) + if (!isEnoent(cause)) throw claimIoError('write', target, cause) + await mkdir(path, { recursive: true }).catch((e: unknown) => { + throw claimIoError('create', path, e) + }) + await write().catch((e: unknown) => { + throw claimIoError('write', target, e) + }) } } From 8d31fcdcc1c65133c4ec623af1c0532034f8313c Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Wed, 18 Nov 2026 13:45:55 +0300 Subject: [PATCH 2/3] fix(lock): protect the replacement model claim during recovery --- ...-a-model-claim-whose-directory-vanished.md | 10 +-- src/lock/model-claim-recovery.test.ts | 37 ++++++++- src/lock/model-claim.ts | 79 ++++++++++++------- 3 files changed, 91 insertions(+), 35 deletions(-) diff --git a/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md b/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md index 22cd92c4..7386e6fb 100644 --- a/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md +++ b/docs/decisions/2026-11-16-re-establish-a-model-claim-whose-directory-vanished.md @@ -15,11 +15,11 @@ title: "Re-establish a model claim whose directory vanished" failed identically until the model was unloaded. Nothing in the core removes a live claim directory: only `ModelClaimHandle.release()` does, serialized against the load or awaited by `releaseAll()` after the runtimes stop. -- **Decision:** In `writeClaim`, treat `ENOENT` from the record write as a vanished directory: - re-create `path` with `mkdir(path, { recursive: true })` and write the record once more. A second - failure, and every other errno, keep the existing `claimIoError('write', …)` wrapping, so a full - volume, a file where the folder belongs, or an unwritable folder stay coded `IO_ERROR`s and still - refuse the load. +- **Decision:** Stage updates under a unique name and check the existing record's claim ID before + replacing it. If the directory is removed after that check, the staged file disappears with it. + On `ENOENT`, use the same exclusive claim-directory creation as initial acquisition and retry the + write once. An incumbent or a process that wins creation makes the update return + `CORE_ALREADY_RUNNING`; other filesystem failures retain their coded `IO_ERROR` and errno cause. - **Consequences:** A load whose engine is up keeps its claim recorded and returns its session instead of failing, and later attaches no longer inherit a broken handle. The retry is bounded to one attempt, so a path that keeps disappearing surfaces as the same `IO_ERROR` instead of looping. diff --git a/src/lock/model-claim-recovery.test.ts b/src/lock/model-claim-recovery.test.ts index 78a62ec8..1b05aed3 100644 --- a/src/lock/model-claim-recovery.test.ts +++ b/src/lock/model-claim-recovery.test.ts @@ -1,10 +1,15 @@ import { readFile, rm, writeFile } from 'node:fs/promises' import { join } from 'node:path' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { makeTmpDataFolder } from '../../test/helpers/tmp-data-folder.js' import type { TmpDataFolder } from '../../test/helpers/tmp-data-folder.js' import { acquireModelClaim } from './model-claim.js' +vi.mock('node:fs/promises', async (importOriginal) => { + const fs = await importOriginal() + return { ...fs, writeFile: vi.fn(fs.writeFile) } +}) + let data: TmpDataFolder beforeEach(async () => { @@ -29,6 +34,36 @@ describe('a model claim whose directory vanished', () => { expect(record).toMatchObject({ claim_id: claim.claim.claim_id, state: 'ready' }) }) + it('refuses to update a claim already acquired by another owner', async () => { + const first = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') + await rm(first.path, { recursive: true, force: true }) + const replacement = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-b') + const record = join(first.path, 'claim.json') + const before = await readFile(record, 'utf8') + + await expect(first.update('ready')).rejects.toMatchObject({ code: 'CORE_ALREADY_RUNNING' }) + expect(first.claim.state).toBe('loading') + await first.release() + expect(await readFile(record, 'utf8')).toBe(before) + await expect(replacement.update('ready')).resolves.toBeUndefined() + }) + + it('refuses recovery when a contender creates the directory after the failed write', async () => { + const first = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') + await rm(first.path, { recursive: true, force: true }) + const record = join(first.path, 'claim.json') + let replacementRecord: string | undefined + vi.mocked(writeFile).mockImplementationOnce(async () => { + await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-b') + replacementRecord = await readFile(record, 'utf8') + throw Object.assign(new Error('Claim directory disappeared'), { code: 'ENOENT' }) + }) + + await expect(first.update('ready')).rejects.toMatchObject({ code: 'CORE_ALREADY_RUNNING' }) + expect(replacementRecord).toBeDefined() + expect(await readFile(record, 'utf8')).toBe(replacementRecord) + }) + it('still reports an unreachable claims folder as a coded IO_ERROR', async () => { const claim = await acquireModelClaim(data.layout, 'llamacpp-upstream', 'demo', 'instance-a') await rm(claim.path, { recursive: true, force: true }) diff --git a/src/lock/model-claim.ts b/src/lock/model-claim.ts index 8c87d991..eac9a70d 100644 --- a/src/lock/model-claim.ts +++ b/src/lock/model-claim.ts @@ -1,7 +1,7 @@ /** Atomic cross-process ownership claim for a model shared by the app and the core. */ import { createHash, randomUUID } from 'node:crypto' -import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' -import { join } from 'node:path' +import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' import { AtomicCoreError } from '../contracts/index.js' import type { DataLayout } from '../config/index.js' import { isProcessAlive, processStartEpoch } from './process-identity.js' @@ -38,6 +38,14 @@ function claimIoError(action: string, path: string, cause: unknown): AtomicCoreE ) } +function alreadyClaimed(modelId: string, path: string, owner?: ModelClaim): AtomicCoreError { + return new AtomicCoreError( + 'CORE_ALREADY_RUNNING', + `Another runtime already owns "${modelId}".`, + owner ? `${owner.owner_kind} pid ${owner.owner_pid} (${owner.state})` : path + ) +} + export function modelClaimKey(provider: string, modelId: string): string { return createHash('sha256').update(provider).update('\0').update(modelId).digest('hex') } @@ -71,7 +79,9 @@ export async function acquireModelClaim( if (isEexist(cause)) throw cause throw claimIoError('create', path, cause) }) - await writeClaim(path, claim) + await writeClaim(path, claim).catch((cause: unknown) => { + throw claimIoError('write', join(path, 'claim.json'), cause) + }) return handle(path, claim) } catch (e) { if (!isEexist(e)) throw e @@ -79,11 +89,7 @@ export async function acquireModelClaim( if (existing?.owner_kind === 'core' && existing.owner_instance_id === ownerInstanceId) return handle(path, existing) if (!existing || !(await claimOwnerIsStale(existing))) { - throw new AtomicCoreError( - 'CORE_ALREADY_RUNNING', - `Another runtime already owns "${modelId}".`, - existing ? `${existing.owner_kind} pid ${existing.owner_pid} (${existing.state})` : path - ) + throw alreadyClaimed(modelId, path, existing) } const stale = `${path}.stale-${randomUUID()}` await rename(path, stale).catch(() => undefined) @@ -108,32 +114,47 @@ async function readClaim(path: string): Promise { } } -/** - * Write the record through a temp file. The claim directory is created before a load starts and the - * record is only written again when the engine reports itself ready, so a directory removed in the - * meantime leaves an owned claim with no record. The load behind it is already up and this process - * still holds the claim, so a missing directory is re-created once and the record written again - * instead of answering a finished load as failed. Nothing inside the core removes a live claim - * directory; a second ENOENT, or any other cause, stays the coded `IO_ERROR` it was. - */ -async function writeClaim(path: string, claim: ModelClaim): Promise { +async function writeClaim(path: string, claim: ModelClaim, updating = false): Promise { const target = join(path, 'claim.json') - const write = async (): Promise => { - const temp = join(path, `claim.${claim.claim_id}.${randomUUID()}.tmp`) + const temp = join(path, `claim.${claim.claim_id}.${randomUUID()}.tmp`) + try { await writeFile(temp, `${JSON.stringify(claim, null, 2)}\n`) + if (updating) { + // Check after staging. If the directory is replaced after this check, the unique temp file + // disappears with it and rename cannot overwrite the replacement owner's record. + const owner = await readClaim(path) + if (owner?.claim_id !== claim.claim_id) { + if (!owner) await stat(path) + throw alreadyClaimed(claim.model_id, path, owner) + } + } await rename(temp, target) + } catch (cause) { + await rm(temp, { force: true }).catch(() => {}) + throw cause } +} + +async function updateClaim(path: string, claim: ModelClaim): Promise { + const target = join(path, 'claim.json') try { - await write() + await writeClaim(path, claim, true) + return } catch (cause) { + if (cause instanceof AtomicCoreError) throw cause if (!isEnoent(cause)) throw claimIoError('write', target, cause) - await mkdir(path, { recursive: true }).catch((e: unknown) => { - throw claimIoError('create', path, e) - }) - await write().catch((e: unknown) => { - throw claimIoError('write', target, e) - }) } + // Only the winner of the same exclusive mkdir used by acquire can restore a missing claim. + await mkdir(dirname(path), { recursive: true }).catch((cause: unknown) => { + throw claimIoError('create the claims folder for', path, cause) + }) + await mkdir(path).catch(async (cause: unknown) => { + if (isEexist(cause)) throw alreadyClaimed(claim.model_id, path, await readClaim(path)) + throw claimIoError('create', path, cause) + }) + await writeClaim(path, claim).catch((cause: unknown) => { + throw claimIoError('write', target, cause) + }) } function handle(path: string, claim: ModelClaim): ModelClaimHandle { @@ -141,9 +162,9 @@ function handle(path: string, claim: ModelClaim): ModelClaimHandle { claim, path, async update(state) { - claim.state = state - claim.updated_at = new Date().toISOString() - await writeClaim(path, claim) + const next = { ...claim, state, updated_at: new Date().toISOString() } + await updateClaim(path, next) + Object.assign(claim, next) }, async release() { const current = await readClaim(path) From 6552c78b2415cea81cec0476ca6bc4f550b6303d Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Wed, 18 Nov 2026 13:49:04 +0300 Subject: [PATCH 3/3] test(lock): use a named filesystem type import --- src/lock/model-claim-recovery.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lock/model-claim-recovery.test.ts b/src/lock/model-claim-recovery.test.ts index 1b05aed3..c72acfbf 100644 --- a/src/lock/model-claim-recovery.test.ts +++ b/src/lock/model-claim-recovery.test.ts @@ -1,4 +1,5 @@ import { readFile, rm, writeFile } from 'node:fs/promises' +import type * as Fs from 'node:fs/promises' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { makeTmpDataFolder } from '../../test/helpers/tmp-data-folder.js' @@ -6,7 +7,7 @@ import type { TmpDataFolder } from '../../test/helpers/tmp-data-folder.js' import { acquireModelClaim } from './model-claim.js' vi.mock('node:fs/promises', async (importOriginal) => { - const fs = await importOriginal() + const fs = await importOriginal() return { ...fs, writeFile: vi.fn(fs.writeFile) } })