diff --git a/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md b/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md new file mode 100644 index 00000000..a492765f --- /dev/null +++ b/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md @@ -0,0 +1,26 @@ +--- +date: 2026-11-17 +title: "Publishing a released instance lock writes nothing" +--- + +# 2026-11-17 — Publishing a released instance lock writes nothing + +- **Context:** `InstanceLock` opens `instance.lock` exclusively, writes a `starting` record and later + rewrites it as `ready` once the control listener is bound. `release()` closes that descriptor and + removes the file. Startup is not atomic with an in-process stop: the app-lease timer can decide to + retire a core whose `createAtomicCore` has not reached `publish()` yet, and `AtomicCore.shutdown()` + releases the lock. Startup then resumes and publishes on the closed handle, `handle.truncate(0)` + throws `EBADF` ("file closed"), the throw escapes `createAtomicCore`, and the daemon reports a + fatal `core startup: unknown` instead of finishing its intended clean stop. `EBADF` is not a + reportable code, so telemetry could not name the cause either. +- **Decision:** Serialize publications on the open descriptor. `release()` stops accepting new + publications and waits for the accepted writes to settle before closing and removing the lock. + Concurrent releases share one close operation. +- **Consequences:** A mid-startup retire becomes the clean stop it was meant to be, on every host, + instead of a fatal startup error. Acquire stays unchanged and a lock that was never released keeps + publishing as before. Shutdown waits for the local record write, so it cannot close the descriptor + between the write's truncate and write calls. A real write failure still reaches the caller that + requested the publication, and release still closes the descriptor. +- **Owner:** team +- **Links:** `src/lock/instance-lock.ts`, `src/lock/instance-lock-publish.test.ts`, + `src/core/create.ts`, `src/core/app-lease.ts`, `src/core/app-daemon.ts` diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 2c72bb5e..9acdd83c 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -263,6 +263,7 @@ add a new one that says which record it supersedes. - [Keep the stream failure that ended a download retry ladder](2026-11-16-downloads-keep-the-stream-failure-that-ended-a-retry-ladder.md) - [Refuse to replace a backend pack a session runs from](2026-11-12-refuse-replacing-in-use-backend-pack.md) - [Browser stack frames keep their served asset path](2026-11-12-keep-browser-asset-paths-in-stack-frames.md) +- [Publishing a released instance lock writes nothing](2026-11-17-publishing-a-released-instance-lock-writes-nothing.md) - [An audio shutdown refusal is not an engine crash](2026-11-12-audio-shutdown-refusal-is-not-a-crash.md) - [Check MLX builds with the engine candidate workflow](2026-11-12-check-mlx-builds-with-the-engine-candidate.md) — `engine=mlx`: the fork's `vX.Y.Z`, conf's `mlxvlm-macos-arm64-` mirror; everything on macos-15, watched folders, folder live models - [Spell the sd.cpp argv for the installed build](2026-11-13-spell-the-sd-cpp-argv-for-the-installed-build.md) — `--backend te=cpu,vae=cpu` and `--llm` everywhere; `--flash-attn`/`--conv-direct` from build 982, the deprecated pair before; no `-M` diff --git a/src/lock/instance-lock-publish.test.ts b/src/lock/instance-lock-publish.test.ts new file mode 100644 index 00000000..28a9c0fd --- /dev/null +++ b/src/lock/instance-lock-publish.test.ts @@ -0,0 +1,42 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { makeTmpDataFolder } from '../../test/helpers/tmp-data-folder.js' +import type { TmpDataFolder } from '../../test/helpers/tmp-data-folder.js' +import { InstanceLock, inspectLock } from './instance-lock.js' + +let data: TmpDataFolder +beforeEach(async () => { + data = await makeTmpDataFolder('atomic-core-lock-publish-') +}) +afterEach(() => data.cleanup()) + +// A shutdown that arrives while `createAtomicCore` is still bringing the endpoint up releases the +// lock and closes its handle; startup then publishes the endpoint anyway. That write must be a +// no-op: the file is gone (and may belong to another owner), and its descriptor is closed. +describe('InstanceLock.publish after release', () => { + it('does not write to the closed lock handle a shutdown already released', async () => { + const lock = await InstanceLock.acquire(data.layout) + await lock.release() + expect(await inspectLock(data.layout)).toEqual({ kind: 'free' }) + await expect(lock.publish('127.0.0.1', 42_424)).resolves.toBeUndefined() + expect(await inspectLock(data.layout)).toEqual({ kind: 'free' }) + }) + + it('still publishes, and keeps publishing, for a lock that was never released', async () => { + const lock = await InstanceLock.acquire(data.layout) + await lock.publish('127.0.0.1', 42_424) + expect(await inspectLock(data.layout)).toMatchObject({ kind: 'owned', record: { state: 'ready' } }) + await lock.release() + }) + + it('finishes an accepted publication before shutdown can close its descriptor', async () => { + const lock = await InstanceLock.acquire(data.layout) + const publishing = lock.publish('127.0.0.1', 42_424) + const releasing = lock.release() + await expect(Promise.all([publishing, releasing, lock.release()])).resolves.toEqual([ + undefined, + undefined, + undefined, + ]) + expect(await inspectLock(data.layout)).toEqual({ kind: 'free' }) + }) +}) diff --git a/src/lock/instance-lock.ts b/src/lock/instance-lock.ts index e6abc44f..68b0be71 100644 --- a/src/lock/instance-lock.ts +++ b/src/lock/instance-lock.ts @@ -149,6 +149,9 @@ export async function inspectLock(layout: DataLayout, deps: LockDeps = {}): Prom } export class InstanceLock { + private publication = Promise.resolve() + private releasing?: Promise + private constructor( private readonly handle: FileHandle, private readonly path: string, @@ -226,14 +229,33 @@ export class InstanceLock { ) } - /** Announce the bound control listener. Until this lands, clients see `state: "starting"`. */ + /** + * Announce the bound control listener. Until this lands, clients see `state: "starting"`. + * + * Stop accepting writes once shutdown starts. A write already accepted must finish before its + * descriptor closes, including the gap between truncating and rewriting the record. + */ async publish(controlHost: string, controlPort: number): Promise { - this.current = { ...this.current, control_host: controlHost, control_port: controlPort, state: 'ready' } - await writeRecord(this.handle, this.current) + if (this.releasing) return + const record: LockRecord = { + ...this.current, + control_host: controlHost, + control_port: controlPort, + state: 'ready', + } + this.current = record + const pending = this.publication.then(() => writeRecord(this.handle, record)) + this.publication = pending.catch(() => {}) + await pending } /** Release only what is still ours: a lock taken over by someone else is left alone. */ - async release(): Promise { + release(): Promise { + return (this.releasing ??= this.close()) + } + + private async close(): Promise { + await this.publication await this.handle.close().catch(() => {}) const onDisk = await readLockRecord(this.path, this.deps).catch(() => undefined) if (onDisk && onDisk !== 'corrupt' && onDisk.instance_id !== this.current.instance_id) return