From f0787c08d3bd59c7729c7aa89de94266eb758750 Mon Sep 17 00:00:00 2001 From: atomic-sentry-bot Date: Tue, 17 Nov 2026 00:51:26 +0000 Subject: [PATCH 1/2] fix: address Sentry issue 7850959954 --- ...a-released-instance-lock-writes-nothing.md | 27 +++++++++++++++++ docs/decisions/INDEX.md | 1 + src/lock/instance-lock-publish.test.ts | 30 +++++++++++++++++++ src/lock/instance-lock.ts | 14 ++++++++- 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md create mode 100644 src/lock/instance-lock-publish.test.ts diff --git a/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md b/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md new file mode 100644 index 00000000..503d26ff --- /dev/null +++ b/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md @@ -0,0 +1,27 @@ +--- +date: 2026-11-17 +title: "Publishing a released instance lock writes nothing" +--- + +# 2026-11-17 — Publishing a released instance lock writes nothing + +- **Context:** `InstanceLock` opens `instance.lock` exclusively, writes a `starting` record and later + rewrites it as `ready` once the control listener is bound. `release()` closes that descriptor and + removes the file. Startup is not atomic with an in-process stop: the app-lease timer can decide to + retire a core whose `createAtomicCore` has not reached `publish()` yet, and `AtomicCore.shutdown()` + releases the lock. Startup then resumes and publishes on the closed handle, `handle.truncate(0)` + throws `EBADF` ("file closed"), the throw escapes `createAtomicCore`, and the daemon reports a + fatal `core startup: unknown` instead of finishing its intended clean stop. `EBADF` is not a + reportable code, so telemetry could not name the cause either. +- **Decision:** `release()` marks the lock released before closing the descriptor, and `publish()` + returns without writing when that flag is set. A released lock has no record to announce: its file + is gone and may already belong to a new owner. +- **Consequences:** A mid-startup retire becomes the clean stop it was meant to be, on every host, + instead of a fatal startup error. Acquire stays unchanged and a lock that was never released keeps + publishing as before. The flag is set before `close()`, so a publish already in flight is the only + remaining window, and it is no wider than the pre-existing ordering between the two calls. Callers + can no longer detect a released lock from a throwing `publish()`; they never did, since the error + was a bare `EBADF`. +- **Owner:** team +- **Links:** `src/lock/instance-lock.ts`, `src/lock/instance-lock-publish.test.ts`, + `src/core/create.ts`, `src/core/app-lease.ts`, `src/core/app-daemon.ts` diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index a8af0842..e6d3495e 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -258,3 +258,4 @@ add a new one that says which record it supersedes. - [An audio shutdown refusal is not an engine crash](2026-11-12-audio-shutdown-refusal-is-not-a-crash.md) - [Check MLX builds with the engine candidate workflow](2026-11-12-check-mlx-builds-with-the-engine-candidate.md) — `engine=mlx`: the fork's `vX.Y.Z`, conf's `mlxvlm-macos-arm64-` mirror; everything on macos-15, watched folders, folder live models - [Spell the sd.cpp argv for the installed build](2026-11-13-spell-the-sd-cpp-argv-for-the-installed-build.md) — `--backend te=cpu,vae=cpu` and `--llm` everywhere; `--flash-attn`/`--conv-direct` from build 982, the deprecated pair before; no `-M` +- [Publishing a released instance lock writes nothing](2026-11-17-publishing-a-released-instance-lock-writes-nothing.md) diff --git a/src/lock/instance-lock-publish.test.ts b/src/lock/instance-lock-publish.test.ts new file mode 100644 index 00000000..c9028f46 --- /dev/null +++ b/src/lock/instance-lock-publish.test.ts @@ -0,0 +1,30 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { makeTmpDataFolder } from '../../test/helpers/tmp-data-folder.js' +import type { TmpDataFolder } from '../../test/helpers/tmp-data-folder.js' +import { InstanceLock, inspectLock } from './instance-lock.js' + +let data: TmpDataFolder +beforeEach(async () => { + data = await makeTmpDataFolder('atomic-core-lock-publish-') +}) +afterEach(() => data.cleanup()) + +// A shutdown that arrives while `createAtomicCore` is still bringing the endpoint up releases the +// lock and closes its handle; startup then publishes the endpoint anyway. That write must be a +// no-op: the file is gone (and may belong to another owner), and its descriptor is closed. +describe('InstanceLock.publish after release', () => { + it('does not write to the closed lock handle a shutdown already released', async () => { + const lock = await InstanceLock.acquire(data.layout) + await lock.release() + expect(await inspectLock(data.layout)).toEqual({ kind: 'free' }) + await expect(lock.publish('127.0.0.1', 42_424)).resolves.toBeUndefined() + expect(await inspectLock(data.layout)).toEqual({ kind: 'free' }) + }) + + it('still publishes, and keeps publishing, for a lock that was never released', async () => { + const lock = await InstanceLock.acquire(data.layout) + await lock.publish('127.0.0.1', 42_424) + expect(await inspectLock(data.layout)).toMatchObject({ kind: 'owned', record: { state: 'ready' } }) + await lock.release() + }) +}) diff --git a/src/lock/instance-lock.ts b/src/lock/instance-lock.ts index e6abc44f..55f1dbc5 100644 --- a/src/lock/instance-lock.ts +++ b/src/lock/instance-lock.ts @@ -149,6 +149,9 @@ export async function inspectLock(layout: DataLayout, deps: LockDeps = {}): Prom } export class InstanceLock { + /** Set by `release()` before the descriptor is closed; see `publish()`. */ + private released = false + private constructor( private readonly handle: FileHandle, private readonly path: string, @@ -226,14 +229,23 @@ export class InstanceLock { ) } - /** Announce the bound control listener. Until this lands, clients see `state: "starting"`. */ + /** + * Announce the bound control listener. Until this lands, clients see `state: "starting"`. + * + * A shutdown that arrives while startup is still finishing releases the lock first, closing its + * descriptor; startup then reaches this call on the way out. A released lock has no record to + * announce — its file is gone and may already belong to a new owner — so the write is skipped + * instead of throwing `EBADF` on the closed descriptor and turning a clean stop into a crash. + */ async publish(controlHost: string, controlPort: number): Promise { + if (this.released) return this.current = { ...this.current, control_host: controlHost, control_port: controlPort, state: 'ready' } await writeRecord(this.handle, this.current) } /** Release only what is still ours: a lock taken over by someone else is left alone. */ async release(): Promise { + this.released = true await this.handle.close().catch(() => {}) const onDisk = await readLockRecord(this.path, this.deps).catch(() => undefined) if (onDisk && onDisk !== 'corrupt' && onDisk.instance_id !== this.current.instance_id) return From 142e7e5de9f6af1ec9f2cc34df07d2121311e25e Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Wed, 18 Nov 2026 13:45:55 +0300 Subject: [PATCH 2/2] fix(lock): drain instance lock writes before release --- ...a-released-instance-lock-writes-nothing.md | 13 ++++---- src/lock/instance-lock-publish.test.ts | 12 +++++++ src/lock/instance-lock.ts | 32 ++++++++++++------- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md b/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md index 503d26ff..a492765f 100644 --- a/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md +++ b/docs/decisions/2026-11-17-publishing-a-released-instance-lock-writes-nothing.md @@ -13,15 +13,14 @@ title: "Publishing a released instance lock writes nothing" throws `EBADF` ("file closed"), the throw escapes `createAtomicCore`, and the daemon reports a fatal `core startup: unknown` instead of finishing its intended clean stop. `EBADF` is not a reportable code, so telemetry could not name the cause either. -- **Decision:** `release()` marks the lock released before closing the descriptor, and `publish()` - returns without writing when that flag is set. A released lock has no record to announce: its file - is gone and may already belong to a new owner. +- **Decision:** Serialize publications on the open descriptor. `release()` stops accepting new + publications and waits for the accepted writes to settle before closing and removing the lock. + Concurrent releases share one close operation. - **Consequences:** A mid-startup retire becomes the clean stop it was meant to be, on every host, instead of a fatal startup error. Acquire stays unchanged and a lock that was never released keeps - publishing as before. The flag is set before `close()`, so a publish already in flight is the only - remaining window, and it is no wider than the pre-existing ordering between the two calls. Callers - can no longer detect a released lock from a throwing `publish()`; they never did, since the error - was a bare `EBADF`. + publishing as before. Shutdown waits for the local record write, so it cannot close the descriptor + between the write's truncate and write calls. A real write failure still reaches the caller that + requested the publication, and release still closes the descriptor. - **Owner:** team - **Links:** `src/lock/instance-lock.ts`, `src/lock/instance-lock-publish.test.ts`, `src/core/create.ts`, `src/core/app-lease.ts`, `src/core/app-daemon.ts` diff --git a/src/lock/instance-lock-publish.test.ts b/src/lock/instance-lock-publish.test.ts index c9028f46..28a9c0fd 100644 --- a/src/lock/instance-lock-publish.test.ts +++ b/src/lock/instance-lock-publish.test.ts @@ -27,4 +27,16 @@ describe('InstanceLock.publish after release', () => { expect(await inspectLock(data.layout)).toMatchObject({ kind: 'owned', record: { state: 'ready' } }) await lock.release() }) + + it('finishes an accepted publication before shutdown can close its descriptor', async () => { + const lock = await InstanceLock.acquire(data.layout) + const publishing = lock.publish('127.0.0.1', 42_424) + const releasing = lock.release() + await expect(Promise.all([publishing, releasing, lock.release()])).resolves.toEqual([ + undefined, + undefined, + undefined, + ]) + expect(await inspectLock(data.layout)).toEqual({ kind: 'free' }) + }) }) diff --git a/src/lock/instance-lock.ts b/src/lock/instance-lock.ts index 55f1dbc5..68b0be71 100644 --- a/src/lock/instance-lock.ts +++ b/src/lock/instance-lock.ts @@ -149,8 +149,8 @@ export async function inspectLock(layout: DataLayout, deps: LockDeps = {}): Prom } export class InstanceLock { - /** Set by `release()` before the descriptor is closed; see `publish()`. */ - private released = false + private publication = Promise.resolve() + private releasing?: Promise private constructor( private readonly handle: FileHandle, @@ -232,20 +232,30 @@ export class InstanceLock { /** * Announce the bound control listener. Until this lands, clients see `state: "starting"`. * - * A shutdown that arrives while startup is still finishing releases the lock first, closing its - * descriptor; startup then reaches this call on the way out. A released lock has no record to - * announce — its file is gone and may already belong to a new owner — so the write is skipped - * instead of throwing `EBADF` on the closed descriptor and turning a clean stop into a crash. + * Stop accepting writes once shutdown starts. A write already accepted must finish before its + * descriptor closes, including the gap between truncating and rewriting the record. */ async publish(controlHost: string, controlPort: number): Promise { - if (this.released) return - this.current = { ...this.current, control_host: controlHost, control_port: controlPort, state: 'ready' } - await writeRecord(this.handle, this.current) + if (this.releasing) return + const record: LockRecord = { + ...this.current, + control_host: controlHost, + control_port: controlPort, + state: 'ready', + } + this.current = record + const pending = this.publication.then(() => writeRecord(this.handle, record)) + this.publication = pending.catch(() => {}) + await pending } /** Release only what is still ours: a lock taken over by someone else is left alone. */ - async release(): Promise { - this.released = true + release(): Promise { + return (this.releasing ??= this.close()) + } + + private async close(): Promise { + await this.publication await this.handle.close().catch(() => {}) const onDisk = await readLockRecord(this.path, this.deps).catch(() => undefined) if (onDisk && onDisk !== 'corrupt' && onDisk.instance_id !== this.current.instance_id) return