From db794f7b4e6a63a9af0fc73eab9100ef9a4d297e Mon Sep 17 00:00:00 2001 From: atomic-sentry-bot Date: Sun, 15 Nov 2026 03:23:11 +0000 Subject: [PATCH 1/2] fix: address Sentry issue 7847994825 --- ...-a-superseded-download-does-not-publish.md | 12 +++ docs/decisions/INDEX.md | 1 + .../downloader-supersede-race.test.ts | 79 +++++++++++++++++++ src/downloads/downloader.ts | 6 ++ 4 files changed, 98 insertions(+) create mode 100644 docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md create mode 100644 src/downloads/downloader-supersede-race.test.ts diff --git a/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md b/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md new file mode 100644 index 00000000..f33b8fa3 --- /dev/null +++ b/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md @@ -0,0 +1,12 @@ +--- +date: 2026-11-15 +title: "A superseded download stands back before it publishes" +--- + +# 2026-11-15 — A superseded download stands back before it publishes + +- **Context:** `Downloader.download` supersedes an earlier call with the same task id by aborting it, and the doc comment promises the superseded run "leaves the files to the newcomer". The abort was only consulted while a chunk arrived, inside the read loop of `downloadOne`. An attempt whose body had already ended — or which was blocked in `reader.read()` and then saw `done` — walked past the loop into the publish block: it renamed the shared `.tmp` to `` and removed `.url`, then failed its own post-transfer verification with `Validation cancelled`. The verification-failure branch of `run` removes the published file and its directory, so the live attempt's next `stat(tmpPath)` threw `ENOENT`, reported as `[disk_device_lost] ENOENT` for a download that had lost nothing of its own. +- **Decision:** Check `signal.aborted` once immediately after the transfer loop closes and before the persisted-size check and the `rename`/`rm(urlPath)`. A superseded or cancelled attempt then throws `DOWNLOAD_CANCELLED` and leaves the `.tmp` partial in place, where the newcomer's write stream or the existing `downloaded === expectedSize` resume branch owns it. Nothing else changes: aborted attempts stay unreported (`download()` only calls `onFailure` for a task whose own signal is not aborted), and every message, tag and public contract is untouched. +- **Consequences:** A retry with the same task id can no longer be killed by the attempt it replaced, so the `[disk_device_lost] ENOENT` warning disappears for that trigger. Other `ENOENT` sources (an actually vanishing volume or staging directory) still report the same tag. The guard covers cancel-after-body-end as well, since a cancelled task also has an aborted signal. Cost: one boolean check per file. +- **Owner:** team. +- **Links:** `src/downloads/downloader.ts` (`downloadOne`), `src/downloads/downloader-supersede-race.test.ts`. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index a8af0842..12eccd2c 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -258,3 +258,4 @@ add a new one that says which record it supersedes. - [An audio shutdown refusal is not an engine crash](2026-11-12-audio-shutdown-refusal-is-not-a-crash.md) - [Check MLX builds with the engine candidate workflow](2026-11-12-check-mlx-builds-with-the-engine-candidate.md) — `engine=mlx`: the fork's `vX.Y.Z`, conf's `mlxvlm-macos-arm64-` mirror; everything on macos-15, watched folders, folder live models - [Spell the sd.cpp argv for the installed build](2026-11-13-spell-the-sd-cpp-argv-for-the-installed-build.md) — `--backend te=cpu,vae=cpu` and `--llm` everywhere; `--flash-attn`/`--conv-direct` from build 982, the deprecated pair before; no `-M` +- [A superseded download stands back before it publishes](2026-11-15-a-superseded-download-does-not-publish.md) diff --git a/src/downloads/downloader-supersede-race.test.ts b/src/downloads/downloader-supersede-race.test.ts new file mode 100644 index 00000000..8730033e --- /dev/null +++ b/src/downloads/downloader-supersede-race.test.ts @@ -0,0 +1,79 @@ +import { mkdtemp, readFile, rm, stat } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { DOWNLOAD_CANCELLED, Downloader } from './downloader.js' + +const folders: string[] = [] + +afterEach(async () => { + await Promise.all(folders.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void + const promise = new Promise((r) => (resolve = r)) + return { promise, resolve } +} + +describe('a second download() with the same task id', () => { + it('stops the superseded attempt before it publishes the staging file the newcomer owns', async () => { + const dataFolder = await mkdtemp(join(tmpdir(), 'atomic-dl-supersede-')) + folders.push(dataFolder) + // Same length, different bytes: the two attempts can be told apart on disk. + const firstBytes = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]) + const secondBytes = new Uint8Array([11, 12, 13, 14, 15, 16, 17, 18]) + const releaseFirst = deferred() + const releaseSecond = deferred() + const firstChunk = deferred() + const secondChunk = deferred() + let calls = 0 + const fetchStub = (async () => { + calls += 1 + const first = calls === 1 + return new Response( + new ReadableStream({ + async pull(controller) { + controller.enqueue(first ? firstBytes : secondBytes) + ;(first ? firstChunk : secondChunk).resolve() + await (first ? releaseFirst : releaseSecond).promise + controller.close() + }, + }), + { status: 200, headers: { 'content-length': '8' } } + ) + }) as unknown as typeof fetch + const dl = new Downloader({ + dataFolder, + platform: 'linux', + fetch: fetchStub, + availableSpace: async () => undefined, + sleep: async () => {}, + emit: () => {}, + }) + const savePath = join(dataFolder, 'backends/pack.bin') + const item = { url: 'http://engine.test/pack.bin', save_path: 'backends/pack.bin', size: 8 } + + // The first attempt has written its chunk and is waiting for the rest of its body. + const first = dl.download('same', [item]) + await firstChunk.promise + await vi.waitFor(async () => expect((await stat(`${savePath}.tmp`)).size).toBe(8)) + + // A client retries with the same task id: the first attempt is superseded, the second owns the file. + const second = dl.download('same', [item]) + await secondChunk.promise + await vi.waitFor(async () => expect([...(await readFile(`${savePath}.tmp`))]).toEqual([...secondBytes])) + + // The superseded attempt's body ends. It must stand back; it used to publish the newcomer's file + // and then remove it, so the newcomer read `ENOENT` from its own staging path. + releaseFirst.resolve() + await expect(first).rejects.toThrow(DOWNLOAD_CANCELLED) + + releaseSecond.resolve() + await expect(second).resolves.toBeUndefined() + expect([...(await readFile(savePath))]).toEqual([...secondBytes]) + // The newcomer published its own bytes: the staging file and the sidecar are gone. + await expect(stat(`${savePath}.tmp`)).rejects.toThrow() + await expect(stat(`${savePath}.url`)).rejects.toThrow() + }) +}) diff --git a/src/downloads/downloader.ts b/src/downloads/downloader.ts index 93843387..44354339 100644 --- a/src/downloads/downloader.ts +++ b/src/downloads/downloader.ts @@ -779,6 +779,12 @@ export class Downloader { await writer.close().catch(() => {}) await reader.cancel().catch(() => {}) } + // The loop above consults `signal` only when a chunk arrives, so a superseded or cancelled + // attempt can reach this point after its body ended without noticing. Publishing now would + // rename the staging file the newcomer is writing and delete its `.url` sidecar, and its own + // verification failure would then remove the newcomer's file and directory. Stand back and + // leave the partial to the newcomer (or to the resume branch above). + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) const persisted = ( await stat(tmpPath).catch((e) => { throw disk(e) From 90657e5d837a8e59a9f3396f73d737035e653c51 Mon Sep 17 00:00:00 2001 From: Biogenic Ooze Date: Wed, 18 Nov 2026 13:45:55 +0300 Subject: [PATCH 2/2] fix(downloads): drain superseded attempts before retrying the same task --- ...-a-superseded-download-does-not-publish.md | 9 +- .../downloader-supersede-race.test.ts | 82 ++++++++++++++++++- src/downloads/downloader.ts | 47 +++++------ 3 files changed, 111 insertions(+), 27 deletions(-) diff --git a/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md b/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md index f33b8fa3..a2f0761f 100644 --- a/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md +++ b/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md @@ -6,7 +6,12 @@ title: "A superseded download stands back before it publishes" # 2026-11-15 — A superseded download stands back before it publishes - **Context:** `Downloader.download` supersedes an earlier call with the same task id by aborting it, and the doc comment promises the superseded run "leaves the files to the newcomer". The abort was only consulted while a chunk arrived, inside the read loop of `downloadOne`. An attempt whose body had already ended — or which was blocked in `reader.read()` and then saw `done` — walked past the loop into the publish block: it renamed the shared `.tmp` to `` and removed `.url`, then failed its own post-transfer verification with `Validation cancelled`. The verification-failure branch of `run` removes the published file and its directory, so the live attempt's next `stat(tmpPath)` threw `ENOENT`, reported as `[disk_device_lost] ENOENT` for a download that had lost nothing of its own. -- **Decision:** Check `signal.aborted` once immediately after the transfer loop closes and before the persisted-size check and the `rename`/`rm(urlPath)`. A superseded or cancelled attempt then throws `DOWNLOAD_CANCELLED` and leaves the `.tmp` partial in place, where the newcomer's write stream or the existing `downloaded === expectedSize` resume branch owns it. Nothing else changes: aborted attempts stay unreported (`download()` only calls `onFailure` for a task whose own signal is not aborted), and every message, tag and public contract is untouched. -- **Consequences:** A retry with the same task id can no longer be killed by the attempt it replaced, so the `[disk_device_lost] ENOENT` warning disappears for that trigger. Other `ENOENT` sources (an actually vanishing volume or staging directory) still report the same tag. The guard covers cancel-after-body-end as well, since a cancelled task also has an aborted signal. Cost: one boolean check per file. +- **Decision:** Abort the superseded attempt and wait for its filesystem operations to settle before + the same task ID starts again. Cancel its response reader to unblock a stalled read. Check for + cancellation before publishing a partial and before cleaning up after verification. +- **Consequences:** An already-submitted rename or cleanup can finish before the next attempt touches + the shared files. Cancellation leaves partials and completed files available to the next attempt, + including when it interrupts the hash check. Other task IDs can still run concurrently, and a + real staging-directory failure retains its existing disk diagnostic. - **Owner:** team. - **Links:** `src/downloads/downloader.ts` (`downloadOne`), `src/downloads/downloader-supersede-race.test.ts`. diff --git a/src/downloads/downloader-supersede-race.test.ts b/src/downloads/downloader-supersede-race.test.ts index 8730033e..9fd957b8 100644 --- a/src/downloads/downloader-supersede-race.test.ts +++ b/src/downloads/downloader-supersede-race.test.ts @@ -31,13 +31,17 @@ describe('a second download() with the same task id', () => { const fetchStub = (async () => { calls += 1 const first = calls === 1 + let cancelled = false return new Response( new ReadableStream({ async pull(controller) { controller.enqueue(first ? firstBytes : secondBytes) ;(first ? firstChunk : secondChunk).resolve() await (first ? releaseFirst : releaseSecond).promise - controller.close() + if (!cancelled) controller.close() + }, + cancel() { + cancelled = true }, }), { status: 200, headers: { 'content-length': '8' } } @@ -60,6 +64,7 @@ describe('a second download() with the same task id', () => { await vi.waitFor(async () => expect((await stat(`${savePath}.tmp`)).size).toBe(8)) // A client retries with the same task id: the first attempt is superseded, the second owns the file. + const firstCancelled = expect(first).rejects.toThrow(DOWNLOAD_CANCELLED) const second = dl.download('same', [item]) await secondChunk.promise await vi.waitFor(async () => expect([...(await readFile(`${savePath}.tmp`))]).toEqual([...secondBytes])) @@ -67,7 +72,7 @@ describe('a second download() with the same task id', () => { // The superseded attempt's body ends. It must stand back; it used to publish the newcomer's file // and then remove it, so the newcomer read `ENOENT` from its own staging path. releaseFirst.resolve() - await expect(first).rejects.toThrow(DOWNLOAD_CANCELLED) + await firstCancelled releaseSecond.resolve() await expect(second).resolves.toBeUndefined() @@ -76,4 +81,77 @@ describe('a second download() with the same task id', () => { await expect(stat(`${savePath}.tmp`)).rejects.toThrow() await expect(stat(`${savePath}.url`)).rejects.toThrow() }) + + it('leaves the completed partial when the final progress callback supersedes the attempt', async () => { + const dataFolder = await mkdtemp(join(tmpdir(), 'atomic-dl-publish-')) + folders.push(dataFolder) + const savePath = join(dataFolder, 'pack.bin') + const item = { url: 'http://engine.test/pack.bin', save_path: 'pack.bin', size: 4 } + let replacement: Promise | undefined + let partial: string | undefined + let calls = 0 + const dl: Downloader = new Downloader({ + dataFolder, + platform: 'linux', + fetch: async () => { + if (++calls === 1) return new Response('old!') + partial = await readFile(`${savePath}.tmp`, 'utf8').catch(() => undefined) + return new Response('new!') + }, + availableSpace: async () => undefined, + emit(event) { + if (event === 'download:progress' && !replacement) replacement = dl.download('same', [item]) + }, + }) + + await expect(dl.download('same', [item])).rejects.toThrow(DOWNLOAD_CANCELLED) + await expect(replacement).resolves.toBeUndefined() + expect(partial).toBe('old!') + expect(await readFile(savePath, 'utf8')).toBe('new!') + }) + + it('keeps the finished file if cancellation interrupts validation before a retry', async () => { + const dataFolder = await mkdtemp(join(tmpdir(), 'atomic-dl-validation-')) + folders.push(dataFolder) + const savePath = join(dataFolder, 'pack.bin') + const sha256 = 'a'.repeat(64) + const item = { url: 'http://engine.test/pack.bin', save_path: 'pack.bin', size: 4, sha256 } + const verifying = deferred() + const finishVerification = deferred() + let calls = 0 + let original: string | undefined + const dl = new Downloader({ + dataFolder, + platform: 'linux', + fetch: async () => { + if (++calls === 1) return new Response('old!') + original = await readFile(savePath, 'utf8').catch(() => undefined) + return new Response('new!') + }, + availableSpace: async () => undefined, + emit: () => {}, + verify: { + fileSize: async (path) => (await stat(path)).size, + sha256: async (_path, signal) => { + if (calls === 1) { + verifying.resolve() + await finishVerification.promise + if (signal?.aborted) throw new Error('Validation cancelled') + } + return sha256 + }, + }, + }) + const firstCancelled = expect(dl.download('same', [item])).rejects.toThrow(DOWNLOAD_CANCELLED) + await verifying.promise + expect(dl.cancel('same')).toBe(true) + expect(dl.active()).toEqual([]) + const second = dl.download('same', [item]) + finishVerification.resolve() + + await firstCancelled + await expect(second).resolves.toBeUndefined() + expect(original).toBe('old!') + expect(await readFile(savePath, 'utf8')).toBe('new!') + }) }) diff --git a/src/downloads/downloader.ts b/src/downloads/downloader.ts index 26627798..c417b261 100644 --- a/src/downloads/downloader.ts +++ b/src/downloads/downloader.ts @@ -94,7 +94,7 @@ type StageReport = (kind: DownloadStage['kind'], attempt: number) => void interface Task { controller: AbortController - superseded: boolean + done: Promise /** The last combined progress this task reported, for `snapshot()`. */ progress?: DownloadProgress } @@ -267,7 +267,7 @@ export class Downloader { /** Task ids currently running. */ active(): string[] { - return [...this.tasks.keys()] + return [...this.tasks].filter(([, task]) => !task.controller.signal.aborted).map(([taskId]) => taskId) } /** @@ -275,16 +275,15 @@ export class Downloader { * reconnects mid-transfer reads from the control snapshot instead of waiting for the next event. */ snapshot(): DownloadProgress[] { - return [...this.tasks.entries()].map( - ([taskId, task]) => task.progress ?? { taskId, transferred: 0, total: 0, percent: 0 } + return this.active().map( + (taskId) => this.tasks.get(taskId)?.progress ?? { taskId, transferred: 0, total: 0, percent: 0 } ) } /** Cancel a running task. Keeps partials and finished files. */ cancel(taskId: string): boolean { const task = this.tasks.get(taskId) - if (!task) return false - this.tasks.delete(taskId) + if (!task || task.controller.signal.aborted) return false task.controller.abort() return true } @@ -294,15 +293,17 @@ export class Downloader { * observes a cancellation but leaves the files to the newcomer). */ async download(taskId: string, items: DownloadItem[], options: DownloadOptions = {}): Promise { - const existing = this.tasks.get(taskId) - if (existing) { - existing.superseded = true - existing.controller.abort() - } - const task: Task = { controller: new AbortController(), superseded: false } + const previous = this.tasks.get(taskId) + const controller = new AbortController() + // An aborted attempt can still have filesystem work in flight. Hand off the shared files only + // once it has closed its writer and settled any publish or validation cleanup. + const drained = previous?.done.catch(() => {}) ?? Promise.resolve() + const done = drained.then(() => this.run(taskId, items, options, controller.signal)) + const task: Task = { controller, done } this.tasks.set(taskId, task) + previous?.controller.abort() try { - await this.run(taskId, items, options, task.controller.signal) + await done } catch (error) { if (!task.controller.signal.aborted) this.deps.onFailure?.(error, { component: 'downloads', operation: 'download' }) @@ -313,6 +314,7 @@ export class Downloader { } private async run(taskId: string, items: DownloadItem[], options: DownloadOptions, signal: AbortSignal) { + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) const { dataFolder, platform, log } = this.deps const resume = options.resume ?? false const headers = options.headers ?? {} @@ -396,6 +398,7 @@ export class Downloader { ) ) if (failed) throw firstError + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) const results = settled.map((outcome) => (outcome as PromiseFulfilledResult).value) if (items.some((it) => it.sha256 != null || it.size != null)) { @@ -409,6 +412,8 @@ export class Downloader { const verifications = await Promise.allSettled( items.map((item, i) => verifyDownloadedFile(item, results[i] as string, this.deps.verify, signal)) ) + // Cancelling validation must keep the finished files, even if a verifier rejected on abort. + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) for (let i = 0; i < verifications.length; i++) { const v = verifications[i] if (v && v.status === 'rejected') { @@ -653,23 +658,22 @@ export class Downloader { let retryCount = 0 let sinceReset = 0 + const stopReading = () => void reader.cancel().catch(() => {}) + signal.addEventListener('abort', stopReading) try { for (;;) { + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) let streamError: string | undefined // The caught object itself, so the report keeps the code that classifies the failure. let streamCause: unknown try { const { done, value } = await reader.read() + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) if (done) { if (expectedSize > 0 && totalTransferred < expectedSize) { streamError = `stream ended after ${totalTransferred} of ${expectedSize} bytes` } else break } else if (value) { - if (signal.aborted) { - await writer.close().catch(() => {}) - this.deps.log('info', `Download cancelled: ${item.url}`) - throw new Error(DOWNLOAD_CANCELLED) - } // Writes are awaited one at a time, so `totalTransferred` is where this chunk starts. await writer.write(value, totalTransferred) delta += value.byteLength @@ -779,14 +783,10 @@ export class Downloader { await writer.close() } finally { + signal.removeEventListener('abort', stopReading) await writer.close().catch(() => {}) await reader.cancel().catch(() => {}) } - // The loop above consults `signal` only when a chunk arrives, so a superseded or cancelled - // attempt can reach this point after its body ended without noticing. Publishing now would - // rename the staging file the newcomer is writing and delete its `.url` sidecar, and its own - // verification failure would then remove the newcomer's file and directory. Stand back and - // leave the partial to the newcomer (or to the resume branch above). if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) const persisted = ( await stat(tmpPath).catch((e) => { @@ -805,6 +805,7 @@ export class Downloader { } progress.set(fileId, totalTransferred) emitProgress() + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) await rename(tmpPath, savePath).catch((e) => { throw disk(e) })