diff --git a/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md b/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md new file mode 100644 index 00000000..a2f0761f --- /dev/null +++ b/docs/decisions/2026-11-15-a-superseded-download-does-not-publish.md @@ -0,0 +1,17 @@ +--- +date: 2026-11-15 +title: "A superseded download stands back before it publishes" +--- + +# 2026-11-15 — A superseded download stands back before it publishes + +- **Context:** `Downloader.download` supersedes an earlier call with the same task id by aborting it, and the doc comment promises the superseded run "leaves the files to the newcomer". The abort was only consulted while a chunk arrived, inside the read loop of `downloadOne`. An attempt whose body had already ended — or which was blocked in `reader.read()` and then saw `done` — walked past the loop into the publish block: it renamed the shared `.tmp` to `` and removed `.url`, then failed its own post-transfer verification with `Validation cancelled`. The verification-failure branch of `run` removes the published file and its directory, so the live attempt's next `stat(tmpPath)` threw `ENOENT`, reported as `[disk_device_lost] ENOENT` for a download that had lost nothing of its own. +- **Decision:** Abort the superseded attempt and wait for its filesystem operations to settle before + the same task ID starts again. Cancel its response reader to unblock a stalled read. Check for + cancellation before publishing a partial and before cleaning up after verification. +- **Consequences:** An already-submitted rename or cleanup can finish before the next attempt touches + the shared files. Cancellation leaves partials and completed files available to the next attempt, + including when it interrupts the hash check. Other task IDs can still run concurrently, and a + real staging-directory failure retains its existing disk diagnostic. +- **Owner:** team. +- **Links:** `src/downloads/downloader.ts` (`downloadOne`), `src/downloads/downloader-supersede-race.test.ts`. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 2c72bb5e..d038f79b 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -261,6 +261,7 @@ add a new one that says which record it supersedes. - [Accept the app's client keys next to the server key](2026-11-12-accept-app-client-keys-next-to-the-server-key.md) — `PUT /atomic/v1/server/client-keys`: in-memory extra keys next to a non-empty `api_key`; keyring wins, keyless stays open - [Downloads report the disk reason they already tag](2026-11-11-downloads-report-their-own-disk-reason.md) - [Keep the stream failure that ended a download retry ladder](2026-11-16-downloads-keep-the-stream-failure-that-ended-a-retry-ladder.md) +- [A superseded download stands back before it publishes](2026-11-15-a-superseded-download-does-not-publish.md) - [Refuse to replace a backend pack a session runs from](2026-11-12-refuse-replacing-in-use-backend-pack.md) - [Browser stack frames keep their served asset path](2026-11-12-keep-browser-asset-paths-in-stack-frames.md) - [An audio shutdown refusal is not an engine crash](2026-11-12-audio-shutdown-refusal-is-not-a-crash.md) diff --git a/src/downloads/downloader-supersede-race.test.ts b/src/downloads/downloader-supersede-race.test.ts new file mode 100644 index 00000000..9fd957b8 --- /dev/null +++ b/src/downloads/downloader-supersede-race.test.ts @@ -0,0 +1,157 @@ +import { mkdtemp, readFile, rm, stat } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { DOWNLOAD_CANCELLED, Downloader } from './downloader.js' + +const folders: string[] = [] + +afterEach(async () => { + await Promise.all(folders.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void + const promise = new Promise((r) => (resolve = r)) + return { promise, resolve } +} + +describe('a second download() with the same task id', () => { + it('stops the superseded attempt before it publishes the staging file the newcomer owns', async () => { + const dataFolder = await mkdtemp(join(tmpdir(), 'atomic-dl-supersede-')) + folders.push(dataFolder) + // Same length, different bytes: the two attempts can be told apart on disk. + const firstBytes = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]) + const secondBytes = new Uint8Array([11, 12, 13, 14, 15, 16, 17, 18]) + const releaseFirst = deferred() + const releaseSecond = deferred() + const firstChunk = deferred() + const secondChunk = deferred() + let calls = 0 + const fetchStub = (async () => { + calls += 1 + const first = calls === 1 + let cancelled = false + return new Response( + new ReadableStream({ + async pull(controller) { + controller.enqueue(first ? firstBytes : secondBytes) + ;(first ? firstChunk : secondChunk).resolve() + await (first ? releaseFirst : releaseSecond).promise + if (!cancelled) controller.close() + }, + cancel() { + cancelled = true + }, + }), + { status: 200, headers: { 'content-length': '8' } } + ) + }) as unknown as typeof fetch + const dl = new Downloader({ + dataFolder, + platform: 'linux', + fetch: fetchStub, + availableSpace: async () => undefined, + sleep: async () => {}, + emit: () => {}, + }) + const savePath = join(dataFolder, 'backends/pack.bin') + const item = { url: 'http://engine.test/pack.bin', save_path: 'backends/pack.bin', size: 8 } + + // The first attempt has written its chunk and is waiting for the rest of its body. + const first = dl.download('same', [item]) + await firstChunk.promise + await vi.waitFor(async () => expect((await stat(`${savePath}.tmp`)).size).toBe(8)) + + // A client retries with the same task id: the first attempt is superseded, the second owns the file. + const firstCancelled = expect(first).rejects.toThrow(DOWNLOAD_CANCELLED) + const second = dl.download('same', [item]) + await secondChunk.promise + await vi.waitFor(async () => expect([...(await readFile(`${savePath}.tmp`))]).toEqual([...secondBytes])) + + // The superseded attempt's body ends. It must stand back; it used to publish the newcomer's file + // and then remove it, so the newcomer read `ENOENT` from its own staging path. + releaseFirst.resolve() + await firstCancelled + + releaseSecond.resolve() + await expect(second).resolves.toBeUndefined() + expect([...(await readFile(savePath))]).toEqual([...secondBytes]) + // The newcomer published its own bytes: the staging file and the sidecar are gone. + await expect(stat(`${savePath}.tmp`)).rejects.toThrow() + await expect(stat(`${savePath}.url`)).rejects.toThrow() + }) + + it('leaves the completed partial when the final progress callback supersedes the attempt', async () => { + const dataFolder = await mkdtemp(join(tmpdir(), 'atomic-dl-publish-')) + folders.push(dataFolder) + const savePath = join(dataFolder, 'pack.bin') + const item = { url: 'http://engine.test/pack.bin', save_path: 'pack.bin', size: 4 } + let replacement: Promise | undefined + let partial: string | undefined + let calls = 0 + const dl: Downloader = new Downloader({ + dataFolder, + platform: 'linux', + fetch: async () => { + if (++calls === 1) return new Response('old!') + partial = await readFile(`${savePath}.tmp`, 'utf8').catch(() => undefined) + return new Response('new!') + }, + availableSpace: async () => undefined, + emit(event) { + if (event === 'download:progress' && !replacement) replacement = dl.download('same', [item]) + }, + }) + + await expect(dl.download('same', [item])).rejects.toThrow(DOWNLOAD_CANCELLED) + await expect(replacement).resolves.toBeUndefined() + expect(partial).toBe('old!') + expect(await readFile(savePath, 'utf8')).toBe('new!') + }) + + it('keeps the finished file if cancellation interrupts validation before a retry', async () => { + const dataFolder = await mkdtemp(join(tmpdir(), 'atomic-dl-validation-')) + folders.push(dataFolder) + const savePath = join(dataFolder, 'pack.bin') + const sha256 = 'a'.repeat(64) + const item = { url: 'http://engine.test/pack.bin', save_path: 'pack.bin', size: 4, sha256 } + const verifying = deferred() + const finishVerification = deferred() + let calls = 0 + let original: string | undefined + const dl = new Downloader({ + dataFolder, + platform: 'linux', + fetch: async () => { + if (++calls === 1) return new Response('old!') + original = await readFile(savePath, 'utf8').catch(() => undefined) + return new Response('new!') + }, + availableSpace: async () => undefined, + emit: () => {}, + verify: { + fileSize: async (path) => (await stat(path)).size, + sha256: async (_path, signal) => { + if (calls === 1) { + verifying.resolve() + await finishVerification.promise + if (signal?.aborted) throw new Error('Validation cancelled') + } + return sha256 + }, + }, + }) + const firstCancelled = expect(dl.download('same', [item])).rejects.toThrow(DOWNLOAD_CANCELLED) + await verifying.promise + expect(dl.cancel('same')).toBe(true) + expect(dl.active()).toEqual([]) + const second = dl.download('same', [item]) + finishVerification.resolve() + + await firstCancelled + await expect(second).resolves.toBeUndefined() + expect(original).toBe('old!') + expect(await readFile(savePath, 'utf8')).toBe('new!') + }) +}) diff --git a/src/downloads/downloader.ts b/src/downloads/downloader.ts index 8571a77e..c417b261 100644 --- a/src/downloads/downloader.ts +++ b/src/downloads/downloader.ts @@ -94,7 +94,7 @@ type StageReport = (kind: DownloadStage['kind'], attempt: number) => void interface Task { controller: AbortController - superseded: boolean + done: Promise /** The last combined progress this task reported, for `snapshot()`. */ progress?: DownloadProgress } @@ -267,7 +267,7 @@ export class Downloader { /** Task ids currently running. */ active(): string[] { - return [...this.tasks.keys()] + return [...this.tasks].filter(([, task]) => !task.controller.signal.aborted).map(([taskId]) => taskId) } /** @@ -275,16 +275,15 @@ export class Downloader { * reconnects mid-transfer reads from the control snapshot instead of waiting for the next event. */ snapshot(): DownloadProgress[] { - return [...this.tasks.entries()].map( - ([taskId, task]) => task.progress ?? { taskId, transferred: 0, total: 0, percent: 0 } + return this.active().map( + (taskId) => this.tasks.get(taskId)?.progress ?? { taskId, transferred: 0, total: 0, percent: 0 } ) } /** Cancel a running task. Keeps partials and finished files. */ cancel(taskId: string): boolean { const task = this.tasks.get(taskId) - if (!task) return false - this.tasks.delete(taskId) + if (!task || task.controller.signal.aborted) return false task.controller.abort() return true } @@ -294,15 +293,17 @@ export class Downloader { * observes a cancellation but leaves the files to the newcomer). */ async download(taskId: string, items: DownloadItem[], options: DownloadOptions = {}): Promise { - const existing = this.tasks.get(taskId) - if (existing) { - existing.superseded = true - existing.controller.abort() - } - const task: Task = { controller: new AbortController(), superseded: false } + const previous = this.tasks.get(taskId) + const controller = new AbortController() + // An aborted attempt can still have filesystem work in flight. Hand off the shared files only + // once it has closed its writer and settled any publish or validation cleanup. + const drained = previous?.done.catch(() => {}) ?? Promise.resolve() + const done = drained.then(() => this.run(taskId, items, options, controller.signal)) + const task: Task = { controller, done } this.tasks.set(taskId, task) + previous?.controller.abort() try { - await this.run(taskId, items, options, task.controller.signal) + await done } catch (error) { if (!task.controller.signal.aborted) this.deps.onFailure?.(error, { component: 'downloads', operation: 'download' }) @@ -313,6 +314,7 @@ export class Downloader { } private async run(taskId: string, items: DownloadItem[], options: DownloadOptions, signal: AbortSignal) { + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) const { dataFolder, platform, log } = this.deps const resume = options.resume ?? false const headers = options.headers ?? {} @@ -396,6 +398,7 @@ export class Downloader { ) ) if (failed) throw firstError + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) const results = settled.map((outcome) => (outcome as PromiseFulfilledResult).value) if (items.some((it) => it.sha256 != null || it.size != null)) { @@ -409,6 +412,8 @@ export class Downloader { const verifications = await Promise.allSettled( items.map((item, i) => verifyDownloadedFile(item, results[i] as string, this.deps.verify, signal)) ) + // Cancelling validation must keep the finished files, even if a verifier rejected on abort. + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) for (let i = 0; i < verifications.length; i++) { const v = verifications[i] if (v && v.status === 'rejected') { @@ -653,23 +658,22 @@ export class Downloader { let retryCount = 0 let sinceReset = 0 + const stopReading = () => void reader.cancel().catch(() => {}) + signal.addEventListener('abort', stopReading) try { for (;;) { + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) let streamError: string | undefined // The caught object itself, so the report keeps the code that classifies the failure. let streamCause: unknown try { const { done, value } = await reader.read() + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) if (done) { if (expectedSize > 0 && totalTransferred < expectedSize) { streamError = `stream ended after ${totalTransferred} of ${expectedSize} bytes` } else break } else if (value) { - if (signal.aborted) { - await writer.close().catch(() => {}) - this.deps.log('info', `Download cancelled: ${item.url}`) - throw new Error(DOWNLOAD_CANCELLED) - } // Writes are awaited one at a time, so `totalTransferred` is where this chunk starts. await writer.write(value, totalTransferred) delta += value.byteLength @@ -779,9 +783,11 @@ export class Downloader { await writer.close() } finally { + signal.removeEventListener('abort', stopReading) await writer.close().catch(() => {}) await reader.cancel().catch(() => {}) } + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) const persisted = ( await stat(tmpPath).catch((e) => { throw disk(e) @@ -799,6 +805,7 @@ export class Downloader { } progress.set(fileId, totalTransferred) emitProgress() + if (signal.aborted) throw new Error(DOWNLOAD_CANCELLED) await rename(tmpPath, savePath).catch((e) => { throw disk(e) })