From 105b84dd95c73eccb3eab83a2c199e0c1413b1f4 Mon Sep 17 00:00:00 2001 From: Afloat Date: Mon, 9 Nov 2026 21:51:35 +0800 Subject: [PATCH] fix: honor backpressure and completion when dumping SSE streams --- src/server/stream-io.ts | 15 +--- tests/sse-dump-backpressure.test.ts | 118 ++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 tests/sse-dump-backpressure.test.ts diff --git a/src/server/stream-io.ts b/src/server/stream-io.ts index 3011a4ed55..8e179bfe4e 100644 --- a/src/server/stream-io.ts +++ b/src/server/stream-io.ts @@ -92,21 +92,12 @@ export async function pipeThrough(stream: ReadableStream, res: http. export async function dumpStreamToFile(stream: ReadableStream, dir: string, name: string): Promise { const { mkdirSync, createWriteStream } = await import("node:fs"); const { join } = await import("node:path"); + const { pipeline } = await import("node:stream/promises"); try { mkdirSync(dir, { recursive: true }); const ws = createWriteStream(join(dir, name)); - ws.on("error", (e) => { logDumpFailure("SSE stream dump", e); }); - const reader = stream.getReader(); - try { - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - ws.write(Buffer.from(value)); - } - } finally { - reader.releaseLock(); - ws.end(() => gcDumpDirIfConfigured(dir)); - } + await pipeline(stream, ws); + gcDumpDirIfConfigured(dir); } catch (err) { logDumpFailure("SSE stream dump", err); } diff --git a/tests/sse-dump-backpressure.test.ts b/tests/sse-dump-backpressure.test.ts new file mode 100644 index 0000000000..f2e1b6d8ed --- /dev/null +++ b/tests/sse-dump-backpressure.test.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { Writable } from "node:stream"; +import test from "node:test"; +import { dumpStreamToFile } from "../src/server/stream-io.ts"; +import { rmrf } from "./tmp-rm.ts"; + +async function within(promise: Promise): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("dump stream test timed out")), 3_000); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +test("SSE dump honors sink backpressure and waits for final flush", async (t) => { + const dir = fs.mkdtempSync(path.join(tmpdir(), "bili-dump-pressure-")); + let firstWrite!: () => void; + const started = new Promise((resolve) => { firstWrite = resolve; }); + let finalEntered!: () => void; + const finalizing = new Promise((resolve) => { finalEntered = resolve; }); + let releaseWrite: (() => void) | undefined; + let releaseFinal: (() => void) | undefined; + let written = 0; + const sink = new Writable({ + highWaterMark: 1024, + write(chunk: Buffer, _encoding, callback) { + written += chunk.length; + if (releaseWrite === undefined && written === 1024) { + releaseWrite = callback; + firstWrite(); + } else callback(); + }, + final(callback) { + releaseFinal = callback; + finalEntered(); + }, + }); + let produced = 0; + const upstream = new ReadableStream({ + pull(controller) { + if (produced === 512) controller.close(); + else { + produced++; + controller.enqueue(Buffer.alloc(1024)); + } + }, + }); + let finished = false; + let dumping: Promise | undefined; + try { + t.mock.method(fs, "createWriteStream", () => sink as fs.WriteStream); + syncBuiltinESMExports(); + dumping = dumpStreamToFile(upstream, dir, "stream.sse").then(() => { finished = true; }); + await within(started); + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(produced < 512, "a blocked sink must not drain the entire source"); + assert.equal(finished, false); + releaseWrite!(); + releaseWrite = undefined; + await within(finalizing); + assert.equal(finished, false, "dump is incomplete until the sink's final callback"); + releaseFinal!(); + releaseFinal = undefined; + await within(dumping); + assert.equal(written, 512 * 1024); + assert.equal(upstream.locked, false); + } finally { + releaseWrite?.(); + releaseFinal?.(); + sink.destroy(); + await dumping?.catch(() => {}); + t.mock.restoreAll(); + syncBuiltinESMExports(); + rmrf(dir); + } +}); + +test("SSE dump cancels the source when the destination fails", async (t) => { + const dir = fs.mkdtempSync(path.join(tmpdir(), "bili-dump-error-")); + let cancelled = false; + let produced = 0; + const upstream = new ReadableStream({ + pull(controller) { + if (produced++ < 512) controller.enqueue(Buffer.alloc(1024)); + else controller.close(); + }, + cancel() { cancelled = true; }, + }); + const sink = new Writable({ + highWaterMark: 1024, + write(_chunk, _encoding, callback) { + callback(Object.assign(new Error("disk full"), { code: "ENOSPC" })); + }, + }); + try { + t.mock.method(fs, "createWriteStream", () => sink as fs.WriteStream); + syncBuiltinESMExports(); + await within(dumpStreamToFile(upstream, dir, "stream.sse")); + assert.equal(cancelled, true); + assert.equal(upstream.locked, false); + } finally { + sink.destroy(); + if (!upstream.locked) await upstream.cancel().catch(() => {}); + t.mock.restoreAll(); + syncBuiltinESMExports(); + rmrf(dir); + } +});