Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions src/server/stream-io.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,21 +92,12 @@ export async function pipeThrough(stream: ReadableStream<Uint8Array>, res: http.
export async function dumpStreamToFile(stream: ReadableStream<Uint8Array>, dir: string, name: string): Promise<void> {
const { mkdirSync, createWriteStream } = await import("node:fs");
const { join } = await import("node:path");
const { pipeline } = await import("node:stream/promises");
try {
mkdirSync(dir, { recursive: true });
const ws = createWriteStream(join(dir, name));
ws.on("error", (e) => { logDumpFailure("SSE stream dump", e); });
const reader = stream.getReader();
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
ws.write(Buffer.from(value));
}
} finally {
reader.releaseLock();
ws.end(() => gcDumpDirIfConfigured(dir));
}
await pipeline(stream, ws);
gcDumpDirIfConfigured(dir);
} catch (err) {
logDumpFailure("SSE stream dump", err);
}
Expand Down
118 changes: 118 additions & 0 deletions tests/sse-dump-backpressure.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import { syncBuiltinESMExports } from "node:module";
import { tmpdir } from "node:os";
import path from "node:path";
import { Writable } from "node:stream";
import test from "node:test";
import { dumpStreamToFile } from "../src/server/stream-io.ts";
import { rmrf } from "./tmp-rm.ts";

async function within<T>(promise: Promise<T>): Promise<T> {
let timer: NodeJS.Timeout | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error("dump stream test timed out")), 3_000);
}),
]);
} finally {
clearTimeout(timer);
}
}

test("SSE dump honors sink backpressure and waits for final flush", async (t) => {
const dir = fs.mkdtempSync(path.join(tmpdir(), "bili-dump-pressure-"));
let firstWrite!: () => void;
const started = new Promise<void>((resolve) => { firstWrite = resolve; });
let finalEntered!: () => void;
const finalizing = new Promise<void>((resolve) => { finalEntered = resolve; });
let releaseWrite: (() => void) | undefined;
let releaseFinal: (() => void) | undefined;
let written = 0;
const sink = new Writable({
highWaterMark: 1024,
write(chunk: Buffer, _encoding, callback) {
written += chunk.length;
if (releaseWrite === undefined && written === 1024) {
releaseWrite = callback;
firstWrite();
} else callback();
},
final(callback) {
releaseFinal = callback;
finalEntered();
},
});
let produced = 0;
const upstream = new ReadableStream<Uint8Array>({
pull(controller) {
if (produced === 512) controller.close();
else {
produced++;
controller.enqueue(Buffer.alloc(1024));
}
},
});
let finished = false;
let dumping: Promise<void> | undefined;
try {
t.mock.method(fs, "createWriteStream", () => sink as fs.WriteStream);
syncBuiltinESMExports();
dumping = dumpStreamToFile(upstream, dir, "stream.sse").then(() => { finished = true; });
await within(started);
await new Promise<void>((resolve) => setImmediate(resolve));
assert.ok(produced < 512, "a blocked sink must not drain the entire source");
assert.equal(finished, false);
releaseWrite!();
releaseWrite = undefined;
await within(finalizing);
assert.equal(finished, false, "dump is incomplete until the sink's final callback");
releaseFinal!();
releaseFinal = undefined;
await within(dumping);
assert.equal(written, 512 * 1024);
assert.equal(upstream.locked, false);
} finally {
releaseWrite?.();
releaseFinal?.();
sink.destroy();
await dumping?.catch(() => {});
t.mock.restoreAll();
syncBuiltinESMExports();
rmrf(dir);
}
});

test("SSE dump cancels the source when the destination fails", async (t) => {
const dir = fs.mkdtempSync(path.join(tmpdir(), "bili-dump-error-"));
let cancelled = false;
let produced = 0;
const upstream = new ReadableStream<Uint8Array>({
pull(controller) {
if (produced++ < 512) controller.enqueue(Buffer.alloc(1024));
else controller.close();
},
cancel() { cancelled = true; },
});
const sink = new Writable({
highWaterMark: 1024,
write(_chunk, _encoding, callback) {
callback(Object.assign(new Error("disk full"), { code: "ENOSPC" }));
},
});
try {
t.mock.method(fs, "createWriteStream", () => sink as fs.WriteStream);
syncBuiltinESMExports();
await within(dumpStreamToFile(upstream, dir, "stream.sse"));
assert.equal(cancelled, true);
assert.equal(upstream.locked, false);
} finally {
sink.destroy();
if (!upstream.locked) await upstream.cancel().catch(() => {});
t.mock.restoreAll();
syncBuiltinESMExports();
rmrf(dir);
}
});
Loading