diff --git a/scripts/bench-zstd-decode.mjs b/scripts/bench-zstd-decode.mjs new file mode 100644 index 0000000000..b645e8050c --- /dev/null +++ b/scripts/bench-zstd-decode.mjs @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import * as zlib from "node:zlib"; +import { decodeRequestBody } from "../src/content-encoding.ts"; + +if (typeof zlib.zstdCompressSync !== "function") { + throw new Error("This benchmark requires a Node version with native zstd compression"); +} +const input = Buffer.alloc(16 * 1024 * 1024); +input.fill("a representative repeated conversation entry; 中文 content.\n"); +const encoded = zlib.zstdCompressSync(input); +const synchronousMs = [], totalMs = []; +for (let iteration = 0; iteration < 6; iteration++) { + const started = performance.now(); + const pending = decodeRequestBody("zstd", encoded, input.length); + const returned = performance.now(); + const decoded = await pending; + const completed = performance.now(); + assert.deepEqual(decoded.body, input); + if (iteration > 0) { + synchronousMs.push(returned - started); + totalMs.push(completed - started); + } +} +const median = (values) => values.toSorted((a, b) => a - b)[Math.floor(values.length / 2)]; +console.log(JSON.stringify({ + node: process.version, bytes: input.length, compressed: encoded.length, + warmup: 1, samples: 5, + medianSynchronousMs: median(synchronousMs), medianTotalMs: median(totalMs), + synchronousMs, totalMs, +})); diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index b606dc43bd..2aec38693a 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -142,6 +142,9 @@ export const LIVE_FILES = [ "tests/pack-install.test.ts", // Runs the Linux janitor shell against per-case local API fixtures; no ports or credentials. "tests/ci-janitor-fork-heads.test.ts", + // Native-zstd absence is simulated in an isolated Node child; it binds + // no sockets and changes only that child's built-in module exports. + "tests/content-encoding-zstd.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/src/content-encoding.ts b/src/content-encoding.ts index 848856e47a..bccd5d2fd1 100644 --- a/src/content-encoding.ts +++ b/src/content-encoding.ts @@ -1,4 +1,4 @@ -import { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from "node:zlib"; +import * as zlib from "node:zlib"; import { DecodedRequestBusyError } from "./request-body-budget.js"; import type { Duplex } from "node:stream"; import { Decompress as ZstdDecompress } from "fzstd"; @@ -68,7 +68,79 @@ function streamDecode(factory: () => Duplex, input: Buffer, max: number, options }); } +/** Frame boundaries from RFC 8878 section 3.1, without decompressing blocks. + * Node 22's native decoder drops later frames in one input buffer and accepts + * truncated frames (nodejs/node#64741, #64592). Validate their extent first, + * then decode complete frames separately; native zstd still checks contents. */ +function* zstdFrames(input: Buffer): Generator { + let offset = 0; + const requireBytes = (count: number): void => { + if (count > input.length - offset) throw new Error("truncated zstd frame"); + }; + do { + const start = offset; + requireBytes(4); + const magic = input.readUInt32LE(offset); + if ((magic & 0xfffffff0) === 0x184d2a50) { + requireBytes(8); + const size = input.readUInt32LE(offset + 4); + offset += 8; + requireBytes(size); + offset += size; + } else { + if (magic !== 0xfd2fb528) throw new Error("invalid zstd frame magic"); + requireBytes(5); + const descriptor = input[offset + 4]; + if (descriptor & 8) throw new Error("reserved zstd frame bit"); + const singleSegment = (descriptor & 32) !== 0; + const dictionaryFlag = descriptor & 3; + const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag; + const sizeFlag = descriptor >>> 6; + const sizeBytes = sizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << sizeFlag; + offset += 5; + const remainingHeader = (singleSegment ? 0 : 1) + dictionaryBytes + sizeBytes; + requireBytes(remainingHeader); + offset += remainingHeader; + let lastBlock: boolean; + do { + requireBytes(3); + const header = input.readUIntLE(offset, 3); + offset += 3; + lastBlock = (header & 1) !== 0; + const blockType = (header >>> 1) & 3; + if (blockType === 3) throw new Error("reserved zstd block type"); + const size = blockType === 1 ? 1 : header >>> 3; + requireBytes(size); + offset += size; + } while (!lastBlock); + if (descriptor & 4) { + requireBytes(4); + offset += 4; + } + } + yield input.subarray(start, offset); + } while (offset < input.length); +} + async function decodeZstd(input: Buffer, max: number, options?: DecodeOptions): Promise { + if (typeof zlib.createZstdDecompress === "function") { + const chunks: Buffer[] = []; + let size = 0; + for (const frame of zstdFrames(input)) { + options?.signal?.throwIfAborted(); + const body = await streamDecode(zlib.createZstdDecompress, frame, max, { + signal: options?.signal, + onBytes: (bytes) => { + const total = size + bytes; + if (total > max) throw new DecompressedTooLargeError(max); + options?.onBytes?.(total); + }, + }); + chunks.push(body); + size += body.length; + } + return chunks.length === 1 ? chunks[0] : Buffer.concat(chunks, size); + } const chunks: Buffer[] = []; let size = 0; const decoder = new ZstdDecompress((chunk) => { @@ -83,15 +155,15 @@ async function decodeZstd(input: Buffer, max: number, options?: DecodeOptions): } async function decodeOne(coding: string, input: Buffer, max: number, options?: DecodeOptions): Promise { - if (coding === "gzip" || coding === "x-gzip") return streamDecode(createGunzip, input, max, options); - if (coding === "br") return streamDecode(createBrotliDecompress, input, max, options); + if (coding === "gzip" || coding === "x-gzip") return streamDecode(zlib.createGunzip, input, max, options); + if (coding === "br") return streamDecode(zlib.createBrotliDecompress, input, max, options); if (coding === "zstd") return decodeZstd(input, max, options); if (coding === "deflate") { try { - return await streamDecode(createInflate, input, max, options); + return await streamDecode(zlib.createInflate, input, max, options); } catch (err) { if (err instanceof DecompressedTooLargeError || err instanceof DecodedRequestBusyError || options?.signal?.aborted) throw err; - return streamDecode(createInflateRaw, input, max, options); + return streamDecode(zlib.createInflateRaw, input, max, options); } } throw new Error(`unsupported request content-encoding: ${coding}`); diff --git a/tests/content-encoding-zstd.test.ts b/tests/content-encoding-zstd.test.ts new file mode 100644 index 0000000000..be7d78ebb4 --- /dev/null +++ b/tests/content-encoding-zstd.test.ts @@ -0,0 +1,154 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import * as zlib from "node:zlib"; +import { spawnSync } from "node:child_process"; +import { decodeRequestBody, DecompressedTooLargeError } from "../src/content-encoding.ts"; +import { DecodedRequestBusyError } from "../src/request-body-budget.ts"; + +const hasNativeZstd = typeof zlib.createZstdDecompress === "function"; +const plain = Buffer.from('{"model":"gpt-5","input":"hello"}'); +const encoded = Buffer.from("KLUv/SAhCQEAeyJtb2RlbCI6ImdwdC01IiwiaW5wdXQiOiJoZWxsbyJ9", "base64"); +const unknownSize = Buffer.from("KLUv/QAACQEAeyJtb2RlbCI6ImdwdC01IiwiaW5wdXQiOiJoZWxsbyJ9", "base64"); +const checksummed = Buffer.from("KLUv/SQhCQEAeyJtb2RlbCI6ImdwdC01IiwiaW5wdXQiOiJoZWxsbyJ9TTP3+Q==", "base64"); +const skippable = Buffer.from("5f2a4d1803000000010203", "hex"); + +test("zstd decoding can be cancelled after starting without processing the whole body", { skip: !hasNativeZstd }, async () => { + const raw = Buffer.alloc(4 * 1024 * 1024, 65); + const input = zlib.zstdCompressSync(raw); + const controller = new AbortController(); + const reason = new Error("client disconnected"); + let bytes = 0; + const pending = decodeRequestBody("zstd", input, raw.length, { + signal: controller.signal, + onBytes: (count) => { bytes = count; }, + }); + controller.abort(reason); + await assert.rejects(pending, (error) => error === reason); + assert.ok(bytes < raw.length); +}); + +test("cancelling one zstd decode does not cancel a concurrent request", { skip: !hasNativeZstd }, async () => { + const controller = new AbortController(); + const cancelled = decodeRequestBody("zstd", encoded, 1024, { signal: controller.signal }); + const completed = decodeRequestBody("zstd", encoded, 1024); + controller.abort(); + await assert.rejects(cancelled, { name: "AbortError" }); + assert.deepEqual((await completed).body, plain); +}); + +test("zstd decoding stops if the byte observer cancels the final output chunk", { skip: !hasNativeZstd }, async () => { + const controller = new AbortController(); + let chunks = 0; + await assert.rejects(decodeRequestBody("zstd", encoded, 1024, { + signal: controller.signal, + onBytes: () => { chunks++; controller.abort(); }, + }), { name: "AbortError" }); + assert.equal(chunks, 1); +}); + +test("zstd decoding preserves exact byte limits and the resource-admission error", async () => { + assert.deepEqual((await decodeRequestBody("zstd", encoded, plain.length)).body, plain); + await assert.rejects(decodeRequestBody("zstd", encoded, plain.length - 1), DecompressedTooLargeError); + const busy = new DecodedRequestBusyError(); + await assert.rejects(decodeRequestBody("zstd", encoded, 1024, { + onBytes: () => { throw busy; }, + }), (error) => error === busy); +}); + +test("zstd decoding supports concatenated frames and stacked content encodings", async () => { + const input = Buffer.concat([encoded, encoded]); + const observed: number[] = []; + assert.deepEqual((await decodeRequestBody("zstd", input, plain.length * 2, { + onBytes: (count) => observed.push(count), + })).body, + Buffer.concat([plain, plain])); + assert.equal(observed.at(-1), plain.length * 2); + assert.ok(observed.every((count, i) => i === 0 || count >= observed[i - 1])); + await assert.rejects(decodeRequestBody("zstd", input, plain.length * 2 - 1), + (error) => error instanceof DecompressedTooLargeError && error.limit === plain.length * 2 - 1); + const busy = new DecodedRequestBusyError(); + await assert.rejects(decodeRequestBody("zstd", input, 1024, { + onBytes: (count) => { if (count > plain.length) throw busy; }, + }), (error) => error === busy); + const controller = new AbortController(); + const reason = new Error("cancelled in the second frame"); + await assert.rejects(decodeRequestBody("zstd", input, 1024, { + signal: controller.signal, + onBytes: (count) => { if (count > plain.length) controller.abort(reason); }, + }), (error) => error === reason); + assert.deepEqual((await decodeRequestBody("zstd, gzip", zlib.gzipSync(encoded), 1024)).body, plain); + await assert.rejects(decodeRequestBody("zstd, gzip", zlib.gzipSync(encoded), plain.length - 1), + DecompressedTooLargeError); +}); + +test("zstd decoding accepts unknown-size and checksummed frames with interleaved skippable data", async () => { + assert.deepEqual((await decodeRequestBody("zstd", unknownSize, plain.length)).body, plain); + assert.deepEqual((await decodeRequestBody("zstd", checksummed, plain.length)).body, plain); + const input = Buffer.concat([skippable, unknownSize, skippable, checksummed, skippable]); + assert.deepEqual((await decodeRequestBody("zstd", input, plain.length * 2)).body, + Buffer.concat([plain, plain])); + assert.deepEqual((await decodeRequestBody("zstd", skippable, 0)).body, Buffer.alloc(0)); +}); + +test("zstd decoding accepts RLE and compressed blocks", async () => { + const rle = Buffer.from("28b52ffd20210b010041", "hex"); + assert.deepEqual((await decodeRequestBody("zstd", rle, 33)).body, Buffer.alloc(33, 65)); + const compressed = Buffer.from("KLUv/WDoAk0AABBBQQEA4yuABQ==", "base64"); + assert.deepEqual((await decodeRequestBody("zstd", compressed, 1000)).body, Buffer.alloc(1000, 65)); +}); + +for (const [name, input] of [ + ["unknown-size block", unknownSize.subarray(0, -1)], + ["checksum", checksummed.subarray(0, -1)], + ["second frame", Buffer.concat([encoded, encoded.subarray(0, -1)])], + ["skippable payload", skippable.subarray(0, -1)], + ["trailing skippable payload", Buffer.concat([encoded, skippable.subarray(0, -1)])], + ["frame header", encoded.subarray(0, 5)], + ["block header", encoded.subarray(0, 8)], +] as const) { + test(`zstd decoding rejects a truncated ${name}`, async () => { + await assert.rejects(decodeRequestBody("zstd", input, 1024)); + }); +} + +test("zstd decoding keeps native checksum validation for later frames", { skip: !hasNativeZstd }, async () => { + const corrupt = Buffer.from(checksummed); + corrupt[corrupt.length - 1] ^= 1; + await assert.rejects(decodeRequestBody("zstd", Buffer.concat([encoded, corrupt]), 1024)); +}); + +test("zstd decoding rejects truncated input and respects pre-aborted requests", async () => { + await assert.rejects(decodeRequestBody("zstd", encoded.subarray(0, -1), 1024)); + await assert.rejects(decodeRequestBody("zstd", Buffer.from("not a zstd frame"), 1024)); + const controller = new AbortController(); + const reason = new Error("already disconnected"); + controller.abort(reason); + await assert.rejects(decodeRequestBody("zstd", encoded, 1024, { signal: controller.signal }), + (error) => error === reason); +}); + +test("zstd decoding still works when node:zlib has no native zstd factory", () => { + const contentEncoding = new URL("../src/content-encoding.ts", import.meta.url).href; + const budget = new URL("../src/request-body-budget.ts", import.meta.url).href; + const child = spawnSync(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", ` + import assert from "node:assert/strict"; + import zlib from "node:zlib"; + import { syncBuiltinESMExports } from "node:module"; + Reflect.set(zlib, "createZstdDecompress", undefined); + syncBuiltinESMExports(); + const { decodeRequestBody, DecompressedTooLargeError } = await import(${JSON.stringify(contentEncoding)}); + const { DecodedRequestBusyError } = await import(${JSON.stringify(budget)}); + const input = Buffer.from(${JSON.stringify(encoded.toString("base64"))}, "base64"); + const expected = Buffer.from(${JSON.stringify(plain.toString())}); + assert.deepEqual((await decodeRequestBody("zstd", input, expected.length)).body, expected); + await assert.rejects(decodeRequestBody("zstd", input, expected.length - 1), DecompressedTooLargeError); + await assert.rejects(decodeRequestBody("zstd", input, 1024, { + onBytes: () => { throw new DecodedRequestBusyError(); }, + }), DecodedRequestBusyError); + const controller = new AbortController(); + controller.abort(); + await assert.rejects(decodeRequestBody("zstd", input, 1024, { signal: controller.signal }), { name: "AbortError" }); + `], { encoding: "utf8", timeout: 20_000 }); + assert.ifError(child.error); + assert.equal(child.status, 0, child.stderr); +}); diff --git a/tests/e2e/e2e-update-state.test.ts b/tests/e2e/e2e-update-state.test.ts index 03f2b1db22..e31baf7659 100644 --- a/tests/e2e/e2e-update-state.test.ts +++ b/tests/e2e/e2e-update-state.test.ts @@ -299,6 +299,11 @@ test("self-update keeps live compression state: folds survive the OLD→NEW hop envBase.BILI_UPDATE_POLICY_URL = "http://127.0.0.1:1/no-policy.json"; envBase.BILI_PERSIST_DEBOUNCE_MS = "200"; envBase.BILI_SESSIONS_DIR = sessionsDir; + // Pure-sqlite default (5a80e2b8e, #6480) turns the JSON envelope + // writer OFF, but this lane asserts on the on-disk .json session file + // (state survival across the update). Pin the writer ON so the + // envelope this test was written for is still produced. + envBase.BILI_STORAGE_JSON_WRITER = "on"; const port = await freePort(); proxy = spawnProxy(installDir, port, envBase);