Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
1bd117e
fix: decode zstd requests asynchronously when supported
Afloat16 Nov 12, 2026
829a785
test: register zstd fallback process coverage
Afloat16 Nov 13, 2026
50926b8
Merge master and preserve CI process registrations
Afloat16 Nov 13, 2026
7341f7d
fix: preserve complete zstd frames on Node 22
Afloat16 Nov 13, 2026
c6369fb
fix(tests): pin pure-sqlite selection in notice-registry suite
Nov 13, 2026
b1bc17b
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in CCR cold-disk-restore …
Nov 14, 2026
958cbbf
Merge remote-tracking branch 'origin/master' into w5p
ranxianglei Nov 14, 2026
cd8bc8a
Merge remote-tracking branch 'origin/master' into ci-fixes
Nov 14, 2026
7492c35
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in native opencode e2e lane
Nov 14, 2026
45f93db
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in native dsh e2e lane
Nov 14, 2026
d67cc43
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in pi codex-responses WS …
Nov 14, 2026
c392237
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in pi package-native e2e …
Nov 14, 2026
974cd4c
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in fork-ccr-adoption suite
Nov 14, 2026
271abba
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in subagent-sessions e2e …
Nov 14, 2026
5d47196
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in dsh-persona e2e lane
Nov 14, 2026
1ca4087
fix(tests): pin BILI_STORAGE_JSON_WRITER=on in update-state e2e lane
Nov 14, 2026
1cd78a3
Merge upstream master and ci-fixes to refresh PR validation
Afloat16 Nov 14, 2026
889a403
Merge master into 2026-11-12_async-zstd-decode
Afloat16 Nov 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions scripts/bench-zstd-decode.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import assert from "node:assert/strict";
import * as zlib from "node:zlib";
import { decodeRequestBody } from "../src/content-encoding.ts";

if (typeof zlib.zstdCompressSync !== "function") {
throw new Error("This benchmark requires a Node version with native zstd compression");
}
const input = Buffer.alloc(16 * 1024 * 1024);
input.fill("a representative repeated conversation entry; 中文 content.\n");
const encoded = zlib.zstdCompressSync(input);
const synchronousMs = [], totalMs = [];
for (let iteration = 0; iteration < 6; iteration++) {
const started = performance.now();
const pending = decodeRequestBody("zstd", encoded, input.length);
const returned = performance.now();
const decoded = await pending;
const completed = performance.now();
assert.deepEqual(decoded.body, input);
if (iteration > 0) {
synchronousMs.push(returned - started);
totalMs.push(completed - started);
}
}
const median = (values) => values.toSorted((a, b) => a - b)[Math.floor(values.length / 2)];
console.log(JSON.stringify({
node: process.version, bytes: input.length, compressed: encoded.length,
warmup: 1, samples: 5,
medianSynchronousMs: median(synchronousMs), medianTotalMs: median(totalMs),
synchronousMs, totalMs,
}));
3 changes: 3 additions & 0 deletions scripts/live-files.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,9 @@ export const LIVE_FILES = [
"tests/pack-install.test.ts",
// Runs the Linux janitor shell against per-case local API fixtures; no ports or credentials.
"tests/ci-janitor-fork-heads.test.ts",
// Native-zstd absence is simulated in an isolated Node child; it binds
// no sockets and changes only that child's built-in module exports.
"tests/content-encoding-zstd.test.ts",
];

/** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry;
Expand Down
82 changes: 77 additions & 5 deletions src/content-encoding.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { createBrotliDecompress, createGunzip, createInflate, createInflateRaw } from "node:zlib";
import * as zlib from "node:zlib";
import { DecodedRequestBusyError } from "./request-body-budget.js";
import type { Duplex } from "node:stream";
import { Decompress as ZstdDecompress } from "fzstd";
Expand Down Expand Up @@ -68,7 +68,79 @@ function streamDecode(factory: () => Duplex, input: Buffer, max: number, options
});
}

/** Frame boundaries from RFC 8878 section 3.1, without decompressing blocks.
* Node 22's native decoder drops later frames in one input buffer and accepts
* truncated frames (nodejs/node#64741, #64592). Validate their extent first,
* then decode complete frames separately; native zstd still checks contents. */
function* zstdFrames(input: Buffer): Generator<Buffer> {
let offset = 0;
const requireBytes = (count: number): void => {
if (count > input.length - offset) throw new Error("truncated zstd frame");
};
do {
const start = offset;
requireBytes(4);
const magic = input.readUInt32LE(offset);
if ((magic & 0xfffffff0) === 0x184d2a50) {
requireBytes(8);
const size = input.readUInt32LE(offset + 4);
offset += 8;
requireBytes(size);
offset += size;
} else {
if (magic !== 0xfd2fb528) throw new Error("invalid zstd frame magic");
requireBytes(5);
const descriptor = input[offset + 4];
if (descriptor & 8) throw new Error("reserved zstd frame bit");
const singleSegment = (descriptor & 32) !== 0;
const dictionaryFlag = descriptor & 3;
const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag;
const sizeFlag = descriptor >>> 6;
const sizeBytes = sizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << sizeFlag;
offset += 5;
const remainingHeader = (singleSegment ? 0 : 1) + dictionaryBytes + sizeBytes;
requireBytes(remainingHeader);
offset += remainingHeader;
let lastBlock: boolean;
do {
requireBytes(3);
const header = input.readUIntLE(offset, 3);
offset += 3;
lastBlock = (header & 1) !== 0;
const blockType = (header >>> 1) & 3;
if (blockType === 3) throw new Error("reserved zstd block type");
const size = blockType === 1 ? 1 : header >>> 3;
requireBytes(size);
offset += size;
} while (!lastBlock);
if (descriptor & 4) {
requireBytes(4);
offset += 4;
}
}
yield input.subarray(start, offset);
} while (offset < input.length);
}

async function decodeZstd(input: Buffer, max: number, options?: DecodeOptions): Promise<Buffer> {
if (typeof zlib.createZstdDecompress === "function") {
const chunks: Buffer[] = [];
let size = 0;
for (const frame of zstdFrames(input)) {
options?.signal?.throwIfAborted();
const body = await streamDecode(zlib.createZstdDecompress, frame, max, {
signal: options?.signal,
onBytes: (bytes) => {
const total = size + bytes;
if (total > max) throw new DecompressedTooLargeError(max);
options?.onBytes?.(total);
},
});
chunks.push(body);
size += body.length;
}
return chunks.length === 1 ? chunks[0] : Buffer.concat(chunks, size);
}
const chunks: Buffer[] = [];
let size = 0;
const decoder = new ZstdDecompress((chunk) => {
Expand All @@ -83,15 +155,15 @@ async function decodeZstd(input: Buffer, max: number, options?: DecodeOptions):
}

async function decodeOne(coding: string, input: Buffer, max: number, options?: DecodeOptions): Promise<Buffer> {
if (coding === "gzip" || coding === "x-gzip") return streamDecode(createGunzip, input, max, options);
if (coding === "br") return streamDecode(createBrotliDecompress, input, max, options);
if (coding === "gzip" || coding === "x-gzip") return streamDecode(zlib.createGunzip, input, max, options);
if (coding === "br") return streamDecode(zlib.createBrotliDecompress, input, max, options);
if (coding === "zstd") return decodeZstd(input, max, options);
if (coding === "deflate") {
try {
return await streamDecode(createInflate, input, max, options);
return await streamDecode(zlib.createInflate, input, max, options);
} catch (err) {
if (err instanceof DecompressedTooLargeError || err instanceof DecodedRequestBusyError || options?.signal?.aborted) throw err;
return streamDecode(createInflateRaw, input, max, options);
return streamDecode(zlib.createInflateRaw, input, max, options);
}
}
throw new Error(`unsupported request content-encoding: ${coding}`);
Expand Down
154 changes: 154 additions & 0 deletions tests/content-encoding-zstd.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
import test from "node:test";
import assert from "node:assert/strict";
import * as zlib from "node:zlib";
import { spawnSync } from "node:child_process";
import { decodeRequestBody, DecompressedTooLargeError } from "../src/content-encoding.ts";
import { DecodedRequestBusyError } from "../src/request-body-budget.ts";

const hasNativeZstd = typeof zlib.createZstdDecompress === "function";
const plain = Buffer.from('{"model":"gpt-5","input":"hello"}');
const encoded = Buffer.from("KLUv/SAhCQEAeyJtb2RlbCI6ImdwdC01IiwiaW5wdXQiOiJoZWxsbyJ9", "base64");
const unknownSize = Buffer.from("KLUv/QAACQEAeyJtb2RlbCI6ImdwdC01IiwiaW5wdXQiOiJoZWxsbyJ9", "base64");
const checksummed = Buffer.from("KLUv/SQhCQEAeyJtb2RlbCI6ImdwdC01IiwiaW5wdXQiOiJoZWxsbyJ9TTP3+Q==", "base64");
const skippable = Buffer.from("5f2a4d1803000000010203", "hex");

test("zstd decoding can be cancelled after starting without processing the whole body", { skip: !hasNativeZstd }, async () => {
const raw = Buffer.alloc(4 * 1024 * 1024, 65);
const input = zlib.zstdCompressSync(raw);
const controller = new AbortController();
const reason = new Error("client disconnected");
let bytes = 0;
const pending = decodeRequestBody("zstd", input, raw.length, {
signal: controller.signal,
onBytes: (count) => { bytes = count; },
});
controller.abort(reason);
await assert.rejects(pending, (error) => error === reason);
assert.ok(bytes < raw.length);
});

test("cancelling one zstd decode does not cancel a concurrent request", { skip: !hasNativeZstd }, async () => {
const controller = new AbortController();
const cancelled = decodeRequestBody("zstd", encoded, 1024, { signal: controller.signal });
const completed = decodeRequestBody("zstd", encoded, 1024);
controller.abort();
await assert.rejects(cancelled, { name: "AbortError" });
assert.deepEqual((await completed).body, plain);
});

test("zstd decoding stops if the byte observer cancels the final output chunk", { skip: !hasNativeZstd }, async () => {
const controller = new AbortController();
let chunks = 0;
await assert.rejects(decodeRequestBody("zstd", encoded, 1024, {
signal: controller.signal,
onBytes: () => { chunks++; controller.abort(); },
}), { name: "AbortError" });
assert.equal(chunks, 1);
});

test("zstd decoding preserves exact byte limits and the resource-admission error", async () => {
assert.deepEqual((await decodeRequestBody("zstd", encoded, plain.length)).body, plain);
await assert.rejects(decodeRequestBody("zstd", encoded, plain.length - 1), DecompressedTooLargeError);
const busy = new DecodedRequestBusyError();
await assert.rejects(decodeRequestBody("zstd", encoded, 1024, {
onBytes: () => { throw busy; },
}), (error) => error === busy);
});

test("zstd decoding supports concatenated frames and stacked content encodings", async () => {
const input = Buffer.concat([encoded, encoded]);
const observed: number[] = [];
assert.deepEqual((await decodeRequestBody("zstd", input, plain.length * 2, {
onBytes: (count) => observed.push(count),
})).body,
Buffer.concat([plain, plain]));
assert.equal(observed.at(-1), plain.length * 2);
assert.ok(observed.every((count, i) => i === 0 || count >= observed[i - 1]));
await assert.rejects(decodeRequestBody("zstd", input, plain.length * 2 - 1),
(error) => error instanceof DecompressedTooLargeError && error.limit === plain.length * 2 - 1);
const busy = new DecodedRequestBusyError();
await assert.rejects(decodeRequestBody("zstd", input, 1024, {
onBytes: (count) => { if (count > plain.length) throw busy; },
}), (error) => error === busy);
const controller = new AbortController();
const reason = new Error("cancelled in the second frame");
await assert.rejects(decodeRequestBody("zstd", input, 1024, {
signal: controller.signal,
onBytes: (count) => { if (count > plain.length) controller.abort(reason); },
}), (error) => error === reason);
assert.deepEqual((await decodeRequestBody("zstd, gzip", zlib.gzipSync(encoded), 1024)).body, plain);
await assert.rejects(decodeRequestBody("zstd, gzip", zlib.gzipSync(encoded), plain.length - 1),
DecompressedTooLargeError);
});

test("zstd decoding accepts unknown-size and checksummed frames with interleaved skippable data", async () => {
assert.deepEqual((await decodeRequestBody("zstd", unknownSize, plain.length)).body, plain);
assert.deepEqual((await decodeRequestBody("zstd", checksummed, plain.length)).body, plain);
const input = Buffer.concat([skippable, unknownSize, skippable, checksummed, skippable]);
assert.deepEqual((await decodeRequestBody("zstd", input, plain.length * 2)).body,
Buffer.concat([plain, plain]));
assert.deepEqual((await decodeRequestBody("zstd", skippable, 0)).body, Buffer.alloc(0));
});

test("zstd decoding accepts RLE and compressed blocks", async () => {
const rle = Buffer.from("28b52ffd20210b010041", "hex");
assert.deepEqual((await decodeRequestBody("zstd", rle, 33)).body, Buffer.alloc(33, 65));
const compressed = Buffer.from("KLUv/WDoAk0AABBBQQEA4yuABQ==", "base64");
assert.deepEqual((await decodeRequestBody("zstd", compressed, 1000)).body, Buffer.alloc(1000, 65));
});

for (const [name, input] of [
["unknown-size block", unknownSize.subarray(0, -1)],
["checksum", checksummed.subarray(0, -1)],
["second frame", Buffer.concat([encoded, encoded.subarray(0, -1)])],
["skippable payload", skippable.subarray(0, -1)],
["trailing skippable payload", Buffer.concat([encoded, skippable.subarray(0, -1)])],
["frame header", encoded.subarray(0, 5)],
["block header", encoded.subarray(0, 8)],
] as const) {
test(`zstd decoding rejects a truncated ${name}`, async () => {
await assert.rejects(decodeRequestBody("zstd", input, 1024));
});
}

test("zstd decoding keeps native checksum validation for later frames", { skip: !hasNativeZstd }, async () => {
const corrupt = Buffer.from(checksummed);
corrupt[corrupt.length - 1] ^= 1;
await assert.rejects(decodeRequestBody("zstd", Buffer.concat([encoded, corrupt]), 1024));
});

test("zstd decoding rejects truncated input and respects pre-aborted requests", async () => {
await assert.rejects(decodeRequestBody("zstd", encoded.subarray(0, -1), 1024));
await assert.rejects(decodeRequestBody("zstd", Buffer.from("not a zstd frame"), 1024));
const controller = new AbortController();
const reason = new Error("already disconnected");
controller.abort(reason);
await assert.rejects(decodeRequestBody("zstd", encoded, 1024, { signal: controller.signal }),
(error) => error === reason);
});

test("zstd decoding still works when node:zlib has no native zstd factory", () => {
const contentEncoding = new URL("../src/content-encoding.ts", import.meta.url).href;
const budget = new URL("../src/request-body-budget.ts", import.meta.url).href;
const child = spawnSync(process.execPath, ["--import", "tsx", "--input-type=module", "--eval", `
import assert from "node:assert/strict";
import zlib from "node:zlib";
import { syncBuiltinESMExports } from "node:module";
Reflect.set(zlib, "createZstdDecompress", undefined);
syncBuiltinESMExports();
const { decodeRequestBody, DecompressedTooLargeError } = await import(${JSON.stringify(contentEncoding)});
const { DecodedRequestBusyError } = await import(${JSON.stringify(budget)});
const input = Buffer.from(${JSON.stringify(encoded.toString("base64"))}, "base64");
const expected = Buffer.from(${JSON.stringify(plain.toString())});
assert.deepEqual((await decodeRequestBody("zstd", input, expected.length)).body, expected);
await assert.rejects(decodeRequestBody("zstd", input, expected.length - 1), DecompressedTooLargeError);
await assert.rejects(decodeRequestBody("zstd", input, 1024, {
onBytes: () => { throw new DecodedRequestBusyError(); },
}), DecodedRequestBusyError);
const controller = new AbortController();
controller.abort();
await assert.rejects(decodeRequestBody("zstd", input, 1024, { signal: controller.signal }), { name: "AbortError" });
`], { encoding: "utf8", timeout: 20_000 });
assert.ifError(child.error);
assert.equal(child.status, 0, child.stderr);
});
5 changes: 5 additions & 0 deletions tests/e2e/e2e-update-state.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,11 @@ test("self-update keeps live compression state: folds survive the OLD→NEW hop
envBase.BILI_UPDATE_POLICY_URL = "http://127.0.0.1:1/no-policy.json";
envBase.BILI_PERSIST_DEBOUNCE_MS = "200";
envBase.BILI_SESSIONS_DIR = sessionsDir;
// Pure-sqlite default (5a80e2b8e, #6480) turns the JSON envelope
// writer OFF, but this lane asserts on the on-disk .json session file
// (state survival across the update). Pin the writer ON so the
// envelope this test was written for is still produced.
envBase.BILI_STORAGE_JSON_WRITER = "on";

const port = await freePort();
proxy = spawnProxy(installDir, port, envBase);
Expand Down
Loading