From fe9fe3f40074c7b3d286b298f3ba7952eb77b6f1 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:21:36 +0800 Subject: [PATCH 01/67] fix: ignore stale keepalive response bodies after rearming (cherry picked from commit 0d2ee1d61af9f353f13abad08abedf6fea9d6995) --- src/cache-keepalive.ts | 1 + tests/cache-keepalive.test.ts | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/cache-keepalive.ts b/src/cache-keepalive.ts index d14606747a..4cb13d0619 100644 --- a/src/cache-keepalive.ts +++ b/src/cache-keepalive.ts @@ -379,6 +379,7 @@ async function firePing(entry: KeepaliveEntry): Promise { entry.pingInFlight = false; if (registry.get(entry.sessionId) !== entry) return; const text = await resp.text().catch(() => ""); + if (registry.get(entry.sessionId) !== entry) return; if (!resp.ok) { if (resp.status === 429) { disarm(entry.sessionId, "rate-limited(429)"); diff --git a/tests/cache-keepalive.test.ts b/tests/cache-keepalive.test.ts index 6c2988ad15..66b060a2f5 100644 --- a/tests/cache-keepalive.test.ts +++ b/tests/cache-keepalive.test.ts @@ -590,3 +590,34 @@ noteSuccessfulForward({ fs.rmSync(tmp, { recursive: true, force: true }); assert.equal(exitCode, 0, `an armed keepalive must not hold the event loop open (got ${exitCode === null ? "TIMEOUT — the cadence timer kept the process alive" : `exit code ${exitCode}`})\n--- child output ---\n${out}`); }); + +for (const reply of [ + { label: "rate limit", status: 429, body: { error: "slow down" } }, + { label: "cache miss", status: 200, body: MISS_BODY }, + { label: "cache hit", status: 200, body: HIT_BODY }, +]) { + test(`ping: a stale ${reply.label} body cannot change a replacement snapshot`, async (t) => { + let finishBody!: (text: string) => void; + let startedBody!: () => void; + const body = new Promise((resolve) => { finishBody = resolve; }); + const started = new Promise((resolve) => { startedBody = resolve; }); + const response = new Response(null, { status: reply.status }); + t.mock.method(response, "text", () => { + startedBody(); + return body; + }); + t.mock.method(globalThis, "fetch", async () => response); + + noteSuccessfulForward(armInput()); + timers.advance(270_000); + await started; + noteSuccessfulForward(armInput({ wireBody: BODY_1H })); + const replacement = _keepaliveEntryForTest(sid); + assert.equal(replacement?.tierLabel, "1h"); + + finishBody(JSON.stringify(reply.body)); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(_keepaliveEntryForTest(sid), replacement); + assert.equal(timers.pendingCount(), 1, "only the replacement snapshot stays scheduled"); + }); +} From 850ce16e88ab2bb6bf4e7960fed215f05f64ffda Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:23:17 +0800 Subject: [PATCH 02/67] fix: collect expired witness rings during record-only traffic (cherry picked from commit 30bbd3dff97935db44986a729944e8adaad27b23) --- src/tool-ring.ts | 8 ++++++++ tests/tool-ring.test.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/tool-ring.ts b/src/tool-ring.ts index e4ef6aec61..2fd9a6b492 100644 --- a/src/tool-ring.ts +++ b/src/tool-ring.ts @@ -34,6 +34,7 @@ interface WitnessEntry { const rings = new Map(); /** witness hash → set of sessionIds holding it (populated on lookup sweep). */ const hashIndex = new Map>(); +let lastRecordSweepAt: number | undefined; /** Fake-clock seam for TTL tests. Undefined = real wall clock. */ let clockForTest: (() => number) | undefined; @@ -119,6 +120,12 @@ export function recordToolWitness(sessionId: string, name: string, args: string const hash = witnessHash(name, args); if (!hash) return; const now = nowMs(); + // Native-id routing may never look up a hash witness. Reclaim other + // sessions on writes too, amortizing the full scan over the TTL window. + if (lastRecordSweepAt === undefined || now - lastRecordSweepAt > WITNESS_TTL_MS || now < lastRecordSweepAt) { + for (const id of rings.keys()) sweep(id, now); + lastRecordSweepAt = now; + } sweep(sessionId, now); const ring = rings.get(sessionId) ?? []; if (ring.length > 0 && ring[ring.length - 1].hash === hash) { @@ -219,6 +226,7 @@ export function lookupToolUseIdSession(toolUseId: string): string | undefined { export function resetToolRingForTest(): void { rings.clear(); hashIndex.clear(); + lastRecordSweepAt = undefined; toolUseIds.clear(); clockForTest = undefined; } diff --git a/tests/tool-ring.test.ts b/tests/tool-ring.test.ts index 7d0c0a9548..057c19acd9 100644 --- a/tests/tool-ring.test.ts +++ b/tests/tool-ring.test.ts @@ -268,4 +268,30 @@ describe("#1685 review: duplicate-hash index safety + TTL", () => { assert.equal(_toolRingSizeForTest(), 0); _setToolRingClockForTest(undefined); }); + + it("record-only traffic reclaims expired sessions without a witness lookup", () => { + const ttl = 10 * 60 * 1000; + let now = 1_700_000_000_000; + _setToolRingClockForTest(() => now); + for (let i = 0; i < 20; i++) { + recordToolWitness(`finished-${i}`, "compress", { summary: `summary ${i}` }); + assert.equal(_toolRingSizeForTest(), 1, "expired sessions must not accumulate on native-id lanes"); + now += ttl + 1; + } + }); + + it("record-time collection preserves fresh witnesses and shared hash memberships", () => { + const ttl = 10 * 60 * 1000; + let now = 1_700_000_000_000; + _setToolRingClockForTest(() => now); + const shared = { summary: "shared witness" }; + recordToolWitness("expired", "compress", shared); + now += 1000; + recordToolWitness("fresh", "compress", shared); + now += ttl - 999; + recordToolWitness("trigger", "compress", { summary: "new witness" }); + assert.equal(_toolRingSizeForTest(), 2); + assert.deepEqual([...lookupToolWitness("compress", shared)], ["fresh"]); + assert.deepEqual([...lookupToolWitness("compress", { summary: "new witness" })], ["trigger"]); + }); }); From c9e9152e9c67079e11589a808cc383c6cc8b935e Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 16:12:01 +0800 Subject: [PATCH 03/67] docs: clarify witness hash index lifecycle (cherry picked from commit e1706ab8fbeeabddedefe62f5728d2f029d7ddb4) --- src/tool-ring.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tool-ring.ts b/src/tool-ring.ts index 2fd9a6b492..5280454b1b 100644 --- a/src/tool-ring.ts +++ b/src/tool-ring.ts @@ -32,7 +32,7 @@ interface WitnessEntry { /** sessionId → ring of recent witness hashes (oldest first, capped). */ const rings = new Map(); -/** witness hash → set of sessionIds holding it (populated on lookup sweep). */ +/** witness hash → set of sessionIds holding it (added on record, pruned on sweep). */ const hashIndex = new Map>(); let lastRecordSweepAt: number | undefined; From 2480d88d00d2057850b473e54ab4c99a6faf5c54 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:33:24 +0800 Subject: [PATCH 04/67] fix: scope image encodes to their resize policy (cherry picked from commit 4bead210c634d784aff79a8f9cad39a886334fa0) --- src/image-rescale.ts | 2 +- tests/issue3400-image-rescale.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/image-rescale.ts b/src/image-rescale.ts index 8fedd121a5..acc495376b 100644 --- a/src/image-rescale.ts +++ b/src/image-rescale.ts @@ -150,7 +150,7 @@ async function rescaleImage(mediaType: string, b64: string, maxDim: number): Pro const dims = decodeImageDims(b64); const gated = (dims !== undefined && Math.max(dims.w, dims.h) > maxDim) || origBytes > RESCALE_TRIGGER_BYTES; if (!gated) return null; - const hash = createHash("sha256").update(b64, "utf8").digest("hex"); + const hash = createHash("sha256").update(`${mime}:${maxDim}:${b64}`, "utf8").digest("hex"); let pending = encodeCache.get(hash); const cacheHit = pending !== undefined; if (!pending) { diff --git a/tests/issue3400-image-rescale.test.ts b/tests/issue3400-image-rescale.test.ts index 1c3cec9815..3395ab0cba 100644 --- a/tests/issue3400-image-rescale.test.ts +++ b/tests/issue3400-image-rescale.test.ts @@ -137,6 +137,31 @@ test("byte-stable: same history → identical outbound bytes across cache states assert.equal(out2, out1, "two independent encodes of the same history must be byte-identical (upstream prefix-cache protection)"); }); +for (const [firstDim, nextDim] of [[1600, 800], [800, 1600]]) { + test(`encode cache respects a changed dimension limit (${firstDim} to ${nextDim})`, async () => { + const { ctx } = ctxFor(`s-cache-dim-${firstDim}`); + const input = [anthropicImageMsg("msg_dim", BIG.b64)]; + const first = await applyImageRescalePass(input, { ...ctx, maxDim: firstDim }); + const next = await applyImageRescalePass(input, { ...ctx, maxDim: nextDim }); + const imageData = (messages: BiliMessage[]) => (messages[0].rawAnthropicBlock as { source: { data: string } }).source.data; + const decoded = await Jimp.read(Buffer.from(imageData(next), "base64")); + assert.equal(decoded.width, nextDim, "cached encodes must use the current policy"); + assert.notEqual(imageData(first), imageData(next)); + assert.deepEqual(await applyImageRescalePass(input, { ...ctx, maxDim: nextDim }), next, "the same policy still reuses byte-identical output"); + assert.equal(logLines.filter((line) => line.includes("[image-rescale]")).length, 2); + }); +} + +test("a cached low-benefit result does not suppress a later stricter resize", async () => { + const { ctx } = ctxFor("s-cache-null-dim"); + const input = [anthropicImageMsg("msg_null_dim", BMP.b64, "image/bmp")]; + assert.equal((await applyImageRescalePass(input, { ...ctx, maxDim: 1600 }))[0], input[0]); + const next = await applyImageRescalePass(input, { ...ctx, maxDim: 800 }); + const block = next[0].rawAnthropicBlock as { source: { data: string } }; + const decoded = await Jimp.read(Buffer.from(block.source.data, "base64")); + assert.equal(decoded.width, 800); +}); + test("openai singular image_url converts to plural parts with inline note", async () => { const { session, ctx } = ctxFor("s-openai"); session.state.messageRefs.byRaw["msg_o"] = "m20001"; From 1eb3843a27bb763e9553f4a518f5f07c4e4759f6 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:33:26 +0800 Subject: [PATCH 05/67] fix: repair incomplete image spills when originals are resent (cherry picked from commit 6f2a546400537a4271a86eea805dde0f78fbe76f) --- src/image-restore.ts | 5 +++-- tests/image-restore.test.ts | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/image-restore.ts b/src/image-restore.ts index fdac387535..cf6fbd8b0f 100644 --- a/src/image-restore.ts +++ b/src/image-restore.ts @@ -17,7 +17,7 @@ // sends no such bytes to the proxy, so those belong to the agent's in-process // recovery (cf. billion-context-pi#594), not here. -import { existsSync, mkdirSync, readdirSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs"; +import { mkdirSync, readdirSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs"; import { createHash } from "node:crypto"; import { join } from "node:path"; import { @@ -438,7 +438,8 @@ export function writeRestoredImage(ref: string, idx: number, img: RestorableImag const digest = createHash("sha256").update(body).digest("hex").slice(0, 8); const path = join(dir, `${safeRef}${idx > 0 ? `-${idx}` : ""}-${digest}.${extFor(img.mediaType)}`); try { - if (existsSync(path)) return path; + const existing = statSync(path, { throwIfNoEntry: false }); + if (existing?.isFile() && existing.size === body.length) return path; mkdirSync(dir, { recursive: true }); writeFileSync(path, body, { mode: 0o600 }); return path; diff --git a/tests/image-restore.test.ts b/tests/image-restore.test.ts index d0611aea23..c846a5c146 100644 --- a/tests/image-restore.test.ts +++ b/tests/image-restore.test.ts @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { existsSync, mkdtempSync, readFileSync, statSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { anthropicToCore, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; @@ -191,6 +191,22 @@ test("writeRestoredImage: a rebase that restarts refs never restores the previou assert.equal(writeRestoredImage("m00001", 0, pngImg, "sess-rebase"), gen1); }); +for (const length of [0, 8]) { + test(`image index repairs an incomplete ${length}-byte spill when the original is resent`, () => { + const body = { model: "gpt", messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: dataUrl } }] }] }; + const sessionId = `sess-incomplete-${length}`; + const original = buildIndex("openai", body, false, sessionId); + const file = [...original.values()][0][0].path; + writeFileSync(file, Buffer.from(PNG, "base64").subarray(0, length)); + const repaired = buildIndex("openai", body, false, sessionId); + assert.equal([...repaired.values()][0][0].path, file, "stable refs retain the same spill path"); + assert.deepEqual(readFileSync(file), Buffer.from(PNG, "base64"), "resending the original must recover a partial write"); + const mtime = statSync(file).mtimeMs; + buildIndex("openai", body, false, sessionId); + assert.equal(statSync(file).mtimeMs, mtime, "complete files retain the skip-write fast path"); + }); +} + test("resolveDecompress({ imageRef }): lists, restores to file, and reports misses", () => { const p = writeRestoredImage("m00042", 0, pngImg, "sess-A")!; const idx = new Map([["m00042", [{ mediaType: "image/png", bytes: Buffer.byteLength(PNG, "base64"), width: 1, height: 1, path: p }]]]); From c7e39ee51a3cc4b4525205367a58dcee06a1c212 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:50:19 +0800 Subject: [PATCH 06/67] fix: separate IPv6 socket hosts from CONNECT authorities (cherry picked from commit 95f6818d92167499c1a8ff7938714e316a6ac703) --- src/upstream-proxy.ts | 9 ++-- tests/fix-78-connect-timeout.test.ts | 66 +++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/upstream-proxy.ts b/src/upstream-proxy.ts index e4031dae8a..fed6ddc4f3 100644 --- a/src/upstream-proxy.ts +++ b/src/upstream-proxy.ts @@ -108,7 +108,7 @@ export function parseHttpProxy(proxy?: string, biliPort?: number): ParsedHttpPro } return { url: url.href, - host: url.hostname, + host: url.hostname.replace(/^\[|\]$/g, ""), port, protocol: url.protocol, ...(proxyAuthorization(url) ? { authorization: proxyAuthorization(url) } : {}), @@ -457,7 +457,7 @@ export function _setConnectFactoryForTest(factory: ((port: number, host: string) * branch's 10s handshake timeout (see #78). */ export function connectDirect(host: string, port: number, timeoutMs: number = CONNECT_TIMEOUT_MS): Promise { return new Promise((resolve, reject) => { - const socket = connectFactory(port, host); + const socket = connectFactory(port, host.replace(/^\[|\]$/g, "")); let settled = false; const finishError = (error: Error) => { if (settled) return; @@ -497,9 +497,10 @@ export function connectThroughProxy(host: string, port: number, proxyUrl: string socket.once("error", finishError); const connectedEvent = proxy.protocol === "https:" ? "secureConnect" : "connect"; socket.once(connectedEvent, () => { + const authority = `${net.isIP(host) === 6 ? `[${host}]` : host}:${port}`; const headers = [ - `CONNECT ${host}:${port} HTTP/1.1`, - `Host: ${host}:${port}`, + `CONNECT ${authority} HTTP/1.1`, + `Host: ${authority}`, "Proxy-Connection: Keep-Alive", ...(proxy.authorization ? [`Proxy-Authorization: ${proxy.authorization}`] : []), "", diff --git a/tests/fix-78-connect-timeout.test.ts b/tests/fix-78-connect-timeout.test.ts index 0bd27fd57c..335821e11a 100644 --- a/tests/fix-78-connect-timeout.test.ts +++ b/tests/fix-78-connect-timeout.test.ts @@ -2,7 +2,7 @@ import assert from "node:assert"; import net from "node:net"; import test from "node:test"; -import { connectDirect, connectThroughProxy, _setConnectFactoryForTest } from "../src/upstream-proxy.ts"; +import { connectDirect, connectThroughProxy, parseHttpProxy, _setConnectFactoryForTest } from "../src/upstream-proxy.ts"; function listen(server: net.Server): Promise { if (server.listening) return Promise.resolve(); @@ -69,3 +69,67 @@ test("direct connect: immediate errors (connection refused) still surface right await assert.rejects(connectDirect("127.0.0.1", port), /ECONNREFUSED/); assert.ok(Date.now() - t0 < 2_000, "refusal is immediate, not a timeout"); }); + +test("IPv6 proxy URLs expose an unbracketed socket hostname", () => { + for (const scheme of ["http", "https"]) { + const proxy = parseHttpProxy(`${scheme}://[::1]:8080`); + assert.equal(proxy?.host, "::1"); + assert.equal(proxy?.url, `${scheme}://[::1]:8080/`, "the URL retains authority brackets"); + } +}); + +for (const proxied of [false, true]) { + test(`IPv6 loopback ${proxied ? "proxy" : "direct"} connection accepts bracketed authorities`, async (t) => { + const peers: net.Socket[] = []; + const server = net.createServer((socket) => { + peers.push(socket); + if (proxied) socket.once("data", () => socket.write("HTTP/1.1 200 Connection Established\r\n\r\n")); + }); + try { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "::1", resolve); + }); + } catch (error) { + if (["EAFNOSUPPORT", "EADDRNOTAVAIL"].includes((error as NodeJS.ErrnoException).code ?? "")) { + t.skip("IPv6 loopback is unavailable on this host"); + return; + } + throw error; + } + try { + const port = (server.address() as net.AddressInfo).port; + const socket = proxied + ? await connectThroughProxy("upstream.example", 443, `http://[::1]:${port}`) + : await connectDirect("[::1]", port); + socket.destroy(); + } finally { + for (const peer of peers) peer.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } + }); +} + +test("CONNECT uses a bracketed authority for a bare IPv6 target", async () => { + let request = ""; + const peers: net.Socket[] = []; + const server = net.createServer((socket) => { + peers.push(socket); + socket.once("data", (data) => { + request = data.toString(); + socket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + }); + }); + server.listen(0, "127.0.0.1"); + await listen(server); + try { + const port = (server.address() as net.AddressInfo).port; + const socket = await connectThroughProxy("2001:db8::1", 443, `http://127.0.0.1:${port}`); + socket.destroy(); + assert.match(request, /^CONNECT \[2001:db8::1\]:443 HTTP\/1\.1\r\n/); + assert.match(request, /\r\nHost: \[2001:db8::1\]:443\r\n/); + } finally { + for (const peer of peers) peer.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); From d016f552842169f5298a5a237542e8bfafa1f388 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:50:22 +0800 Subject: [PATCH 07/67] fix: select the requested prompt pack from npm dependencies (cherry picked from commit 3a9ceb48ef87453b58588fb27ad5ef7ac64cbb41) --- src/pack-install.ts | 43 +++++++++++++------------------------- tests/pack-install.test.ts | 36 +++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/src/pack-install.ts b/src/pack-install.ts index e91ceed746..3455529109 100644 --- a/src/pack-install.ts +++ b/src/pack-install.ts @@ -19,7 +19,7 @@ import { execFileSync } from "node:child_process"; import { safePrefix } from "./text-safe.js"; -import { closeSync, existsSync, fsyncSync, mkdirSync, mkdtempSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs"; +import { closeSync, existsSync, fsyncSync, mkdirSync, mkdtempSync, openSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { randomUUID } from "node:crypto"; @@ -69,34 +69,21 @@ function atomicWriteFile(file: string, content: string): void { } } -/** Find the package directory npm just installed under prefix/node_modules. */ +/** Find the directly requested package, never a hoisted dependency. */ function findInstalledPkg(prefix: string): { dir: string; manifest: Record } | undefined { - const nmRoot = path.join(prefix, "node_modules"); - if (!existsSync(nmRoot)) return undefined; - const candidates: string[] = []; - const push = (dir: string) => candidates.push(dir); - for (const entry of readdirSync(nmRoot)) { - if (entry.startsWith(".")) continue; - const full = path.join(nmRoot, entry); - if (!statSync(full).isDirectory()) continue; - if (entry.startsWith("@")) { - for (const sub of readdirSync(full)) { - push(path.join(full, sub)); - } - } else { - push(full); - } - } - let best: { dir: string; mtimeMs: number; manifest: Record } | undefined; - for (const dir of candidates) { - const manifestPath = path.join(dir, "package.json"); - if (!existsSync(manifestPath)) continue; - let manifest: Record; - try { manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; } catch { continue; } - const mtimeMs = statSync(dir).mtimeMs; - if (!best || mtimeMs > best.mtimeMs) best = { dir, mtimeMs, manifest }; + try { + const root = JSON.parse(readFileSync(path.join(prefix, "package.json"), "utf8")) as { dependencies?: unknown }; + const dependencies = root?.dependencies; + if (!dependencies || typeof dependencies !== "object" || Array.isArray(dependencies)) return undefined; + const names = Object.keys(dependencies); + if (names.length !== 1) return undefined; + const dir = path.join(prefix, "node_modules", names[0]); + const manifest: unknown = JSON.parse(readFileSync(path.join(dir, "package.json"), "utf8")); + if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) return undefined; + return { dir, manifest: manifest as Record }; + } catch { + return undefined; } - return best ? { dir: best.dir, manifest: best.manifest } : undefined; } /** @@ -126,7 +113,7 @@ export function installPackFromNpm(spec: string, options: PackInstallOptions = { const npmExec = resolveNpmExec(); execFileSync(npmExec.file, [ ...npmExec.argsPrefix, - "install", "--prefix", tmp, "--no-save", "--no-audit", "--no-fund", trimmed, + "install", "--prefix", tmp, "--save", "--save-prod", "--no-audit", "--no-fund", trimmed, ], { timeout: NPM_TIMEOUT_MS, stdio: ["ignore", "pipe", "pipe"] }); } catch (error) { const err = error as { stderr?: unknown; message?: string }; diff --git a/tests/pack-install.test.ts b/tests/pack-install.test.ts index 2e1296fc50..d68b4dc171 100644 --- a/tests/pack-install.test.ts +++ b/tests/pack-install.test.ts @@ -5,6 +5,7 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -72,3 +73,38 @@ test("packDirFor distinguishes project vs user targets", () => { assert.equal(packDirFor(false), userPacksDir); assert.equal(packDirFor(true), path.join(process.cwd(), ".billion-context", "packs")); }); + +for (const dependencyIsPack of [false, true]) { + test(`install selects the requested package when a newer dependency ${dependencyIsPack ? "also contains a pack" : "has no pack"}`, () => { + const dependency = fixturePkg(dependencyIsPack, { name: "zz-decoy-pack", promptSections: { philosophy: "dependency" } }); + writeFileSync(path.join(dependency, "package.json"), JSON.stringify({ name: "zz-pack-helper", version: "1.0.0" })); + const requested = fixturePkg(true, { name: "zz-requested-pack", promptSections: { philosophy: "requested" } }); + writeFileSync(path.join(requested, "package.json"), JSON.stringify({ + name: "zz-requested-package", + version: "1.0.0", + dependencies: { "zz-pack-helper": `file:${dependency}` }, + scripts: { postinstall: "node touch-dependency.cjs" }, + })); + writeFileSync(path.join(requested, "touch-dependency.cjs"), ` + const fs = require("node:fs"); + const path = require("node:path"); + const future = new Date(Date.now() + 60_000); + fs.utimesSync(path.join(__dirname, "..", "zz-pack-helper"), future, future); + `); + const npm = process.platform === "win32" + ? path.join(path.dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js") + : undefined; + execFileSync(npm ? process.execPath : "npm", [ + ...(npm ? [npm] : []), "pack", requested, "--pack-destination", requested, "--ignore-scripts", + ], { stdio: "pipe", timeout: 30_000 }); + const tarball = path.join(requested, "zz-requested-package-1.0.0.tgz"); + const installed = installPackFromNpm(tarball); + try { + assert.equal(installed.name, "zz-requested-pack"); + assert.match(readFileSync(installed.file, "utf8"), /requested/); + assert.ok(!existsSync(path.join(userPacksDir, "zz-decoy-pack.json"))); + } finally { + removePackFile(installed.name); + } + }); +} From ba57b829aabbd8286dc7b89873899564d94120bd Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 16:29:02 +0800 Subject: [PATCH 08/67] test: register pack installer subprocess suite (cherry picked from commit 981fea274a331fc50214c6416923408641e20a80) --- scripts/live-files.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index 014f5c22d4..91e5d49457 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -134,6 +134,9 @@ export const LIVE_FILES = [ // child binds no port and does no network I/O, so no zone band is needed — // the entry is for the tripwire alone, like #3890/#5114 "tests/integrity-worker-stderr.test.ts", + // Real npm pack/install children use mkdtemp package fixtures and the + // preload's per-process XDG roots; they bind no listener ports. + "tests/pack-install.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; From 6ce4cdba90c4bfd955a113010e01dc8e33121144 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:50:25 +0800 Subject: [PATCH 09/67] fix: refresh upstream CA cache on configured path changes (cherry picked from commit b0a7c7ff2b632c6d6d28baa48202de597bdee9e0) --- src/upstream-ca.ts | 4 +- tests/issue3776-tls-mitm-diagnostics.test.ts | 61 ++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/upstream-ca.ts b/src/upstream-ca.ts index 40f5fd3623..b021f135dc 100644 --- a/src/upstream-ca.ts +++ b/src/upstream-ca.ts @@ -37,7 +37,9 @@ export function upstreamCaIdentity(now = Date.now()): string { lastIdentity = "none"; return "none"; } - if (now - lastStatAt < IDENTITY_TTL_MS && lastIdentity !== "none") return lastIdentity; + // Throttle file-change probes only for the same configured path. A live + // config switch must not reuse the previous file's dispatcher/trust cache. + if (now - lastStatAt < IDENTITY_TTL_MS && lastIdentity.startsWith(`${caFile}\u0000`)) return lastIdentity; let mtimeMs = 0; try { mtimeMs = statSync(caFile).mtimeMs; } catch { /* missing file => mtime 0 */ } lastIdentity = `${caFile}\u0000${mtimeMs}`; diff --git a/tests/issue3776-tls-mitm-diagnostics.test.ts b/tests/issue3776-tls-mitm-diagnostics.test.ts index 71d1599456..30ed47f8c0 100644 --- a/tests/issue3776-tls-mitm-diagnostics.test.ts +++ b/tests/issue3776-tls-mitm-diagnostics.test.ts @@ -172,6 +172,67 @@ test("#3776 sub-defect 2: NODE_EXTRA_CA_CERTS entries are folded into the merge } }); +for (const missingFirst of [false, true]) { + test(`configured CA path changes bypass the mtime throttle (previous file missing=${missingFirst})`, (t) => { + const dir = mkTmp("bili-ca-switch-"); + const first = path.join(dir, "first.pem"); + const second = path.join(dir, "second.pem"); + const firstPem = "-----BEGIN CERTIFICATE-----\nFIRST\n-----END CERTIFICATE-----\n"; + const secondPem = "-----BEGIN CERTIFICATE-----\nSECOND\n-----END CERTIFICATE-----\n"; + if (!missingFirst) fs.writeFileSync(first, firstPem); + fs.writeFileSync(second, secondPem); + const prev = process.env.BILI_CA_FILE; + t.mock.method(Date, "now", () => 1_800_000_000_000); + try { + _resetUpstreamCaForTest(); + process.env.BILI_CA_FILE = first; + const firstIdentity = upstreamCaIdentity(); + const firstRoots = upstreamExtraCa(); + assert.equal(firstRoots!.includes(firstPem), !missingFirst); + + process.env.BILI_CA_FILE = second; + assert.notEqual(upstreamCaIdentity(), firstIdentity, "an explicit new path gets its own dispatcher identity immediately"); + const secondRoots = upstreamExtraCa(); + assert.ok(secondRoots!.includes(secondPem), "trust comes from the newly selected file within the same TTL window"); + assert.ok(!secondRoots!.includes(firstPem), "the old configured root is no longer retained"); + assert.ok(secondRoots!.includes(tls.rootCertificates[0]), "built-in trust is still merged"); + } finally { + if (prev === undefined) delete process.env.BILI_CA_FILE; else process.env.BILI_CA_FILE = prev; + _resetUpstreamCaForTest(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +} + +test("unchanged CA paths retain the existing throttled file-rotation behavior", (t) => { + const dir = mkTmp("bili-ca-rotation-"); + const caFile = path.join(dir, "root.pem"); + const before = "original CA material"; + const after = "rotated CA material"; + fs.writeFileSync(caFile, before); + const initialMtime = fs.statSync(caFile).mtimeMs; + const prev = process.env.BILI_CA_FILE; + let now = 1_800_000_000_000; + t.mock.method(Date, "now", () => now); + try { + _resetUpstreamCaForTest(); + process.env.BILI_CA_FILE = caFile; + const initialIdentity = upstreamCaIdentity(); + const initialRoots = upstreamExtraCa(); + fs.writeFileSync(caFile, after); + fs.utimesSync(caFile, new Date(initialMtime + 1000), new Date(initialMtime + 1000)); + assert.equal(upstreamCaIdentity(), initialIdentity); + assert.equal(upstreamExtraCa(), initialRoots, "same-path probes stay cached inside the 30-second window"); + now += 30_001; + assert.notEqual(upstreamCaIdentity(), initialIdentity); + assert.ok(upstreamExtraCa()!.includes(after), "same-path rotation is picked up after the existing TTL"); + } finally { + if (prev === undefined) delete process.env.BILI_CA_FILE; else process.env.BILI_CA_FILE = prev; + _resetUpstreamCaForTest(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + // --- e2e: a self-signed MITM upstream, through a real in-process bili proxy --- function close(server: http.Server | https.Server): Promise { From 9664b820afdd8e7a13e276cea44bd28d70235a53 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:50:29 +0800 Subject: [PATCH 10/67] fix: strip credentials from loopback diagnostic URLs (cherry picked from commit 866f4964b88587cb2915d0f5972384f0307a1115) --- src/log-mask.ts | 14 ++++++++------ tests/log-mask.test.ts | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/log-mask.ts b/src/log-mask.ts index efef437a85..4e37ef3db2 100644 --- a/src/log-mask.ts +++ b/src/log-mask.ts @@ -207,13 +207,15 @@ export function maskUrlForDiagnostics(url: string): string { try { const u = new URL(url); // Node keeps the brackets in hostname for bracketed IPv6 ([::1]). - // Loopback is returned VERBATIM (not re-serialized): URL.pathname - // normalizes pathless URLs by appending "/", which would drift - // byte-pinned local-origin lines for no security benefit — loopback - // identifies bili's own proxy, and secrets are scrubbed separately - // by redactSecretsInText upstream of every call site. + // Keep clean loopback URLs verbatim: URL.pathname would add "/" to + // pathless local-origin lines. The exemption covers host visibility, + // never userinfo/query/hash; generic text redaction cannot recognize + // arbitrary passwords or every query parameter carrying a secret. const h = u.hostname.replace(/^\[|\]$/g, "").toLowerCase(); - if (h === "localhost" || isLoopbackIp(h)) return url; + if (h === "localhost" || isLoopbackIp(h)) { + if (!u.username && !u.password && !u.search && !u.hash) return url; + return `${u.protocol}//${u.host}${u.pathname}`; + } } catch { // unparseable — fall through to the generic masker } diff --git a/tests/log-mask.test.ts b/tests/log-mask.test.ts index b6686fd5dc..249dc140ae 100644 --- a/tests/log-mask.test.ts +++ b/tests/log-mask.test.ts @@ -13,6 +13,8 @@ import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; import { closeLogger, configureLogger, log as loggerLog, setLogCapture } from "../src/logger.ts"; import { formatUpstreamError } from "../src/upstream-proxy.ts"; +import { logHostEvent } from "../src/agent/host-log.js"; +import { defaultLogFile } from "../src/paths.js"; import { isMaskHostsEnabled, isPublicApiHost, @@ -140,6 +142,43 @@ test("maskUrlsForDiagnostics: text-level twin — loopback URLs kept, others mas assert.equal(maskUrlsForDiagnostics("junk http:// trailing stays"), "junk http:// trailing stays"); }); +for (const origin of ["http://localhost:9090", "ws://127.0.0.1:9000", "socks5://[::1]:1080"]) { + test(`loopback URL visibility never exposes credentials: ${origin}`, () => { + const u = new URL(origin); + const sensitive = `${u.protocol}//alice:local-secret@${u.host}/proxy?q=private-query#private-fragment`; + const safe = `${u.protocol}//${u.host}/proxy`; + assert.equal(maskUrlForDiagnostics(sensitive), safe); + assert.equal(maskUrlsForDiagnostics(`dial ${sensitive} failed`), `dial ${safe} failed`); + assert.equal(maskUrlForDiagnostics(origin), origin, "clean pathless local origins remain byte-identical"); + const prev = isMaskHostsEnabled(); + try { + setMaskHostsEnabled(false); + assert.equal(maskUrlForDiagnostics(sensitive), safe, "host visibility cannot disable credential masking"); + } finally { + setMaskHostsEnabled(prev); + } + }); +} + +test("host-event file output drops loopback URL secrets while retaining the local endpoint", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-loopback-log-")); + const prev = process.env.XDG_STATE_HOME; + process.env.XDG_STATE_HOME = dir; + try { + logHostEvent("info", "[test-host]", "dial http://alice:local-secret@localhost:9090/proxy?q=private-query#private-fragment failed"); + logHostEvent("info", "[test-host]", "healthy http://localhost:9090"); + const output = fs.readFileSync(defaultLogFile(), "utf8"); + assert.match(output, /dial http:\/\/localhost:9090\/proxy failed/); + assert.match(output, /healthy http:\/\/localhost:9090\n/); + for (const secret of ["alice", "local-secret", "private-query", "private-fragment"]) { + assert.ok(!output.includes(secret), `${secret} must not reach the paste-able log`); + } + } finally { + if (prev === undefined) delete process.env.XDG_STATE_HOME; else process.env.XDG_STATE_HOME = prev; + await rmrf(dir); + } +}); + test("maskHostPortForLog: CONNECT targets", () => { assert.equal(maskHostPortForLog("relay.internal:8443"), ":8443"); assert.equal(maskHostPortForLog("api.anthropic.com:443"), "api.anthropic.com:443"); From 1c9c015146b11349dd2d7209d15f6164ce4f03a5 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:50:32 +0800 Subject: [PATCH 11/67] fix: recognize legal JPEG marker fill during image billing (cherry picked from commit 83689e9d288cefd877c5f040cdc4ab96737ac542) --- src/image-tokens.ts | 1 + tests/issue767-pixel-billing.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/image-tokens.ts b/src/image-tokens.ts index 91d85e16b9..c68eece005 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -119,6 +119,7 @@ function jpegDims(b: Buffer): Dims | undefined { while (off + 4 <= b.length) { if (b[off] !== 0xff) { off += 1; continue; } const marker = b[off + 1]; + if (marker === 0xff) { off += 1; continue; } // T.81 B.1.1.2: marker fill, not a length-bearing segment if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) { off += 2; continue; } if (marker === 0xda) break; // SOS with no SOF before it — malformed const segLen = b.readUInt16BE(off + 2); diff --git a/tests/issue767-pixel-billing.test.ts b/tests/issue767-pixel-billing.test.ts index 6cf2958767..a5202e0387 100644 --- a/tests/issue767-pixel-billing.test.ts +++ b/tests/issue767-pixel-billing.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import http from "node:http"; import { once } from "node:events"; import test from "node:test"; +import { Jimp } from "jimp"; process.env.NODE_ENV = "test"; @@ -145,6 +146,32 @@ test("#767 unit: decodeImageDims parses container headers without full decode", assert.equal(decodeImageDims(Buffer.from([0x89, 0x50, 0x4e, 0x47]).toString("base64")), undefined, "truncated header → fallback"); }); +for (const before of ["APP0", "SOF0"] as const) { + test(`JPEG marker fill before ${before} retains dimensions and pixel billing`, async () => { + const original = await new Jimp({ width: 40, height: 24, color: 0x123456ff }).getBuffer("image/jpeg"); + const offset = before === "APP0" ? 2 : original.findIndex((byte, i) => byte === 0xff && original[i + 1] === 0xc0); + assert.ok(offset >= 2); + // T.81 B.1.1.2 allows FF fill bytes before any marker. Keep the + // encoded image intact, adding one or several bytes at that boundary. + const padded = Buffer.concat([original.subarray(0, offset), Buffer.alloc(before === "APP0" ? 1 : 3, 0xff), original.subarray(offset)]); + const decoded = await Jimp.read(padded); + assert.equal(decoded.bitmap.width, 40, "independent full decoder accepts the padded JPEG"); + assert.equal(decoded.bitmap.height, 24); + const b64 = padded.toString("base64"); + assert.deepEqual(decodeImageDims(b64), { w: 40, h: 24 }); + const body = { input: [{ role: "user", content: [{ type: "input_image", image_url: `data:image/jpeg;base64,${b64}` }] }] }; + assert.equal(imageTokensInParsedBody("responses", body, "pixels"), pixelTileEstimate(40, 24), "legal marker fill must not trigger the 16K fallback"); + assert.deepEqual(decodeImageDims(original.toString("base64")), { w: 40, h: 24 }, "ordinary marker parsing is unchanged"); + }); +} + +test("JPEG fill without a complete frame header still falls back within the scan budget", () => { + const onlyFill = Buffer.concat([Buffer.from([0xff, 0xd8]), Buffer.alloc(300_000, 0xff)]); + assert.equal(decodeImageDims(onlyFill.toString("base64")), undefined); + const truncatedSof = Buffer.from([0xff, 0xd8, 0xff, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x00]); + assert.equal(decodeImageDims(truncatedSof.toString("base64")), undefined); +}); + test("#767 unit: pixelTileEstimate matches the OpenAI high-detail tile model", () => { assert.equal(pixelTileEstimate(100, 100), 765, "small square scales up to 768×768 = 4 tiles"); assert.equal(pixelTileEstimate(2048, 2048), 2805, "max-size square = 16 tiles (the ceiling)"); From f04031c23cf63ccd57f578761b3342750ceb6ae9 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 11:50:36 +0800 Subject: [PATCH 12/67] fix: parse summary SSE data fields as complete events (cherry picked from commit 7ae67d6189de09bc6cd87ac34b49c8e69dde41a1) --- src/external-summary-http.ts | 7 +++---- src/preflight.ts | 15 +++----------- src/sse-util.ts | 21 ++++++++++++++++++++ tests/external-summary-http.test.ts | 26 +++++++++++++++++++++++++ tests/preflight-sse-extraction.test.ts | 27 ++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/external-summary-http.ts b/src/external-summary-http.ts index dd6f7433c6..9dbc6bb6f7 100644 --- a/src/external-summary-http.ts +++ b/src/external-summary-http.ts @@ -6,7 +6,7 @@ import { fetchWithTimeout, isTransientUpstreamError, sleep } from "./fetch-util. import { extractSummaryFromSse, extractSummaryText, summaryPayload, type PreflightProtocol } from "./preflight.js"; import { log as loggerLog } from "./logger.js"; import { maskUrlForLog, maskUrlsInText } from "./log-mask.js"; -import { finalizeSseLineEndings } from "./sse-util.js"; +import { bufferedSseEvents, finalizeSseLineEndings } from "./sse-util.js"; import { proxyDispatcher } from "./upstream-proxy.js"; import { PACKAGE_NAME, VERSION } from "./version.js"; @@ -172,9 +172,8 @@ function parseSummary(protocol: PreflightProtocol, text: string): string { } else { const normalized = finalizeSseLineEndings(text); const events: unknown[] = []; - for (const line of normalized.split("\n")) { - if (!line.startsWith("data:")) continue; - const data = line.slice(5).trim(); + for (const event of bufferedSseEvents(normalized)) { + const data = event.data.trim(); if (!data || data === "[DONE]") continue; try { events.push(JSON.parse(data)); } catch { throw new Error("External summary response contains an invalid frame"); } } diff --git a/src/preflight.ts b/src/preflight.ts index 88b32d94c4..d4283ac6f2 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -12,6 +12,7 @@ import { } from "acp-kernel"; import { createHash } from "node:crypto"; import { yieldToEventLoop } from "./yield-loop.js"; +import { bufferedSseEvents } from "./sse-util.js"; import { buildCompressSystemPrompt, parseCompressInput } from "./compress-tool.js"; import { IMAGE_PLACEHOLDER, imagePlaceholdersForSummary } from "./image-note.js"; import { applyAbsorbView } from "./absorb.js"; @@ -950,18 +951,8 @@ export function extractSummaryFromSse(protocol: PreflightProtocol, text: string) let completed = false; let invalid = false; let badFrame = false; - let eventType = ""; - for (const line of text.split("\n")) { - if (line.trim() === "") { - eventType = ""; - continue; - } - if (line.startsWith("event:")) { - eventType = line.slice(6).trim(); - continue; - } - if (!line.startsWith("data:")) continue; - const payload = line.slice(5).trim(); + for (const { event: eventType, data } of bufferedSseEvents(text)) { + const payload = data.trim(); if (!payload || payload === "[DONE]") continue; let obj: unknown; try { diff --git a/src/sse-util.ts b/src/sse-util.ts index 72a98a7aa4..dcd7887230 100644 --- a/src/sse-util.ts +++ b/src/sse-util.ts @@ -30,3 +30,24 @@ export function finalizeSseLineEndings(buf: string): string { if (buf.indexOf("\r") === -1) return buf; return buf.replace(/\r\n|\r/g, "\n"); } + +/** Read buffered SSE event payloads. Multiple data fields belong to ONE + * event and are joined with LF before a consumer parses JSON. Keep the final + * residual available: callers retain their own framing/terminal validity + * checks (Responses may trust a completed terminal without a blank line). */ +export function* bufferedSseEvents(text: string): Generator<{ event: string; data: string }> { + let event = ""; + let data: string[] = []; + for (const line of finalizeSseLineEndings(text).split("\n")) { + if (line.trim() === "") { + if (data.length) yield { event, data: data.join("\n") }; + event = ""; + data = []; + } else if (line.startsWith("event:")) { + event = line.slice(6).trim(); + } else if (line.startsWith("data:")) { + data.push(line.slice(5).replace(/^ /, "")); + } + } + if (data.length) yield { event, data: data.join("\n") }; +} diff --git a/tests/external-summary-http.test.ts b/tests/external-summary-http.test.ts index 2adde87494..7a2c78a3ec 100644 --- a/tests/external-summary-http.test.ts +++ b/tests/external-summary-http.test.ts @@ -106,6 +106,32 @@ for (const protocol of ["responses", "anthropic", "openai", "google"] as const) } } +for (const protocol of ["responses", "anthropic", "openai", "google"] as const) { + test(`external summary HTTP: ${protocol} accepts a JSON event split across multiple data fields`, async () => { + const multiline = streamCompletion(protocol).replace(/^data: (.+)$/gm, (line, data: string) => data === "[DONE]" ? line : + JSON.stringify(JSON.parse(data), null, 2).split("\n").map((part) => `data: ${part}`).join("\n: keepalive\n")); + await upstream((_req, res) => { + res.setHeader("content-type", "text/event-stream"); + res.end(multiline.replace(/\n/g, "\r\n")); + }, async (url) => { + const candidate = createSummaryHttpCandidate(target(url, protocol, true), 8192); + const result = await new ExternalSummaryExecutor(1).execute(work, [candidate], budget); + assert.equal(result.status, "success"); + if (result.status === "success") assert.equal(result.summary, summary); + }); + }); +} + +test("external summary HTTP: malformed multiline JSON still rejects the entire summary", async () => { + await upstream((_req, res) => { + res.setHeader("content-type", "text/event-stream"); + res.end(streamCompletion("responses") + 'data: {\ndata: invalid json\ndata: }\n\n'); + }, async (url) => { + const candidate = createSummaryHttpCandidate(target(url, "responses", true), 8192); + await assert.rejects(candidate.summarize(work, AbortSignal.timeout(4000)), { message: "External summary response contains an invalid frame" }); + }); +}); + for (const status of [400, 401, 403]) { test(`external summary HTTP: ${status} fails fast to the backup without leaking errors`, async () => { const paths: string[] = []; diff --git a/tests/preflight-sse-extraction.test.ts b/tests/preflight-sse-extraction.test.ts index 77d25022db..a30ff9867c 100644 --- a/tests/preflight-sse-extraction.test.ts +++ b/tests/preflight-sse-extraction.test.ts @@ -19,6 +19,33 @@ const R_COMPLETED = (text?: string): string => const A_DELTA = (text: string): string => `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text } })}\n\n`; const O_DELTA = (content: string): string => `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content } }] })}\n\n`; +for (const [protocol, body] of [ + ["responses", R_DELTA("complete summary") + R_COMPLETED()], + ["anthropic", A_DELTA("complete summary")], + ["openai", O_DELTA("complete summary") + "data: [DONE]\n\n"], + ["google", `data: ${JSON.stringify({ candidates: [{ content: { parts: [{ text: "complete summary" }] }, finishReason: "STOP" }] })}\n\n`], +] as const) { + test(`summary extraction joins multiline ${protocol} SSE data before parsing JSON`, () => { + const multiline = body.replace(/^data: (.+)$/gm, (line, data: string) => data === "[DONE]" ? line : + JSON.stringify(JSON.parse(data), null, 2).split("\n").map((part) => `data: ${part}`).join("\n: keepalive\n")); + assert.equal(extractSummaryFromSse(protocol, multiline), "complete summary"); + assert.equal(extractSummaryFromSse(protocol, multiline.replace(/\n/g, "\r\n")), "complete summary"); + }); +} + +test("multiline SSE keeps frame-level event types and failure terminals", () => { + const delta = 'event: response.output_text.delta\ndata: {\ndata: "delta":"retained"\ndata: }\n\n'; + const complete = 'event: response.completed\ndata: {\ndata: "response":{"status":"completed","output":[]}\ndata: }\n\n'; + assert.equal(extractSummaryFromSse("responses", delta + complete), "retained"); + const failed = 'event: response.failed\ndata: {\ndata: "response":{"status":"failed"}\ndata: }\n\n'; + assert.equal(extractSummaryFromSse("responses", delta + failed), ""); +}); + +test("joining SSE data does not accept malformed JSON or merge adjacent events", () => { + assert.equal(extractSummaryFromSse("openai", O_DELTA("partial") + 'data: {\ndata: invalid json\ndata: }\n\n'), ""); + assert.equal(extractSummaryFromSse("openai", 'data: {"choices":\n\ndata: []}\n\n'), ""); +}); + test("#780 responses: complete stream, completed full text is authoritative", () => { assert.equal(extractSummaryFromSse("responses", R_DELTA("Hello ") + R_DELTA("world.") + R_COMPLETED("Hello world.")), "Hello world."); }); From 9d0f00aa0dc86d572c7c67d833b9ac424249f453 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:08:02 +0800 Subject: [PATCH 13/67] fix: preserve runaway guard frame boundaries across chunks (cherry picked from commit a3d29b5b208859203c5f349df876400dccefbd7a) --- src/runaway-guard.ts | 8 +++- tests/runaway-frame-boundaries.test.ts | 61 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 tests/runaway-frame-boundaries.test.ts diff --git a/src/runaway-guard.ts b/src/runaway-guard.ts index b579f1e0ad..418b4724ad 100644 --- a/src/runaway-guard.ts +++ b/src/runaway-guard.ts @@ -269,6 +269,7 @@ export function wrapStreamWithRunawayGuard( // #3260 text-run lane state. let frameBuf = ""; // unprocessed tail: complete frames + at most one partial frame + let ndjson = false; let walkTripped = false; let tripVerdict: RunawayVerdict | null = null; @@ -337,9 +338,11 @@ export function wrapStreamWithRunawayGuard( // newlines and never emit a blank line. Engages only once the buffer is large // enough that standard SSE framing is clearly absent (a real SSE body separates // every frame long before NDJSON_FALLBACK_MIN). - if (frameBuf.length >= NDJSON_FALLBACK_MIN && frameBuf.includes("\n") && !HAS_FRAME_SEP_RE.test(frameBuf)) { + if (ndjson || (frameBuf.length >= NDJSON_FALLBACK_MIN && /^[{\[]/.test(frameBuf) && frameBuf.includes("\n") && !HAS_FRAME_SEP_RE.test(frameBuf))) { + ndjson = true; const lines = frameBuf.split("\n"); - frameBuf = ""; + frameBuf = lines.pop() ?? ""; + if (frameBuf.length > FRAME_BUF_CAP) frameBuf = ""; for (const lineRaw of lines) { const line = lineRaw.endsWith("\r") ? lineRaw.slice(0, -1) : lineRaw; if (line.length === 0 || (line.charAt(0) !== "{" && line.charAt(0) !== "[")) continue; @@ -363,6 +366,7 @@ export function wrapStreamWithRunawayGuard( while ((m = FRAME_SEP_RE.exec(frameBuf)) !== null) { const frame = frameBuf.slice(0, m.index); frameBuf = frameBuf.slice(m.index + m[0].length); + FRAME_SEP_RE.lastIndex = 0; processSseFrame(frame); if (walkTripped) return; if (frameBuf.length > FRAME_BUF_CAP) { diff --git a/tests/runaway-frame-boundaries.test.ts b/tests/runaway-frame-boundaries.test.ts new file mode 100644 index 0000000000..141c7fc262 --- /dev/null +++ b/tests/runaway-frame-boundaries.test.ts @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { wrapStreamWithRunawayGuard, type RunawayVerdict } from "../src/runaway-guard.ts"; + +const encoder = new TextEncoder(); +const delta = (content: string) => JSON.stringify({ choices: [{ delta: { content } }] }); +const frame = (content: string, eol = "\n") => `data: ${delta(content)}${eol}${eol}`; + +async function inspect(parts: string[]) { + const verdicts: RunawayVerdict[] = []; + const chunks: Uint8Array[] = []; + const source = new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(encoder.encode(part)); + controller.close(); + }, + }); + for await (const chunk of wrapStreamWithRunawayGuard(source, (v) => verdicts.push(v))) chunks.push(chunk); + return { verdicts, output: Buffer.concat(chunks).toString("utf8") }; +} + +for (const eol of ["\n", "\r\n"]) { + test(`runaway guard inspects every SSE frame in one chunk (${JSON.stringify(eol)})`, async () => { + const result = await inspect([frame("=".repeat(100), eol) + frame("=".repeat(100), eol)]); + assert.equal(result.verdicts.length, 1); + assert.equal(result.verdicts[0].reason, "text-run"); + assert.equal(result.output, ""); + }); + + test(`batched SSE preserves a reset between separate near-threshold runs (${JSON.stringify(eol)})`, async () => { + const body = frame("=".repeat(199), eol) + frame("different", eol) + frame("=".repeat(199), eol); + const result = await inspect([body]); + assert.deepEqual(result.verdicts, []); + assert.equal(result.output, body); + }); +} + +test("NDJSON fallback preserves a partial last event until its next chunk", async () => { + const prefix = `${delta("ab".repeat(33_000))}\n`; + const run = `${delta("=".repeat(200))}\n`; + const tail = `${delta("finished")}\n`; + const result = await inspect([prefix + run.slice(0, 60), run.slice(60) + tail]); + assert.equal(result.verdicts.length, 1); + assert.equal(result.verdicts[0].reason, "text-run"); + assert.equal(result.output, prefix + run.slice(0, 60)); +}); + +test("a large SSE event split before its final newline stays in SSE mode", async () => { + const body = frame("ab".repeat(33_000)); + const result = await inspect([body.slice(0, -1), body.slice(-1) + frame("=".repeat(200))]); + assert.equal(result.verdicts.length, 1); + assert.equal(result.verdicts[0].reason, "text-run"); +}); + +test("batched SSE keeps tool arguments out of the text-run counter", async () => { + const args = JSON.stringify({ choices: [{ delta: { tool_calls: [{ function: { arguments: delta("=".repeat(200)) } }] } }] }); + const body = frame("normal reply") + `data: ${args}\n\n` + frame(" done"); + const result = await inspect([body]); + assert.deepEqual(result.verdicts, []); + assert.equal(result.output, body); +}); From ecd228a6901f22186039827ad3e68f46dcfbcf53 Mon Sep 17 00:00:00 2001 From: "awork[bot]" Date: Wed, 11 Nov 2026 01:53:48 +0800 Subject: [PATCH 14/67] test: pin the NDJSON opening-token gate with a discriminating case The existing split-SSE case passes on unpatched source too (its large frame carries no trip-worthy run), so the /^[{\[]/ gate added alongside the frame-boundary fix had no regression coverage. This frame is a single >=64KB SSE event whose pretty-printed payload spans many data: lines and ends in a 200-codepoint run: without the gate the NDJSON fallback engages on the first chunk and drops every unparseable line, missing the trip. Reviewer addition for #6128. (cherry picked from commit b8e6d9d8736eec86e4fcb3663a1c24361956da25) --- tests/runaway-frame-boundaries.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/runaway-frame-boundaries.test.ts b/tests/runaway-frame-boundaries.test.ts index 141c7fc262..6096cf8352 100644 --- a/tests/runaway-frame-boundaries.test.ts +++ b/tests/runaway-frame-boundaries.test.ts @@ -45,6 +45,18 @@ test("NDJSON fallback preserves a partial last event until its next chunk", asyn assert.equal(result.output, prefix + run.slice(0, 60)); }); +test("NDJSON fallback does not swallow a large multi-line data SSE frame", async () => { + // >=64KB single SSE frame whose payload spans many data: lines (pretty-printed + // JSON). Without the /^[{\[]/ opening-token gate the fallback engages on the + // first chunk, drops every unparseable "data:" line and never counts the run. + const json = JSON.stringify({ choices: [{ delta: { content: "ab".repeat(33_000) + "=".repeat(200) } }] }, null, 2); + const body = json.split("\n").map((l) => `data: ${l}`).join("\n") + "\n\n"; + const result = await inspect([body.slice(0, -1), body.slice(-1)]); + assert.equal(result.verdicts.length, 1); + assert.equal(result.verdicts[0].reason, "text-run"); + assert.equal(result.output, body.slice(0, -1)); +}); + test("a large SSE event split before its final newline stays in SSE mode", async () => { const body = frame("ab".repeat(33_000)); const result = await inspect([body.slice(0, -1), body.slice(-1) + frame("=".repeat(200))]); From ff1a5c50ff5efe783cddf1a34957a7fe73fa575c Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 11 Nov 2026 07:48:13 +0800 Subject: [PATCH 15/67] fix: harden runaway guard NDJSON mode selection; document framing rules Review follow-up for the frame-boundary fix (issue #6127): - Tolerate leading blank lines/whitespace before the first NDJSON line (gate /^[{\[]/ -> /^\s*[{\[]/): an NDJSON stream opening with a stray blank line would otherwise never engage the fallback and lose text-run detection for the whole stream. SSE field lines never open with whitespace, so oversized split SSE events stay protected by the gate. - Document the non-obvious framing invariants at their sites: the cursor re-anchor after each consumed frame, sticky NDJSON mode, retained partial tail, and why the JSON-token gate exists. Adds a regression test: leading-blank-line NDJSON trips once its repeated delta completes across chunks (fails on the pre-change gate). (cherry picked from commit 3b150bff929ded8e7fed5592a9326e734bab6c74) --- src/runaway-guard.ts | 13 +++++++++++-- tests/runaway-frame-boundaries.test.ts | 11 ++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/runaway-guard.ts b/src/runaway-guard.ts index 418b4724ad..d11056d1ee 100644 --- a/src/runaway-guard.ts +++ b/src/runaway-guard.ts @@ -337,8 +337,14 @@ export function wrapStreamWithRunawayGuard( // NDJSON fallback: some OpenAI-compatible servers separate events with bare // newlines and never emit a blank line. Engages only once the buffer is large // enough that standard SSE framing is clearly absent (a real SSE body separates - // every frame long before NDJSON_FALLBACK_MIN). - if (ndjson || (frameBuf.length >= NDJSON_FALLBACK_MIN && /^[{\[]/.test(frameBuf) && frameBuf.includes("\n") && !HAS_FRAME_SEP_RE.test(frameBuf))) { + // every frame long before NDJSON_FALLBACK_MIN) and its first non-blank content + // is a JSON token ({ / [): a single oversized SSE event split across chunks + // also reaches this size with no blank line yet, but opens with an SSE field + // ("data: …"), so the token gate keeps it in SSE mode until the real separator + // arrives. Once engaged the mode is sticky for the rest of the stream (re-arming + // per chunk would forget it on small chunks), and the unfinished last line stays + // in frameBuf for the chunk that completes it. + if (ndjson || (frameBuf.length >= NDJSON_FALLBACK_MIN && /^\s*[{\[]/.test(frameBuf) && frameBuf.includes("\n") && !HAS_FRAME_SEP_RE.test(frameBuf))) { ndjson = true; const lines = frameBuf.split("\n"); frameBuf = lines.pop() ?? ""; @@ -366,6 +372,9 @@ export function wrapStreamWithRunawayGuard( while ((m = FRAME_SEP_RE.exec(frameBuf)) !== null) { const frame = frameBuf.slice(0, m.index); frameBuf = frameBuf.slice(m.index + m[0].length); + // Re-anchor the shared global regex: the buffer shrank under it, so without + // this the next exec resumes at the stale offset (past the end of the new + // string) and silently skips every remaining frame in the chunk. FRAME_SEP_RE.lastIndex = 0; processSseFrame(frame); if (walkTripped) return; diff --git a/tests/runaway-frame-boundaries.test.ts b/tests/runaway-frame-boundaries.test.ts index 6096cf8352..05461c12f9 100644 --- a/tests/runaway-frame-boundaries.test.ts +++ b/tests/runaway-frame-boundaries.test.ts @@ -47,7 +47,7 @@ test("NDJSON fallback preserves a partial last event until its next chunk", asyn test("NDJSON fallback does not swallow a large multi-line data SSE frame", async () => { // >=64KB single SSE frame whose payload spans many data: lines (pretty-printed - // JSON). Without the /^[{\[]/ opening-token gate the fallback engages on the + // JSON). Without the opening-token gate the fallback engages on the // first chunk, drops every unparseable "data:" line and never counts the run. const json = JSON.stringify({ choices: [{ delta: { content: "ab".repeat(33_000) + "=".repeat(200) } }] }, null, 2); const body = json.split("\n").map((l) => `data: ${l}`).join("\n") + "\n\n"; @@ -57,6 +57,15 @@ test("NDJSON fallback does not swallow a large multi-line data SSE frame", async assert.equal(result.output, body.slice(0, -1)); }); +test("NDJSON fallback tolerates leading blank lines before the first event", async () => { + const prefix = `\n${delta("ab".repeat(33_000))}\n`; + const run = `${delta("=".repeat(200))}\n`; + const result = await inspect([prefix + run.slice(0, 60), run.slice(60)]); + assert.equal(result.verdicts.length, 1); + assert.equal(result.verdicts[0].reason, "text-run"); + assert.equal(result.output, prefix + run.slice(0, 60)); +}); + test("a large SSE event split before its final newline stays in SSE mode", async () => { const body = frame("ab".repeat(33_000)); const result = await inspect([body.slice(0, -1), body.slice(-1) + frame("=".repeat(200))]); From 267d3b24e01a9bd45c9f4df834b78019a407c2bc Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:16:48 +0800 Subject: [PATCH 16/67] fix: avoid recursive runaway guard traversal of provider JSON (cherry picked from commit 84d84ed2e4efa8eab1aae3f5dfa02ee6ff4c57ef) --- src/runaway-guard.ts | 48 +++++++++++++++------------------ tests/runaway-deep-json.test.ts | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 tests/runaway-deep-json.test.ts diff --git a/src/runaway-guard.ts b/src/runaway-guard.ts index d11056d1ee..f8b7431800 100644 --- a/src/runaway-guard.ts +++ b/src/runaway-guard.ts @@ -273,32 +273,28 @@ export function wrapStreamWithRunawayGuard( let walkTripped = false; let tripVerdict: RunawayVerdict | null = null; - // Recursive walker over a parsed frame: feeds ONLY text-family leaves that sit - // outside excluded branches into the guard's run tracker. - const collectText = (node: unknown, key: string, excluded: boolean): void => { - if (walkTripped) return; - if (typeof node === "string") { - if (!excluded && TEXT_LEAF_KEYS.has(key)) { - const v = guard.feedText(node); - if (v.tripped) { - walkTripped = true; - tripVerdict = v; + // Walk parsed text leaves outside excluded branches without consuming the call stack. + const collectText = (root: unknown): void => { + const pending: Array<{ node: unknown; key: string }> = [{ node: root, key: "" }]; + while (pending.length > 0 && !walkTripped) { + const { node, key } = pending.pop()!; + if (EXCLUDED_BRANCHES.has(key)) continue; + if (typeof node === "string") { + if (TEXT_LEAF_KEYS.has(key)) { + const v = guard.feedText(node); + if (v.tripped) { + walkTripped = true; + tripVerdict = v; + } + } + } else if (Array.isArray(node)) { + for (let i = node.length - 1; i >= 0; i--) pending.push({ node: node[i], key }); + } else if (typeof node === "object" && node !== null) { + const entries = Object.entries(node as Record); + for (let i = entries.length - 1; i >= 0; i--) { + const [childKey, child] = entries[i]; + pending.push({ node: child, key: childKey }); } - } - return; - } - if (Array.isArray(node)) { - for (const item of node) { - collectText(item, key, excluded); - if (walkTripped) return; - } - return; - } - if (typeof node === "object" && node !== null) { - const ex = excluded || EXCLUDED_BRANCHES.has(key); - for (const [k, v] of Object.entries(node as Record)) { - collectText(v, k, ex); - if (walkTripped) return; } } }; @@ -312,7 +308,7 @@ export function wrapStreamWithRunawayGuard( } walkTripped = false; tripVerdict = null; - collectText(obj, "", false); + collectText(obj); }; const processSseFrame = (frame: string): void => { diff --git a/tests/runaway-deep-json.test.ts b/tests/runaway-deep-json.test.ts new file mode 100644 index 0000000000..fddd161bff --- /dev/null +++ b/tests/runaway-deep-json.test.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { wrapStreamWithRunawayGuard, type RunawayVerdict } from "../src/runaway-guard.ts"; + +const encoder = new TextEncoder(); +const nested = (value: string) => '{"nested":'.repeat(20_000) + value + "}".repeat(20_000); + +async function inspect(json: string) { + const input = `data: ${json}\n\n`; + const verdicts: RunawayVerdict[] = []; + const chunks: Uint8Array[] = []; + const source = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(input)); + controller.close(); + }, + }); + for await (const chunk of wrapStreamWithRunawayGuard(source, (v) => verdicts.push(v))) chunks.push(chunk); + return { input, verdicts, output: Buffer.concat(chunks).toString("utf8") }; +} + +test("deep non-runaway JSON is forwarded without exhausting the call stack", async () => { + const result = await inspect(nested('{"content":"normal text"}')); + assert.deepEqual(result.verdicts, []); + assert.equal(result.output, result.input); +}); + +test("deep model text still triggers the text-run guard", async () => { + const result = await inspect(nested(JSON.stringify({ content: "=".repeat(200) }))); + assert.equal(result.verdicts.length, 1); + assert.equal(result.verdicts[0].reason, "text-run"); + assert.equal(result.output, ""); +}); + +test("deep tool-argument objects are skipped without traversing their text leaves", async () => { + const result = await inspect(`{"tool_calls":[${nested(JSON.stringify({ content: "=".repeat(200) }))}]}`); + assert.deepEqual(result.verdicts, []); + assert.equal(result.output, result.input); +}); + +test("iterative traversal preserves text order across nested arrays and objects", async () => { + const result = await inspect(JSON.stringify({ + choices: [{ delta: { content: ["=".repeat(100), { text: "reset" }, "=".repeat(100)] } }], + text: "=".repeat(100), + })); + assert.equal(result.verdicts.length, 1); + assert.equal(result.verdicts[0].reason, "text-run"); +}); From 4a94a8f81210a8deb219fe8a617ce4058eca1360 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 11 Nov 2026 07:07:27 +0800 Subject: [PATCH 17/67] test: pin leaf feed order in the deep-JSON runaway-guard tests Reviewer alignment for PR #6131 (fixes #6130): test-only, no behavior change. The four original cases passed under every leaf permutation, so the one named "preserves text order" pinned no ordering; renamed it to what it actually asserts (mixed array/object visitation plus cross-leaf run continuity) and added one positive and one negative wire-order case (separated vs contiguous 150-runs against the 200 default threshold). Both new cases fail against a forward-push (reversed-order) variant of the walker and pass on this implementation. (cherry picked from commit 5aa1d6bdf61d52ce7a5f22714be20990e38825a7) --- tests/runaway-deep-json.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/runaway-deep-json.test.ts b/tests/runaway-deep-json.test.ts index fddd161bff..8952c53fee 100644 --- a/tests/runaway-deep-json.test.ts +++ b/tests/runaway-deep-json.test.ts @@ -38,7 +38,7 @@ test("deep tool-argument objects are skipped without traversing their text leave assert.equal(result.output, result.input); }); -test("iterative traversal preserves text order across nested arrays and objects", async () => { +test("iterative traversal visits every text leaf in mixed arrays and objects", async () => { const result = await inspect(JSON.stringify({ choices: [{ delta: { content: ["=".repeat(100), { text: "reset" }, "=".repeat(100)] } }], text: "=".repeat(100), @@ -46,3 +46,15 @@ test("iterative traversal preserves text order across nested arrays and objects" assert.equal(result.verdicts.length, 1); assert.equal(result.verdicts[0].reason, "text-run"); }); + +test("leaf feed follows wire order: separated runs stay below the threshold", async () => { + const result = await inspect(JSON.stringify({ content: "=".repeat(150), delta: `x${"=".repeat(150)}` })); + assert.deepEqual(result.verdicts, []); + assert.equal(result.output, result.input); +}); + +test("leaf feed follows wire order: contiguous runs reach the threshold", async () => { + const result = await inspect(JSON.stringify({ content: "=".repeat(150), delta: `${"=".repeat(150)}x` })); + assert.equal(result.verdicts.length, 1); + assert.equal(result.verdicts[0].reason, "text-run"); +}); From cafe4143532404ddcf0f3ac801899cd2518128d4 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 12:37:28 +0800 Subject: [PATCH 18/67] fix: propagate upstream error-body failures without replay (cherry picked from commit 70f175b545220fd4439d283786d77d960a422900) --- src/fetch-util.ts | 8 +++- tests/retry-error-body.test.ts | 82 ++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 tests/retry-error-body.test.ts diff --git a/src/fetch-util.ts b/src/fetch-util.ts index 9634d9ede8..4242d2853a 100644 --- a/src/fetch-util.ts +++ b/src/fetch-util.ts @@ -549,8 +549,12 @@ export async function fetchWithRetry( continue; } if (result.response.ok) return result; - const errText = await result.response.text().catch(() => "upstream error"); - result.clearTimer(); + let errText: string; + try { + errText = await result.response.text(); + } finally { + result.clearTimer(); + } const lastAttempt = attempt >= maxAttempts; if (!lastAttempt && isTransientUpstreamError(result.response.status, errText)) { const delayMs = replayBackoffMs(attempt); diff --git a/tests/retry-error-body.test.ts b/tests/retry-error-body.test.ts new file mode 100644 index 0000000000..66d1888098 --- /dev/null +++ b/tests/retry-error-body.test.ts @@ -0,0 +1,82 @@ +import assert from "node:assert/strict"; +import { afterEach, test } from "node:test"; +import { withFetchTransport } from "../src/fetch-transport.ts"; +import { _liveUpstreamTimersForTest, _resetFetchUtilForTest, fetchWithRetry, type ReplayRetryInfo } from "../src/fetch-util.ts"; + +const priorMax = process.env.BILI_REPLAY_RETRY_MAX; +const priorDelay = process.env.BILI_REPLAY_RETRY_BASE_MS; + +afterEach(() => { + const live = _liveUpstreamTimersForTest(); + _resetFetchUtilForTest(); + if (priorMax === undefined) delete process.env.BILI_REPLAY_RETRY_MAX; + else process.env.BILI_REPLAY_RETRY_MAX = priorMax; + if (priorDelay === undefined) delete process.env.BILI_REPLAY_RETRY_BASE_MS; + else process.env.BILI_REPLAY_RETRY_BASE_MS = priorDelay; + assert.equal(live, 0); +}); + +for (const status of [400, 503]) { + test(`error-body failure at HTTP ${status} preserves the transport error without replay`, async () => { + process.env.BILI_REPLAY_RETRY_MAX = "3"; + process.env.BILI_REPLAY_RETRY_BASE_MS = "0"; + const failure = Object.assign(new Error("body timeout"), { code: "UND_ERR_BODY_TIMEOUT" }); + let sends = 0; + const retries: ReplayRetryInfo[] = []; + await withFetchTransport(async () => { + sends++; + return new Response(new ReadableStream({ + start(controller) { controller.error(failure); }, + }), { status }); + }, async () => { + await assert.rejects( + fetchWithRetry("https://upstream.example/api", {}, undefined, undefined, (info) => retries.push(info)), + (error) => error === failure, + ); + }); + assert.equal(sends, 1); + assert.deepEqual(retries, []); + }); +} + +test("aborting while an HTTP error body is read preserves cancellation", async () => { + process.env.BILI_REPLAY_RETRY_MAX = "3"; + process.env.BILI_REPLAY_RETRY_BASE_MS = "0"; + const controller = new AbortController(); + const failure = new DOMException("client disconnected", "AbortError"); + let sends = 0; + const retries: ReplayRetryInfo[] = []; + await withFetchTransport(async () => { + sends++; + return new Response(new ReadableStream({ + pull(stream) { + controller.abort(failure); + stream.error(failure); + }, + }), { status: 503 }); + }, async () => { + await assert.rejects( + fetchWithRetry("https://upstream.example/api", {}, undefined, controller.signal, (info) => retries.push(info)), + (error) => error === failure, + ); + }); + assert.equal(sends, 1); + assert.deepEqual(retries, []); +}); + +test("complete transient error bodies still retry and recover", async () => { + process.env.BILI_REPLAY_RETRY_MAX = "3"; + process.env.BILI_REPLAY_RETRY_BASE_MS = "0"; + let sends = 0; + const retries: ReplayRetryInfo[] = []; + await withFetchTransport(async () => ++sends === 1 + ? new Response("temporarily unavailable", { status: 503 }) + : new Response("recovered"), async () => { + const result = await fetchWithRetry("https://upstream.example/api", {}, undefined, undefined, (info) => retries.push(info)); + try { assert.equal(await result.response.text(), "recovered"); } + finally { result.clearTimer(); } + }); + assert.equal(sends, 2); + assert.equal(retries.length, 1); + assert.equal(retries[0].detail, "temporarily unavailable"); +}); From 269b821bf98f7866b6f61656be16847d74864fe6 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:19:02 +0800 Subject: [PATCH 19/67] fix: honor native compaction JSON terminal status (cherry picked from commit 5caa9a1df4273e68906591e348ac8d819a2c138e) --- src/stream-terminal.ts | 4 +++- tests/native-compact-terminal.test.ts | 16 ++++++++++++++++ tests/stream-terminal.test.ts | 24 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/stream-terminal.ts b/src/stream-terminal.ts index 7bfc69694b..1b08ede935 100644 --- a/src/stream-terminal.ts +++ b/src/stream-terminal.ts @@ -103,7 +103,9 @@ async function observeJsonTerminalState(stream: ReadableStream): Pro } if (typeof body !== "object" || body === null) return "unknown"; const rec = body as Record; - if (rec.error !== undefined) return "failed"; + if (rec.error !== undefined && rec.error !== null) return "failed"; + if (rec.status === "failed" || rec.status === "incomplete" || rec.status === "cancelled") return "failed"; + if (rec.status !== undefined && rec.status !== "completed") return "unknown"; if (Array.isArray(rec.output)) return "completed"; return "unknown"; } diff --git a/tests/native-compact-terminal.test.ts b/tests/native-compact-terminal.test.ts index 9b8a7ca65c..55f60da26b 100644 --- a/tests/native-compact-terminal.test.ts +++ b/tests/native-compact-terminal.test.ts @@ -54,6 +54,11 @@ test("e2e #321 PR-C: rebase only on completed; trigger request forwarded clean", res.end(JSON.stringify({ error: { message: "compact failed" } })); return; } + if (mode?.startsWith("status-")) { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ status: mode.slice(7), error: null, output: [] })); + return; + } if (mode === "empty") { res.writeHead(200, { "content-type": "application/json" }); res.end(); @@ -187,6 +192,17 @@ test("e2e #321 PR-C: rebase only on completed; trigger request forwarded clean", await r5.text(); assert.equal(pendingRebase("term-empty"), undefined, "empty body must not schedule a rebase"); + for (const status of ["failed", "incomplete", "cancelled", "queued", "in_progress"]) { + const label = `term-json-${status}`; + const response = await postResponses(`/responses/compact?mode=status-${status}`, { model: "gpt-resp", session_id: label }); + assert.equal(response.status, 200); + await response.text(); + assert.equal(pendingRebase(label), undefined, `${status} JSON output must not schedule a rebase`); + } + const completed = await postResponses("/responses/compact?mode=status-completed", { model: "gpt-resp", session_id: "term-null-error" }); + await completed.text(); + assert.equal(pendingRebase("term-null-error"), true, "a null error must not suppress completed compaction"); + // 6. Endpoint form, JSON with output → rebase (the #249 happy path). const r6 = await postResponses("/responses/compact", { model: "gpt-resp", session_id: "term-jsonok" }); assert.equal(r6.status, 200); diff --git a/tests/stream-terminal.test.ts b/tests/stream-terminal.test.ts index 9b592f298e..c1cc7b4c46 100644 --- a/tests/stream-terminal.test.ts +++ b/tests/stream-terminal.test.ts @@ -95,6 +95,30 @@ test("JSON: error field present → failed", async () => { assert.equal(await observeResponsesTerminalState(stream, false), "failed"); }); +for (const status of ["failed", "incomplete", "cancelled"]) { + test(`JSON: ${status} output never permits a native-compaction rebase`, async () => { + const stream = sseStream([JSON.stringify({ status, output: [{ type: "compaction" }] })]); + assert.equal(await observeResponsesTerminalState(stream, false), "failed"); + }); +} + +for (const status of ["queued", "in_progress", "future_status", null, 1]) { + test(`JSON: nonterminal or unknown status ${JSON.stringify(status)} stays unknown`, async () => { + const stream = sseStream([JSON.stringify({ status, output: [] })]); + assert.equal(await observeResponsesTerminalState(stream, false), "unknown"); + }); +} + +test("JSON: completed output with a null error is successful", async () => { + const stream = sseStream([JSON.stringify({ status: "completed", error: null, output: [] })]); + assert.equal(await observeResponsesTerminalState(stream, false), "completed"); +}); + +test("JSON: compact endpoint output without a status remains supported", async () => { + const stream = sseStream([JSON.stringify({ object: "response.compaction", output: [] })]); + assert.equal(await observeResponsesTerminalState(stream, false), "completed"); +}); + test("JSON: unparseable body → unknown", async () => { assert.equal(await observeResponsesTerminalState(sseStream(["502 bad gateway"]), false), "unknown"); }); From be1b8a7e6d5c77bd1d4633e6e777670a65f268d6 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 11 Nov 2026 02:38:20 +0800 Subject: [PATCH 20/67] test(#6135): pin that a completed claim carrying a real error still fails Additive review follow-up on PR #6136: the fix checks the error payload before honoring status=completed, but no test pinned that ordering. A {"status":"completed","error":{...}} body must stay failed so a contradictory completion claim cannot schedule a rebase. (cherry picked from commit e5235f1ef89df6d7536f0e03552b71e26a20fdc4) --- tests/stream-terminal.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/stream-terminal.test.ts b/tests/stream-terminal.test.ts index c1cc7b4c46..14cb3f8c9e 100644 --- a/tests/stream-terminal.test.ts +++ b/tests/stream-terminal.test.ts @@ -114,6 +114,11 @@ test("JSON: completed output with a null error is successful", async () => { assert.equal(await observeResponsesTerminalState(stream, false), "completed"); }); +test("JSON: a completed claim carrying a real error still fails", async () => { + const stream = sseStream([JSON.stringify({ status: "completed", error: { message: "boom" }, output: [] })]); + assert.equal(await observeResponsesTerminalState(stream, false), "failed"); +}); + test("JSON: compact endpoint output without a status remains supported", async () => { const stream = sseStream([JSON.stringify({ object: "response.compaction", output: [] })]); assert.equal(await observeResponsesTerminalState(stream, false), "completed"); From bfda2f16b11f4eef2e9d77beb5e7cc883ca00eda Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:24:22 +0800 Subject: [PATCH 21/67] fix: classify native compaction only from complete SSE events (cherry picked from commit d48b0a93c982975fe6ab16b66464538fc108157d) --- src/stream-terminal.ts | 63 +++++++++++++++++------------- tests/terminal-sse-framing.test.ts | 37 ++++++++++++++++++ 2 files changed, 73 insertions(+), 27 deletions(-) create mode 100644 tests/terminal-sse-framing.test.ts diff --git a/src/stream-terminal.ts b/src/stream-terminal.ts index 1b08ede935..41597d91f3 100644 --- a/src/stream-terminal.ts +++ b/src/stream-terminal.ts @@ -4,6 +4,8 @@ // never happened discards valid blocks (#249). "unknown" is the safe default // (no rebase; the next request self-heals via kernel message-id deactivation). +import { finalizeSseLineEndings, normalizeSseLineEndings } from "./sse-util.js"; + type TerminalState = "completed" | "failed" | "unknown"; const SUCCEEDED = "response.completed"; @@ -14,24 +16,24 @@ export async function observeResponsesTerminalState(stream: ReadableStream): Prom const decoder = new TextDecoder(); let buf = ""; let state: TerminalState = "unknown"; + const consumeFrames = () => { + let boundary: number; + while ((boundary = buf.indexOf("\n\n")) >= 0) { + const frame = buf.slice(0, boundary); + buf = buf.slice(boundary + 2); + const next = terminalFromSseFrame(frame); + if (next !== null) { + state = next; + break; + } + } + }; try { for (;;) { const { done, value } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - let nl: number; - while ((nl = buf.indexOf("\n")) >= 0) { - const line = buf.slice(0, nl).replace(/\r$/, ""); - buf = buf.slice(nl + 1); - const next = terminalFromSseLine(line); - if (next !== null) { - state = next; - break; - } + if (done) { + buf = finalizeSseLineEndings(buf + decoder.decode()); + consumeFrames(); + break; } + buf = normalizeSseLineEndings(buf + decoder.decode(value, { stream: true })); + consumeFrames(); if (state !== "unknown") { // Release this tee branch; the client branch keeps draining the source. await reader.cancel().catch(() => {}); diff --git a/tests/terminal-sse-framing.test.ts b/tests/terminal-sse-framing.test.ts new file mode 100644 index 0000000000..a7bec20b19 --- /dev/null +++ b/tests/terminal-sse-framing.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { observeResponsesTerminalState } from "../src/stream-terminal.ts"; + +function stream(parts: string[]) { + const encoder = new TextEncoder(); + return new ReadableStream({ + start(controller) { + for (const part of parts) controller.enqueue(encoder.encode(part)); + controller.close(); + }, + }); +} + +for (const tail of [ + "event: response.completed\n", + 'event: response.completed\ndata: {"type":', + 'data: {"type":"response.completed"}\n', + "event: response.completed\n\n", + "event: response.completed\ndata: {broken}\n\n", +]) { + test(`an undispatched or invalid terminal SSE event is not completed: ${JSON.stringify(tail)}`, async () => { + assert.equal(await observeResponsesTerminalState(stream([tail]), true), "unknown"); + }); +} + +for (const eol of ["\n", "\r\n", "\r"]) { + test(`multiline data with ${JSON.stringify(eol)} separators is assembled before classification`, async () => { + const input = ['data: {"type":', 'data: "response.completed"}', "", ""].join(eol); + assert.equal(await observeResponsesTerminalState(stream([...input]), true), "completed"); + }); +} + +test("a terminal header cannot override a failure in its data", async () => { + const input = 'event: response.completed\ndata: {"type":"response.failed"}\n\n'; + assert.equal(await observeResponsesTerminalState(stream([input]), true), "failed"); +}); From c31fd99fedefaac6e33610fe55650e44f44f3a41 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:35:24 +0800 Subject: [PATCH 22/67] fix: repair overlapping tool groups with a linear sweep (cherry picked from commit 4f04a7a9f52a48edbce08c5b2719101ff493e311) --- src/tool-pair-order.ts | 28 ++++++++++++-------- tests/tool-pair-overlap.test.ts | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 tests/tool-pair-overlap.test.ts diff --git a/src/tool-pair-order.ts b/src/tool-pair-order.ts index 7164c011a4..7b7b2e17b8 100644 --- a/src/tool-pair-order.ts +++ b/src/tool-pair-order.ts @@ -40,21 +40,27 @@ export function hoistTrappedToolItems(items: T[]): T[] { } if (callPos.size === 0) return items; - // For each non-tool item p, find the earliest call c < p whose output o > p. - // Such an item is "trapped" between that call and its output. + const groups: Array<{ start: number; end: number }> = []; + for (const [id, start] of callPos) { + const end = outPos.get(id); + if (end === undefined || end <= start) continue; + const previous = groups.at(-1); + if (previous && start < previous.end) previous.end = Math.max(previous.end, end); + else groups.push({ start, end }); + } + if (groups.length === 0) return items; + + // Overlapping pairs form one group: hoisting into an earlier pair would create a new trap. const hoistBefore = new Array(n).fill(null); let anyTrapped = false; + let groupIndex = 0; for (let p = 0; p < n; p++) { const t = items[p].type; if (isCall(t) || isOutput(t)) continue; - let target = -1; - for (const [id, c] of callPos) { - const o = outPos.get(id); - if (o === undefined || !(c < p && o > p)) continue; - if (target === -1 || c < target) target = c; - } - if (target >= 0) { - hoistBefore[p] = target; + while (groupIndex < groups.length && groups[groupIndex].end <= p) groupIndex++; + const group = groups[groupIndex]; + if (group && group.start < p) { + hoistBefore[p] = group.start; anyTrapped = true; } } @@ -73,7 +79,7 @@ export function hoistTrappedToolItems(items: T[]): T[] { for (let p = 0; p < n; p++) { if (hoistBefore[p] !== null) continue; const batch = pendingByTarget.get(p); - if (batch) result.push(...batch); + if (batch) for (const item of batch) result.push(item); result.push(items[p]); } return result; diff --git a/tests/tool-pair-overlap.test.ts b/tests/tool-pair-overlap.test.ts new file mode 100644 index 0000000000..4303c1d15b --- /dev/null +++ b/tests/tool-pair-overlap.test.ts @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { hoistTrappedToolItems } from "../src/tool-pair-order.ts"; + +const call = (id: string) => ({ type: "function_call", call_id: id }); +const output = (id: string) => ({ type: "function_call_output", call_id: id }); +const message = (id: string) => ({ type: "message", id }); + +test("crossing tool pairs hoist all intervening messages before the complete call group", () => { + const a = call("a"), b = call("b"), ao = output("a"), bo = output("b"); + const first = message("first"), second = message("second"); + const result = hoistTrappedToolItems([a, b, first, ao, second, bo]); + assert.deepEqual(result, [first, second, a, b, ao, bo]); +}); + +test("overlap propagation spans three pairs without moving a later completed exchange", () => { + const a = call("a"), b = call("b"), c = call("c"), d = call("d"); + const ao = output("a"), bo = output("b"), co = output("c"), od = output("d"); + const first = message("first"), second = message("second"), between = message("between"); + const result = hoistTrappedToolItems([a, b, ao, first, c, bo, second, co, between, d, od]); + assert.deepEqual(result, [first, second, a, b, ao, c, bo, co, between, d, od]); +}); + +test("nested pairs preserve original non-tool order", () => { + const a = call("a"), b = call("b"), ao = output("a"), bo = output("b"); + const first = message("first"), second = message("second"); + assert.deepEqual(hoistTrappedToolItems([a, first, b, second, bo, ao]), [first, second, a, b, bo, ao]); +}); + +test("a large trapped batch does not overflow the function argument stack", () => { + const a = call("a"), ao = output("a"); + const messages = Array.from({ length: 150_000 }, (_, i) => message(String(i))); + const input: Array<{ type: string; call_id?: string; id?: string }> = [a]; + for (const item of messages) input.push(item); + input.push(ao); + const result = hoistTrappedToolItems(input); + assert.equal(result.length, input.length); + for (let i = 0; i < messages.length; i++) assert.equal(result[i], messages[i]); + assert.equal(result.at(-2), a); + assert.equal(result.at(-1), ao); +}); + +test("long healthy histories preserve the original array and item identities", () => { + const input: Array<{ type: string; call_id?: string; id?: string }> = []; + for (let i = 0; i < 5_000; i++) input.push(call(String(i)), output(String(i)), message(String(i))); + assert.equal(hoistTrappedToolItems(input), input); +}); From ed11b64936909942a435600e7f7ddda5cda1e095 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:37:02 +0800 Subject: [PATCH 23/67] fix(storage): retain malformed records during session garbage collection (cherry picked from commit bf3509d29062f461a480e9ee74b394c84ce7375e) --- src/session-gc.ts | 15 ++++++++++++--- tests/session-gc.test.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/session-gc.ts b/src/session-gc.ts index 6df3fb915c..544ab4292e 100644 --- a/src/session-gc.ts +++ b/src/session-gc.ts @@ -113,16 +113,24 @@ export function viewFromParsed(parsed: unknown): FileView | null { const savedAt = num(top.savedAt ?? rec.savedAt); if (savedAt === null || savedAt <= 0) return null; const state = asRecord(rec.state); + if (rec.state !== undefined && state === null) return null; + if (state?.blocks !== undefined && !Array.isArray(state.blocks)) return null; const blocks = Array.isArray(state?.blocks) ? (state!.blocks as unknown[]) : []; // Any block — active OR inactive — or any stored fold content means the // session was compressed at some point: its summaries cannot be rebuilt // losslessly from a client re-send, so the file is never GC-eligible. const blockContents = asRecord(rec.blockContents); + if (rec.blockContents !== undefined && blockContents === null) return null; const everCompressed = blocks.length > 0 || (blockContents !== null && Object.keys(blockContents).length > 0); const stats = asRecord(rec.stats); - const contextTokens = num(stats?.contextTokens ?? rec.contextTokens) ?? 0; + if (rec.stats !== undefined && stats === null) return null; const metadata = asRecord(rec.metadata); - const rawInputTokens = num(metadata?.rawInputTokens); + const contextValue = stats?.contextTokens ?? rec.contextTokens; + const rawValue = metadata?.rawInputTokens; + if ([contextValue, rawValue].some((value) => value !== undefined && value !== null + && (typeof value !== "number" || !Number.isFinite(value) || value < 0))) return null; + const contextTokens = num(contextValue) ?? 0; + const rawInputTokens = num(rawValue); const id = typeof top.id === "string" ? top.id : typeof rec.id === "string" ? rec.id : null; @@ -143,7 +151,8 @@ export function contentStoreTokens(parsed: unknown): number | null { if (!byHash || !byRef) return null; let tokens = 0; for (const text of Object.values(byHash)) { - if (typeof text === "string") tokens += defaultCountTokens(text); + if (typeof text !== "string") return null; + tokens += defaultCountTokens(text); } return tokens; } diff --git a/tests/session-gc.test.ts b/tests/session-gc.test.ts index da91a4ba36..a58ddb3ad7 100644 --- a/tests/session-gc.test.ts +++ b/tests/session-gc.test.ts @@ -200,6 +200,46 @@ test("viewFromParsed: envelope v3, legacy flat, corrupt input", () => { assert.equal(viewFromParsed({ savedAt: -5 }), null); }); +test("viewFromParsed: malformed compression state and token counts are never treated as empty", () => { + for (const extra of [ + { state: "unreadable state" }, + { state: { blocks: { b1: { summary: "irreplaceable" } } } }, + { blockContents: ["irreplaceable"] }, + { stats: { contextTokens: "large" } }, + { stats: { contextTokens: -1 } }, + { metadata: { rawInputTokens: -1 } }, + { metadata: { rawInputTokens: "large" } }, + ]) { + assert.equal(viewFromParsed(envelope("malformed", 1234, extra)), null, JSON.stringify(extra)); + } +}); + +test("contentStoreTokens: malformed content entries are not silently counted as zero", () => { + for (const value of [null, 1, {}, ["preserved bytes"]]) { + assert.equal(contentStoreTokens({ version: 1, byHash: { h1: value }, byRef: {} }), null); + } +}); + +test("gcSessionFiles: keep valid JSON with malformed state or malformed companion content", async () => { + const dir = tmpDir("bili-gc-malformed-shapes-"); + await withEnv({ BILI_SESSION_GC: "1" }, async () => { + const store = new SessionStore({ dir, debounceMs: 500 }); + const old = Date.now() - 10 * DAY; + const malformed = writeFile(dir, "anthropic/malformed.json", JSON.stringify(envelope("gc-malformed", old, { + state: { blocks: { b1: { summary: "preserve this" } } }, + })), 10); + const session = writeFile(dir, "anthropic/companion.json", JSON.stringify(envelope("gc-malformed-companion", old)), 10); + const companion = writeFile(dir, "anthropic/companion.content-store.json", JSON.stringify({ + version: 1, byHash: { h1: { text: "preserve unknown representation" } }, byRef: {}, + }), 10); + const result = await gcSessionFiles({ dir, store, now: Date.now() }); + assert.equal(result.removed, 0); + assert.equal(result.companionsRemoved, 0); + assert.equal(result.unreadable, 1); + for (const file of [malformed, session, companion]) assert.ok(existsSync(file), file); + }); +}); + test("gcConfigFromEnv: disabled by default, opt-in only", async () => { await withEnv( { BILI_SESSION_GC: undefined, BILI_SESSION_GC_MAX_AGE_DAYS: undefined, BILI_SESSION_GC_MAX_TOKENS: undefined, BILI_SESSION_GC_INTERVAL_MS: undefined }, From f19df89ef50467ddd4ca8560546b497384dccc56 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:43:49 +0800 Subject: [PATCH 24/67] fix(server): validate catalog identity before replaying 304 responses (cherry picked from commit 70bc835daf7bbc17aba63bc66e9a6981cc321ebc) --- src/server/codex-catalog-cache.ts | 41 +++++++++-- tests/codex-catalog-cache.test.ts | 2 +- tests/codex-catalog-revalidation.test.ts | 86 ++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 tests/codex-catalog-revalidation.test.ts diff --git a/src/server/codex-catalog-cache.ts b/src/server/codex-catalog-cache.ts index b2ade6a59d..ea15629a73 100644 --- a/src/server/codex-catalog-cache.ts +++ b/src/server/codex-catalog-cache.ts @@ -5,7 +5,7 @@ import { cacheDir } from "../paths.js"; import type { FetchOptions, fetchWithTimeout } from "../fetch-util.js"; type FetchResult = Awaited>; -type Entry = { key: string; savedAt: number; body: string }; +type Entry = { key: string; savedAt: number; body: string; etag?: string; lastModified?: string }; type Log = (level: "info" | "warn", message: string) => void; const MAX_BYTES = 1024 * 1024; const MAX_ENTRIES = 16; @@ -66,6 +66,29 @@ function transient(error: unknown): boolean { return false; } +function sameEntityTag(left: string, right: unknown): boolean { + if (typeof right !== "string") return false; + const tag = /^(?:W\/)?("[^"]*")$/; + const a = tag.exec(left.trim()); + const b = tag.exec(right.trim()); + return a !== null && b !== null && a[1] === b[1]; +} + +function revalidates(entry: Entry, request: Headers, response: Headers): boolean { + const returnedTag = response.get("etag"); + if (returnedTag !== null) return sameEntityTag(returnedTag, entry.etag); + const requestedTag = request.get("if-none-match"); + // A list or wildcard can match a different representation. Without the + // response's own ETag, only one matching tag proves this body unchanged. + if (requestedTag !== null) return sameEntityTag(requestedTag, entry.etag); + const modified = request.get("if-modified-since"); + if (modified === null || typeof entry.lastModified !== "string") return false; + const storedTime = Date.parse(entry.lastModified); + if (!Number.isFinite(storedTime) || Date.parse(modified) !== storedTime) return false; + const returnedTime = response.get("last-modified"); + return returnedTime === null || Date.parse(returnedTime) === storedTime; +} + export function isCodexCatalogRequest(url: string, init: FetchOptions): boolean { try { const parsed = new URL(url); @@ -217,8 +240,9 @@ export function createCodexCatalogFetch( // A 304 only means "not modified" when THIS request was // conditional — the wrapper itself never sends a validator, // but the end client may have added if-none-match / - // if-modified-since and that negotiated 304 replays the - // cached 200 legitimately. An UNCONDITIONAL 304 is off-spec + // if-modified-since. The validator must also identify the + // stored representation before it can replay a cached 200. + // An UNCONDITIONAL 304 is off-spec // (some transparent middleboxes answer it anyway): treat it // like any other unexpected status — drop the cache, pass // the oddity through — instead of extending freshness on a @@ -233,6 +257,10 @@ export function createCodexCatalogFetch( await invalidate(); return result; } + if (!revalidates(entry, headers, result.response.headers)) { + await invalidate(); + return result; + } await remember({ ...entry, savedAt: clock() }); discard(result); return replay(entry, false); @@ -263,7 +291,12 @@ export function createCodexCatalogFetch( } const text = complete ? Buffer.concat(chunks).toString("utf8") : ""; if (validCatalog(text)) { - await remember({ key, savedAt: clock(), body: text }); + const etag = result.response.headers.get("etag"); + const lastModified = result.response.headers.get("last-modified"); + await remember({ key, savedAt: clock(), body: text, + ...(etag !== null ? { etag } : {}), + ...(lastModified !== null ? { lastModified } : {}), + }); log("info", "[model-catalog] complete catalog refreshed; offline protection ready"); } else { await invalidate(); diff --git a/tests/codex-catalog-cache.test.ts b/tests/codex-catalog-cache.test.ts index a89024ab6f..a9472d0925 100644 --- a/tests/codex-catalog-cache.test.ts +++ b/tests/codex-catalog-cache.test.ts @@ -129,7 +129,7 @@ test("malformed/empty/oversized successful catalog is forwarded but never rescue test("temporary HTTP outages recover; a NEGOTIATED 304 revalidation is a complete 200 catalog", async () => { const f = createCodexCatalogFetch({ directory: null }); - await send(f, async () => result()); + await send(f, async () => result(text, 200, { etag: '"synthetic-v1"' })); for (const status of [500, 502, 503, 504]) assert.equal(await (await send(f, async () => result("outage", status))).response.text(), text); // The client revalidated (conditional header on the request), so the // 304 replays the cached catalog as a full 200. diff --git a/tests/codex-catalog-revalidation.test.ts b/tests/codex-catalog-revalidation.test.ts new file mode 100644 index 0000000000..22ff981e1a --- /dev/null +++ b/tests/codex-catalog-revalidation.test.ts @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createCodexCatalogFetch } from "../src/server/codex-catalog-cache.js"; + +const url = "https://chatgpt.com/backend-api/codex/models"; +const catalog = JSON.stringify({ models: [{ slug: "synthetic", context_window: 10000, model_messages: {} }] }); +const headers = { authorization: "Bearer SYNTHETIC" }; +const result = (body: string | null, status = 200, extra: Record = {}) => ({ + response: new Response(body, { status, headers: extra }), clearTimer() {}, stopIdleTimer() {}, +}); +const offline = async () => { throw Object.assign(new Error("offline"), { code: "ENETUNREACH" }); }; + +test("a 304 for another client validator cannot renew or replay the stored catalog", async () => { + const f = createCodexCatalogFetch({ directory: null }); + await f(async () => result(catalog, 200, { etag: '"old"' }), url, { headers }); + const revalidated = await f(async () => result(null, 304), url, { headers: { ...headers, "if-none-match": '"new"' } }); + assert.equal(revalidated.response.status, 304); + await assert.rejects(() => f(offline, url, { headers }), /offline/); +}); + +test("a cached response without validators is not revalidated by an arbitrary conditional request", async () => { + const f = createCodexCatalogFetch({ directory: null }); + await f(async () => result(catalog), url, { headers }); + const revalidated = await f(async () => result(null, 304), url, { headers: { ...headers, "if-none-match": '"unknown"' } }); + assert.equal(revalidated.response.status, 304); +}); + +test("a matching weak entity tag revalidates the complete catalog", async () => { + const f = createCodexCatalogFetch({ directory: null }); + await f(async () => result(catalog, 200, { etag: '"same"' }), url, { headers }); + const revalidated = await f(async () => result(null, 304), url, { headers: { ...headers, "if-none-match": 'W/"same"' } }); + assert.equal(revalidated.response.status, 200); + assert.equal(await revalidated.response.text(), catalog); +}); + +test("304 response validators disambiguate lists and reject a different representation", async () => { + for (const [responseTag, expected] of [['W/"same"', 200], ['"other"', 304]] as const) { + const f = createCodexCatalogFetch({ directory: null }); + await f(async () => result(catalog, 200, { etag: '"same"' }), url, { headers }); + const revalidated = await f(async () => result(null, 304, { etag: responseTag }), url, { + headers: { ...headers, "if-none-match": '"same", "other"' }, + }); + assert.equal(revalidated.response.status, expected); + } +}); + +test("an ambiguous entity-tag list or wildcard does not prove the stored representation unchanged", async () => { + for (const condition of ['"same", "other"', '*']) { + const f = createCodexCatalogFetch({ directory: null }); + await f(async () => result(catalog, 200, { etag: '"same"' }), url, { headers }); + const revalidated = await f(async () => result(null, 304), url, { headers: { ...headers, "if-none-match": condition } }); + assert.equal(revalidated.response.status, 304); + } +}); + +test("Last-Modified revalidation requires the stored timestamp and honors ETag precedence", async () => { + const modified = "Wed, 21 Oct 2015 07:28:00 GMT"; + for (const [condition, extra, expected] of [ + [modified, {}, 200], + ["Thu, 22 Oct 2015 07:28:00 GMT", {}, 304], + [modified, { "if-none-match": '"different"' }, 304], + ] as const) { + const f = createCodexCatalogFetch({ directory: null }); + await f(async () => result(catalog, 200, { "last-modified": modified, etag: '"same"' }), url, { headers }); + const revalidated = await f(async () => result(null, 304), url, { + headers: { ...headers, "if-modified-since": condition, ...extra }, + }); + assert.equal(revalidated.response.status, expected); + } +}); + +test("validator identity survives a private disk-cache restart", async () => { + const directory = await mkdtemp(join(tmpdir(), "bili-catalog-validator-")); + try { + await createCodexCatalogFetch({ directory })(async () => result(catalog, 200, { etag: '"disk"' }), url, { headers }); + const f = createCodexCatalogFetch({ directory }); + const revalidated = await f(async () => result(null, 304), url, { headers: { ...headers, "if-none-match": '"disk"' } }); + assert.equal(revalidated.response.status, 200); + assert.equal(await revalidated.response.text(), catalog); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); From 9faf34b1fef69e3efbd992f51f0bcb9b7b3cdc63 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:50:34 +0800 Subject: [PATCH 25/67] fix(google): account for images nested in tool responses (cherry picked from commit 9bfdd9224fabf81fe4bcc563c8709bef112f2716) --- src/image-note.ts | 20 ++++++++- src/image-tokens.ts | 18 ++++++-- tests/google-tool-image-accounting.test.ts | 51 ++++++++++++++++++++++ tests/preflight-google-summary.test.ts | 46 +++++++++++++++++++ 4 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 tests/google-tool-image-accounting.test.ts diff --git a/src/image-note.ts b/src/image-note.ts index 301c8b2335..0488c67a55 100644 --- a/src/image-note.ts +++ b/src/image-note.ts @@ -12,7 +12,7 @@ import type { CoreMessage } from "acp-kernel"; import { parseDataUrl, type BiliMessage } from "acp-kernel/wire"; -import { decodeImageDims } from "./image-tokens.js"; +import { decodeImageDims, googleMediaParts } from "./image-tokens.js"; import { responsesToolImageParts } from "./responses-tool-output.js"; /** The codec's own placeholder for an anthropic image block (wire/index.js). @@ -93,6 +93,24 @@ export function messageImages(m: CoreMessage): ImageRef[] { } if (refs.length > 0) return refs; } + // Gemini tool responses keep their images in rawGoogleParts rather than + // the singular image sidecar; inspect all parts before that fallback. + if (Array.isArray(mm.rawGoogleParts)) { + const refs: ImageRef[] = []; + for (const part of googleMediaParts(mm.rawGoogleParts)) { + const inline = part.inlineData; + if (isObj(inline) && typeof inline.mimeType === "string" && inline.mimeType.startsWith("image/") + && typeof inline.data === "string") { + refs.push({ mediaType: inline.mimeType, b64: inline.data }); + continue; + } + const file = part.fileData; + if (isObj(file) && typeof file.mimeType === "string" && file.mimeType.startsWith("image/")) { + refs.push(refFromDataUrl(file.fileUri) ?? { mediaType: file.mimeType }); + } + } + if (refs.length > 0) return refs; + } // OpenAI chat: a single data-URL image lands in imageBase64 (+media type), // multi-image messages in rawOpenaiContentParts. rawOpenaiContent can also // be a developer-role message object — gate on the part shape. Checked diff --git a/src/image-tokens.ts b/src/image-tokens.ts index c68eece005..161b881b9e 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -225,6 +225,18 @@ function opaquePartCost(part: Record, configuredCap?: number): return applyCap(Math.ceil(JSON.stringify(part).length / 4), configuredCap); } +/** Gemini tool responses carry media beside their JSON result, in parts. */ +export function* googleMediaParts(parts: unknown[]): Generator> { + for (const part of parts) { + if (!isObj(part)) continue; + yield part; + const response = part.functionResponse; + if (isObj(response) && Array.isArray(response.parts)) { + for (const nested of response.parts) if (isObj(nested)) yield nested; + } + } +} + export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "responses" | "google", body: unknown, billing: ResolvedImageBilling = "bytes", configuredCap?: number): number { if (!isObj(body)) return 0; let total = 0; @@ -236,8 +248,7 @@ export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "resp if (!Array.isArray(contents)) return 0; for (const c of contents) { if (!isObj(c) || !Array.isArray(c.parts)) continue; - for (const part of c.parts) { - if (!isObj(part)) continue; + for (const part of googleMediaParts(c.parts)) { const inline = part.inlineData; if (isObj(inline) && typeof inline.data === "string") { total += base64ImageCost(inline.data, billing, configuredCap); continue; } const file = part.fileData; @@ -396,8 +407,7 @@ export function countImagesInParsedBody(protocol: "anthropic" | "openai" | "resp if (!Array.isArray(contents)) return 0; for (const c of contents) { if (!isObj(c) || !Array.isArray(c.parts)) continue; - for (const part of c.parts) { - if (!isObj(part)) continue; + for (const part of googleMediaParts(c.parts)) { const inline = part.inlineData; if (isObj(inline) && typeof inline.data === "string") { n += 1; continue; } const file = part.fileData; diff --git a/tests/google-tool-image-accounting.test.ts b/tests/google-tool-image-accounting.test.ts new file mode 100644 index 0000000000..6ae4eb536e --- /dev/null +++ b/tests/google-tool-image-accounting.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { googleToCore, type GoogleRequestBody } from "acp-kernel/wire"; +import { imagePlaceholders, imagePlaceholdersForSummary } from "../src/image-note.js"; +import { countImagesInParsedBody, countImagesInRawBody, imageTokensInParsedBody, imageTokensInRawBody, pixelTileEstimate, REMOTE_IMAGE_TOKENS } from "../src/image-tokens.js"; + +const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; +const image = { inlineData: { mimeType: "image/png", data: png } }; +const remote = { fileData: { mimeType: "image/jpeg", fileUri: "https://example.com/screenshot.jpg" } }; +const toolBody: GoogleRequestBody = { + contents: [{ role: "user", parts: [{ functionResponse: { + name: "screenshot", response: { output: "Captured two views" }, parts: [image, image], + } }] }], +}; + +test("Google tool-result image notes retain every image and its recovery ref", () => { + const message = googleToCore(toolBody).msgs[0]; + assert.equal(message.text, "Captured two views"); + assert.deepEqual(imagePlaceholders(message), ["[image: png 1x1]", "[image: png 1x1]"]); + assert.deepEqual(imagePlaceholdersForSummary(message, "m00002"), ["[image: png 1x1 · m00002]", "[image: png 1x1 · m00002]"]); + assert.ok(!imagePlaceholdersForSummary(message, "m00002").join(" ").includes(png)); +}); + +test("Google remote image notes are retained without fetching or exposing the URL", () => { + const message = googleToCore({ contents: [{ role: "user", parts: [remote] }] }).msgs[0]; + assert.deepEqual(imagePlaceholders(message), ["[image: jpeg]"]); +}); + +test("Google nested image bytes and pixels are counted in both parsed and raw entry points", () => { + const bytes = 2 * Math.ceil(png.length / 4); + const pixels = 2 * pixelTileEstimate(1, 1); + assert.equal(imageTokensInParsedBody("google", toolBody, "bytes"), bytes); + assert.equal(imageTokensInRawBody("google", JSON.stringify(toolBody), "bytes"), bytes); + assert.equal(imageTokensInParsedBody("google", toolBody, "pixels"), pixels); + assert.equal(imageTokensInRawBody("google", JSON.stringify(toolBody), "pixels"), pixels); + assert.equal(countImagesInParsedBody("google", toolBody), 2); + assert.equal(countImagesInRawBody("google", JSON.stringify(toolBody)), 2); + assert.equal(imageTokensInParsedBody("google", toolBody, "pixels", 100), 200); +}); + +test("Google tool images share top-level media billing without charging the result text twice", () => { + const body: GoogleRequestBody = { contents: [{ role: "user", parts: [image, { functionResponse: { + name: "screenshot", response: { output: "result text ".repeat(1000) }, parts: [remote, image], + } }] }] }; + assert.equal(countImagesInParsedBody("google", body), 3); + assert.equal(imageTokensInParsedBody("google", body, "pixels"), 2 * pixelTileEstimate(1, 1) + REMOTE_IMAGE_TOKENS); + const textOnly: GoogleRequestBody = { contents: [{ role: "user", parts: [{ functionResponse: { name: "text", response: { output: "plain" } } }] }] }; + assert.equal(imageTokensInParsedBody("google", textOnly), 0); + assert.equal(countImagesInParsedBody("google", textOnly), 0); + assert.deepEqual(imagePlaceholders(googleToCore(textOnly).msgs[0]), []); +}); diff --git a/tests/preflight-google-summary.test.ts b/tests/preflight-google-summary.test.ts index 56ee02cf27..7e0b307aca 100644 --- a/tests/preflight-google-summary.test.ts +++ b/tests/preflight-google-summary.test.ts @@ -6,6 +6,7 @@ import test from "node:test"; process.env.NODE_ENV = "test"; import { defaultConfig } from "acp-kernel"; +import type { GoogleRequestBody } from "acp-kernel/wire"; import { startServer } from "../src/server.ts"; import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; @@ -193,3 +194,48 @@ test("e2e preflight (#829): the JSON summary of a non-streaming Gemini call is r await once(upstream, "close"); } }); + + +test("Gemini tool screenshots reach the real preflight summary as recovery notes", async () => { + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + const calls: Call[] = []; + const upstream = makeGeminiUpstream(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + try { + const contents: NonNullable = longGeminiContents(); + contents[1].parts!.push({ functionCall: { name: "screenshot", args: {} } }); + contents[2].parts!.unshift({ functionResponse: { + name: "screenshot", response: { output: "Captured two views" }, + parts: [ + { inlineData: { mimeType: "image/png", data: png } }, + { inlineData: { mimeType: "image/png", data: png } }, + ], + } }); + const response = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1beta/models/gemini-test:streamGenerateContent?alt=sse`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "google-tool-image-notes" }, + body: JSON.stringify({ contents, systemInstruction: { parts: [{ text: "You are a test assistant." }] }, generationConfig: { maxOutputTokens: 4096 } }), + }); + const body = await response.text(); + assert.equal(response.status, 200, body.slice(0, 300)); + const summaries = calls.filter((call) => call.summary).map((call) => call.raw).join("\n"); + assert.ok(summaries.includes("Captured two views"), "the tool result belongs to the folded range"); + assert.equal((summaries.match(/\[image: png 1x1 · m\d{5}\]/g) ?? []).length, 2); + assert.ok(!summaries.includes(png), "summary input carries notes instead of image bytes"); + const forwards = calls.filter((call) => !call.summary).map((call) => call.raw); + assert.ok(forwards.some((raw) => raw.includes(SUMMARY_TEXT.slice(0, 40))), "the real proxy applied the fold"); + assert.ok(forwards.every((raw) => !raw.includes(png)), "folded pixels leave the model request"); + } finally { + proxy.close(); + proxy.closeIdleConnections?.(); + await once(proxy, "close"); + upstream.close(); + upstream.closeIdleConnections?.(); + await once(upstream, "close"); + } +}); From fb85a2211df92c8bd89540cfad3c4f167c791dfc Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 11 Nov 2026 07:04:55 +0800 Subject: [PATCH 26/67] test(google): pin the image/* note gate negative direction for nested tool media (#6146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review supplement to PR #6147: #3617 gate discipline requires both directions of a gate. The image/* mime filter on the Gemini note path had only its positive case pinned. Adds a nested application/pdf case asserting it is billed and counted but earns no recovery note — the note carries a decompress({imageRef}) recovery ref, and the restore channel indexes nested google media for image/* mimes only (src/image-restore.ts), so a note there would promise a restore it cannot deliver. Billing stays mime-agnostic. Also drops a stray double blank line in tests/preflight-google-summary.test.ts. (cherry picked from commit 5f3d479dbfa222a68e16bfa5ddf4358d027afd42) --- tests/google-tool-image-accounting.test.ts | 15 +++++++++++++++ tests/preflight-google-summary.test.ts | 1 - 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/google-tool-image-accounting.test.ts b/tests/google-tool-image-accounting.test.ts index 6ae4eb536e..ca02e6b9db 100644 --- a/tests/google-tool-image-accounting.test.ts +++ b/tests/google-tool-image-accounting.test.ts @@ -49,3 +49,18 @@ test("Google tool images share top-level media billing without charging the resu assert.equal(countImagesInParsedBody("google", textOnly), 0); assert.deepEqual(imagePlaceholders(googleToCore(textOnly).msgs[0]), []); }); + +test("Google nested non-image media is billed and counted but earns no recovery note", () => { + const data = "JVBERi0xLjQKJcOkw7zDtsOfCg=="; + const body: GoogleRequestBody = { contents: [{ role: "user", parts: [{ functionResponse: { + name: "export_doc", response: { output: "written" }, parts: [{ inlineData: { mimeType: "application/pdf", data } }], + } }] }] }; + assert.equal(countImagesInParsedBody("google", body), 1); + assert.equal(imageTokensInParsedBody("google", body, "bytes"), Math.ceil(data.length / 4)); + // Negative direction of the image/* note gate (#3617): the note carries a + // decompress({ imageRef }) recovery ref, and the restore channel indexes + // NESTED google media only for image/* mimes (src/image-restore.ts) — an + // note here would promise a restore it cannot deliver. Billing stays + // mime-agnostic: every forwarded byte reaches upstream. + assert.deepEqual(imagePlaceholders(googleToCore(body).msgs[0]), []); +}); diff --git a/tests/preflight-google-summary.test.ts b/tests/preflight-google-summary.test.ts index 7e0b307aca..058a9e0476 100644 --- a/tests/preflight-google-summary.test.ts +++ b/tests/preflight-google-summary.test.ts @@ -195,7 +195,6 @@ test("e2e preflight (#829): the JSON summary of a non-streaming Gemini call is r } }); - test("Gemini tool screenshots reach the real preflight summary as recovery notes", async () => { const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; const calls: Call[] = []; From 01e8159a4fc59f2e260e3b23837c50f0006d96f9 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 12:39:45 +0800 Subject: [PATCH 27/67] fix: index Gemini inline images referenced by function responses (cherry picked from commit 378ba55151ab257e8b638e1b688ae1f04260317d) --- src/image-restore.ts | 21 +++++++-------------- tests/image-restore.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/image-restore.ts b/src/image-restore.ts index cf6fbd8b0f..4282aeb106 100644 --- a/src/image-restore.ts +++ b/src/image-restore.ts @@ -155,20 +155,13 @@ export function messageImageBytes(m: CoreMessage): RestorableImage[] { } const fr = part.functionResponse; if (isObj(fr) && Array.isArray(fr.parts)) { - // Review ⑤: mirror the strip side's Gemini-3 $ref guard (from the - // removed kernel strip-images.ts; kept for parity) — when - // `response` points at a part via - // {"$ref": displayName} the bytes are not inline, - // so there is nothing to restore: skip indexing (harmless waste, - // not corruption). - const refGuarded = JSON.stringify(fr.response ?? null).includes('"$ref"'); - if (!refGuarded) { - for (const np of fr.parts) { - if (!isObj(np)) continue; - const ni = np.inlineData; - if (isObj(ni) && typeof ni.data === "string" && typeof ni.mimeType === "string" && ni.mimeType.startsWith("image/")) { - out.push({ mediaType: ni.mimeType, b64: ni.data, bytes: b64Bytes(ni.data) }); - } + // Gemini's $ref names an inlineData part in this same response: + // https://ai.google.dev/api/generate-content#FunctionResponse + for (const np of fr.parts) { + if (!isObj(np)) continue; + const ni = np.inlineData; + if (isObj(ni) && typeof ni.data === "string" && typeof ni.mimeType === "string" && ni.mimeType.startsWith("image/")) { + out.push({ mediaType: ni.mimeType, b64: ni.data, bytes: b64Bytes(ni.data) }); } } } diff --git a/tests/image-restore.test.ts b/tests/image-restore.test.ts index c846a5c146..5dc702d26d 100644 --- a/tests/image-restore.test.ts +++ b/tests/image-restore.test.ts @@ -14,6 +14,7 @@ import { type RestorableImage, } from "../src/image-restore.ts"; import { resolveDecompress, type ProxyToolCtx } from "../src/decompress-shared.ts"; +import { imagePlaceholdersForSummary } from "../src/image-note.ts"; import type { WireProtocol } from "../src/util.js"; // Isolate stateDir() (hence restoreExportDir()) under a throwaway dir before any @@ -316,3 +317,29 @@ test("buildIncomingImageIndex: end-to-end nested tool_result recovery (anthropic assert.ok(existsSync(imgs[0].path), "nested image spilled at index time"); assert.deepEqual(readFileSync(imgs[0].path), Buffer.from(PNG, "base64"), "nested pixels round-trip"); }); + +test("buildIncomingImageIndex: Gemini $ref image notes resolve to their inline pixels", () => { + const body = { contents: [{ role: "user", parts: [{ functionResponse: { + name: "screenshot", response: { image_ref: { $ref: "capture.png" } }, + parts: [{ inlineData: { mimeType: "image/png", displayName: "capture.png", data: PNG } }], + } }] }] }; + const original = JSON.stringify(body); + const message = googleToCore(body).msgs[0]; + const index = buildIndex("google", body); + assert.equal(index.size, 1); + const [ref, images] = [...index.entries()][0]; + assert.deepEqual(imagePlaceholdersForSummary(message, ref), [`[image: png 1x1 · ${ref}]`]); + assert.match(resolveDecompress({ imageRef: ref }, ctxWith(index)).text, /Restored 1 image\(s\)/); + assert.deepEqual(readFileSync(images[0].path), Buffer.from(PNG, "base64")); + assert.equal(JSON.stringify(body), original, "indexing preserves the complete wire response"); +}); + +test("buildIncomingImageIndex: a Gemini $ref without inline bytes adds no image", () => { + const body = { contents: [{ role: "user", parts: [{ functionResponse: { + name: "screenshot", response: { image_ref: { $ref: "capture.png" } }, + } }] }] }; + const message = googleToCore(body).msgs[0]; + assert.deepEqual(messageImageBytes(message), []); + assert.deepEqual(imagePlaceholdersForSummary(message, "m00001"), []); + assert.equal(buildIndex("google", body).size, 0); +}); From 42ec722aa576c051d7816b8d8b482967549e9180 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 12:38:35 +0800 Subject: [PATCH 28/67] fix: stabilize identity for requests carrying multiple credentials (cherry picked from commit a10e38f71d5541012a14940f2f3f7d88c777ee29) --- src/cache-ledger.ts | 15 ++++-- tests/credential-set-fingerprint.test.ts | 59 ++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 tests/credential-set-fingerprint.test.ts diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index bf6ad6ce62..3393e4ae5e 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -658,7 +658,8 @@ export const CREDENTIAL_HEADER_NAMES = ["authorization", "x-api-key", "api-key", * returns undefined when the lane sent no recognized credential header. */ export function credentialFingerprint(headers: Record | undefined | null): string | undefined { if (!headers) return undefined; - let cred: string | undefined; + const values = new Set(); + const fields = new Set(); for (const [name, value] of Object.entries(headers)) { const n = name.toLowerCase(); if ((CREDENTIAL_HEADER_NAMES as readonly string[]).includes(n)) { @@ -677,12 +678,18 @@ export function credentialFingerprint(headers: Record | undefine // per request. The STABLE credential on that line is the // Access key id — fingerprint only that. const m = /^SDK-HMAC-SHA256\s+Access=([^\s,]+)/i.exec(v); - cred = m !== null ? `sdk-hmac:${m[1]!}` : v; - break; + const credential = m !== null ? `sdk-hmac:${m[1]!}` : v; + values.add(credential); + fields.add(JSON.stringify([n, credential])); } } } - if (cred === undefined) return undefined; + if (values.size === 0) return undefined; + // Keep existing single-key identities, including aliases carrying the + // same key. Distinct credentials form an order-independent identity so + // header iteration order cannot bypass or cross-clear a throttle hold. + const cred = values.size === 1 ? values.values().next().value! + : `credential-set:${JSON.stringify([...fields].sort())}`; return "sha256:" + createHash("sha256").update(cred, "utf8").digest("hex").slice(0, 12); } diff --git a/tests/credential-set-fingerprint.test.ts b/tests/credential-set-fingerprint.test.ts new file mode 100644 index 0000000000..493c661a6a --- /dev/null +++ b/tests/credential-set-fingerprint.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { credentialFingerprint } from "../src/cache-ledger.js"; +import { _resetUpstreamThrottleForTest, noteUpstreamThrottle429, noteUpstreamThrottleResolved, upstreamThrottleGate } from "../src/upstream-throttle.js"; + +const first = { authorization: "Bearer gateway-key", "x-api-key": "account-a" }; +const reordered = { "X-Api-Key": "account-a", Authorization: "Bearer gateway-key" }; +const other = { authorization: "Bearer gateway-key", "x-api-key": "account-b" }; + +test("credential-set identity is independent of header insertion order and casing", () => { + assert.equal(credentialFingerprint(first), credentialFingerprint(reordered)); +}); + +test("every distinct credential contributes to identity", () => { + assert.notEqual(credentialFingerprint(first), credentialFingerprint(other)); + assert.notEqual(credentialFingerprint(first), credentialFingerprint({ authorization: "Bearer account-a", "x-api-key": "gateway-key" })); +}); + +test("duplicate aliases for one credential preserve the single-key fingerprint", () => { + const single = credentialFingerprint({ authorization: "Bearer same" }); + assert.equal(single, credentialFingerprint({ "x-api-key": "same", Authorization: " Bearer same ", "x-goog-api-key": "same" })); + assert.equal(single, credentialFingerprint({ authorization: "Bearer same", "x-api-key": "", "content-type": "application/json" })); + assert.equal(credentialFingerprint({ "x-api-key": " " }), undefined); +}); + +test("multi-header SDK-HMAC fingerprints ignore request signatures while preserving account isolation", () => { + const signed = (key: string, signature: string) => `SDK-HMAC-SHA256 Access=${key}, SignedHeaders=content-type;x-sdk-date, Signature=${signature}`; + const a = credentialFingerprint({ authorization: signed("AK_A", "first"), "api-key": "gateway" }); + const b = credentialFingerprint({ "Api-Key": "gateway", Authorization: signed("AK_A", "second") }); + const c = credentialFingerprint({ authorization: signed("AK_B", "second"), "api-key": "gateway" }); + assert.equal(a, b); + assert.notEqual(a, c); +}); + +test("a 429 hold survives header reordering and cannot block or be cleared by another credential set", () => { + const keys = ["BILI_THROTTLE_429", "BILI_THROTTLE_429_BASE_DELAY_MS", "BILI_THROTTLE_429_MAX_DELAY_MS", "BILI_THROTTLE_429_MAX_CONSECUTIVE"] as const; + const saved = keys.map((key) => process.env[key]); + try { + process.env.BILI_THROTTLE_429 = "1"; + process.env.BILI_THROTTLE_429_BASE_DELAY_MS = "2000"; + process.env.BILI_THROTTLE_429_MAX_DELAY_MS = "30000"; + process.env.BILI_THROTTLE_429_MAX_CONSECUTIVE = "5"; + _resetUpstreamThrottleForTest(); + const url = "https://multi-key.example/v1/messages"; + noteUpstreamThrottle429(url, first, null, 1000000); + assert.equal(upstreamThrottleGate(url, reordered, 1000001).reject?.status, 429); + assert.equal(upstreamThrottleGate(url, other, 1000001).reject, undefined); + noteUpstreamThrottleResolved(url, other, 200, 1000002); + assert.equal(upstreamThrottleGate(url, first, 1000003).reject?.status, 429); + noteUpstreamThrottleResolved(url, reordered, 200, 1000004); + assert.equal(upstreamThrottleGate(url, first, 1000005).reject, undefined); + } finally { + _resetUpstreamThrottleForTest(); + keys.forEach((key, index) => { + if (saved[index] === undefined) delete process.env[key]; + else process.env[key] = saved[index]; + }); + } +}); From cceb76cc4cb0c35b9601d3f82bbb810014b68d46 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 13:11:23 +0800 Subject: [PATCH 29/67] fix: bracket IPv6 addresses in port-holder health probes (cherry picked from commit 9af4b7f56c2dfab4157d0d878c30890acc71a96d) --- src/port-owner.ts | 3 ++- tests/port-owner.test.ts | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/port-owner.ts b/src/port-owner.ts index bf8ff148fe..436ddeefa0 100644 --- a/src/port-owner.ts +++ b/src/port-owner.ts @@ -226,7 +226,8 @@ export const HOLDER_PROBE_TIMEOUT_MS = 2_000; /** Wildcard bind hosts are probed through loopback instead. */ export function holderProbeOrigin(host: string, port: number): string { const h = host === "0.0.0.0" || host === "::" ? "127.0.0.1" : host; - return `http://${h}:${port}`; + const authority = h.includes(":") && !h.startsWith("[") ? `[${h}]` : h; + return `http://${authority}:${port}`; } /** Pure hint composition for the terminal EADDRINUSE log line — the probe diff --git a/tests/port-owner.test.ts b/tests/port-owner.test.ts index b254263a44..a2b7b6b4a2 100644 --- a/tests/port-owner.test.ts +++ b/tests/port-owner.test.ts @@ -4,6 +4,7 @@ import http from "node:http"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { Agent, fetch as directFetch } from "undici"; import { findForeignOwner, holderHintFragment, holderProbeOrigin, mayClobberPointer, probeHolderForHint, probeOwnerInfo, resolvePortClaim, type HealthFetcher } from "../src/port-owner.ts"; import { atomicWriteInstanceFile, isPidAlive, readProxyInstanceFile, type ProxyInstanceFile } from "../src/instance.ts"; import { rmrf } from "./tmp-rm.ts"; @@ -266,6 +267,39 @@ test("holderProbeOrigin: wildcard binds probe through loopback (#4762)", () => { assert.equal(holderProbeOrigin("::", 18789), "http://127.0.0.1:18789"); }); +test("holderProbeOrigin: IPv6 literals remain valid URL authorities", () => { + assert.equal(holderProbeOrigin("::1", 18789), "http://[::1]:18789"); + assert.equal(holderProbeOrigin("2001:db8::1", 18789), "http://[2001:db8::1]:18789"); + assert.equal(holderProbeOrigin("[::1]", 18789), "http://[::1]:18789"); + assert.equal(holderProbeOrigin("localhost", 18789), "http://localhost:18789"); +}); + +test("probeHolderForHint: a healthy IPv6 listener is not diagnosed as wedged", async () => { + const server = http.createServer((req, res) => { + assert.equal(req.url, "/__bili/health"); + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify({ ok: true, pid: FOREIGN_PID, instanceId: "ipv6-holder" })); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "::1", resolve); + }); + const port = (server.address() as { port: number }).port; + const dispatcher = new Agent(); + try { + const hint = await probeHolderForHint({ + origin: holderProbeOrigin("::1", port), + fetchImpl: (url, init) => directFetch(url, { signal: init?.signal, dispatcher }), + }); + assert.match(hint, /answered \/__bili\/health \(pid 4242424\)/); + assert.doesNotMatch(hint, /wedged or dead|kill it/); + } finally { + await dispatcher.close(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); + test("holderHintFragment: serving vs wedged/dead wording, with and without a pid (#4762)", () => { const serving = holderHintFragment(true, 4242); assert.match(serving, /answered \/__bili\/health \(pid 4242\)/); From 6b00739e2e604cbf19cbe114b19e2312de2d6e4d Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 13:11:51 +0800 Subject: [PATCH 30/67] fix: give every automatic decompression spill a unique file (cherry picked from commit 68429e1704a5f4a607669b76deb4793771a83b72) --- src/decompress-shared.ts | 13 ++++---- tests/fix-decompress-tofile.test.ts | 49 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/src/decompress-shared.ts b/src/decompress-shared.ts index 74a8896108..5e6d009fc5 100644 --- a/src/decompress-shared.ts +++ b/src/decompress-shared.ts @@ -17,6 +17,7 @@ import { type SearchDoc, type SearchResult, } from "acp-kernel"; +import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, unlinkSync, writeFileSync } from "node:fs"; import { dirname, join, resolve as resolvePath } from "node:path"; import { tmpdir } from "node:os"; @@ -247,11 +248,11 @@ export function resolveDecompress( const toFileOut = toFilePointer(args, ctx, header, body); if (toFileOut !== null) return toFileOut; const safeBlockId = blockId.replace(/[^a-zA-Z0-9_-]/g, "-"); - const outPath = body.length > 10000 ? join(tmpdir(), `acp-decompress-${safeBlockId}-${Date.now()}.txt`) : null; + const outPath = body.length > 10000 ? join(tmpdir(), `acp-decompress-${safeBlockId}-${randomUUID()}.txt`) : null; if (outPath) { try { mkdirSync(dirname(outPath), { recursive: true }); - writeFileSync(outPath, body, { encoding: "utf8", mode: 0o600 }); + writeFileSync(outPath, body, { encoding: "utf8", mode: 0o600, flag: "wx" }); trackedTempFiles.push({ path: outPath, mtimeMs: Date.now() }); reapTempFiles(); return toolOk(`${header}\nContent (${body.length} chars) written to: ${outPath}\nUse the read tool to access it.`); @@ -451,13 +452,11 @@ function resolveDecompressRange(args: Record, ctx: ProxyToolCtx injText = toFileOut.text; } else if (body.length > 10000) { const safeBlockId = block.blockId.replace(/[^a-zA-Z0-9_-]/g, "-"); - // [#1207 review F4] Span in the filename (two spans of one block in the - // same millisecond must not clobber each other) and 0600 (folded - // conversation content is not world-readable on multi-user hosts). - const outPath = join(tmpdir(), `acp-decompress-${safeBlockId}-${startRaw}-${endRaw}-${Date.now()}.txt`); + // Each spill owns its path, even when sessions reuse block and span ids in the same millisecond. + const outPath = join(tmpdir(), `acp-decompress-${safeBlockId}-${startRaw}-${endRaw}-${randomUUID()}.txt`); try { mkdirSync(dirname(outPath), { recursive: true }); - writeFileSync(outPath, body, { encoding: "utf8", mode: 0o600 }); + writeFileSync(outPath, body, { encoding: "utf8", mode: 0o600, flag: "wx" }); trackedTempFiles.push({ path: outPath, mtimeMs: Date.now() }); reapTempFiles(); injText = `${header}\nContent (${body.length} chars) written to: ${outPath}\nUse the read tool to access it.`; diff --git a/tests/fix-decompress-tofile.test.ts b/tests/fix-decompress-tofile.test.ts index bbdf6bdcfa..61b346605e 100644 --- a/tests/fix-decompress-tofile.test.ts +++ b/tests/fix-decompress-tofile.test.ts @@ -176,3 +176,52 @@ test("#1691 range toFile: writes the span to the caller path and queues only a l rmrf(scratch); } }); + +test("automatic whole-block spills keep different views written in one millisecond", (t) => { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "bili-tofile-unique-")); + const tempKey = process.platform === "win32" ? "TEMP" : "TMPDIR"; + const previousTmp = process.env[tempKey]; + process.env[tempKey] = scratch; + try { + const { ctx, session, blockId } = foldBlock("unique-views", 8); + const one = "one-level view\n".repeat(1_000); + const full = "full original view\n".repeat(1_000); + session.blockContents.set(blockId, { one: { text: one, count: 1 }, full: { text: full, count: 2 } }); + const now = Date.now(); + t.mock.method(Date, "now", () => now); + const first = resolveDecompress({ blockId }, ctx).text.match(/written to: (.+)$/m)?.[1]; + const second = resolveDecompress({ blockId, full: true }, ctx).text.match(/written to: (.+)$/m)?.[1]; + assert.ok(first); + assert.ok(second); + assert.notEqual(first, second, "each export owns its file even when the clock has not advanced"); + assert.equal(fs.readFileSync(first, "utf8"), one); + assert.equal(fs.readFileSync(second, "utf8"), full); + } finally { + if (previousTmp === undefined) delete process.env[tempKey]; + else process.env[tempKey] = previousTmp; + rmrf(scratch); + } +}); + +test("automatic range spills do not register the same path twice in one millisecond", (t) => { + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "bili-tofile-range-unique-")); + const tempKey = process.platform === "win32" ? "TEMP" : "TMPDIR"; + const previousTmp = process.env[tempKey]; + process.env[tempKey] = scratch; + try { + const f = foldCcr(); + const ctx = { core: f.core, config: f.config, messages: f.msgs, session: f.session, log: () => {} }; + const now = Date.now(); + t.mock.method(Date, "now", () => now); + for (let i = 0; i < 2; i++) { + assert.equal(resolveDecompress({ blockId: f.blockId, startId: "m00001", endId: "m00007" }, ctx).outcome, "success"); + } + const files = fs.readdirSync(scratch).filter((name) => name.startsWith("acp-decompress-")); + assert.equal(files.length, 2, "each retention entry refers to a distinct file"); + for (const file of files) assert.match(fs.readFileSync(path.join(scratch, file), "utf8"), /Historical detail 0\./); + } finally { + if (previousTmp === undefined) delete process.env[tempKey]; + else process.env[tempKey] = previousTmp; + rmrf(scratch); + } +}); From 368f29e20bb7fa2aa30f1f6ff7705a21ad0f5505 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 11 Nov 2026 16:40:56 +0800 Subject: [PATCH 31/67] docs: restore #1207-F4 provenance and 0600 rationale in the range-spill comment Review supplement for #6183 (fix #6175): the collision fix replaced the range-spill comment and dropped its [#1207 review F4] provenance marker and the 0600 rationale. Restored both alongside the new uniqueness note; no code change. (cherry picked from commit 9179dc7fa392e941328eb3c1cbea7d3a2d4f209b) --- src/decompress-shared.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/decompress-shared.ts b/src/decompress-shared.ts index 5e6d009fc5..b1b3d13064 100644 --- a/src/decompress-shared.ts +++ b/src/decompress-shared.ts @@ -452,7 +452,9 @@ function resolveDecompressRange(args: Record, ctx: ProxyToolCtx injText = toFileOut.text; } else if (body.length > 10000) { const safeBlockId = block.blockId.replace(/[^a-zA-Z0-9_-]/g, "-"); - // Each spill owns its path, even when sessions reuse block and span ids in the same millisecond. + // [#1207 review F4] Span in the filename; each spill owns its path even when sessions + // reuse block and span ids in the same millisecond (#6175). 0600: folded conversation + // content is not world-readable on multi-user hosts. const outPath = join(tmpdir(), `acp-decompress-${safeBlockId}-${startRaw}-${endRaw}-${randomUUID()}.txt`); try { mkdirSync(dirname(outPath), { recursive: true }); From 69a547c09c5233769f69f965530fe2e5ae7e846a Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 11 Nov 2026 17:47:24 +0800 Subject: [PATCH 32/67] docs: point the range-spill comment at #6191 (was #6175, an unrelated slow-ingest issue) (cherry picked from commit 69ab7eec2cf970f0606a1f9dbeb75fe26c45cb0c) --- src/decompress-shared.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decompress-shared.ts b/src/decompress-shared.ts index b1b3d13064..7249947014 100644 --- a/src/decompress-shared.ts +++ b/src/decompress-shared.ts @@ -453,7 +453,7 @@ function resolveDecompressRange(args: Record, ctx: ProxyToolCtx } else if (body.length > 10000) { const safeBlockId = block.blockId.replace(/[^a-zA-Z0-9_-]/g, "-"); // [#1207 review F4] Span in the filename; each spill owns its path even when sessions - // reuse block and span ids in the same millisecond (#6175). 0600: folded conversation + // reuse block and span ids in the same millisecond (#6191). 0600: folded conversation // content is not world-readable on multi-user hosts. const outPath = join(tmpdir(), `acp-decompress-${safeBlockId}-${startRaw}-${endRaw}-${randomUUID()}.txt`); try { From ee8100581dcca2d39a68913c0ca712093da807cd Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 13:12:10 +0800 Subject: [PATCH 33/67] fix: limit Google inline media shedding to declared images (cherry picked from commit ee78ddb1b2dd796dfa8ab4a539859309f008b6a7) --- src/egress-shed.ts | 3 ++- tests/egress-shed-media.test.ts | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 tests/egress-shed-media.test.ts diff --git a/src/egress-shed.ts b/src/egress-shed.ts index 0744bc8feb..6d98d7afbd 100644 --- a/src/egress-shed.ts +++ b/src/egress-shed.ts @@ -168,8 +168,9 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] { if (!isObj(part)) return; const inline = part.inlineData; if (!isObj(inline) || typeof inline.data !== "string" || inline.data.length === 0) return; + if (typeof inline.mimeType !== "string" || !/^image\//i.test(inline.mimeType)) return; if (matchesOriginalImageReceipt(inline.data, parts)) return; - const mediaType = typeof inline.mimeType === "string" ? inline.mimeType : "image/png"; + const mediaType = inline.mimeType; occ.push({ entryIndex, b64: inline.data, diff --git a/tests/egress-shed-media.test.ts b/tests/egress-shed-media.test.ts new file mode 100644 index 0000000000..cabd461be4 --- /dev/null +++ b/tests/egress-shed-media.test.ts @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { shedWireImages, shedWireImagesToCount } from "../src/egress-shed.ts"; + +const data = Buffer.alloc(3000, 7).toString("base64"); +const media = (mimeType?: string) => ({ inlineData: { ...(mimeType === undefined ? {} : { mimeType }), data } }); +const tail = Array.from({ length: 8 }, (_, i) => ({ role: "user", parts: [{ text: `tail ${i}` }] })); + +for (const mime of ["audio/wav", "video/mp4", "application/pdf", "application/octet-stream", undefined]) { + test(`image byte shedding preserves inline media declared as ${mime ?? "unknown"}`, async () => { + const body = JSON.stringify({ contents: [{ role: "user", parts: [media(mime)] }, ...tail] }); + const result = await shedWireImages(body, "google", 100); + assert.equal(result.body, body); + assert.equal(result.stats.changed, false); + }); +} + +test("image count limits do not count or archive non-image inline parts", async () => { + const parts = [media("audio/wav"), media("application/pdf"), media("video/mp4"), media()]; + const body = JSON.stringify({ contents: [{ role: "user", parts }] }); + const result = await shedWireImagesToCount(body, "google", 0); + assert.equal(result.body, body); + assert.equal(result.stats.archived, 0); +}); + +test("a mixed-media byte budget archives images while retaining the document", async () => { + const document = media("application/pdf"); + const body = JSON.stringify({ contents: [{ role: "user", parts: [document, media("image/png")] }, ...tail] }); + const result = await shedWireImages(body, "google", 5000); + const parts = JSON.parse(result.body).contents[0].parts; + assert.deepEqual(parts[0], document); + assert.match(parts[1].text, /omitted for size limit/); + assert.equal(result.stats.archived, 1); + assert.ok(result.stats.finalBytes <= 5000); +}); + +test("a mixed-media count cap archives only the oldest surplus image", async () => { + const audio = media("audio/wav"); + const newest = media("image/jpeg"); + const body = JSON.stringify({ contents: [{ role: "user", parts: [audio, media("image/png"), newest] }] }); + const result = await shedWireImagesToCount(body, "google", 1); + const parts = JSON.parse(result.body).contents[0].parts; + assert.deepEqual(parts[0], audio); + assert.match(parts[1].text, /omitted for size limit/); + assert.deepEqual(parts[2], newest); + assert.equal(result.stats.archived, 1); +}); From 6c6b803d8e45d27b6ae176ab58286dc23b5b9880 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 13:12:26 +0800 Subject: [PATCH 34/67] fix: select image downscale work after archiving (cherry picked from commit fe435e92a59db80bab696a63acd1ae4971e4c7ff) --- src/egress-shed.ts | 11 +++----- tests/egress-shed-downscale-tail.test.ts | 32 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 tests/egress-shed-downscale-tail.test.ts diff --git a/src/egress-shed.ts b/src/egress-shed.ts index 6d98d7afbd..9221848377 100644 --- a/src/egress-shed.ts +++ b/src/egress-shed.ts @@ -280,11 +280,6 @@ export async function shedWireImages( let est = originalBytes; let archived = 0; let downscaled = 0; - // Rung B's downscale pass decodes + re-encodes through jimp, so it stays capped at - // MAX_SHED_IMAGES to bound per-request CPU. Rung A (archiving, below) is a cheap string - // swap and must walk EVERY occurrence — see #5745. - const work = occurrences.slice(0, MAX_SHED_IMAGES); - // Rung A: archive oldest non-protected images to placeholder notes until under target. // Walks every occurrence (not just the first MAX_SHED_IMAGES): archiving is a string swap, // so the jimp CPU bound does not apply here — capping it left any session with more than @@ -304,12 +299,12 @@ export async function shedWireImages( // Rung B: downscale whatever still rides the wire (protected tail included) through // the dim ladder; stop as soon as the estimate fits. if (est > targetBytes) { + // Only live occurrences consume the decode/re-encode budget; archived + // prefixes must not keep the surviving protected tail out of the pass. + const work = occurrences.filter((o) => !o.archived).slice(0, MAX_SHED_IMAGES); outer: for (const dim of DOWNSCALE_DIM_LADDER) { for (const o of work) { if (est <= targetBytes) break; - // an archived occurrence's part was already replaced by the note — shrinking - // it would write into a detached object and double-count savings in est - if (o.archived) continue; const dims = decodeImageDims(o.b64); if (dims !== undefined && Math.max(dims.w, dims.h) <= dim) continue; const smaller = await shedDownscaleImage(o.mediaType, o.b64, dim); diff --git a/tests/egress-shed-downscale-tail.test.ts b/tests/egress-shed-downscale-tail.test.ts new file mode 100644 index 0000000000..2f9017cde6 --- /dev/null +++ b/tests/egress-shed-downscale-tail.test.ts @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { Jimp } from "jimp"; +import { shedWireImages } from "../src/egress-shed.ts"; + +test("archived images do not consume the downscale work budget of the surviving tail", async () => { + const image = new Jimp({ width: 1024, height: 1024 }); + let seed = 1977; + const bytes = image.bitmap.data; + for (let i = 0; i < bytes.length; i++) { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; + bytes[i] = i % 4 === 3 ? 255 : seed >>> 24; + } + const original = (await image.getBuffer("image/png")).toString("base64"); + const imageItem = (b64: string) => ({ type: "message", role: "user", content: [{ type: "input_image", image_url: `data:image/png;base64,${b64}` }] }); + const input = [ + ...Array.from({ length: 64 }, () => imageItem(Buffer.alloc(3000, 7).toString("base64"))), + imageItem(original), + ...Array.from({ length: 7 }, () => ({ type: "message", role: "user", content: [{ type: "input_text", text: "tail" }] })), + ]; + const body = JSON.stringify({ model: "m", input }); + const target = 1_000_000; + const result = await shedWireImages(body, "responses", target); + assert.equal(result.stats.archived, 64); + assert.ok(result.stats.downscaled > 0, "the surviving recent image is still eligible for downscaling"); + assert.ok(result.stats.finalBytes <= target, `${result.stats.finalBytes} exceeds ${target}`); + const parts = JSON.parse(result.body).input[64].content; + assert.equal(parts[0].type, "input_image", "the protected tail is resized, never archived"); + const restored = await Jimp.read(Buffer.from(parts[0].image_url.split(",")[1], "base64")); + assert.ok(restored.width < image.width); + assert.equal(JSON.stringify({ model: "m", input }), body); +}); From 4bb1216210ada7f2c2879bb471a7e21d40ff85ac Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 13:12:56 +0800 Subject: [PATCH 35/67] fix: decode BMP dimensions by DIB header layout (cherry picked from commit 4d823114e4c7dc3ed6601add15a54c8760f33047) --- src/image-tokens.ts | 10 +++++++- tests/issue767-pixel-billing.test.ts | 38 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/image-tokens.ts b/src/image-tokens.ts index 161b881b9e..bdbfe3f854 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -85,7 +85,15 @@ function gifDims(b: Buffer): Dims | undefined { function bmpDims(b: Buffer): Dims | undefined { if (b.length < 26 || b[0] !== 0x42 || b[1] !== 0x4d) return undefined; // "BM" - const w = b.readUInt32LE(18), h = Math.abs(b.readInt32LE(22)); + const headerSize = b.readUInt32LE(14); + // BITMAPCOREHEADER stores two WORD dimensions; BITMAPINFOHEADER and + // its extensions store two LONGs at different offsets. + if (headerSize === 12) { + const w = b.readUInt16LE(18), h = b.readUInt16LE(20); + return w > 0 && h > 0 ? { w, h } : undefined; + } + if (headerSize < 40) return undefined; + const w = b.readInt32LE(18), h = Math.abs(b.readInt32LE(22)); return w > 0 && h > 0 ? { w, h } : undefined; } diff --git a/tests/issue767-pixel-billing.test.ts b/tests/issue767-pixel-billing.test.ts index a5202e0387..a6b07df7f1 100644 --- a/tests/issue767-pixel-billing.test.ts +++ b/tests/issue767-pixel-billing.test.ts @@ -65,6 +65,20 @@ function bmpB64(w: number, h: number): string { return b.toString("base64"); } +function bmpCoreB64(w: number, h: number): string { + const stride = Math.ceil(w * 3 / 4) * 4; + const b = Buffer.alloc(26 + stride * h); + b.write("BM", 0, "ascii"); + b.writeUInt32LE(b.length, 2); + b.writeUInt32LE(26, 10); + b.writeUInt32LE(12, 14); + b.writeUInt16LE(w, 18); + b.writeUInt16LE(h, 20); + b.writeUInt16LE(1, 22); + b.writeUInt16LE(24, 24); + return b.toString("base64"); +} + function webpVp8B64(w: number, h: number): string { const b = Buffer.alloc(32); b.write("RIFF", 0, "ascii"); @@ -179,6 +193,30 @@ test("#767 unit: pixelTileEstimate matches the OpenAI high-detail tile model", ( assert.equal(pixelTileEstimate(1000, 200), 765, "double scale: up short side, then cap long side"); }); +test("BMP core headers use 16-bit dimensions without reading plane/depth fields as height", () => { + assert.deepEqual(decodeImageDims(bmpCoreB64(64, 128)), { w: 64, h: 128 }); + assert.deepEqual(decodeImageDims(bmpCoreB64(1, 1)), { w: 1, h: 1 }); +}); + +test("BMP core pixel billing uses the actual aspect ratio", () => { + const b64 = bmpCoreB64(64, 128); + const body = { messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: `data:image/bmp;base64,${b64}` } }] }] }; + assert.equal(imageTokensInParsedBody("openai", body, "pixels"), pixelTileEstimate(64, 128)); +}); + +test("BMP info headers reject negative width while preserving top-down height", () => { + assert.equal(decodeImageDims(bmpB64(-640, 480)), undefined); + assert.deepEqual(decodeImageDims(bmpB64(640, -480)), { w: 640, h: 480 }); +}); + +test("BMP headers with an unsupported layout or zero dimensions use the fallback", () => { + const unsupported = Buffer.from(bmpB64(640, 480), "base64"); + unsupported.writeUInt32LE(13, 14); + assert.equal(decodeImageDims(unsupported.toString("base64")), undefined); + assert.equal(decodeImageDims(bmpCoreB64(0, 128)), undefined); + assert.equal(decodeImageDims(bmpCoreB64(64, 0)), undefined); +}); + test("#767 unit: resolveImageBilling — explicit bytes wins, everything else resolves to pixels (#1843)", () => { assert.equal(resolveImageBilling("pixels", undefined), "pixels"); assert.equal(resolveImageBilling("bytes", "https://api.openai.com/v1"), "bytes"); From b0f263bf177f6411428b49ae4585051da80dccd8 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 11 Nov 2026 19:48:14 +0800 Subject: [PATCH 36/67] fix: correct CORE header offsets so real spec-valid BMPs parse (#6179) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit read BITMAPCOREHEADER dims at offsets 18/20 and detected the header with readUInt32LE(14). Both are off by one field from the BITMAPCOREHEADER layout (bfSize is a WORD at 14; bfWidth@16; bfHeight@18): - readUInt32LE(14) spans bfSize(WORD)+bfWidth(WORD), so for any real CORE BMP with width>=1 it yields 12 + width*65536 != 12 — the ===12 guard never matched and the reader fell through to the INFO branch, reproducing the original {w:65664,h:24} misparse for a 64x128 image. - the dim reads at 18/20 landed on height/planes even where the guard matched. The test passed only because its own fixture wrote dims at those same wrong offsets (self-consistent, but not a valid BMP per MS-Win32 BITMAPCOREHEADER). Read bfSize as a WORD (readUInt16LE(14)) and CORE dims at 16/18; make the fixture spec-valid (dims@16/18, planes@20, bitcount@22) so the regression now exercises a real encodable CORE header. A spec-correct 64x128 CORE BMP now decodes to {w:64,h:128} and bills pixelTileEstimate(64,128)=1105. (cherry picked from commit baf94196d4721bad043748c343fb3ef22d3a6100) --- src/image-tokens.ts | 10 ++++++---- tests/issue767-pixel-billing.test.ts | 10 +++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/image-tokens.ts b/src/image-tokens.ts index bdbfe3f854..db112fcfbc 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -85,11 +85,13 @@ function gifDims(b: Buffer): Dims | undefined { function bmpDims(b: Buffer): Dims | undefined { if (b.length < 26 || b[0] !== 0x42 || b[1] !== 0x4d) return undefined; // "BM" - const headerSize = b.readUInt32LE(14); - // BITMAPCOREHEADER stores two WORD dimensions; BITMAPINFOHEADER and - // its extensions store two LONGs at different offsets. + // bfSize is a WORD (2 bytes): BITMAPCOREHEADER==12, every INFO/V2–V5 variant + // is >=40. Reading it as a WORD keeps the size field from fusing with the + // bfWidth that immediately follows at offset 16. + const headerSize = b.readUInt16LE(14); if (headerSize === 12) { - const w = b.readUInt16LE(18), h = b.readUInt16LE(20); + // CORE: bfWidth@16 / bfHeight@18 (WORD each), then planes@20, bitcount@22. + const w = b.readUInt16LE(16), h = b.readUInt16LE(18); return w > 0 && h > 0 ? { w, h } : undefined; } if (headerSize < 40) return undefined; diff --git a/tests/issue767-pixel-billing.test.ts b/tests/issue767-pixel-billing.test.ts index a6b07df7f1..f96792b30d 100644 --- a/tests/issue767-pixel-billing.test.ts +++ b/tests/issue767-pixel-billing.test.ts @@ -71,11 +71,11 @@ function bmpCoreB64(w: number, h: number): string { b.write("BM", 0, "ascii"); b.writeUInt32LE(b.length, 2); b.writeUInt32LE(26, 10); - b.writeUInt32LE(12, 14); - b.writeUInt16LE(w, 18); - b.writeUInt16LE(h, 20); - b.writeUInt16LE(1, 22); - b.writeUInt16LE(24, 24); + b.writeUInt16LE(12, 14); // bfSize (WORD)=12 → CORE + b.writeUInt16LE(w, 16); // bfWidth + b.writeUInt16LE(h, 18); // bfHeight + b.writeUInt16LE(1, 20); // planes + b.writeUInt16LE(24, 22); // bit count return b.toString("base64"); } From 3cb1158b34d3b04f431d6bd5702a7fed233bb6ec Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 21:24:24 +0800 Subject: [PATCH 37/67] fix: ground BMP CORE dimensions in encoder output (cherry picked from commit a3eadba6e96e0ba6c3cb2a6579a079a9fc0c4153) --- src/image-tokens.ts | 8 ++------ tests/issue767-pixel-billing.test.ts | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/image-tokens.ts b/src/image-tokens.ts index db112fcfbc..09160f0f84 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -85,13 +85,9 @@ function gifDims(b: Buffer): Dims | undefined { function bmpDims(b: Buffer): Dims | undefined { if (b.length < 26 || b[0] !== 0x42 || b[1] !== 0x4d) return undefined; // "BM" - // bfSize is a WORD (2 bytes): BITMAPCOREHEADER==12, every INFO/V2–V5 variant - // is >=40. Reading it as a WORD keeps the size field from fusing with the - // bfWidth that immediately follows at offset 16. - const headerSize = b.readUInt16LE(14); + const headerSize = b.readUInt32LE(14); if (headerSize === 12) { - // CORE: bfWidth@16 / bfHeight@18 (WORD each), then planes@20, bitcount@22. - const w = b.readUInt16LE(16), h = b.readUInt16LE(18); + const w = b.readUInt16LE(18), h = b.readUInt16LE(20); return w > 0 && h > 0 ? { w, h } : undefined; } if (headerSize < 40) return undefined; diff --git a/tests/issue767-pixel-billing.test.ts b/tests/issue767-pixel-billing.test.ts index f96792b30d..9932b1f8f3 100644 --- a/tests/issue767-pixel-billing.test.ts +++ b/tests/issue767-pixel-billing.test.ts @@ -71,11 +71,11 @@ function bmpCoreB64(w: number, h: number): string { b.write("BM", 0, "ascii"); b.writeUInt32LE(b.length, 2); b.writeUInt32LE(26, 10); - b.writeUInt16LE(12, 14); // bfSize (WORD)=12 → CORE - b.writeUInt16LE(w, 16); // bfWidth - b.writeUInt16LE(h, 18); // bfHeight - b.writeUInt16LE(1, 20); // planes - b.writeUInt16LE(24, 22); // bit count + b.writeUInt32LE(12, 14); + b.writeUInt16LE(w, 18); + b.writeUInt16LE(h, 20); + b.writeUInt16LE(1, 22); + b.writeUInt16LE(24, 24); return b.toString("base64"); } @@ -198,6 +198,14 @@ test("BMP core headers use 16-bit dimensions without reading plane/depth fields assert.deepEqual(decodeImageDims(bmpCoreB64(1, 1)), { w: 1, h: 1 }); }); +test("BMP core dimensions and billing match an independently encoded image", () => { + // ImageMagick 7.1.1-43: magick -size 1x2 xc:black BMP2:core.bmp + const b64 = "Qk0iAAAAAAAAABoAAAAMAAAAAQACAAEAGAAAAAAAAAAAAA=="; + assert.deepEqual(decodeImageDims(b64), { w: 1, h: 2 }); + const body = { messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: `data:image/bmp;base64,${b64}` } }] }] }; + assert.equal(imageTokensInParsedBody("openai", body, "pixels"), pixelTileEstimate(1, 2)); +}); + test("BMP core pixel billing uses the actual aspect ratio", () => { const b64 = bmpCoreB64(64, 128); const body = { messages: [{ role: "user", content: [{ type: "image_url", image_url: { url: `data:image/bmp;base64,${b64}` } }] }] }; From 217070e409b40c8b61e6640f6104d4da31f63766 Mon Sep 17 00:00:00 2001 From: ework-daemon Date: Wed, 11 Nov 2026 22:34:52 +0800 Subject: [PATCH 38/67] docs: record BMP DIB field offsets in bmpDims (#6179) Both offset flip-flops in the #6187 review happened because the DIB field layout was only implicit. Pin it inline: bcSize is a DWORD at 14 in every variant (BITMAPCOREHEADER=12 keeps dims as WORDs at 18/20; BITMAPINFOHEADER/V2-V5 >=40 use LONGs at 18/22) per MS-Win32, verified against encoder output. Comment only, no behavior change. (cherry picked from commit 60b29d124883b17f589ef3503f1cb6e4117f62a6) --- src/image-tokens.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/image-tokens.ts b/src/image-tokens.ts index 09160f0f84..515b6f8d00 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -85,6 +85,9 @@ function gifDims(b: Buffer): Dims | undefined { function bmpDims(b: Buffer): Dims | undefined { if (b.length < 26 || b[0] !== 0x42 || b[1] !== 0x4d) return undefined; // "BM" + // DIB size is a DWORD @14 in every variant (MS-Win32 BITMAPCOREHEADER / + // BITMAPINFOHEADER): CORE (=12) keeps dims as WORDs @18/20; INFO/V2-V5 + // (>=40) use LONGs @18/22. Offsets pinned to encoder output, #6179. const headerSize = b.readUInt32LE(14); if (headerSize === 12) { const w = b.readUInt16LE(18), h = b.readUInt16LE(20); From d300f4be9ca54b9f7685a4b648f8be4dc14b7397 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 13:13:12 +0800 Subject: [PATCH 39/67] fix(kimi): preserve TOML integer model limits (cherry picked from commit 9c197621cb75966e50f1b6e477379d310a36e221) --- src/kimi/toml-edit.ts | 15 +++++++++++++-- tests/kimi-native.test.ts | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/kimi/toml-edit.ts b/src/kimi/toml-edit.ts index b8cdbc6cef..33f5427f68 100644 --- a/src/kimi/toml-edit.ts +++ b/src/kimi/toml-edit.ts @@ -50,13 +50,24 @@ function headerParts(line: string): string[] | undefined { return tomlPathParts(m[1]); } +function nonNegativeTomlInteger(value: string): number | undefined { + // Model limits accept TOML integer spelling, not a numeric prefix of a + // float/date. Underscores may only separate digits of the selected base. + if (!/^(?:\+?(?:0|[1-9](?:_?[0-9])*)|0x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*)$/.test(value)) return undefined; + const parsed = Number(value.replace(/_/g, "")); + return Number.isSafeInteger(parsed) ? parsed : undefined; +} + function keyValue(line: string): { key: string; strVal?: string; numVal?: number } | undefined { const l = line.trim(); if (!l || l.startsWith("#") || l.startsWith("[")) return undefined; const strMatch = /^([A-Za-z0-9_.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/.exec(l); if (strMatch) return { key: strMatch[1], strVal: strMatch[2] !== undefined ? strMatch[2] : strMatch[3] }; - const numMatch = /^([A-Za-z0-9_.-]+)\s*=\s*([0-9]+)\b/.exec(l); - if (numMatch) return { key: numMatch[1], numVal: Number(numMatch[2]) }; + const numMatch = /^([A-Za-z0-9_.-]+)\s*=\s*([^\s#]+)\s*(?:#.*)?$/.exec(l); + if (numMatch) { + const numVal = nonNegativeTomlInteger(numMatch[2]); + if (numVal !== undefined) return { key: numMatch[1], numVal }; + } // Inline tables / arrays ({ x-bili-plugin = "kimi" }, ["a"]) have no scalar // value — key-only, so callers can still locate/replace the line. const tblMatch = /^([A-Za-z0-9_.-]+)\s*=\s*[{[]/.exec(l); diff --git a/tests/kimi-native.test.ts b/tests/kimi-native.test.ts index cefbf7f6eb..e766f97274 100644 --- a/tests/kimi-native.test.ts +++ b/tests/kimi-native.test.ts @@ -348,6 +348,42 @@ test("scanKimiModelWindows keys windows by model id across all sections (#2954)" assert.deepEqual(scanKimiModelWindows(SAMPLE_CONFIG), {}, "no windows declared → empty map, nothing to feed"); }); +for (const [literal, expected] of [ + ["128_000", 128000], + ["+131072", 131072], + ["0x2_0000", 131072], + ["0o400_000", 131072], + ["0b10_0000_0000_0000_0000", 131072], + ["65536 # context limit", 65536], +] as const) { + test(`Kimi preserves TOML integer limit ${literal} through routing and unroute`, () => { + const config = THINKING_CONFIG.replace("max_context_size = 65536", `max_context_size = ${literal}`) + .replace("max_output_size = 8192", "max_output_size = 8_192"); + assert.deepEqual(scanKimiModelWindows(config), { "deepseek-r1": expected }); + const route = okOf(resolveKimiRoute(config, {})); + assert.equal(route.contextWindow, expected); + assert.equal(route.maxOutput, 8192); + const applied = applyKimiManagedConfig(config, stateOf(route)); + const reread = okOf(resolveKimiRoute(applied, {})); + assert.equal(reread.contextWindow, expected); + assert.equal(reread.maxOutput, 8192); + assert.equal(applyKimiManagedConfig(applied, stateOf(reread)), applied); + assert.ok(unrouteKimiConfig(applied).includes(`max_context_size = ${literal}`), "the user's literal survives uninstall"); + }); +} + +for (const literal of ["1.5", "1e6", "2026-01-01", "128__000", "128_", "_128", "01", "0x_1", "1 trailing", "9007199254740993"]) { + test(`Kimi does not mistake ${literal} for an integer limit`, () => { + const config = THINKING_CONFIG.replace("max_context_size = 65536", `max_context_size = ${literal}`) + .replace("max_output_size = 8192", `max_output_size = ${literal}`); + assert.deepEqual(scanKimiModelWindows(config), {}); + const route = okOf(resolveKimiRoute(config, {})); + assert.equal(route.contextWindow, undefined); + assert.equal(route.maxOutput, undefined); + assert.ok(!aliasSection(applyKimiManagedConfig(config, stateOf(route))).includes("max_")); + }); +} + test("kimiConfigModelWindows hands both kimi proxy paths ONE windows source (#3436/#3557)", () => { const home = path.join(tmpdir(), `bili-kimi-win-${process.pid}-${Date.now()}`); mkdirSync(home, { recursive: true }); From b8f3053207df99862e93f2074f801bc1688e9367 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:20:23 +0800 Subject: [PATCH 40/67] fix(config): apply strict TOML integers to sibling model readers (cherry picked from commit c8b7ad075a5f75f248a6b39234d4d6e82e029a1f) --- src/client-config.ts | 15 +++-- src/kimi/toml-edit.ts | 9 +-- src/toml-integer.ts | 8 +++ tests/client-config-toml-integers.test.ts | 68 +++++++++++++++++++++++ 4 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 src/toml-integer.ts create mode 100644 tests/client-config-toml-integers.test.ts diff --git a/src/client-config.ts b/src/client-config.ts index 39a090e7a4..5fce9ce87a 100644 --- a/src/client-config.ts +++ b/src/client-config.ts @@ -7,6 +7,7 @@ import os from "node:os"; import path from "node:path"; import { parse as parseJsonc, type ParseError } from "jsonc-parser"; import { vertexEnabled } from "./vertex.js"; +import { nonNegativeTomlInteger } from "./toml-integer.js"; export interface ClaudeSettings { anthropicBaseUrl?: string; @@ -720,7 +721,7 @@ export function parseCodexToml(text: string): CodexConfig { continue; } const strMatch = /^([A-Za-z0-9_.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/.exec(line); - const numMatch = /^([A-Za-z0-9_.-]+)\s*=\s*([0-9]+)\b/.exec(line); + const numMatch = /^([A-Za-z0-9_.-]+)\s*=\s*([^\s#]+)\s*(?:#.*)?$/.exec(line); if (strMatch) { const key = strMatch[1]; const val = strMatch[2] !== undefined ? strMatch[2] : strMatch[3]; @@ -733,9 +734,11 @@ export function parseCodexToml(text: string): CodexConfig { result.providers[curProvider].baseUrl = val; } } else if (numMatch && table === "") { - if (numMatch[1] === "model_context_window") codexContextWindow = Number(numMatch[2]); - else if (numMatch[1] === "model_auto_compact_token_limit") codexAutoCompactLimit = Number(numMatch[2]); - else if (numMatch[1] === "model_max_output_tokens") codexMaxOutput = Number(numMatch[2]); + const numVal = nonNegativeTomlInteger(numMatch[2]); + if (numVal === undefined) continue; + if (numMatch[1] === "model_context_window") codexContextWindow = numVal; + else if (numMatch[1] === "model_auto_compact_token_limit") codexAutoCompactLimit = numVal; + else if (numMatch[1] === "model_max_output_tokens") codexMaxOutput = numVal; } } if (codexModel) result.model = codexModel; @@ -1079,11 +1082,11 @@ export function parseKimiToml(text: string): KimiConfig { continue; } const strMatch = /^([A-Za-z0-9_.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/.exec(line); - const numMatch = /^([A-Za-z0-9_.-]+)\s*=\s*([0-9]+)\b/.exec(line); + const numMatch = /^([A-Za-z0-9_.-]+)\s*=\s*([^\s#]+)\s*(?:#.*)?$/.exec(line); if (!strMatch && !numMatch) continue; const key = (strMatch ?? numMatch)![1]; const strVal = strMatch ? (strMatch[2] !== undefined ? strMatch[2] : strMatch[3]) : undefined; - const numVal = numMatch ? Number(numMatch[2]) : undefined; + const numVal = numMatch ? nonNegativeTomlInteger(numMatch[2]) : undefined; if (parts.length === 0) { if (key === "default_model" && strVal !== undefined) result.defaultModel = strVal; } else if (parts[0] === "providers") { diff --git a/src/kimi/toml-edit.ts b/src/kimi/toml-edit.ts index 33f5427f68..0d30096f3e 100644 --- a/src/kimi/toml-edit.ts +++ b/src/kimi/toml-edit.ts @@ -3,6 +3,7 @@ // own writeback is line-surgical per domain — our block survives login/logout/ // catalog refresh as long as it uses its own provider name + model alias // (the refresh orchestrator preserves user-owned entries). +import { nonNegativeTomlInteger } from "../toml-integer.js"; export const KIMI_PROVIDER = "bili"; export const KIMI_ALIAS = "bili-kimi"; @@ -50,14 +51,6 @@ function headerParts(line: string): string[] | undefined { return tomlPathParts(m[1]); } -function nonNegativeTomlInteger(value: string): number | undefined { - // Model limits accept TOML integer spelling, not a numeric prefix of a - // float/date. Underscores may only separate digits of the selected base. - if (!/^(?:\+?(?:0|[1-9](?:_?[0-9])*)|0x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*)$/.test(value)) return undefined; - const parsed = Number(value.replace(/_/g, "")); - return Number.isSafeInteger(parsed) ? parsed : undefined; -} - function keyValue(line: string): { key: string; strVal?: string; numVal?: number } | undefined { const l = line.trim(); if (!l || l.startsWith("#") || l.startsWith("[")) return undefined; diff --git a/src/toml-integer.ts b/src/toml-integer.ts new file mode 100644 index 0000000000..02173fe562 --- /dev/null +++ b/src/toml-integer.ts @@ -0,0 +1,8 @@ +/** Parse the non-negative TOML integer subset used for model token limits. */ +export function nonNegativeTomlInteger(value: string): number | undefined { + // Reject numeric prefixes of floats/dates. Underscores only separate digits + // of the selected base, and model limits must remain exactly representable. + if (!/^(?:\+?(?:0|[1-9](?:_?[0-9])*)|0x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*)$/.test(value)) return undefined; + const parsed = Number(value.replace(/_/g, "")); + return Number.isSafeInteger(parsed) ? parsed : undefined; +} diff --git a/tests/client-config-toml-integers.test.ts b/tests/client-config-toml-integers.test.ts new file mode 100644 index 0000000000..003121dda0 --- /dev/null +++ b/tests/client-config-toml-integers.test.ts @@ -0,0 +1,68 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { parseCodexToml, parseKimiToml } from "../src/client-config.ts"; + +const valid: ReadonlyArray = [ + ["128_000", 128000], + ["+262_144", 262144], + ["0x1_f400", 128000], + ["0o372_000", 128000], + ["0b1_1111_0100_0000_0000", 128000], + ["65536", 65536], +]; + +for (const [literal, expected] of valid) { + test(`Codex TOML model limits retain the integer ${literal}`, () => { + const config = parseCodexToml([ + 'model = "custom-model"', + `model_context_window = ${literal} # context`, + `model_auto_compact_token_limit = ${literal}`, + `model_max_output_tokens = ${literal}`, + ].join("\n")); + assert.equal(config.contextWindow, expected); + assert.equal(config.autoCompactLimit, expected); + assert.equal(config.maxOutput, expected); + assert.deepEqual(config.modelWindows, [{ id: "custom-model", contextWindow: expected, maxOutput: expected }]); + }); + + test(`Kimi launcher model and override limits retain the integer ${literal}`, () => { + const config = parseKimiToml([ + "[models.base]", + 'model = "base-model"', + `max_context_size = ${literal} # context`, + "[models.overridden]", + 'model = "overridden-model"', + "max_context_size = 1234", + "[models.overridden.overrides]", + `max_context_size = ${literal}`, + ].join("\n")); + assert.deepEqual(config.models, [ + { id: "base-model", contextWindow: expected }, + { id: "overridden-model", contextWindow: expected }, + ]); + }); +} + +for (const literal of ["1.5", "2026-01-01", "01", "1 trailing", "9007199254740993", "1_", "1__0", "0x_10", "+0x10", "-1"]) { + test(`client TOML readers reject the non-limit ${literal} without using its prefix`, () => { + const codex = parseCodexToml([ + 'model = "custom-model"', + `model_context_window = ${literal}`, + `model_auto_compact_token_limit = ${literal}`, + `model_max_output_tokens = ${literal}`, + ].join("\n")); + assert.equal(codex.contextWindow, undefined); + assert.equal(codex.autoCompactLimit, undefined); + assert.equal(codex.maxOutput, undefined); + assert.equal(codex.modelWindows, undefined); + const kimi = parseKimiToml([ + "[models.invalid]", + `max_context_size = ${literal}`, + "[models.valid]", + "max_context_size = 65536", + "[models.valid.overrides]", + `max_context_size = ${literal}`, + ].join("\n")); + assert.deepEqual(kimi.models, [{ id: "valid", contextWindow: 65536 }]); + }); +} From baf19a141153abc290af603bd57cd795f20aa269 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:25:25 +0800 Subject: [PATCH 41/67] docs: register shared TOML integer reader in architecture map (cherry picked from commit 46a95888c2471906287a9454c8805968a91a7e2b) --- reference/architecture.md | 1 + 1 file changed, 1 insertion(+) diff --git a/reference/architecture.md b/reference/architecture.md index 732d166a27..338edddc75 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -183,6 +183,7 @@ billion-context/ │ ├── kiro.ts # Kiro CLI: model-gateway hosts, built-in developer/system→user roles, kiro:conversation_id identity │ ├── launcher.ts # `bili ` launcher (auto-proxies both schemes) │ ├── client-config.ts # Read-only discovery of each client's own upstream config +│ ├── toml-integer.ts # Shared strict TOML integer reader for model token limits │ ├── plugin.ts # Cooperative plugin protocol (x-bili-plugin lane, legacy marker #920) │ ├── plugin-install.ts # `bili plugin install` driving host channels │ ├── dsh-channel.ts # dsh profile bundle management (refreshDshProfileCopy, #1196) From 78c6c22e5c7aa3fa05fa1170b5e3b38390b1accf Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 15:35:30 +0800 Subject: [PATCH 42/67] fix: restore archive content ownership per reference (cherry picked from commit 4fae3b0a25580f453a05b503c88fda111c41013c) --- src/storage/mysql.ts | 24 +++++- src/storage/store.ts | 29 ++++++- tests/archive-reference-counts.test.ts | 113 +++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 tests/archive-reference-counts.test.ts diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index b3005bb5f4..798514d9a9 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1871,7 +1871,29 @@ export class MysqlBackend implements StorageBackend { if (!dump.session || dump.session.sessionId !== sessionId) return { ok: false, reason: "export-mismatch" }; const sess = dump.session; await this.tx(async (c) => { - for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash); + const references = new Map(); + const addRef = (hash: unknown): void => { + if (typeof hash === "string") references.set(hash, (references.get(hash) ?? 0) + 1); + }; + for (const m of dump.messages) addRef(m.content_hash); + for (const mm of (dump as { mediaMaps?: Record[] }).mediaMaps ?? []) addRef(mm.media_hash); + for (const b of dump.blocks) { + const [twins] = await c.query("SELECT 1 FROM blocks WHERE session_id = ? AND block_id = ?", [sessionId, b.block_id]); + if ((twins as Row[]).length > 0) continue; + addRef(b.summary_hash); + addRef(b.one_hash); + } + for (const e of dump.ccrEntries) addRef(e.content_hash); + for (const cp of dump.checkpoints) { + const [twins] = await c.query("SELECT 1 FROM checkpoints WHERE cp_id = ?", [cp.cp_id]); + if ((twins as Row[]).length === 0) addRef(cp.state_hash); + } + for (const [hash, count] of references) { + const body = dump.content[hash] ?? await this.getContent(c, hash); + if (body === null || body === undefined) throw new Error(`archive content missing: ${hash}`); + await this.putContent(c, body, hash); + if (count > 1) await c.query("UPDATE content SET refcount = refcount + ? WHERE hash = ?", [count - 1, hash]); + } for (const m of dump.messages) { // #4665: pre-fix dumps lack both fields — recompute identity // from the CAS body (same formula as the v8 migration diff --git a/src/storage/store.ts b/src/storage/store.ts index 97c674f438..25a7bdc33e 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2154,7 +2154,27 @@ export class UnifiedStore { if (!dump.session || dump.session.sessionId !== sessionId) return { ok: false, reason: "export-mismatch" }; const sess: NonNullable = dump.session; this.db.transaction(() => { - for (const [hash, body] of Object.entries(dump.content)) this.putContent(Buffer.from(body, "utf8"), hash); + const references = new Map(); + const addRef = (hash: unknown): void => { + if (typeof hash === "string") references.set(hash, (references.get(hash) ?? 0) + 1); + }; + for (const m of dump.messages) addRef(m.content_hash); + for (const mm of (dump as { mediaMaps?: Record[] }).mediaMaps ?? []) addRef(mm.media_hash); + for (const b of dump.blocks) { + if (this.db.prepare("SELECT 1 FROM blocks WHERE session_id = ? AND block_id = ?").get(sessionId, b.block_id)) continue; + addRef(b.summary_hash); + addRef(b.one_hash); + } + for (const e of dump.ccrEntries) addRef(e.content_hash); + for (const cp of dump.checkpoints) { + if (!this.getCheckpoint(cp.cp_id as string)) addRef(cp.state_hash); + } + for (const [hash, count] of references) { + const body = dump.content[hash] ?? this.getContent(hash); + if (body === null || body === undefined) throw new Error(`archive content missing: ${hash}`); + this.putContent(body, hash); + if (count > 1) this.db.prepare("UPDATE content SET refcount = refcount + ? WHERE hash = ?").run(count - 1, hash); + } const insMsg = this.db.prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at, ts) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); for (const m of dump.messages) { // #4665: pre-fix dumps lack both fields — recompute identity @@ -2287,7 +2307,12 @@ export class UnifiedStore { "SELECT 1 AS one FROM blocks WHERE session_id = ? AND block_id = ?", ).get(sessionId, blockId); if (twin) return; - for (const [hash, body] of Object.entries(dump.content)) this.putContent(Buffer.from(body, "utf8"), hash); + for (const hash of [dump.block.summary_hash, dump.block.one_hash]) { + if (typeof hash !== "string") continue; + const body = dump.content[hash]; + if (body === undefined) throw new Error(`archive content missing: ${hash}`); + this.putContent(body, hash); + } this.db.prepare( `INSERT INTO blocks (block_id, session_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(session_id, block_id) DO NOTHING`, diff --git a/tests/archive-reference-counts.test.ts b/tests/archive-reference-counts.test.ts new file mode 100644 index 0000000000..cd41656fcc --- /dev/null +++ b/tests/archive-reference-counts.test.ts @@ -0,0 +1,113 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore, sha256Hex } from "../src/storage/store.ts"; +import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; + +function fixture() { + const dir = mkdtempSync(join(tmpdir(), "archive-refs-")); + const store = UnifiedStore.open(join(dir, "index.db")); + const plan = planIngest({ + id: "session", createdAt: 1, lastSeen: 2, meta: { protocol: "openai" }, + stats: {}, metadata: {}, state: { blocks: [], nextBlockId: 1, nextRunId: 1, + messageRefs: { byRaw: {}, byRef: {} } }, blockContents: new Map(), + lastMessages: [], lastMessagesFolded: false, + } as unknown as LegacySessionLike); + store.ingestSession(plan); + return { store, plan, archiveDir: join(dir, "archive"), close() { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } }; +} + +test("restored identical checkpoints each retain ownership of their CAS body", () => { + const f = fixture(); + try { + for (const id of ["first", "second"]) f.store.insertCheckpoint({ + cpId: id, sessionId: "session", seqAt: 0, blockSetHash: "", kind: "manual", + pinned: false, createdAt: 1, stateJson: '{"shared-checkpoint":true}', + }); + assert.equal(f.store.contentRefCountOfBody("shared-checkpoint"), 2); + assert.equal(f.store.archiveSession("session", f).ok, true); + assert.equal(f.store.restoreArchived("session", f).ok, true); + assert.equal(f.store.contentRefCountOfBody("shared-checkpoint"), 2); + assert.equal(f.store.deleteCheckpoint("first"), true); + assert.equal(f.store.checkpointState("second").stateJson, '{"shared-checkpoint":true}'); + assert.equal(f.store.deleteCheckpoint("second"), true); + assert.equal(f.store.contentRefCountOfBody("shared-checkpoint"), 0); + } finally { f.close(); } +}); + +test("already-live pinned checkpoints are not counted twice during restore", () => { + const f = fixture(); + try { + for (const id of ["pinned", "first", "second"]) f.store.insertCheckpoint({ + cpId: id, sessionId: "session", seqAt: 0, blockSetHash: "", kind: "manual", + pinned: id === "pinned", createdAt: 1, stateJson: '{"pinned-shared":true}', + }); + assert.equal(f.store.archiveSession("session", f).ok, true); + assert.equal(f.store.contentRefCountOfBody("pinned-shared"), 1); + assert.equal(f.store.restoreArchived("session", f).ok, true); + assert.equal(f.store.contentRefCountOfBody("pinned-shared"), 3); + f.store.deleteCheckpoint("first"); + f.store.deleteCheckpoint("second"); + assert.equal(f.store.checkpointState("pinned").stateJson, '{"pinned-shared":true}'); + assert.equal(f.store.contentRefCountOfBody("pinned-shared"), 1); + } finally { f.close(); } +}); + +for (const wholeSession of [true, false]) test(`block summary and one-line body share two references (${wholeSession ? "session" : "block"} restore)`, () => { + const f = fixture(); + try { + const body = '{"text":"shared-block-body","count":5}'; + f.plan.blocks.push({ blockId: "b1", ord: 0, startSeq: 1, endSeq: 2, startRef: 1, + endRef: 2, tier: 1, summary: body, summaryHash: sha256Hex(body), one: body, + oneHash: sha256Hex(body), supersedesBlockId: null, active: true, createdAt: 1, + coveredHash: "", payloadJson: "{}" }); + f.store.ingestSession(f.plan); + assert.equal(f.store.contentRefCountOfBody("shared-block-body"), 2); + if (wholeSession) { + assert.equal(f.store.archiveSession("session", f).ok, true); + assert.equal(f.store.restoreArchived("session", f).ok, true); + } else { + assert.equal(f.store.archiveBlock("session", "b1", f).ok, true); + assert.equal(f.store.restoreBlock("session", "b1", f).ok, true); + } + assert.equal(f.store.contentRefCountOfBody("shared-block-body"), 2); + f.store.deleteSession("session"); + assert.equal(f.store.contentRefCountOfBody("shared-block-body"), 0); + } finally { f.close(); } +}); + +test("MySQL restore charges each new checkpoint and preserves an existing pinned owner", async () => { + const f = fixture(); + try { + for (const id of ["pinned", "first", "second"]) f.store.insertCheckpoint({ + cpId: id, sessionId: "session", seqAt: 0, blockSetHash: "", kind: "manual", + pinned: id === "pinned", createdAt: 1, stateJson: '{"mysql-shared":true}', + }); + assert.equal(f.store.archiveSession("session", f).ok, true); + let references = 1; + let committed = false; + const connection = { + async query(sql: string, args: unknown[]) { + if (sql.startsWith("SELECT 1 FROM checkpoints")) return [args[0] === "pinned" ? [{}] : []]; + if (sql.startsWith("SELECT")) return [[]]; + if (sql.startsWith("INSERT INTO content")) references++; + if (sql.startsWith("UPDATE content SET refcount = refcount + ?")) references += Number(args[0]); + return [{ affectedRows: 1 }]; + }, + async beginTransaction() {}, async commit() { committed = true; }, + async rollback() {}, release() {}, + }; + const backend = new MysqlBackend({}, { + createPool: () => ({ async getConnection() { return connection; } }), + } as unknown as Mysql2Module); + assert.equal((await backend.restoreArchived("session", f)).ok, true); + assert.equal(committed, true); + assert.equal(references, 3); + } finally { f.close(); } +}); From 5adb837f8b939047da5a1ab5ce6d4fce0d356d71 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 15:59:00 +0800 Subject: [PATCH 43/67] fix: replace archive logs atomically after restore (cherry picked from commit ff1657b3bef27ebd7782ca5e563d9bebfd68e694) --- src/storage/mysql.ts | 6 +-- src/storage/store.ts | 27 ++++++++-- tests/atomic-archive-rewrite.test.ts | 77 ++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 7 deletions(-) create mode 100644 tests/atomic-archive-rewrite.test.ts diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 798514d9a9..e264a07d76 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -36,10 +36,10 @@ */ import type { Pool, PoolConnection, PoolOptions } from "mysql2/promise"; -import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync } from "node:fs"; +import { mkdirSync, appendFileSync, readFileSync, existsSync } from "node:fs"; import { join } from "node:path"; import { log as loggerLog } from "../logger.js"; -import { sha256Hex, viewPointerSeqs } from "./store.js"; +import { sha256Hex, viewPointerSeqs, rewriteArchiveLog } from "./store.js"; import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; import type { StorageBackend, StorageReadView } from "./backend.js"; @@ -1961,7 +1961,7 @@ export class MysqlBackend implements StorageBackend { } kept.push(l); } - writeFileSync(exportPath, kept.length > 0 ? kept.join("\n") + "\n" : ""); + rewriteArchiveLog(exportPath, kept.length > 0 ? kept.join("\n") + "\n" : ""); loggerLog("info", `[#3931] restored archived session ${sessionId} from ${exportPath} (line removed from the archive log)`); return { ok: true }; } diff --git a/src/storage/store.ts b/src/storage/store.ts index 25a7bdc33e..a7ab32f34b 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -16,8 +16,8 @@ * field, always guarded by those assertions. */ -import { createHash } from "node:crypto"; -import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync, unlinkSync, statSync } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync, unlinkSync, statSync, openSync, fsyncSync, fchmodSync, closeSync, renameSync, realpathSync } from "node:fs"; import { dirname, join } from "node:path"; import { log as loggerLog } from "../logger.js"; import { openSqliteDatabase, type SqliteDatabase, type SqliteEngineName, type SqliteEngineSource } from "./driver.js"; @@ -59,6 +59,25 @@ const GC_ZERO_REF_BATCH_SIZE = 2000; * session's distinct released hashes exceed it only pathologically, and * the loop simply continues with the next chunk. */ const ZERO_REF_SWEEP_CHUNK = 500; + +export function rewriteArchiveLog(file: string, text: string): void { + const target = realpathSync(file); + const tmp = `${target}.${randomUUID()}.tmp`; + let fd: number | undefined; + try { + const mode = statSync(target).mode & 0o777; + fd = openSync(tmp, "wx", mode); + fchmodSync(fd, mode); + writeFileSync(fd, text); + fsyncSync(fd); + closeSync(fd); + fd = undefined; + renameSync(tmp, target); + } finally { + if (fd !== undefined) closeSync(fd); + try { unlinkSync(tmp); } catch { /* rename already consumed the temporary file */ } + } +} /** #4739 test instrumentation: message rows materialized by getMessages / * getMessagesAt (each row = one CAS read + media hydration). Production * code never reads this; tests assert ordinary loads stop decoding the @@ -2223,7 +2242,7 @@ export class UnifiedStore { } catch { continue; } kept.push(l); } - writeFileSync(exportPath, kept.length > 0 ? kept.join("\n") + "\n" : ""); + rewriteArchiveLog(exportPath, kept.length > 0 ? kept.join("\n") + "\n" : ""); loggerLog("info", `[#3901] restored archived session ${sessionId} from ${exportPath} (line removed from the archive log)`); return { ok: true }; } @@ -2332,7 +2351,7 @@ export class UnifiedStore { } catch { continue; } kept.push(l); } - writeFileSync(exportPath, kept.length > 0 ? kept.join("\n") + "\n" : ""); + rewriteArchiveLog(exportPath, kept.length > 0 ? kept.join("\n") + "\n" : ""); loggerLog("info", `[#5312] restored block ${blockId} into ${sessionId} from ${exportPath}`); return { ok: true }; } diff --git a/tests/atomic-archive-rewrite.test.ts b/tests/atomic-archive-rewrite.test.ts new file mode 100644 index 0000000000..d53f7bcb59 --- /dev/null +++ b/tests/atomic-archive-rewrite.test.ts @@ -0,0 +1,77 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +function session(id: string) { + return planIngest({ + id, createdAt: 1, lastSeen: 2, meta: {}, stats: {}, metadata: {}, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(), lastMessages: [{ id: `m-${id}`, role: "user", contentType: "text", text: id }], + lastMessagesFolded: false, + } as unknown as LegacySessionLike); +} + +test("a partial archive cleanup write cannot destroy other archived sessions", (t) => { + const dir = fs.mkdtempSync(join(tmpdir(), "atomic-archive-")); + const store = UnifiedStore.open(join(dir, "index.db")); + try { + const archiveDir = join(dir, "archive"); + for (const id of ["restore-me", "retain-me"]) { + store.ingestSession(session(id)); + assert.equal(store.archiveSession(id, { archiveDir }).ok, true); + } + const file = join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); + const before = fs.readFileSync(file); + const write = fs.writeFileSync; + const injected = t.mock.method(fs, "writeFileSync", (...args: Parameters) => { + const [target, data, options] = args; + if (typeof target === "number" || target === file) { + write(target, String(data).slice(0, 8), options); + throw Object.assign(new Error("injected partial write"), { code: "ENOSPC" }); + } + return write(...args); + }); + syncBuiltinESMExports(); + try { + assert.throws(() => store.restoreArchived("restore-me", { archiveDir }), /injected partial write/); + } finally { + injected.mock.restore(); + syncBuiltinESMExports(); + } + assert.deepEqual(fs.readFileSync(file), before); + assert.deepEqual(fs.readdirSync(archiveDir), [UnifiedStore.ARCHIVE_LOG_NAME]); + assert.equal(store.restoreArchived("retain-me", { archiveDir }).ok, true); + assert.equal(store.getMessages("retain-me")[0].msgId, "m-retain-me"); + } finally { + store.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("successful cleanup preserves archive permissions under a restrictive umask", () => { + const previousUmask = process.umask(0o077); + const dir = fs.mkdtempSync(join(tmpdir(), "atomic-archive-ok-")); + const store = UnifiedStore.open(join(dir, "index.db")); + try { + const archiveDir = join(dir, "archive"); + for (const id of ["first", "second"]) { + store.ingestSession(session(id)); + store.archiveSession(id, { archiveDir }); + } + const file = join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); + fs.chmodSync(file, 0o640); + assert.equal(store.restoreArchived("first", { archiveDir }).ok, true); + assert.equal(JSON.parse(fs.readFileSync(file, "utf8")).sessionId, "second"); + if (process.platform !== "win32") assert.equal(fs.statSync(file).mode & 0o777, 0o640); + assert.deepEqual(fs.readdirSync(archiveDir), [UnifiedStore.ARCHIVE_LOG_NAME]); + } finally { + process.umask(previousUmask); + store.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); From 468b9f520fe4d0affb1aa35afda9f88570164c3c Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:10:20 +0800 Subject: [PATCH 44/67] test: scope archive permission checks to POSIX (cherry picked from commit 18837a0dab6fec02b6e8c4ff8518ba5688c11edf) --- tests/atomic-archive-rewrite.test.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/atomic-archive-rewrite.test.ts b/tests/atomic-archive-rewrite.test.ts index d53f7bcb59..e782da77ed 100644 --- a/tests/atomic-archive-rewrite.test.ts +++ b/tests/atomic-archive-rewrite.test.ts @@ -53,8 +53,7 @@ test("a partial archive cleanup write cannot destroy other archived sessions", ( } }); -test("successful cleanup preserves archive permissions under a restrictive umask", () => { - const previousUmask = process.umask(0o077); +test("successful cleanup preserves records and POSIX archive permissions", () => { const dir = fs.mkdtempSync(join(tmpdir(), "atomic-archive-ok-")); const store = UnifiedStore.open(join(dir, "index.db")); try { @@ -64,13 +63,18 @@ test("successful cleanup preserves archive permissions under a restrictive umask store.archiveSession(id, { archiveDir }); } const file = join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); - fs.chmodSync(file, 0o640); - assert.equal(store.restoreArchived("first", { archiveDir }).ok, true); + const posixPermissions = process.platform !== "win32"; + if (posixPermissions) fs.chmodSync(file, 0o640); + const previousUmask = posixPermissions ? process.umask(0o077) : undefined; + try { + assert.equal(store.restoreArchived("first", { archiveDir }).ok, true); + } finally { + if (previousUmask !== undefined) process.umask(previousUmask); + } assert.equal(JSON.parse(fs.readFileSync(file, "utf8")).sessionId, "second"); - if (process.platform !== "win32") assert.equal(fs.statSync(file).mode & 0o777, 0o640); + if (posixPermissions) assert.equal(fs.statSync(file).mode & 0o777, 0o640); assert.deepEqual(fs.readdirSync(archiveDir), [UnifiedStore.ARCHIVE_LOG_NAME]); } finally { - process.umask(previousUmask); store.close(); fs.rmSync(dir, { recursive: true, force: true }); } From 851c57f7d7c56661e329752b941a2f7ea52fd7ce Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:01:49 +0800 Subject: [PATCH 45/67] fix: enforce extension storage validation across backends (cherry picked from commit 43b32df46696e656c7f33b56db1717d31e57a897) --- src/storage/mysql.ts | 21 ++++++- src/storage/store.ts | 6 +- tests/extension-storage-contract.test.ts | 71 ++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 tests/extension-storage-contract.test.ts diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index e264a07d76..9c68f41c6a 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -39,7 +39,7 @@ import type { Pool, PoolConnection, PoolOptions } from "mysql2/promise"; import { mkdirSync, appendFileSync, readFileSync, existsSync } from "node:fs"; import { join } from "node:path"; import { log as loggerLog } from "../logger.js"; -import { sha256Hex, viewPointerSeqs, rewriteArchiveLog } from "./store.js"; +import { sha256Hex, viewPointerSeqs, rewriteArchiveLog, validateExtKey, serializeExtValue } from "./store.js"; import { identityOfStoredContent } from "./identity.js"; import type { MessageContentStore } from "acp-kernel"; import type { StorageBackend, StorageReadView } from "./backend.js"; @@ -2137,7 +2137,9 @@ export class MysqlBackend implements StorageBackend { // --- session_ext / kv -------------------------------------------------- async setSessionExt(sessionId: string, key: string, value: unknown): Promise { - const valueJson = JSON.stringify(value); + validateExtKey(sessionId, "session id"); + validateExtKey(key, "key"); + const valueJson = serializeExtValue(value); await this.pool.query( "INSERT INTO session_ext (session_id, ext_key, value_json, updated_at) VALUES (?, ?, ?, ?) " + "ON DUPLICATE KEY UPDATE value_json = VALUES(value_json), updated_at = VALUES(updated_at)", @@ -2146,6 +2148,8 @@ export class MysqlBackend implements StorageBackend { } async getSessionExt(sessionId: string, key: string): Promise { + validateExtKey(sessionId, "session id"); + validateExtKey(key, "key"); const [rows] = (await this.pool.query( "SELECT value_json FROM session_ext WHERE session_id = ? AND ext_key = ?", [sessionId, key], @@ -2154,6 +2158,7 @@ export class MysqlBackend implements StorageBackend { } async listSessionExt(sessionId: string): Promise<{ key: string; value: unknown; updatedAt: number }[]> { + validateExtKey(sessionId, "session id"); const [rows] = (await this.pool.query( "SELECT ext_key, value_json, updated_at FROM session_ext WHERE session_id = ? ORDER BY ext_key", [sessionId], @@ -2162,6 +2167,8 @@ export class MysqlBackend implements StorageBackend { } async deleteSessionExt(sessionId: string, key?: string): Promise { + validateExtKey(sessionId, "session id"); + if (key !== undefined) validateExtKey(key, "key"); if (key === undefined) { const [r] = (await this.pool.query("DELETE FROM session_ext WHERE session_id = ?", [sessionId])) as unknown as [{ affectedRows: number }, unknown]; return r.affectedRows; @@ -2171,7 +2178,9 @@ export class MysqlBackend implements StorageBackend { } async setKv(namespace: string, key: string, value: unknown): Promise { - const valueJson = JSON.stringify(value); + validateExtKey(namespace, "namespace"); + validateExtKey(key, "key"); + const valueJson = serializeExtValue(value); await this.pool.query( "INSERT INTO kv_store (namespace, kv_key, value_json, updated_at) VALUES (?, ?, ?, ?) " + "ON DUPLICATE KEY UPDATE value_json = VALUES(value_json), updated_at = VALUES(updated_at)", @@ -2180,6 +2189,8 @@ export class MysqlBackend implements StorageBackend { } async getKv(namespace: string, key: string): Promise { + validateExtKey(namespace, "namespace"); + validateExtKey(key, "key"); const [rows] = (await this.pool.query( "SELECT value_json FROM kv_store WHERE namespace = ? AND kv_key = ?", [namespace, key], @@ -2191,10 +2202,12 @@ export class MysqlBackend implements StorageBackend { const where: string[] = []; const args: unknown[] = []; if (namespace !== undefined) { + validateExtKey(namespace, "namespace"); where.push("namespace = ?"); args.push(namespace); } if (keyPrefix !== undefined && keyPrefix !== "") { + validateExtKey(keyPrefix, "key prefix"); where.push("kv_key LIKE CONCAT(?, '%')"); args.push(keyPrefix.replace(/[\\%_]/g, (ch) => "\\" + ch)); } @@ -2207,6 +2220,8 @@ export class MysqlBackend implements StorageBackend { } async deleteKv(namespace: string, key?: string): Promise { + validateExtKey(namespace, "namespace"); + if (key !== undefined) validateExtKey(key, "key"); if (key === undefined) { const [r] = (await this.pool.query("DELETE FROM kv_store WHERE namespace = ?", [namespace])) as unknown as [{ affectedRows: number }, unknown]; return r.affectedRows; diff --git a/src/storage/store.ts b/src/storage/store.ts index a7ab32f34b..9560b08d8d 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -359,7 +359,7 @@ export interface IngestSessionInput { const EXT_KEY_MAX = 256; const EXT_VALUE_MAX_BYTES = 4 * 1024 * 1024; -function validateExtKey(value: string, what: string): void { +export function validateExtKey(value: string, what: string): void { if (typeof value !== "string" || value.length === 0 || value.length > EXT_KEY_MAX) { throw new Error(`[storage] extension ${what} must be a non-empty string of at most ${EXT_KEY_MAX} characters`); } @@ -371,12 +371,12 @@ function validateExtKey(value: string, what: string): void { } } -function serializeExtValue(value: unknown): string { +export function serializeExtValue(value: unknown): string { const json = JSON.stringify(value); if (json === undefined) { throw new Error("[storage] extension values must be JSON-serializable (undefined/functions are not)"); } - if (json.length > EXT_VALUE_MAX_BYTES) { + if (Buffer.byteLength(json, "utf8") > EXT_VALUE_MAX_BYTES) { throw new Error(`[storage] extension value exceeds ${EXT_VALUE_MAX_BYTES} bytes — store large payloads in the content CAS instead`); } return json; diff --git a/tests/extension-storage-contract.test.ts b/tests/extension-storage-contract.test.ts new file mode 100644 index 0000000000..fe32f698fc --- /dev/null +++ b/tests/extension-storage-contract.test.ts @@ -0,0 +1,71 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; + +const LIMIT = 4 * 1024 * 1024; + +test("SQLite extension limits measure UTF-8 bytes before changing the existing value", () => { + const dir = mkdtempSync(join(tmpdir(), "extension-bytes-")); + const store = UnifiedStore.open(join(dir, "index.db")); + try { + store.setKv("test", "value", "keep"); + const multibyte = "界".repeat(Math.ceil(LIMIT / 3)); + assert.throws(() => store.setKv("test", "value", multibyte), /exceeds.*bytes/); + assert.equal(store.getKv("test", "value"), "keep"); + store.setKv("test", "boundary", "a".repeat(LIMIT - 2)); + assert.equal((store.getKv("test", "boundary") as string).length, LIMIT - 2); + assert.throws(() => store.setKv("test", "boundary", "a".repeat(LIMIT - 1)), /exceeds.*bytes/); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +function mysqlFixture() { + const queries: unknown[][] = []; + const backend = new MysqlBackend({}, { + createPool: () => ({ async query(sql: string, args: unknown[]) { + queries.push([sql, ...args]); + return [sql.startsWith("SELECT") ? [] : { affectedRows: 1 }]; + } }), + } as unknown as Mysql2Module); + return { backend, queries }; +} + +for (const invalid of ["", "x".repeat(257), "bad\u0000key", "bad\u007fkey"]) { + test(`MySQL extension operations reject invalid keys before SQL (${JSON.stringify(invalid.slice(0, 20))})`, async () => { + const { backend: b, queries } = mysqlFixture(); + for (const run of [ + () => b.setKv(invalid, "key", 1), () => b.setKv("ns", invalid, 1), + () => b.getKv(invalid, "key"), () => b.getKv("ns", invalid), + () => b.deleteKv(invalid), () => b.deleteKv("ns", invalid), + () => b.listKv(invalid), + () => b.setSessionExt(invalid, "key", 1), () => b.setSessionExt("session", invalid, 1), + () => b.getSessionExt(invalid, "key"), () => b.getSessionExt("session", invalid), + () => b.listSessionExt(invalid), () => b.deleteSessionExt(invalid), + () => b.deleteSessionExt("session", invalid), + ]) await assert.rejects(run, /extension/); + if (invalid !== "") await assert.rejects(() => b.listKv("ns", invalid), /extension/); + assert.equal(queries.length, 0); + }); +} + +test("MySQL extension writes reject undefined and over-budget values consistently", async () => { + const { backend, queries } = mysqlFixture(); + for (const value of [undefined, () => 1, "界".repeat(Math.ceil(LIMIT / 3))]) { + await assert.rejects(() => backend.setKv("ns", "key", value), /JSON-serializable|exceeds.*bytes/); + await assert.rejects(() => backend.setSessionExt("session", "key", value), /JSON-serializable|exceeds.*bytes/); + } + assert.equal(queries.length, 0); + await backend.setKv("ns", "key", { valid: "界" }); + await backend.setSessionExt("session", "key", null); + assert.equal(queries[0][3], '{"valid":"界"}'); + assert.equal(queries[1][3], "null"); + await backend.listKv("ns", ""); + await backend.deleteKv("ns"); + await backend.deleteSessionExt("session"); +}); From 81724e18a5d09c82daa383f55c23b73fe36a065b Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:03:04 +0800 Subject: [PATCH 46/67] fix: accept completed summaries with empty tool call arrays (cherry picked from commit ec11f43f5dd756b731a06c0502b1cc39be376185) --- src/external-summary-http.ts | 5 +++- tests/summary-empty-tool-calls.test.ts | 37 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/summary-empty-tool-calls.test.ts diff --git a/src/external-summary-http.ts b/src/external-summary-http.ts index 9dbc6bb6f7..88c2b60902 100644 --- a/src/external-summary-http.ts +++ b/src/external-summary-http.ts @@ -122,7 +122,10 @@ function completed(protocol: PreflightProtocol, events: unknown[], streaming: bo if (streaming ? item.type === "message_stop" && anthropicStopReason : anthropicStopReason) terminal = true; } else if (protocol === "openai") { const choice = Array.isArray(item.choices) ? record(item.choices[0]) : undefined; - if (record(choice?.message)?.tool_calls || record(choice?.delta)?.tool_calls) return false; + if ([record(choice?.message), record(choice?.delta)].some((payload) => { + const calls = payload?.tool_calls; + return Array.isArray(calls) ? calls.length > 0 : !!calls; + })) return false; if (choice?.finish_reason) { if (choice.finish_reason !== "stop") return false; terminal = true; diff --git a/tests/summary-empty-tool-calls.test.ts b/tests/summary-empty-tool-calls.test.ts new file mode 100644 index 0000000000..9237710b2b --- /dev/null +++ b/tests/summary-empty-tool-calls.test.ts @@ -0,0 +1,37 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createSummaryHttpCandidate } from "../src/external-summary-http.ts"; +import { withFetchTransport } from "../src/fetch-transport.ts"; +import { _liveUpstreamTimersForTest } from "../src/fetch-util.ts"; + +const text = "Preserved decisions and unfinished objectives."; +const work = { content: "Evidence to retain", reference: "Current task", instructions: "Keep decisions" }; + +async function summarize(stream: boolean, calls: unknown, finishReason = "stop"): Promise { + const choice = stream + ? { delta: { content: text, tool_calls: calls }, finish_reason: finishReason } + : { message: { content: text, tool_calls: calls }, finish_reason: finishReason }; + const body = stream ? `data: ${JSON.stringify({ choices: [choice] })}\n\ndata: [DONE]\n\n` : JSON.stringify({ choices: [choice] }); + const candidate = createSummaryHttpCandidate({ + url: "https://summary.invalid/v1/chat/completions", protocol: "openai", model: "test", + headers: {}, stream, + }, 4096); + return withFetchTransport(async () => new Response(body, { + headers: { "content-type": stream ? "text/event-stream" : "application/json" }, + }), () => candidate.summarize(work, AbortSignal.timeout(4000))); +} + +for (const stream of [false, true]) { + for (const calls of [undefined, null, []]) test(`completed ${stream ? "SSE" : "JSON"} summaries accept ${JSON.stringify(calls)} tool_calls`, async () => { + assert.equal(await summarize(stream, calls), text); + assert.equal(_liveUpstreamTimersForTest(), 0); + }); + for (const calls of [[{ id: "call_1", type: "function" }], { malformed: true }]) test(`${stream ? "SSE" : "JSON"} summaries still reject nonempty tool requests`, async () => { + await assert.rejects(() => summarize(stream, calls), /did not complete/); + assert.equal(_liveUpstreamTimersForTest(), 0); + }); + test(`${stream ? "SSE" : "JSON"} summaries with empty tool_calls still require a complete finish reason`, async () => { + await assert.rejects(() => summarize(stream, [], "length"), /did not complete/); + assert.equal(_liveUpstreamTimersForTest(), 0); + }); +} From 7b615ed1a4ec56692c57a0e2c5c14a222eff0728 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:04:27 +0800 Subject: [PATCH 47/67] fix: bound uncovered message range queries (cherry picked from commit 0f415e00ad9a34f4c8d784a9009fc4b4e30309b8) --- src/storage/store.ts | 35 +++++++++++++---- tests/uncovered-range-bounds.test.ts | 56 ++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 tests/uncovered-range-bounds.test.ts diff --git a/src/storage/store.ts b/src/storage/store.ts index 9560b08d8d..502233c85e 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -3180,17 +3180,36 @@ export class UnifiedStore { /** #D5-5 render windowing: full rows (content + media) ONLY for seqs * outside the covered intervals. Inclusive [start, end] pairs; overlaps - * are harmless (each pair contributes an outside-of-pair conjunct). + * are merged before querying the uncovered gaps in bounded batches. * Empty interval list = every live row. */ uncoveredRows(sessionId: string, intervals: readonly [number, number][]): (MessageRow & { contentHash: string })[] { - let sql = "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts FROM messages WHERE session_id = ? AND retired_at IS NULL"; - const params: unknown[] = [sessionId]; - for (const [a, b] of intervals) { - sql += " AND (seq < ? OR seq > ?)"; - params.push(a, b); + const merged: [number, number][] = []; + for (const [start, end] of [...intervals].sort((a, b) => a[0] - b[0])) { + if (start > end) continue; + const last = merged.at(-1); + if (last && start <= last[1] + 1) last[1] = Math.max(last[1], end); + else merged.push([start, end]); + } + const gaps: Array<[number | undefined, number | undefined]> = []; + let lower: number | undefined; + for (const [start, end] of merged) { + gaps.push([lower, start]); + lower = end; + } + gaps.push([lower, undefined]); + const rows: Record[] = []; + for (let offset = 0; offset < gaps.length; offset += 200) { + const params: unknown[] = [sessionId]; + const clauses = gaps.slice(offset, offset + 200).map(([lo, hi]) => { + const bounds: string[] = []; + if (lo !== undefined) { bounds.push("seq > ?"); params.push(lo); } + if (hi !== undefined) { bounds.push("seq < ?"); params.push(hi); } + return bounds.length > 0 ? `(${bounds.join(" AND ")})` : "1"; + }); + const sql = "SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts FROM messages WHERE session_id = ? AND retired_at IS NULL" + + ` AND (${clauses.join(" OR ")}) ORDER BY seq`; + for (const row of this.db.prepare(sql).all(...params) as Record[]) rows.push(row); } - sql += " ORDER BY seq"; - const rows = this.db.prepare(sql).all(...params) as Record[]; return rows.map((r) => ({ seq: r.seq as number, msgId: r.msg_id as string, diff --git a/tests/uncovered-range-bounds.test.ts b/tests/uncovered-range-bounds.test.ts new file mode 100644 index 0000000000..f31062c16f --- /dev/null +++ b/tests/uncovered-range-bounds.test.ts @@ -0,0 +1,56 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +function fixture(count: number) { + const dir = mkdtempSync(join(tmpdir(), "uncovered-ranges-")); + const store = UnifiedStore.open(join(dir, "index.db")); + const plan = planIngest({ + id: "session", createdAt: 1, lastSeen: 2, meta: {}, stats: {}, metadata: {}, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(), lastMessages: Array.from({ length: count }, (_, i) => ({ + id: `message-${i + 1}`, role: "user", contentType: "text", text: `body ${i + 1}`, + })), lastMessagesFolded: false, + } as unknown as LegacySessionLike); + store.ingestSession(plan, undefined, { completeLog: true }); + return { store, plan, close() { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } }; +} + +test("over a thousand disjoint folded ranges remain queryable without SQLite expression limits", () => { + const f = fixture(2003); + try { + const ranges: [number, number][] = Array.from({ length: 1001 }, (_, i) => [2 * i + 1, 2 * i + 1]); + ranges.reverse(); + const before = JSON.stringify(ranges); + const rows = f.store.uncoveredRows("session", ranges); + assert.deepEqual(rows.map((row) => row.seq), Array.from({ length: 1002 }, (_, i) => 2 * i)); + assert.equal(JSON.parse(rows.at(-1)!.content).text, "body 2003"); + assert.equal(JSON.stringify(ranges), before); + assert.equal(new Set(rows.map((row) => row.seq)).size, rows.length); + } finally { f.close(); } +}); + +test("bounded range queries preserve inclusive overlap, empty and reversed-range behavior", () => { + const f = fixture(12); + try { + const cases: [Array<[number, number]>, number[]][] = [ + [[], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]], + [[[4, 8], [2, 5], [8, 9], [4, 8]], [0, 1, 10, 11]], + [[[2, 2], [3, 3], [4, 4]], [0, 1, 5, 6, 7, 8, 9, 10, 11]], + [[[7, 3]], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]], + [[[-100, 100]], []], + [[[-5, -1], [12, 15]], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]], + ]; + for (const [ranges, expected] of cases) assert.deepEqual(f.store.uncoveredRows("session", ranges).map((row) => row.seq), expected); + f.plan.messages = f.plan.messages.slice(1); + f.store.ingestSession(f.plan, undefined, { completeLog: true }); + assert.deepEqual(f.store.uncoveredRows("session", [[3, 12]]).map((row) => row.seq), [1, 2]); + } finally { f.close(); } +}); From 6407af809c0774eba3e707339f7bc1d81a5b92bc Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:05:35 +0800 Subject: [PATCH 48/67] fix: honor output paths for JSONL exports (cherry picked from commit 195c732f82b006e42fc094b036afa73875c231a5) --- src/export.ts | 157 +++++++++++++++--------------- tests/export-jsonl-output.test.ts | 67 +++++++++++++ 2 files changed, 147 insertions(+), 77 deletions(-) create mode 100644 tests/export-jsonl-output.test.ts diff --git a/src/export.ts b/src/export.ts index 29f70f095f..f467b76f36 100644 --- a/src/export.ts +++ b/src/export.ts @@ -408,6 +408,81 @@ export function renderSessionJsonlFromSession(s: Session, full: boolean, content return `${lines.join("\n")}\n`; } +async function renderSelectedSessionJsonl(store: SessionStore, session: Session, full: boolean): Promise { + // Backend-aware open (#3388 step 2): storage.backend="mysql" + a DSN + // exports straight from MySQL; everything else (and every fallback) + // stays on the local sqlite index. A mysql failure degrades to the + // legacy-tail renderer exactly like a corrupt index does. + const backendName = resolveBackendName(process.env, fileStorageBackend()); + if (backendName === "mysql") { + const dsn = resolveMysqlDsn(process.env, fileStorageMysqlDsn()); + if (dsn.length > 0) { + try { + const backend = await openBackend("mysql", dsn); + try { + await backend.init?.(); + if (await backend.getSessionRow(session.id)) { + return await renderSessionJsonlFromTables(backend, session.id, full); + } + } finally { + await backend.close(); + } + } catch (err) { + process.stderr.write(`[export] mysql backend unreadable (${String(err)}) — falling back to the local index +`); + } + } + } + const index = path.join(store.dir, "index.db"); + const encrypted = process.env.BILI_ENCRYPTION_KEY !== undefined && process.env.BILI_ENCRYPTION_KEY !== ""; + if (existsSync(index) && !encrypted) { + // #3931: the selection is enforced at open time, mirroring the lane-boot + // split — a registered-but-unshipped choice (mysql, step 2) fails here + // with its documented backend error before any file access, so a deliberate + // selection never silently reads sqlite; a typo'd name warns once and reads sqlite. + const selected = selectedBackendRaw(process.env, fileStorageBackend()); + if (selected && selected !== "sqlite") { + if (listBackendNames().includes(selected)) { + // #3931 step 2: open is async now. The contract from the + // #3932 review holds — a deliberate selection never + // silently reads sqlite: without a DSN the open rejects + // with the documented error (bubbles out of the export); + // with a DSN the probe opens + closes cleanly (the real + // mysql export above already ran; if it degraded, its + // stderr notice announced the sqlite fallback). + const probe = await openBackend(selected, selected === "mysql" ? resolveMysqlDsn(process.env, fileStorageMysqlDsn()) : index); + await probe.close(); + } else { + const notice = backendSelectionNotice(selected); + if (notice) process.stderr.write(`[export] ${notice}\n`); + } + } + // Guarded open: a corrupt/foreign index.db must degrade to the + // legacy-tail renderer, not fail the export (#2690 review notes). + // Encryption gates the read for the same reason the ingest side + // does — index.db holds cleartext payloads, so an encrypted lane + // must never read from it. + try { + // #3931 step 2: openBackend is async (network backends + // open a pool + DDL init); the sqlite path resolves + // immediately. + const backend = await openBackend("sqlite", index); + try { + if (await backend.getSessionRow(session.id)) { + const disk = store.loadContentStore(session); + const extra = disk ? new Map(Object.entries(disk.byRef).map(([ref, e]) => [ref, disk.byHash[e.hash] ?? ""] as const)) : undefined; + return await renderSessionJsonlFromTables(backend, session.id, full, extra); + } + } finally { + await backend.close(); + } + } catch (err) { + process.stderr.write(`[export] unified index unreadable (${String(err)}) — falling back to the legacy-tail renderer\n`); + } + } + return renderSessionJsonlFromSession(session, full, store.loadContentStore(session)); +} + export async function exportSession(selector: string | undefined, opts: ExportOptions = {}): Promise { const store = new SessionStore({ dir: opts.dir, enabled: true }); try { @@ -430,87 +505,15 @@ export async function exportSession(selector: string | undefined, opts: ExportOp const ids = matches.map((s) => s.id).join(", "); throw new Error(`selector "${selector}" matches ${matches.length} sessions (${ids}); use the full session id`); } - if (opts.jsonl) { - // Backend-aware open (#3388 step 2): storage.backend="mysql" + a DSN - // exports straight from MySQL; everything else (and every fallback) - // stays on the local sqlite index. A mysql failure degrades to the - // legacy-tail renderer exactly like a corrupt index does. - const backendName = resolveBackendName(process.env, fileStorageBackend()); - if (backendName === "mysql") { - const dsn = resolveMysqlDsn(process.env, fileStorageMysqlDsn()); - if (dsn.length > 0) { - try { - const backend = await openBackend("mysql", dsn); - try { - await backend.init?.(); - if (await backend.getSessionRow(matches[0]!.id)) { - return await renderSessionJsonlFromTables(backend, matches[0]!.id, opts.full ?? false); - } - } finally { - await backend.close(); - } - } catch (err) { - process.stderr.write(`[export] mysql backend unreadable (${String(err)}) — falling back to the local index -`); - } - } - } - const index = path.join(store.dir, "index.db"); - const encrypted = process.env.BILI_ENCRYPTION_KEY !== undefined && process.env.BILI_ENCRYPTION_KEY !== ""; - if (existsSync(index) && !encrypted) { - // #3931: the selection is enforced at open time, mirroring the lane-boot - // split — a registered-but-unshipped choice (mysql, step 2) fails here - // with its documented backend error before any file access, so a deliberate - // selection never silently reads sqlite; a typo'd name warns once and reads sqlite. - const selected = selectedBackendRaw(process.env, fileStorageBackend()); - if (selected && selected !== "sqlite") { - if (listBackendNames().includes(selected)) { - // #3931 step 2: open is async now. The contract from the - // #3932 review holds — a deliberate selection never - // silently reads sqlite: without a DSN the open rejects - // with the documented error (bubbles out of the export); - // with a DSN the probe opens + closes cleanly (the real - // mysql export above already ran; if it degraded, its - // stderr notice announced the sqlite fallback). - const probe = await openBackend(selected, selected === "mysql" ? resolveMysqlDsn(process.env, fileStorageMysqlDsn()) : index); - await probe.close(); - } else { - const notice = backendSelectionNotice(selected); - if (notice) process.stderr.write(`[export] ${notice}\n`); - } - } - // Guarded open: a corrupt/foreign index.db must degrade to the - // legacy-tail renderer, not fail the export (#2690 review notes). - // Encryption gates the read for the same reason the ingest side - // does — index.db holds cleartext payloads, so an encrypted lane - // must never read from it. - try { - // #3931 step 2: openBackend is async (network backends - // open a pool + DDL init); the sqlite path resolves - // immediately. - const backend = await openBackend("sqlite", index); - try { - if (await backend.getSessionRow(matches[0]!.id)) { - const disk = store.loadContentStore(matches[0]!); - const extra = disk ? new Map(Object.entries(disk.byRef).map(([ref, e]) => [ref, disk.byHash[e.hash] ?? ""] as const)) : undefined; - return await renderSessionJsonlFromTables(backend, matches[0]!.id, opts.full ?? false, extra); - } - } finally { - await backend.close(); - } - } catch (err) { - process.stderr.write(`[export] unified index unreadable (${String(err)}) — falling back to the legacy-tail renderer\n`); - } - } - return renderSessionJsonlFromSession(matches[0]!, opts.full ?? false, store.loadContentStore(matches[0]!)); - } - const markdown = renderHandoff(matches[0]!, opts.full ?? false); + const content = opts.jsonl + ? await renderSelectedSessionJsonl(store, matches[0]!, opts.full ?? false) + : renderHandoff(matches[0]!, opts.full ?? false); if (opts.output) { mkdirSync(path.dirname(path.resolve(opts.output)), { recursive: true }); - writeFileSync(opts.output, markdown, "utf8"); + writeFileSync(opts.output, content, "utf8"); return `written to ${opts.output}`; } - return markdown; + return content; } finally { // The one-shot store holds index.db open in unified modes; close it or // Windows callers cannot delete their temp dirs (EPERM). diff --git a/tests/export-jsonl-output.test.ts b/tests/export-jsonl-output.test.ts new file mode 100644 index 0000000000..b423235dba --- /dev/null +++ b/tests/export-jsonl-output.test.ts @@ -0,0 +1,67 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInitialState } from "acp-kernel"; +import { exportSession } from "../src/export.ts"; +import { SessionStore } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; + +function session(): Session { + return { + id: "jsonl-output", meta: { title: "Export output", protocol: "responses" }, + stats: { requests: 1, tokensSaved: 0, inputTokens: 10, cachedTokens: 0, outputTokens: 0, + cacheSamples: 0, lastInputTokens: 10, contextTokens: 10, compressCreditTokens: 0, + retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + metadata: {}, createdAt: 1, lastSeen: 2, state: createInitialState(), pendingRetrievals: [], + blockContents: new Map(), inFlight: 0, persisted: false, lastMessagesFolded: false, + lastMessages: [{ id: "message", role: "user", contentType: "text", text: "original prompt" }], + }; +} + +for (const mode of ["legacy", "full"]) { + test(`JSONL output writes exactly the exported records in ${mode} storage`, async () => { + const previous = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = mode; + const dir = mkdtempSync(join(tmpdir(), "jsonl-output-")); + const store = new SessionStore({ dir, enabled: true, debounceMs: 0 }); + try { + await store.writeNow(session()); + for (const full of [false, true]) { + const expected = await exportSession("jsonl-output", { dir, jsonl: true, full }); + const output = join(dir, "exports", `session-${full}.jsonl`); + const result = await exportSession("jsonl-output", { dir, jsonl: true, full, output }); + assert.equal(result, `written to ${output}`); + assert.equal(readFileSync(output, "utf8"), expected); + const records = expected.trimEnd().split("\n").map((line) => JSON.parse(line)); + assert.equal(records[0].type, "session"); + assert.equal(records[0].source, mode === "full" ? "unified" : "legacy-tail"); + assert.equal(records[0].sessionId, "jsonl-output"); + assert.ok(records.some((record) => record.type === "message")); + } + } finally { + store.closeQuietly(); + rmSync(dir, { recursive: true, force: true }); + if (previous === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = previous; + } + }); +} + +test("JSONL destination failures propagate and a session listing creates no output file", async () => { + const dir = mkdtempSync(join(tmpdir(), "jsonl-destination-")); + const store = new SessionStore({ dir, enabled: true, debounceMs: 0 }); + try { + await store.writeNow(session()); + const output = join(dir, "existing-directory"); + mkdirSync(output); + await assert.rejects(() => exportSession("jsonl-output", { dir, jsonl: true, output }), { code: "EISDIR" }); + const listingOutput = join(dir, "listing.jsonl"); + assert.match(await exportSession(undefined, { dir, jsonl: true, output: listingOutput }), /Persisted sessions:/); + assert.equal(existsSync(listingOutput), false); + } finally { + store.closeQuietly(); + rmSync(dir, { recursive: true, force: true }); + } +}); From 0064fb4b70e0ce7b2b4e3541371a2b9799584cd3 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:12:25 +0800 Subject: [PATCH 49/67] test: accept platform-specific export destination errors (cherry picked from commit bc721f29f57e140b4a1e1aebe48d226a7710d123) --- tests/export-jsonl-output.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/export-jsonl-output.test.ts b/tests/export-jsonl-output.test.ts index b423235dba..e67250fc73 100644 --- a/tests/export-jsonl-output.test.ts +++ b/tests/export-jsonl-output.test.ts @@ -56,7 +56,8 @@ test("JSONL destination failures propagate and a session listing creates no outp await store.writeNow(session()); const output = join(dir, "existing-directory"); mkdirSync(output); - await assert.rejects(() => exportSession("jsonl-output", { dir, jsonl: true, output }), { code: "EISDIR" }); + await assert.rejects(() => exportSession("jsonl-output", { dir, jsonl: true, output }), (error: unknown) => + error instanceof Error && "code" in error && ["EISDIR", "EPERM", "EACCES"].includes(String(error.code))); const listingOutput = join(dir, "listing.jsonl"); assert.match(await exportSession(undefined, { dir, jsonl: true, output: listingOutput }), /Persisted sessions:/); assert.equal(existsSync(listingOutput), false); From 9c09ed2cdd81797fb76e8a2b6f6d716013a3de9a Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:07:00 +0800 Subject: [PATCH 50/67] fix: honor leading Windows proxy bypass wildcards (cherry picked from commit 4334bbfd9785cc2a9273d72ce45b7a77ab6d512c) --- src/upstream-proxy.ts | 4 +- tests/windows-bypass-leading-globs.test.ts | 48 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/windows-bypass-leading-globs.test.ts diff --git a/src/upstream-proxy.ts b/src/upstream-proxy.ts index fed6ddc4f3..ebe3947506 100644 --- a/src/upstream-proxy.ts +++ b/src/upstream-proxy.ts @@ -178,7 +178,7 @@ function parseFallbackProxy(proxy: string | undefined, biliPort?: number): Parse } type NoProxyOptions = { - /** Windows ProxyOverride grammar (#2502): mid-token `*` entries (Clash's default + /** Windows ProxyOverride grammar (#2502): `*` entries (Clash's default * bypass list is `localhost;127.*;192.168.*;…;`) glob-match the hostname — * `.` stays literal, `*` matches any characters. NO_PROXY lists keep their * conventional grammar where `*` only appears as a standalone item. */ @@ -214,7 +214,7 @@ export function matchesNoProxy(target: URL, noProxy: string | undefined, options } } if (tokenPort !== undefined && tokenPort !== port) continue; - if (options.windowsBypass && !token.startsWith("*") && token.includes("*") && wildcardHostMatch(host, token)) return true; + if (options.windowsBypass && token.includes("*") && wildcardHostMatch(host, token)) return true; const suffix = token.startsWith("*.") ? token.slice(1) : token.startsWith(".") ? token : undefined; if (suffix ? host.endsWith(suffix) || host === suffix.slice(1) : host === token) return true; } diff --git a/tests/windows-bypass-leading-globs.test.ts b/tests/windows-bypass-leading-globs.test.ts new file mode 100644 index 0000000000..d5c29395bf --- /dev/null +++ b/tests/windows-bypass-leading-globs.test.ts @@ -0,0 +1,48 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { matchesNoProxy, resolveProxyDecision } from "../src/upstream-proxy.ts"; + +const cases: Array<[string, string, boolean]> = [ + ["*int*", "https://internet/", true], + ["*int*", "https://painter/", true], + ["*int*", "https://api.external.example/", false], + ["*internal", "https://build.internal/", true], + ["*internal", "https://internal.example/", false], + ["*.corp.*", "https://api.corp.example/", true], + ["*.corp.*", "https://apiXcorpYexample/", false], + ["*example.com", "https://sub.example.com/", true], + ["*example.com", "https://example.com.evil/", false], + ["*INTERNAL", "https://build.internal/", true], + ["*int*:8443", "https://painter:8443/", true], + ["*int*:8443", "https://painter/", false], + ["*.example.com", "https://example.com/", true], + ["*.example.com", "https://sub.example.com/", true], +]; + +for (const [pattern, url, expected] of cases) { + test(`Windows bypass ${pattern} matches ${url}: ${expected}`, () => { + assert.equal(matchesNoProxy(new URL(url), pattern, { windowsBypass: true }), expected); + }); +} + +test("Windows leading wildcards do not widen conventional NO_PROXY matching", () => { + for (const [pattern, url] of cases.filter(([pattern, , expected]) => expected && pattern !== "*.example.com")) { + assert.equal(matchesNoProxy(new URL(url), pattern), false); + } + assert.equal(matchesNoProxy(new URL("https://api.example.com"), "*.example.com"), true); + assert.equal(matchesNoProxy(new URL("https://example.com"), "*.example.com"), true); +}); + +test("documented WinINet leading globs select direct routing only for matching upstreams", () => { + const systemProxy = { enabled: true, https: "http://system-proxy.example:8080", bypass: "*int*" }; + assert.deepEqual(resolveProxyDecision({}, undefined, "https://painter/v1", { systemProxy }), { source: "windows-bypass" }); + assert.deepEqual(resolveProxyDecision({}, undefined, "https://api.external.example/v1", { systemProxy }), { + source: "windows-system", proxy: "http://system-proxy.example:8080/", + }); + assert.deepEqual(resolveProxyDecision({}, undefined, "https://painter/v1", { + systemProxy, httpsProxy: "http://environment-proxy.example:8080", + }), { source: "HTTPS_PROXY", proxy: "http://environment-proxy.example:8080/" }); + assert.deepEqual(resolveProxyDecision({}, "http://explicit-proxy.example:8080", "https://painter/v1", { + systemProxy, + }), { source: "global", proxy: "http://explicit-proxy.example:8080/" }); +}); From b27b5fe8cec35d195dbb72a035185bc9f295cbd2 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:08:28 +0800 Subject: [PATCH 51/67] fix: preserve deferred WebSocket listener options and close state (cherry picked from commit 1cbb17d6c696edcace8b700ab48daa0a0c38522c) --- src/agent/native-ws-intercept.ts | 49 ++++++-- tests/native-ws-deferred-events.test.ts | 159 ++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 tests/native-ws-deferred-events.test.ts diff --git a/src/agent/native-ws-intercept.ts b/src/agent/native-ws-intercept.ts index 0e17c2b20f..3bf940af93 100644 --- a/src/agent/native-ws-intercept.ts +++ b/src/agent/native-ws-intercept.ts @@ -61,6 +61,15 @@ interface EventListenerObject { } type WsEventTargetOptions = { capture?: boolean; once?: boolean; passive?: boolean; signal?: AbortSignal } | boolean; +interface DeferredListener { + el: EventListener | EventListenerObject; + capture: boolean; + once: boolean; + passive?: boolean; + signal?: AbortSignal; + onAbort?: () => void; +} + const WS_INTERCEPT_FLAG = Symbol.for("billion-context.native-ws-intercept"); const OWN_MARKER = Symbol.for("billion-context.native-ws-intercept.own"); @@ -219,7 +228,7 @@ class DeferredWebSocket { private cancelled = false; private closed = false; private settled = false; - private listeners = new Map>(); + private listeners = new Map(); private handlers: Partial> = {}; private binaryTypeOverride: string | undefined; @@ -260,8 +269,14 @@ class DeferredWebSocket { if (this.real !== undefined) return; this.real = real; for (const [type, entries] of this.listeners) { - for (const entry of entries) real.addEventListener(type, entry.el, { once: entry.once }); + for (const entry of entries) { + if (entry.onAbort) entry.signal?.removeEventListener("abort", entry.onAbort); + real.addEventListener(type, entry.el, { + capture: entry.capture, once: entry.once, passive: entry.passive, signal: entry.signal, + }); + } } + this.listeners.clear(); for (const key of ["onopen", "onerror", "onclose", "onmessage"] as const) { const handler = this.handlers[key]; if (handler !== undefined) real[key] = handler; @@ -275,22 +290,36 @@ class DeferredWebSocket { return; } if (listener === null) return; - const once = typeof options === "object" && options !== null ? options.once === true : options === true; + const detail = typeof options === "object" && options !== null ? options : undefined; + if (detail?.signal?.aborted) return; + const capture = typeof options === "boolean" ? options : detail?.capture === true; const list = this.listeners.get(type) ?? []; - list.push({ el: listener, once }); + if (list.some(entry => entry.el === listener && entry.capture === capture)) return; + const entry: DeferredListener = { + el: listener, capture, once: detail?.once === true, passive: detail?.passive, signal: detail?.signal, + }; + if (entry.signal) { + entry.onAbort = () => this.removeEventListener(type, listener, { capture }); + entry.signal.addEventListener("abort", entry.onAbort, { once: true }); + } + list.push(entry); this.listeners.set(type, list); } - removeEventListener(type: string, listener: EventListener | EventListenerObject | null, _options?: WsEventTargetOptions): void { + removeEventListener(type: string, listener: EventListener | EventListenerObject | null, options?: WsEventTargetOptions): void { if (this.real !== undefined) { - this.real.removeEventListener(type, listener, _options); + this.real.removeEventListener(type, listener, options); return; } if (listener === null) return; + const capture = typeof options === "boolean" ? options : options?.capture === true; const list = this.listeners.get(type); if (list !== undefined) { - const at = list.findIndex((entry) => entry.el === listener); - if (at >= 0) list.splice(at, 1); + const at = list.findIndex((entry) => entry.el === listener && entry.capture === capture); + if (at >= 0) { + const [entry] = list.splice(at, 1); + if (entry.onAbort) entry.signal?.removeEventListener("abort", entry.onAbort); + } if (list.length === 0) this.listeners.delete(type); } } @@ -298,9 +327,10 @@ class DeferredWebSocket { dispatchEvent(event: Event): boolean { if (this.real !== undefined) return this.real.dispatchEvent(event); for (const entry of [...(this.listeners.get(event.type) ?? [])]) { + if (!this.listeners.get(event.type)?.includes(entry)) continue; + if (entry.once) this.removeEventListener(event.type, entry.el, { capture: entry.capture }); if (typeof entry.el === "function") entry.el.call(this, event); else entry.el.handleEvent(event); - if (entry.once) this.removeEventListener(event.type, entry.el); } // A real EventTarget fires the on* handler property alongside listeners — // a client registered via ws.onclose = fn must not miss a window-phase @@ -326,6 +356,7 @@ class DeferredWebSocket { if (this.cancelled || this.closed) return; this.cancelled = true; queueMicrotask(() => { + this.closed = true; const event = new Event("close"); Object.assign(event, { code: 1006, reason: "", wasClean: false }); this.dispatchEvent(event); diff --git a/tests/native-ws-deferred-events.test.ts b/tests/native-ws-deferred-events.test.ts new file mode 100644 index 0000000000..895afc2d3d --- /dev/null +++ b/tests/native-ws-deferred-events.test.ts @@ -0,0 +1,159 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { setImmediate as tick } from "node:timers/promises"; +import { getEventListeners } from "node:events"; +import { installNativeWebSocketIntercept, _resetForTest } from "../src/agent/native-ws-intercept.ts"; + +class EventSocket extends EventTarget { + static CONNECTING = 0; + static OPEN = 1; + static CLOSING = 2; + static CLOSED = 3; + static instances: EventSocket[] = []; + readonly url: string; + readyState = 0; + bufferedAmount = 0; + protocol = ""; + extensions = ""; + binaryType = "blob"; + onopen: ((event: Event) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + onclose: ((event: Event) => void) | null = null; + onmessage: ((event: Event) => void) | null = null; + constructor(url: string) { + super(); + this.url = url; + EventSocket.instances.push(this); + } + send(): void {} + close(): void { this.readyState = EventSocket.CLOSED; } +} + +async function withDeferred(run: (socket: EventSocket, attach: () => Promise) => Promise): Promise { + const saved = Object.getOwnPropertyDescriptor(globalThis, "WebSocket"); + _resetForTest(); + EventSocket.instances = []; + Object.defineProperty(globalThis, "WebSocket", { value: EventSocket, configurable: true, writable: true }); + let resolve!: (origin: string | undefined) => void; + const ready = new Promise((done) => { resolve = done; }); + try { + installNativeWebSocketIntercept({ origin: undefined, ready }); + const Constructor = globalThis.WebSocket as unknown as new (url: string) => EventSocket; + const socket = new Constructor("wss://chatgpt.com/backend-api/codex/responses"); + await run(socket, async () => { + resolve("http://127.0.0.1:8787"); + await tick(); + await tick(); + assert.equal(EventSocket.instances.length, 1); + return EventSocket.instances[0]; + }); + } finally { + resolve(undefined); + await tick(); + _resetForTest(); + if (saved) Object.defineProperty(globalThis, "WebSocket", saved); + else Reflect.deleteProperty(globalThis, "WebSocket"); + } +} + +test("deferred boolean listener options preserve capture without enabling once", async () => { + await withDeferred(async (socket, attach) => { + let count = 0; + socket.addEventListener("message", () => count++, true); + const real = await attach(); + real.dispatchEvent(new Event("message")); + real.dispatchEvent(new Event("message")); + assert.equal(count, 2); + }); +}); + +test("deferred listeners honor already-aborted and pre-attach abort signals", async () => { + await withDeferred(async (socket, attach) => { + let count = 0; + const aborted = new AbortController(); + aborted.abort(); + const pending = new AbortController(); + socket.addEventListener("message", () => count++, { signal: aborted.signal }); + socket.addEventListener("message", () => count++, { signal: pending.signal }); + pending.abort(); + const real = await attach(); + real.dispatchEvent(new Event("message")); + assert.equal(count, 0); + assert.equal(getEventListeners(pending.signal, "abort").length, 0); + }); +}); + +test("deferred listener signal transfers to the real socket", async () => { + await withDeferred(async (socket, attach) => { + const controller = new AbortController(); + let count = 0; + socket.addEventListener("message", () => count++, { signal: controller.signal }); + const real = await attach(); + real.dispatchEvent(new Event("message")); + controller.abort(); + real.dispatchEvent(new Event("message")); + assert.equal(count, 1); + }); +}); + +test("deferred listener removal matches capture and detaches its abort hook", async () => { + await withDeferred(async (socket, attach) => { + const controller = new AbortController(); + let count = 0; + const listener = (): void => { count++; }; + socket.addEventListener("message", listener, { capture: false, signal: controller.signal }); + socket.addEventListener("message", listener, { capture: true }); + socket.removeEventListener("message", listener, { capture: true }); + const real = await attach(); + real.dispatchEvent(new Event("message")); + assert.equal(count, 1); + controller.abort(); + real.dispatchEvent(new Event("message")); + assert.equal(count, 1); + }); +}); + +test("deferred duplicate registration dispatches once and explicit removal clears signal", async () => { + await withDeferred(async (socket) => { + let count = 0; + const listener = (): void => { count++; }; + socket.addEventListener("message", listener); + socket.addEventListener("message", listener); + socket.dispatchEvent(new Event("message")); + assert.equal(count, 1); + const controller = new AbortController(); + socket.addEventListener("close", listener, { signal: controller.signal }); + socket.removeEventListener("close", listener); + assert.equal(getEventListeners(controller.signal, "abort").length, 0); + socket.close(); + }); +}); + +test("deferred once listener is removed before a reentrant dispatch", async () => { + await withDeferred(async (socket) => { + let count = 0; + socket.addEventListener("message", () => { + count++; + if (count === 1) socket.dispatchEvent(new Event("message")); + }, { once: true }); + socket.dispatchEvent(new Event("message")); + assert.equal(count, 1); + socket.close(); + }); +}); + +test("deferred close reaches CLOSED before notifying listeners and cannot open later", async () => { + await withDeferred(async (socket) => { + const states: number[] = []; + socket.addEventListener("close", () => states.push(socket.readyState)); + socket.close(); + assert.equal(socket.readyState, EventSocket.CLOSING); + await tick(); + assert.deepEqual(states, [EventSocket.CLOSED]); + assert.equal(socket.readyState, EventSocket.CLOSED); + socket.close(); + await tick(); + assert.equal(states.length, 1); + assert.equal(EventSocket.instances.length, 0); + }); +}); From 339072464b4439ff7884806dee3a76833db45eaa Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:09:43 +0800 Subject: [PATCH 52/67] fix: expand Responses WebSocket history in linear time (cherry picked from commit 7d2b483e57cfb78d99c8c680d389ba1e6e19038a) --- scripts/bench-ws-history.ts | 27 ++++++++ src/responses-ws.ts | 21 +++--- tests/responses-ws-history-linear.test.ts | 83 +++++++++++++++++++++++ 3 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 scripts/bench-ws-history.ts create mode 100644 tests/responses-ws-history-linear.test.ts diff --git a/scripts/bench-ws-history.ts b/scripts/bench-ws-history.ts new file mode 100644 index 0000000000..f924f2495e --- /dev/null +++ b/scripts/bench-ws-history.ts @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; +import { ResponsesWsHistory } from "../src/responses-ws.ts"; + +const turns = 6_000; +const history = new ResponsesWsHistory(); +const expected: unknown[] = []; +for (let i = 0; i < turns; i++) { + const tail = [{ type: "message", role: "user", content: `question-${i}` }]; + const output = [{ type: "message", role: "assistant", content: `answer-${i}` }]; + expected.push(...tail, ...output); + history.commit({ model: "m", input: tail }, { id: `r${i}`, status: "completed", output }, + i === 0 ? undefined : { continuedId: `r${i - 1}`, tail }); +} +const frame = { type: "response.create", previous_response_id: `r${turns - 1}`, input: [] }; +assert.deepEqual(history.expand(frame).input, expected); +const milliseconds: number[] = []; +for (let sample = 0; sample < 7; sample++) { + const start = process.hrtime.bigint(); + const expanded = history.expand(frame); + milliseconds.push(Number(process.hrtime.bigint() - start) / 1e6); + assert.deepEqual(expanded.input, expected); +} +console.log(JSON.stringify({ + turns, outputItems: expected.length, milliseconds, + medianMs: [...milliseconds].sort((a, b) => a - b)[3], + exactOrderChecked: true, +}, null, 2)); diff --git a/src/responses-ws.ts b/src/responses-ws.ts index b6c97f5fc4..b728054545 100644 --- a/src/responses-ws.ts +++ b/src/responses-ws.ts @@ -237,32 +237,33 @@ export class ResponsesWsHistory { if (!Array.isArray(request.input)) throw new Error("response.create requires an input array"); if (previous !== undefined && previous !== null) { if (previous !== this.checkpoint?.id) throw new Error("previous_response_not_found"); - // Walk the checkpoint chain oldest→newest. The base level always + // Replay the checkpoint chain oldest→newest. The base level always // carries a complete reconstruction in request.input; every later // level contributes only its own tail + output (its augmented body // would double-count the echoed previous output otherwise). const levels: Checkpoint[] = []; let cursor: Checkpoint | undefined = this.checkpoint; while (cursor) { - levels.unshift(cursor); + levels.push(cursor); cursor = cursor.base; } - const base = levels[0]; + const base = levels[levels.length - 1]; if (!base || !Array.isArray(base.request.input)) throw new Error("previous_response_not_found"); // Per-level contribution in conversation order: the base's complete // input, then each later level's own tail followed by its output. - let input: unknown[] = []; - for (let i = 0; i < levels.length; i++) { + const input: unknown[] = []; + for (let i = levels.length - 1; i >= 0; i--) { const level = levels[i]; - if (i === 0 && Array.isArray(level.request.input)) input = [...input, ...level.request.input]; - else if (Array.isArray(level.tail)) input = [...input, ...level.tail]; - input = [...input, ...level.output]; + const added = i === levels.length - 1 ? level.request.input : level.tail; + if (Array.isArray(added)) for (const item of added) input.push(item); + for (const item of level.output) input.push(item); } + for (const item of request.input) input.push(item); // #2966: generate is a per-request action control, not conversation // state — a prewarm's generate:false must not leak into a continuation // that omits the field (the upstream would then refuse to generate). - const { generate: _priorGenerate, ...topRequest } = levels[levels.length - 1].request; - return { ...topRequest, ...request, input: [...input, ...request.input] }; + const { generate: _priorGenerate, ...topRequest } = levels[0].request; + return { ...topRequest, ...request, input }; } return request; } diff --git a/tests/responses-ws-history-linear.test.ts b/tests/responses-ws-history-linear.test.ts new file mode 100644 index 0000000000..5ed3846397 --- /dev/null +++ b/tests/responses-ws-history-linear.test.ts @@ -0,0 +1,83 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { ResponsesWsHistory } from "../src/responses-ws.ts"; + +type Item = Record; + +function chainedHistory(turns: number): { history: ResponsesWsHistory; expected: Item[] } { + const history = new ResponsesWsHistory(); + const expected: Item[] = []; + let priorOutput: Item[] = []; + for (let i = 0; i < turns; i++) { + const tail = [{ type: "message", role: "user", content: `question-${i}` }]; + const output = i % 3 === 0 ? [] : [{ type: "message", role: "assistant", content: `answer-${i}` }]; + history.commit( + { model: `model-${i}`, input: [...priorOutput, ...tail], generate: false }, + { id: `response-${i}`, status: "completed", output }, + i === 0 ? undefined : { continuedId: `response-${i - 1}`, tail }, + ); + expected.push(...tail, ...output); + priorOutput = output; + } + return { history, expected }; +} + +test("checkpoint expansion visits history items linearly and preserves exact order", () => { + const turns = 256; + const { history, expected } = chainedHistory(turns); + const next = { type: "message", role: "user", content: "continue" }; + const descriptor = Object.getOwnPropertyDescriptor(Array.prototype, Symbol.iterator)!; + const original = Array.prototype[Symbol.iterator]; + let visits = 0; + let expanded: Item; + Object.defineProperty(Array.prototype, Symbol.iterator, { + ...descriptor, + value: function* (this: unknown[]): Generator { + const iterator = original.call(this); + for (;;) { + const step = iterator.next(); + if (step.done) return; + visits++; + yield step.value; + } + }, + }); + try { + expanded = history.expand({ type: "response.create", previous_response_id: `response-${turns - 1}`, input: [next] }); + } finally { + Object.defineProperty(Array.prototype, Symbol.iterator, descriptor); + } + assert.deepEqual(expanded, { model: `model-${turns - 1}`, input: [...expected, next] }); + assert.ok(visits <= 8 * (expected.length + turns), `visited ${visits} array elements for ${expected.length} history items`); +}); + +test("long chained fallback preserves fields, output gaps and explicit per-turn actions", () => { + const { history, expected } = chainedHistory(2_048); + const expanded = history.expand({ type: "response.create", previous_response_id: "response-2047", input: "next", model: "override", generate: false }); + assert.deepEqual(expanded, { + model: "override", generate: false, + input: [...expected, { type: "message", role: "user", content: "next" }], + }); + assert.throws(() => history.expand({ type: "response.create", previous_response_id: "unknown", input: [] }), /previous_response_not_found/); +}); + +test("a full replay rebaselines a checkpoint chain without mutating retained arrays", () => { + const { history } = chainedHistory(16); + const input = Object.freeze([{ type: "message", role: "user", content: "new branch" }]); + const output = Object.freeze([{ type: "message", role: "assistant", content: "new answer" }]); + history.commit({ model: "new", input }, { id: "rebase", status: "completed", output }); + const expanded = history.expand({ type: "response.create", previous_response_id: "rebase", input: [] }); + assert.deepEqual(expanded, { model: "new", input: [...input, ...output] }); + assert.equal(input.length, 1); + assert.equal(output.length, 1); + assert.equal(history.current()?.base, undefined); +}); + +test("a single large checkpoint does not exceed a function argument limit", () => { + const history = new ResponsesWsHistory(); + const input = Array.from({ length: 150_000 }, (_, i) => ({ type: "message", role: "user", content: String(i) })); + history.commit({ model: "m", input }, { id: "large", status: "completed", output: [] }); + const expanded = history.expand({ type: "response.create", previous_response_id: "large", input: [] }); + assert.deepEqual(expanded.input, input); + assert.notEqual(expanded.input, input); +}); From f72e13ad7e55a24708d74af4fc32bcc0394b6a49 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 16:11:25 +0800 Subject: [PATCH 53/67] fix: recover Responses turns with string input (cherry picked from commit 8177a9332700286b646128dc2dc702ee51fa132a) --- src/wire-body.ts | 6 +- tests/responses-string-recovery.test.ts | 90 +++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 tests/responses-string-recovery.test.ts diff --git a/src/wire-body.ts b/src/wire-body.ts index b11f683613..bec70cdf62 100644 --- a/src/wire-body.ts +++ b/src/wire-body.ts @@ -10,7 +10,7 @@ import { normalizeGoogleUserPartOrder, type GooglePart } from "acp-kernel/wire"; * MERGED into the last user turn when there is one and a fresh user turn is * started only when there is not. * - * Returns null when the body is unparseable or carries no turn array — the + * Returns null when the body is unparseable or carries no supported turn input — the * caller skips the mutation rather than sending a body it has mangled. */ export function appendTrailingUserText(protocol: WireProtocol, body: string | Buffer, text: string, model?: string): string | null { const raw = typeof body === "string" ? body : body.toString("utf8"); @@ -22,6 +22,10 @@ export function appendTrailingUserText(protocol: WireProtocol, body: string | Bu } if (protocol === "responses") { const input = obj.input; + if (typeof input === "string") { + obj.input = `${input}\n\n${text}`; + return JSON.stringify(obj); + } if (!Array.isArray(input)) return null; const arr = input as Record[]; const last = arr[arr.length - 1]; diff --git a/tests/responses-string-recovery.test.ts b/tests/responses-string-recovery.test.ts new file mode 100644 index 0000000000..464539e3d0 --- /dev/null +++ b/tests/responses-string-recovery.test.ts @@ -0,0 +1,90 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { DEGENERATE_RETRY_NUDGE, injectContinuationNudge, makeContinuationRefetch } from "../src/degenerate-retry.ts"; +import { FAKE_COMPLETION_HINT, injectFakeCompletionHint } from "../src/fake-completion.ts"; +import { withFetchTransport } from "../src/fetch-transport.ts"; +import { _resetFetchUtilForTest } from "../src/fetch-util.ts"; + +const builders = [ + { name: "continuation", build: injectContinuationNudge, hint: DEGENERATE_RETRY_NUDGE }, + { name: "fake completion", build: injectFakeCompletionHint, hint: FAKE_COMPLETION_HINT }, +]; + +for (const { name, build, hint } of builders) { + for (const buffer of [false, true]) { + test(`${name} recovery preserves Responses string input (${buffer ? "Buffer" : "string"} body)`, () => { + for (const prompt of ["hello", "first\n中文 😀 \\\"quoted\\\"", ""]) { + const request = { + model: "m", input: prompt, instructions: "Keep this instruction", + previous_response_id: "prior", store: false, temperature: 0.3, + metadata: { test: "unchanged" }, tools: [{ type: "function", name: "lookup", parameters: { type: "object" } }], + }; + const serialized = JSON.stringify(request); + const result = build("responses", buffer ? Buffer.from(serialized) : serialized); + assert.notEqual(result, null); + assert.deepEqual(JSON.parse(result!), { ...request, input: `${prompt}\n\n${hint}` }); + } + }); + } + + test(`${name} recovery keeps existing Responses array behavior`, () => { + const input = [{ role: "user", content: [{ type: "input_text", text: "prompt" }, { type: "input_image", image_url: "data:image/png;base64,AA==" }] }]; + const result = build("responses", JSON.stringify({ model: "m", input })); + assert.deepEqual(JSON.parse(result!).input, [{ role: "user", content: [...input[0].content, { type: "input_text", text: hint }] }]); + const assistant = [{ role: "assistant", content: "answer" }]; + assert.deepEqual(JSON.parse(build("responses", JSON.stringify({ input: assistant }))!).input, + [...assistant, { role: "user", content: [{ type: "input_text", text: hint }] }]); + }); +} + +test("recovery still declines absent, malformed and unsupported input shapes", () => { + for (const { build } of builders) { + for (const body of ["{broken", "{}", '{"input":null}', '{"input":42}', '{"input":{}}']) { + assert.equal(build("responses", body), null); + } + } +}); + +for (const buffer of [false, true]) { + test(`string-input continuation actually reaches the transport (${buffer ? "Buffer" : "string"} body)`, async () => { + const request = { model: "m", input: "original prompt 😀", instructions: "preserve", store: false }; + const serialized = JSON.stringify(request); + const controller = new AbortController(); + const bytes = 'event: response.completed\ndata: {"type":"response.completed","response":{"status":"completed"}}\n\n'; + let calls = 0; + try { + const body = await withFetchTransport(async (url, options) => { + calls++; + assert.equal(url, "https://recovery.invalid/v1/responses"); + assert.equal(options.method, "POST"); + assert.equal(typeof options.body, "string"); + assert.deepEqual(JSON.parse(options.body as string), { ...request, input: `${request.input}\n\n${DEGENERATE_RETRY_NUDGE}` }); + return new Response(bytes, { headers: { "content-type": "text/event-stream" } }); + }, () => makeContinuationRefetch({ + protocol: "responses", body: buffer ? Buffer.from(serialized) : serialized, + upstreamUrl: "https://recovery.invalid/v1/responses", reqHeaders: {}, + signal: controller.signal, label: "string-input", log: () => {}, + })()); + assert.equal(calls, 1); + assert.ok(body); + assert.equal(await new Response(body).text(), bytes); + } finally { + controller.abort(); + _resetFetchUtilForTest(); + } + }); +} + +test("an already-aborted string-input continuation sends nothing", async () => { + let calls = 0; + const body = await withFetchTransport(async () => { + calls++; + return new Response("unexpected"); + }, () => makeContinuationRefetch({ + protocol: "responses", body: JSON.stringify({ input: "prompt" }), + upstreamUrl: "https://recovery.invalid/v1/responses", reqHeaders: {}, + signal: AbortSignal.abort(), label: "aborted", log: () => {}, + })()); + assert.equal(body, null); + assert.equal(calls, 0); +}); From 185ad09206f74a6c08b673cce096f9d74b547bab Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:50:17 +0800 Subject: [PATCH 54/67] fix(storage): include prototype-named JSON fields in verification digests (cherry picked from commit 4cbd978c56650c9aa8c810168d4a9be792d4bc8b) --- src/storage/canonical.ts | 10 ++++-- tests/storage-canonical-proto-key.test.ts | 37 +++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/storage-canonical-proto-key.test.ts diff --git a/src/storage/canonical.ts b/src/storage/canonical.ts index d585e2027e..f8da114fc1 100644 --- a/src/storage/canonical.ts +++ b/src/storage/canonical.ts @@ -67,7 +67,12 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno // DIGEST_SKIPPED_KEYS for why presence, not value, is the problem. // hasOwn, not `in` — a "__proto__" key must not read as skipped. if (Object.hasOwn(DIGEST_SKIPPED_KEYS, k)) continue; - out[k] = canonicalizeWithContext(v, k); + const normalized = canonicalizeWithContext(v, k); + if (k === "__proto__") { + Object.defineProperty(out, k, { value: normalized, enumerable: true, writable: true, configurable: true }); + } else { + out[k] = normalized; + } } return out; } @@ -155,8 +160,7 @@ export function canonicalDigest(value: unknown): string { const orderedKeys = keys.filter(isArrayIndexKey).sort((a, b) => Number(a) - Number(b)) .concat(keys.filter((k) => !isArrayIndexKey(k)).sort()); for (const k of orderedKeys) { - // Assigning __proto__ to canonicalize's plain output creates no JSON property. - if (Object.hasOwn(DIGEST_SKIPPED_KEYS, k) || k === "__proto__") continue; + if (Object.hasOwn(DIGEST_SKIPPED_KEYS, k)) continue; const v = (current as Record)[k]; if (v === undefined || typeof v === "function" || typeof v === "symbol") continue; write(separator); diff --git a/tests/storage-canonical-proto-key.test.ts b/tests/storage-canonical-proto-key.test.ts new file mode 100644 index 0000000000..f05d3610d4 --- /dev/null +++ b/tests/storage-canonical-proto-key.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { canonicalDigest, canonicalize, sha256 } from "../src/storage/canonical.ts"; + +test("canonical storage views preserve an own __proto__ field without changing the prototype", () => { + const input: unknown = JSON.parse('{"text":"kept","__proto__":{"text":"original"}}'); + const output = canonicalize(input) as Record; + assert.equal(Object.hasOwn(output, "__proto__"), true); + assert.equal(Object.getPrototypeOf(output), Object.prototype); + assert.equal(JSON.stringify(output), '{"__proto__":{"text":"original"},"text":"kept"}'); +}); + +test("storage digests distinguish changes and deletion of __proto__ content", () => { + const before: unknown = JSON.parse('{"__proto__":{"value":"original"},"text":"kept"}'); + const changed: unknown = JSON.parse('{"__proto__":{"value":"changed"},"text":"kept"}'); + const missing = { text: "kept" }; + assert.notEqual(canonicalDigest(before), canonicalDigest(changed)); + assert.notEqual(canonicalDigest(before), canonicalDigest(missing)); + for (const value of [before, changed, missing]) { + assert.equal(canonicalDigest(value), sha256(JSON.stringify(canonicalize(value)))); + } +}); + +test("scalar and nested prototype-named fields remain part of retained tool content", () => { + const makeView = (value: string) => ({ + blockContents: new Map([["b1", { full: JSON.parse(`{"tool":{"arguments":{"__proto__":${value}}}}`) as unknown }]]), + }); + const digests = ["null", "false", "0", '"original"', '{"nested":true}'].map((value) => canonicalDigest(makeView(value))); + assert.equal(new Set(digests).size, digests.length); +}); + +test("ordinary canonical key order and excluded storage provenance stay unchanged", () => { + const first = { text: "same", constructor: "data", nested: { b: 2, a: 1 }, storageMode: "full", tailGaps: 4 }; + const second = { nested: { a: 1, b: 2 }, constructor: "data", text: "same", storageMode: "shadow", tailGaps: 0 }; + assert.equal(canonicalDigest(first), canonicalDigest(second)); + assert.equal(canonicalDigest(first), sha256('{"constructor":"data","nested":{"a":1,"b":2},"text":"same"}')); +}); From 6250e07f36e1add5c687f58542c281a862a16f10 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:50:39 +0800 Subject: [PATCH 55/67] fix: redact JSON-escaped home paths in published diagnostics (cherry picked from commit 5931bfb4b5f8c977e1bf96cbf6bfb22797b879e1) --- src/log-mask.ts | 5 ++- tests/publish-json-home-paths.test.ts | 59 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/publish-json-home-paths.test.ts diff --git a/src/log-mask.ts b/src/log-mask.ts index 4e37ef3db2..06d51a7d28 100644 --- a/src/log-mask.ts +++ b/src/log-mask.ts @@ -179,7 +179,10 @@ function publishHomeRoots(): string[] { export function maskPathsForPublish(text: string): string { if (!text) return text; - const roots = publishHomeRoots(); + const roots = [...new Set(publishHomeRoots().flatMap((root) => [ + root, + JSON.stringify(root).slice(1, -1), + ]))].sort((a, b) => b.length - a.length); if (roots.length === 0) return text; const re = new RegExp("(? void): void { + const previous = process.env.USERPROFILE; + process.env.USERPROFILE = profile; + try { + run(); + } finally { + if (previous === undefined) delete process.env.USERPROFILE; + else process.env.USERPROFILE = previous; + } +} + +test("publish redaction removes Windows homes from serialized configuration values", () => { + withProfile(String.raw`C:\Users\ExampleUser`, () => { + const config = JSON.stringify({ network: { caFile: String.raw`C:\Users\ExampleUser\certs\relay.pem` } }); + const masked = redactForPublish(config); + assert.equal(masked.includes("ExampleUser"), false); + assert.deepEqual(JSON.parse(masked), { network: { caFile: String.raw`~\certs\relay.pem` } }); + }); +}); + +test("raw and JSON-escaped home spellings can occur in the same report", () => { + withProfile(String.raw`C:\Users\ExampleUser`, () => { + const raw = String.raw`C:\Users\ExampleUser\cache\index.db`; + const serialized = JSON.stringify({ file: raw }); + assert.equal(maskPathsForPublish(`open ${raw}\n${serialized}`), `open ~\\cache\\index.db\n${JSON.stringify({ file: String.raw`~\cache\index.db` })}`); + }); +}); + +test("JSON-escaped quotes in a home name do not escape the publish boundary", () => { + withProfile('/srv/users/example"user', () => { + const serialized = JSON.stringify({ file: '/srv/users/example"user/cache/index.db' }); + const masked = maskPathsForPublish(serialized); + assert.deepEqual(JSON.parse(masked), { file: "~/cache/index.db" }); + }); +}); + +test("escaped home matching retains sibling and relative path boundaries", () => { + withProfile(String.raw`C:\Users\ExampleUser`, () => { + const siblings = JSON.stringify({ + sibling: String.raw`C:\Users\ExampleUserOther\index.db`, + relative: String.raw`prefixC:\Users\ExampleUser\index.db`, + other: String.raw`D:\Users\ExampleUser\index.db`, + }); + assert.equal(maskPathsForPublish(siblings), siblings); + }); +}); + +test("publish path redaction remains idempotent and keeps ordinary URL paths", () => { + withProfile("/home/exampleuser", () => { + const text = 'https://api.openai.com/home/exampleuser/file.json {"file":"/home/exampleuser/cache/index.db"}'; + const masked = maskPathsForPublish(text); + assert.equal(masked, 'https://api.openai.com/home/exampleuser/file.json {"file":"~/cache/index.db"}'); + assert.equal(maskPathsForPublish(masked), masked); + }); +}); From cad59f30b4ec457e68fd0fbc220352209e857212 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:51:30 +0800 Subject: [PATCH 56/67] fix: recognize valid JSON layouts in raw media accounting (cherry picked from commit c57bc926cc8a1b1e0b9da71fadd01e973fb9e1a7) --- src/image-tokens.ts | 11 ++--- tests/raw-media-json-layout.test.ts | 65 +++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 tests/raw-media-json-layout.test.ts diff --git a/src/image-tokens.ts b/src/image-tokens.ts index 515b6f8d00..737daf69d6 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -370,18 +370,19 @@ export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "resp } // Cheap gate: most bodies carry no images — skip the JSON parse entirely then. -// prepared.body is bili's own compact JSON.stringify, but client raw buffers -// may carry spaces, so probe both forms. +// prepared.body is compact JSON.stringify; client buffers may use any JSON +// whitespace or escape field names and discriminators with Unicode escapes. function bodyHasImagesProbe(protocol: "anthropic" | "openai" | "responses" | "google", s: string): boolean { + if (s.includes("\\u")) return true; // #3398: custom-shape media carries no recognizable literal — on the // wires where content arrays are the multi-part marker (openai/responses) // the array form itself is the gate; anthropic adds its two opaque block // types. Array-content bodies are the minority case on these wires, so the // extra parses stay rare (#2824's 170ms/32MiB worst case is unchanged). return protocol === "google" ? s.includes("inlineData") || s.includes("fileData") - : protocol === "responses" ? s.includes("input_image") || s.includes('"content":[') || s.includes('"content": [') - : protocol === "openai" ? s.includes("image_url") || s.includes('"type":"file"') || s.includes('"type": "file"') || s.includes('"content":[') || s.includes('"content": [') - : s.includes('"type":"image"') || s.includes('"type": "image"') || s.includes('"type":"redacted_thinking"') || s.includes('"type":"document"'); + : protocol === "responses" ? s.includes("input_image") || /"content"[ \t\r\n]*:[ \t\r\n]*\[/.test(s) + : protocol === "openai" ? s.includes("image_url") || /"type"[ \t\r\n]*:[ \t\r\n]*"file"/.test(s) || /"content"[ \t\r\n]*:[ \t\r\n]*\[/.test(s) + : /"type"[ \t\r\n]*:[ \t\r\n]*"(?:image|redacted_thinking|document)"/.test(s); } export function imageTokensInRawBody(protocol: "anthropic" | "openai" | "responses" | "google", raw: string | Buffer, billing: ResolvedImageBilling = "bytes", configuredCap?: number): number { diff --git a/tests/raw-media-json-layout.test.ts b/tests/raw-media-json-layout.test.ts new file mode 100644 index 0000000000..6793fc2a8b --- /dev/null +++ b/tests/raw-media-json-layout.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { countImagesInParsedBody, countImagesInRawBody, imageTokensInParsedBody, imageTokensInRawBody } from "../src/image-tokens.js"; + +type Protocol = Parameters[0]; + +const cases: Array<{ name: string; protocol: Protocol; raw: string }> = [ + { + name: "Anthropic image with JSON newline and tab separators", + protocol: "anthropic", + raw: '{"messages":[{"role":"user","content":[{"type"\t:\n"image","source":{"type":"base64","media_type":"image/png","data":"QUJDRA=="}}]}]}', + }, + { + name: "pretty-printed Anthropic opaque thinking payload", + protocol: "anthropic", + raw: JSON.stringify({ messages: [{ role: "assistant", content: [{ type: "redacted_thinking", data: "QUJDRA==" }] }] }, null, 2), + }, + { + name: "OpenAI custom attachment after a multiline content separator", + protocol: "openai", + raw: '{"messages":[{"role":"user","content"\n:\t[{"type":"image","attachment":{"data":"QUJDRA=="}}]}]}', + }, + { + name: "Responses custom attachment after a multiline content separator", + protocol: "responses", + raw: '{"input":[{"type":"message","role":"user","content"\r\n:\t[{"type":"image","attachment":{"data":"QUJDRA=="}}]}]}', + }, + { + name: "Google inline data with a Unicode-escaped field name", + protocol: "google", + raw: '{"contents":[{"role":"user","parts":[{"inline\\u0044ata":{"mimeType":"image/png","data":"QUJDRA=="}}]}]}', + }, + { + name: "Anthropic image with a Unicode-escaped discriminator", + protocol: "anthropic", + raw: '{"messages":[{"role":"user","content":[{"type":"im\\u0061ge","source":{"type":"base64","media_type":"image/png","data":"QUJDRA=="}}]}]}', + }, +]; + +for (const { name, protocol, raw } of cases) { + test(`raw media accounting matches parsed accounting: ${name}`, () => { + const parsed: unknown = JSON.parse(raw); + const expectedTokens = imageTokensInParsedBody(protocol, parsed, "bytes"); + const expectedCount = countImagesInParsedBody(protocol, parsed); + assert.ok(expectedTokens > 0); + assert.equal(expectedCount, 1); + for (const body of [raw, Buffer.from(raw)]) { + assert.equal(imageTokensInRawBody(protocol, body, "bytes"), expectedTokens); + assert.equal(countImagesInRawBody(protocol, body), expectedCount); + } + }); +} + +test("compact media accounting and configured image caps stay unchanged", () => { + const body = { messages: [{ role: "user", content: [{ type: "image", source: { type: "base64", data: "QUJDRA==" } }] }] }; + assert.equal(imageTokensInRawBody("anthropic", JSON.stringify(body), "bytes", 1), 1); + assert.equal(countImagesInRawBody("anthropic", JSON.stringify(body)), 1); +}); + +test("plain text with Unicode escapes and malformed probe hits stay uncharged", () => { + for (const raw of ['{"messages":[{"role":"user","content":"\\u4e2d文 image"}]}', '{"messages":[{"content"\n:[broken'] ) { + assert.equal(imageTokensInRawBody("openai", raw, "bytes"), 0); + assert.equal(countImagesInRawBody("openai", raw), 0); + } +}); From e8952552b6ec6cab6bbc7c1591951c64a49c32ff Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:52:25 +0800 Subject: [PATCH 57/67] fix(storage): preserve extension timestamps across archive restore (cherry picked from commit 06c7a80fe7f5232a3e9bf47e00f4fff09d8c5596) --- src/storage/mysql.ts | 4 +- src/storage/store.ts | 8 +- ...orage-archive-extension-timestamps.test.ts | 81 +++++++++++++++++++ 3 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 tests/storage-archive-extension-timestamps.test.ts diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 9c68f41c6a..d2f0dfe391 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1777,7 +1777,7 @@ export class MysqlBackend implements StorageBackend { const refsRows = await q("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord"); const deadRefs = await q("SELECT ord, ref, ref_num, recorded_at FROM dead_refs WHERE session_id = ? ORDER BY ord"); const ccr = await q("SELECT ord, ccr_key, content_hash FROM ccr_entries WHERE session_id = ? ORDER BY ord"); - const ext = await q("SELECT ext_key AS `key`, value_json FROM session_ext WHERE session_id = ?"); + const ext = await q("SELECT ext_key AS `key`, value_json, updated_at FROM session_ext WHERE session_id = ?"); const checkpoints = await q("SELECT cp_id, seq_at, block_set_hash, kind, created_at, pinned, label, state_hash FROM checkpoints WHERE session_id = ?"); const wanted = new Set(); for (const m of messages) wanted.add(str(m.content_hash)); @@ -1929,7 +1929,7 @@ export class MysqlBackend implements StorageBackend { await c.query( "INSERT INTO session_ext (session_id, ext_key, value_json, updated_at) VALUES (?, ?, ?, ?) " + "ON DUPLICATE KEY UPDATE value_json = VALUES(value_json), updated_at = VALUES(updated_at)", - [sessionId, x.key, x.value_json, Date.now()], + [sessionId, x.key, x.value_json, typeof x.updated_at === "number" && Number.isFinite(x.updated_at) ? x.updated_at : Date.now()], ); } for (const cp of dump.checkpoints) { diff --git a/src/storage/store.ts b/src/storage/store.ts index 502233c85e..7890da378e 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2080,7 +2080,7 @@ export class UnifiedStore { const refsRows = (this.db.prepare("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); const deadRefs = (this.db.prepare("SELECT ord, ref, ref_num, recorded_at FROM dead_refs WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); const ccr = (this.db.prepare("SELECT ord, ccr_key, content_hash FROM ccr_entries WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); - const ext = (this.db.prepare("SELECT key, value_json FROM session_ext WHERE session_id = ?").all(sessionId) as Record[]); + const ext = (this.db.prepare("SELECT key, value_json, updated_at FROM session_ext WHERE session_id = ?").all(sessionId) as Record[]); const checkpoints = (this.db.prepare("SELECT cp_id, seq_at, block_set_hash, kind, created_at, pinned, label, state_hash FROM checkpoints WHERE session_id = ?").all(sessionId) as Record[]); // --- collect the distinct CAS bodies referenced by those rows --- const wanted = new Set(); @@ -2215,8 +2215,10 @@ export class UnifiedStore { for (const d of dump.deadRefs) insDead.run(sessionId, d.ord, d.ref, d.ref_num, d.recorded_at); const insCcr = this.db.prepare("INSERT INTO ccr_entries (session_id, ord, ccr_key, content_hash) VALUES (?, ?, ?, ?)"); for (const e of dump.ccrEntries) insCcr.run(sessionId, e.ord, e.ccr_key, e.content_hash); - const insExt = this.db.prepare("INSERT INTO session_ext (session_id, key, value_json) VALUES (?, ?, ?)"); - for (const x of dump.sessionExt) insExt.run(sessionId, x.key, x.value_json); + const insExt = this.db.prepare("INSERT INTO session_ext (session_id, key, value_json, updated_at) VALUES (?, ?, ?, ?)"); + for (const x of dump.sessionExt) { + insExt.run(sessionId, x.key, x.value_json, typeof x.updated_at === "number" && Number.isFinite(x.updated_at) ? x.updated_at : Date.now()); + } const insCp = this.db.prepare("INSERT INTO checkpoints (cp_id, session_id, seq_at, block_set_hash, kind, created_at, pinned, label, state_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(cp_id) DO NOTHING"); for (const cp of dump.checkpoints) insCp.run(cp.cp_id, sessionId, cp.seq_at, cp.block_set_hash, cp.kind, cp.created_at, cp.pinned, cp.label ?? null, cp.state_hash); this.db diff --git a/tests/storage-archive-extension-timestamps.test.ts b/tests/storage-archive-extension-timestamps.test.ts new file mode 100644 index 0000000000..c03a4b63a3 --- /dev/null +++ b/tests/storage-archive-extension-timestamps.test.ts @@ -0,0 +1,81 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.js"; +import { UnifiedStore } from "../src/storage/store.js"; + +function session(id: string): LegacySessionLike { + return { + id, createdAt: 1_000, lastSeen: 2_000, meta: { protocol: "openai" }, stats: {}, metadata: {}, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(), + lastMessages: [{ id: "message-1", role: "user", contentType: "text", text: "retained history" }], + }; +} + +function withStore(run: (store: UnifiedStore, archiveDir: string) => void): void { + const root = mkdtempSync(path.join(os.tmpdir(), "bili-archive-ext-")); + let store: UnifiedStore | undefined; + try { + store = UnifiedStore.open(path.join(root, "index.db"), "node:sqlite"); + run(store, path.join(root, "archive")); + } finally { + store?.close(); + rmSync(root, { recursive: true, force: true }); + } +} + +test("archiving a session with extension state remains restorable", () => { + withStore((store, archiveDir) => { + store.ingestSession(planIngest(session("with-ext"))); + store.setSessionExt("with-ext", "plugin-state", { installed: true, revision: 2 }); + const before = store.listSessionExt("with-ext"); + assert.equal(store.archiveSession("with-ext", { archiveDir }).ok, true); + assert.deepEqual(store.listSessionExt("with-ext"), []); + assert.deepEqual(store.restoreArchived("with-ext", { archiveDir }), { ok: true }); + assert.deepEqual(store.listSessionExt("with-ext"), before); + assert.equal(store.getMessages("with-ext")[0]?.content.includes("retained history"), true); + }); +}); + +test("archive exports retain extension timestamps including zero", (t) => { + withStore((store, archiveDir) => { + store.ingestSession(planIngest(session("timestamp-ext"))); + t.mock.method(Date, "now", () => 0); + store.setSessionExt("timestamp-ext", "first", { value: "first" }); + t.mock.restoreAll(); + const before = store.listSessionExt("timestamp-ext"); + const archived = store.archiveSession("timestamp-ext", { archiveDir }); + assert.ok(archived.exportPath); + const dump = JSON.parse(readFileSync(archived.exportPath, "utf8")) as { sessionExt: Array<{ updated_at?: number }> }; + assert.equal(dump.sessionExt[0]?.updated_at, 0); + assert.deepEqual(store.restoreArchived("timestamp-ext", { archiveDir }), { ok: true }); + assert.deepEqual(store.listSessionExt("timestamp-ext"), before); + }); +}); + +test("old archives without extension timestamps restore using a finite write time", () => { + withStore((store, archiveDir) => { + store.ingestSession(planIngest(session("legacy-ext"))); + store.setSessionExt("legacy-ext", "legacy", { value: "preserved" }); + const archived = store.archiveSession("legacy-ext", { archiveDir }); + assert.ok(archived.exportPath); + const dump = JSON.parse(readFileSync(archived.exportPath, "utf8")) as { sessionExt: Array<{ updated_at?: number }> }; + for (const row of dump.sessionExt) delete row.updated_at; + writeFileSync(archived.exportPath, JSON.stringify(dump) + "\n"); + assert.deepEqual(store.restoreArchived("legacy-ext", { archiveDir }), { ok: true }); + assert.deepEqual(store.getSessionExt("legacy-ext", "legacy"), { value: "preserved" }); + assert.equal(Number.isFinite(store.listSessionExt("legacy-ext")[0]?.updatedAt), true); + }); +}); + +test("sessions without extension rows keep their archive round trip", () => { + withStore((store, archiveDir) => { + store.ingestSession(planIngest(session("no-ext"))); + assert.equal(store.archiveSession("no-ext", { archiveDir }).ok, true); + assert.deepEqual(store.restoreArchived("no-ext", { archiveDir }), { ok: true }); + assert.deepEqual(store.listSessionExt("no-ext"), []); + }); +}); From 56d979bca3d5f3d6850577305db18b27c9d23898 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:49:52 +0800 Subject: [PATCH 58/67] fix: exclude tool data from keepalive TTL detection (cherry picked from commit 7ed10dddb189f18f01176d3d8a59183922133c46) --- src/cache-keepalive.ts | 11 ++-- tests/keepalive-breakpoint-scope.test.ts | 68 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 tests/keepalive-breakpoint-scope.test.ts diff --git a/src/cache-keepalive.ts b/src/cache-keepalive.ts index 4cb13d0619..a1483a609d 100644 --- a/src/cache-keepalive.ts +++ b/src/cache-keepalive.ts @@ -114,8 +114,9 @@ export function resolveCacheKeepalive(routes: ProviderRoutes | undefined, upstre }; } -/** Pure: the TTL tier declared by the snapshot's cache_control breakpoints — - * any "1h" wins over "5m"; undefined when the body carries none. */ +/** Pure: the TTL tier declared by the snapshot's cache_control breakpoints. + * Tool inputs and schema examples are data, not directives. Any "1h" wins + * over "5m"; undefined when the body carries no breakpoint. */ export function detectKeepaliveTier(body: unknown): "5m" | "1h" | undefined { let found: "5m" | "1h" | undefined; const walk = (n: unknown): void => { @@ -137,7 +138,11 @@ export function detectKeepaliveTier(body: unknown): "5m" | "1h" | undefined { // explicit "1h" lifts the tier. found = "5m"; } - for (const v of Object.values(o)) walk(v); + for (const [key, value] of Object.entries(o)) { + if (key === "input_schema") continue; + if (key === "input" && (o.type === "tool_use" || o.type === "server_tool_use")) continue; + walk(value); + } }; walk(body); return found; diff --git a/tests/keepalive-breakpoint-scope.test.ts b/tests/keepalive-breakpoint-scope.test.ts new file mode 100644 index 0000000000..d779451ca8 --- /dev/null +++ b/tests/keepalive-breakpoint-scope.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import { afterEach, test } from "node:test"; +import { detectKeepaliveTier, keepaliveStatusFor, noteSuccessfulForward, _resetKeepaliveForTest, _setKeepaliveSchedulerForTest } from "../src/cache-keepalive.js"; + +const breakpoint = { type: "ephemeral", ttl: "1h" }; +const toolInput = { cache_control: breakpoint, nested: { cache_control: breakpoint } }; + +const lookalikeOnly = { + model: "claude-test", + max_tokens: 1024, + tools: [{ name: "read_settings", input_schema: { type: "object", examples: [toolInput] } }], + messages: [{ role: "assistant", content: [{ type: "tool_use", id: "call-1", name: "read_settings", input: toolInput }] }], +}; + +afterEach(() => _resetKeepaliveForTest()); + +test("tool arguments and JSON-schema examples are data, not cache breakpoints", () => { + assert.equal(detectKeepaliveTier(lookalikeOnly), undefined); + assert.equal(detectKeepaliveTier({ messages: [{ role: "assistant", content: [{ type: "server_tool_use", id: "call-server", name: "web_search", input: toolInput }] }] }), undefined); +}); + +test("a fake one-hour directive cannot override a real five-minute breakpoint", () => { + assert.equal(detectKeepaliveTier({ ...lookalikeOnly, system: [{ type: "text", text: "system", cache_control: { type: "ephemeral" } }] }), "5m"); +}); + +test("actual request, tool, system and message cache-control locations remain eligible", () => { + const controls = [ + { cache_control: breakpoint, messages: [{ role: "user", content: "hi" }] }, + { tools: [{ name: "read_settings", input_schema: { type: "object" }, cache_control: breakpoint }] }, + { system: [{ type: "text", text: "system", cache_control: breakpoint }] }, + { messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: breakpoint }] }] }, + { messages: [{ role: "user", content: [{ type: "tool_result", tool_use_id: "call-1", content: [{ type: "text", text: "result" }], cache_control: breakpoint }] }] }, + ]; + for (const body of controls) assert.equal(detectKeepaliveTier(body), "1h"); +}); + +test("nested tool-result content keeps its genuine block breakpoints", () => { + assert.equal(detectKeepaliveTier({ messages: [{ role: "user", content: [{ + type: "tool_result", tool_use_id: "call-1", content: [{ + type: "tool_reference", tool_name: "read_settings", cache_control: breakpoint, + }], + }] }] }), "1h"); +}); + +test("lookalikes do not arm an otherwise enabled idle-ping schedule", () => { + const scheduled: number[] = []; + _setKeepaliveSchedulerForTest({ schedule(_fn, ms) { scheduled.push(ms); return ms; }, cancel() {} }); + noteSuccessfulForward({ + sessionId: "scope-no-breakpoint", protocol: "anthropic", reqMethod: "POST", sidePassthrough: false, resignLane: false, + wireBody: JSON.stringify(lookalikeOnly), upstreamUrl: "https://api.anthropic.com/v1/messages", headers: {}, dispatcher: undefined, + routes: { "https://api.anthropic.com": { cacheKeepalive: { enabled: true } } }, + }); + assert.equal(keepaliveStatusFor("scope-no-breakpoint"), undefined); + assert.deepEqual(scheduled, []); +}); + +test("a real breakpoint still arms keepalive with the correct tier", () => { + const scheduled: number[] = []; + _setKeepaliveSchedulerForTest({ schedule(_fn, ms) { scheduled.push(ms); return ms; }, cancel() {} }); + noteSuccessfulForward({ + sessionId: "scope-real-breakpoint", protocol: "anthropic", reqMethod: "POST", sidePassthrough: false, resignLane: false, + wireBody: JSON.stringify({ ...lookalikeOnly, system: [{ type: "text", text: "system", cache_control: { type: "ephemeral" } }] }), + upstreamUrl: "https://api.anthropic.com/v1/messages", headers: {}, dispatcher: undefined, + routes: { "https://api.anthropic.com": { cacheKeepalive: { enabled: true } } }, + }); + assert.equal(keepaliveStatusFor("scope-real-breakpoint")?.tierLabel, "5m"); + assert.deepEqual(scheduled, [270_000]); +}); From 4e7e3b122db29c852fbdc14ca712ff8bbbd49b32 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 18:47:00 +0800 Subject: [PATCH 59/67] fix(storage): treat malformed integrity caches as misses (cherry picked from commit edb8848b1dacdaa5455df805b8ddbbbbb5067391) --- src/storage/integrity-check.ts | 17 +++++++- tests/storage-integrity-cache.test.ts | 57 +++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/storage/integrity-check.ts b/src/storage/integrity-check.ts index 86e3e87ab3..1c4bb8faa0 100644 --- a/src/storage/integrity-check.ts +++ b/src/storage/integrity-check.ts @@ -89,14 +89,27 @@ function statKeyOf(dbPath: string): { db: StatKeyPart | null; wal: StatKeyPart | return { db: statPart(dbPath), wal: statPart(`${dbPath}-wal`) }; } +function isStatKeyPart(value: unknown): value is StatKeyPart { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const part = value as Partial; + return typeof part.s === "number" && Number.isSafeInteger(part.s) && part.s >= 0 + && typeof part.m === "number" && Number.isFinite(part.m); +} + /** Read + validate the sidecar. Malformed/absent → null (a plain cache * miss — never a boot failure). */ export function readIntegrityCache(dbPath: string): IntegrityCache | null { const file = integrityCachePath(dbPath); if (!existsSync(file)) return null; try { - const parsed = JSON.parse(readFileSync(file, "utf8")) as Partial; - if (parsed.v !== 1 || typeof parsed.ok !== "boolean") return null; + const value: unknown = JSON.parse(readFileSync(file, "utf8")); + if (value === null || typeof value !== "object" || Array.isArray(value)) return null; + const parsed = value as Partial; + if (parsed.v !== 1 || typeof parsed.ok !== "boolean" + || typeof parsed.schema !== "number" || !Number.isSafeInteger(parsed.schema) || parsed.schema < 0 + || !isStatKeyPart(parsed.db) || (parsed.wal !== null && !isStatKeyPart(parsed.wal)) + || typeof parsed.at !== "number" || !Number.isFinite(parsed.at) || parsed.at < 0 + || (parsed.detail !== undefined && typeof parsed.detail !== "string")) return null; return parsed as IntegrityCache; } catch { return null; diff --git a/tests/storage-integrity-cache.test.ts b/tests/storage-integrity-cache.test.ts index 9b09322478..3c566ea2a2 100644 --- a/tests/storage-integrity-cache.test.ts +++ b/tests/storage-integrity-cache.test.ts @@ -168,3 +168,60 @@ test("a corrupt small store still fails open synchronously and writes no ok-cach rmSync(dir, { recursive: true, force: true }); } }); + +test("malformed verdict records are cache misses", () => { + const dir = tmpDir(); + try { + const dbPath = join(dir, "index.db"); + const store = UnifiedStore.open(dbPath); + store.close(); + const valid = readIntegrityCache(dbPath); + assert.ok(valid !== null); + const malformed: unknown[] = [ + null, + [], + { v: 1, ok: true }, + { ...valid, db: null }, + { ...valid, db: [] }, + { ...valid, db: { ...valid.db, s: -1 } }, + { ...valid, db: { ...valid.db, s: "0" } }, + { ...valid, db: { ...valid.db, m: null } }, + { ...valid, wal: {} }, + { ...valid, wal: { s: 1, m: "today" } }, + { ...valid, schema: "11" }, + { ...valid, at: null }, + { ...valid, detail: {} }, + ]; + for (const value of malformed) { + writeFileSync(integrityCachePath(dbPath), JSON.stringify(value)); + assert.equal(readIntegrityCache(dbPath), null, JSON.stringify(value)); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a damaged verdict cache cannot prevent a healthy store from reopening", () => { + resetIntegrityCheckStateForTests(); + setIntegritySyncThresholdForTests(null); + const dir = tmpDir(); + try { + const dbPath = join(dir, "index.db"); + const store = UnifiedStore.open(dbPath); + store.setKv("test", "kept", { message: "still here" }); + store.close(); + const before = integrityCheckRunsForTests(); + writeFileSync(integrityCachePath(dbPath), JSON.stringify({ v: 1, ok: true, db: null })); + + const again = UnifiedStore.open(dbPath); + try { + assert.deepEqual(again.getKv("test", "kept"), { message: "still here" }); + assert.equal(integrityCheckRunsForTests(), before + 1); + } finally { + again.close(); + } + assert.equal(readIntegrityCache(dbPath)?.ok, true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From 2d145b4cda0e720edb11e636e211aa21023f9fd6 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:47:57 +0800 Subject: [PATCH 60/67] fix: enforce readable byte limit before credential store replacement (cherry picked from commit cbdd841b6558a21c6489f3c4872c251fc4357c36) --- src/external-summary-credentials.ts | 8 ++- tests/summary-credential-byte-limit.test.ts | 57 +++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/summary-credential-byte-limit.test.ts diff --git a/src/external-summary-credentials.ts b/src/external-summary-credentials.ts index b07ba7e24e..e0e78352eb 100644 --- a/src/external-summary-credentials.ts +++ b/src/external-summary-credentials.ts @@ -5,6 +5,8 @@ import { configFile } from "./paths.js"; import { loadConfigFile, loadNamedProviders, loadRoutes, type NamedProviderRecipe } from "./config.js"; import { expandExternalSummaryChain, parseExternalSummaryChain, validSummaryCredentialName } from "./external-summary-settings.js"; +const MAX_CREDENTIAL_STORE_BYTES = 256 * 1024; + function validKey(value: unknown): value is string { return typeof value === "string" && value.length > 0 && value.length <= 8192 && !/[\x00-\x20\x7f]/.test(value); } @@ -34,7 +36,7 @@ export class SummaryCredentialStore { try { const stat = lstatSync(this.path); if (!stat.isFile() || stat.isSymbolicLink() || (process.platform !== "win32" && (stat.mode & 0o077) !== 0)) throw new Error(); - if (stat.size > 256 * 1024) throw new Error(); + if (stat.size > MAX_CREDENTIAL_STORE_BYTES) throw new Error(); const data: unknown = JSON.parse(readFileSync(this.path, "utf8")); if (!data || typeof data !== "object" || Array.isArray(data)) throw new Error(); const result = Object.create(null) as Record; @@ -88,8 +90,10 @@ export class SummaryCredentialStore { if (key === null) delete values[name]; else values[name] = key; if (Object.keys(values).length > 16) throw new Error("External summary credential store is full"); + const serialized = JSON.stringify(values) + "\n"; + if (Buffer.byteLength(serialized, "utf8") > MAX_CREDENTIAL_STORE_BYTES) throw new Error("External summary credential store is full"); descriptor = openSync(temp, "wx", 0o600); - writeFileSync(descriptor, JSON.stringify(values) + "\n", "utf8"); + writeFileSync(descriptor, serialized, "utf8"); fsyncSync(descriptor); closeSync(descriptor); descriptor = undefined; diff --git a/tests/summary-credential-byte-limit.test.ts b/tests/summary-credential-byte-limit.test.ts new file mode 100644 index 0000000000..ff242b6d71 --- /dev/null +++ b/tests/summary-credential-byte-limit.test.ts @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { SummaryCredentialStore } from "../src/external-summary-credentials.js"; + +function withStore(run: (store: SummaryCredentialStore, file: string, dir: string) => void): void { + const dir = mkdtempSync(join(tmpdir(), "summary-credential-bytes-")); + const file = join(dir, "credentials.json"); + try { run(new SummaryCredentialStore(file), file, dir); } + finally { rmSync(dir, { recursive: true, force: true }); } +} + +test("a JSON-escaped key rotation cannot make all existing credentials unreadable", () => { + withStore((store, file, dir) => { + const escaped = "\\".repeat(8192); + for (let i = 0; i < 15; i++) store.set(`key${i}`, escaped); + store.set("rotating", "x".repeat(8192)); + const before = readFileSync(file); + assert.ok(before.byteLength < 256 * 1024); + + assert.throws(() => store.set("rotating", escaped), /store is full/); + assert.deepEqual(readFileSync(file), before); + assert.deepEqual(readdirSync(dir), ["credentials.json"]); + assert.equal(store.status("secret:key0"), "configured"); + assert.equal(store.resolve("secret:rotating"), "x".repeat(8192)); + + store.set("key0", null); + store.set("rotating", escaped); + assert.equal(store.resolve("secret:rotating"), escaped); + }); +}); + +test("the write ceiling counts UTF-8 bytes rather than JavaScript string length", () => { + withStore((store, file, dir) => { + const unicode = "界".repeat(8192); + for (let i = 0; i < 10; i++) store.set(`key${i}`, unicode); + const before = readFileSync(file); + assert.ok(before.byteLength < 256 * 1024); + assert.throws(() => store.set("newkey", unicode), /store is full/); + assert.deepEqual(readFileSync(file), before); + assert.equal(store.resolve("secret:key9"), unicode); + assert.equal(store.status("secret:newkey"), "missing"); + assert.deepEqual(readdirSync(dir), ["credentials.json"]); + }); +}); + +test("sixteen maximum-length ASCII credentials remain valid below the byte ceiling", () => { + withStore((store, file) => { + const key = "x".repeat(8192); + for (let i = 0; i < 16; i++) store.set(`key${i}`, key); + assert.ok(statSync(file).size < 256 * 1024); + const reopened = new SummaryCredentialStore(file); + for (let i = 0; i < 16; i++) assert.equal(reopened.resolve(`secret:key${i}`), key); + }); +}); From 8e7c804cc35f640de414e9d6113a3192e0d4cbe4 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 18:54:25 +0800 Subject: [PATCH 61/67] perf(storage): append split media without copying prior entries (cherry picked from commit f304049813ccf6b5bfebd94015023a99760eb91d) --- scripts/bench-media-split.mjs | 35 +++++++++++++++++++++++ src/storage/identity.ts | 4 +-- tests/storage-media-sidecar-split.test.ts | 33 +++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 scripts/bench-media-split.mjs diff --git a/scripts/bench-media-split.mjs b/scripts/bench-media-split.mjs new file mode 100644 index 0000000000..fa63b5eba3 --- /dev/null +++ b/scripts/bench-media-split.mjs @@ -0,0 +1,35 @@ +// Run with: node --expose-gc --import tsx scripts/bench-media-split.mjs [count] +import { createHash } from "node:crypto"; +import { performance } from "node:perf_hooks"; +import { splitMediaContent } from "../src/storage/identity.ts"; + +const count = Number(process.argv[2] ?? 20_000); +if (!Number.isSafeInteger(count) || count < 1 || count > 0xffff_ffff) { + throw new Error("count must be a positive 32-bit integer"); +} +const urls = Array.from({ length: count }, (_, i) => { + const prefix = Buffer.alloc(6); + prefix.writeUInt32BE(i, 2); + return `data:image/png;base64,${prefix.toString("base64")}${"A".repeat(2040)}`; +}); +const content = JSON.stringify({ rawResponsesItem: urls }); +const timings = []; +let digest; +for (let i = 0; i < 4; i++) { + global.gc?.(); + const started = performance.now(); + const result = splitMediaContent(content); + const ms = performance.now() - started; + if (result.media.length !== count) throw new Error(`expected ${count} media rows`); + const signature = createHash("sha256").update(JSON.stringify(result)).digest("hex"); + if (digest && digest !== signature) throw new Error("non-deterministic output"); + digest = signature; + if (i > 0) timings.push(ms); +} +console.log(JSON.stringify({ + count, + bytes: Buffer.byteLength(content), + timings_ms: timings, + median_ms: [...timings].sort((a, b) => a - b)[1], + digest, +})); diff --git a/src/storage/identity.ts b/src/storage/identity.ts index 4276971175..51b99d3b4f 100644 --- a/src/storage/identity.ts +++ b/src/storage/identity.ts @@ -38,7 +38,7 @@ const MEDIA_URL_MARKER_RE = new RegExp( ); export function splitMediaContent(content: string): { content: string; hash: string; media: { hash: string; bytes: string }[] } { - let media: { hash: string; bytes: string }[] = []; + const media: { hash: string; bytes: string }[] = []; let split = content; if (content.includes("\"imageBase64\"")) { let parsed: { imageBase64?: unknown }; @@ -68,7 +68,7 @@ export function splitMediaContent(content: string): { content: string; hash: str const mediaHash = sha256Hex(payload); if (!byHash.has(mediaHash)) { byHash.set(mediaHash, payload); - media = [...media, { hash: mediaHash, bytes: payload }]; + media.push({ hash: mediaHash, bytes: payload }); } return `data:${mime};base64,{$media:${mediaHash}}`; }); diff --git a/tests/storage-media-sidecar-split.test.ts b/tests/storage-media-sidecar-split.test.ts index 83dfae911b..734a4681f0 100644 --- a/tests/storage-media-sidecar-split.test.ts +++ b/tests/storage-media-sidecar-split.test.ts @@ -125,3 +125,36 @@ test("#4069: small payloads (<2 KiB) stay inline — no CAS row, no join", () => rmSync(dir, { recursive: true, force: true }); } }); + +test("many distinct sidecar payloads retain first-seen order, deduplication and byte-exact hydration", () => { + const payloads = Array.from({ length: 512 }, (_, i) => { + const prefix = Buffer.alloc(6); + prefix.writeUInt32BE(i, 2); + return prefix.toString("base64") + "A".repeat(2040); + }); + const original = JSON.stringify({ + id: "m00001", + role: "user", + contentType: "text", + text: "many images", + imageBase64: payloads[17], + rawOpenaiContentParts: [...payloads, payloads[0], payloads[17], payloads[511]].map((payload) => ({ + type: "image_url", + image_url: { url: `data:image/png;base64,${payload}` }, + })), + }); + const dir = mkdtempSync(join(tmpdir(), "msp-many-")); + const store = UnifiedStore.open(join(dir, "index.db")); + try { + const plan = planIngest(sessionOf("media-split-many", [JSON.parse(original)])); + const entry = plan.messages[0]; + const expected = [payloads[17], ...payloads.filter((_, i) => i !== 17)]; + assert.deepEqual(entry.media?.map((m) => m.hash), expected.map(sha256)); + assert.deepEqual(entry.media?.map((m) => m.bytes), expected); + store.ingestSession(plan); + assert.equal(store.getMessages("media-split-many")[0].content, original); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); From c6fa0c930426f90d80459667fd53c02cb7257304 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:55:28 +0800 Subject: [PATCH 62/67] fix(summary): keep exponential backoff for blank Retry-After headers (cherry picked from commit ada232da6cdb141823021f538fda03888276aa7e) --- src/external-summary-http.ts | 2 +- tests/external-summary-http.test.ts | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/external-summary-http.ts b/src/external-summary-http.ts index 88c2b60902..303d165d4f 100644 --- a/src/external-summary-http.ts +++ b/src/external-summary-http.ts @@ -46,7 +46,7 @@ function backoffDelayMs(attempt: number): number { } function retryAfterDelayMs(header: string | null, fallbackMs: number): number { - if (header !== null) { + if (header !== null && header.trim() !== "") { const seconds = Number(header); if (Number.isFinite(seconds) && seconds >= 0) return Math.min(Math.ceil(seconds * 1000), SUMMARY_RETRY_AFTER_CAP_MS); const dateMs = Date.parse(header); diff --git a/tests/external-summary-http.test.ts b/tests/external-summary-http.test.ts index 7a2c78a3ec..e4a73f3435 100644 --- a/tests/external-summary-http.test.ts +++ b/tests/external-summary-http.test.ts @@ -196,6 +196,27 @@ test("external summary HTTP: a transient 429 with Retry-After recovers on the se assert.deepEqual(_summaryRetryDelaysForTest(), [7000]); }); +for (const [label, header, delays] of [ + ["empty", "", [1000, 2000]], + ["whitespace-only", " \t ", [1000, 2000]], + ["explicit zero", "0", [0, 0]], +] as const) { + test(`external summary HTTP: ${label} Retry-After preserves the intended retry delay`, async () => { + let hits = 0; + _setSummaryRetryDelayForTest(0); + await upstream((_req, res) => { + hits++; + if (hits < 3) { res.writeHead(429, { "retry-after": header }); res.end("rate limited"); return; } + res.end(JSON.stringify(completion("responses"))); + }, async (url) => { + const result = await new ExternalSummaryExecutor(1).execute(work, [createSummaryHttpCandidate(target(url), 4096)], budget); + assert.equal(result.status, "success"); + }); + assert.equal(hits, 3); + assert.deepEqual(_summaryRetryDelaysForTest(), delays); + }); +} + test("external summary HTTP: a hostile Retry-After is capped so it cannot eat the attempt budget", async () => { let hits = 0; _setSummaryRetryDelayForTest(0); From 23bcd2e135f38c628a94ab91f2c65c9fb30e81a7 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:56:54 +0800 Subject: [PATCH 63/67] fix(storage): skip non-object JSONL entries when locating archives (cherry picked from commit a76d5031e545e87e06d0f054b1ec99126e043958) --- src/storage/mysql.ts | 6 +- src/storage/store.ts | 12 +-- tests/storage-archive-scalar-lines.test.ts | 109 +++++++++++++++++++++ 3 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 tests/storage-archive-scalar-lines.test.ts diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index d2f0dfe391..e58d62b749 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1848,13 +1848,13 @@ export class MysqlBackend implements StorageBackend { rawLines = readFileSync(exportPath, "utf8").split("\n"); for (let i = 0; i < rawLines.length; i++) { if (!rawLines[i].trim()) continue; - let candidate: { sessionId?: string }; + let candidate: unknown; try { - candidate = JSON.parse(rawLines[i]) as { sessionId?: string }; + candidate = JSON.parse(rawLines[i]) as unknown; } catch { continue; } - if (candidate.sessionId === sessionId) { + if (candidate && typeof candidate === "object" && "sessionId" in candidate && candidate.sessionId === sessionId) { lineNo = i; break; } diff --git a/src/storage/store.ts b/src/storage/store.ts index 7890da378e..3414133b8b 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2153,13 +2153,13 @@ export class UnifiedStore { rawLines = readFileSync(exportPath, "utf8").split("\n"); for (let i = 0; i < rawLines.length; i++) { if (!rawLines[i].trim()) continue; - let candidate: { sessionId?: string }; + let candidate: unknown; try { - candidate = JSON.parse(rawLines[i]) as { sessionId?: string }; + candidate = JSON.parse(rawLines[i]) as unknown; } catch { continue; } - if (candidate.sessionId === sessionId) { lineNo = i; break; } + if (candidate && typeof candidate === "object" && "sessionId" in candidate && candidate.sessionId === sessionId) { lineNo = i; break; } } } catch { return { ok: false, reason: "export-corrupt" }; @@ -2312,9 +2312,9 @@ export class UnifiedStore { let lineNo = -1; for (let i = 0; i < rawLines.length; i++) { if (!rawLines[i].trim()) continue; - let head: { sessionId?: string; blockId?: string }; - try { head = JSON.parse(rawLines[i]) as typeof head; } catch { continue; } - if (head.sessionId === sessionId && head.blockId === blockId) { lineNo = i; break; } + let head: unknown; + try { head = JSON.parse(rawLines[i]) as unknown; } catch { continue; } + if (head && typeof head === "object" && "sessionId" in head && "blockId" in head && head.sessionId === sessionId && head.blockId === blockId) { lineNo = i; break; } } if (lineNo === -1) return { ok: false, reason: "not-found" }; let dump: { block: Record; content: Record }; diff --git a/tests/storage-archive-scalar-lines.test.ts b/tests/storage-archive-scalar-lines.test.ts new file mode 100644 index 0000000000..d8ab5ade0d --- /dev/null +++ b/tests/storage-archive-scalar-lines.test.ts @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "full"; + +function session(id: string): LegacySessionLike { + return { + id, createdAt: 1000, lastSeen: 2000, meta: { protocol: "openai" }, stats: {}, metadata: {}, + state: { + blocks: [{ blockId: "b1", runId: 1, active: true, createdAt: 1000, tier: 1, + summary: "retained summary", kind: "fold", reason: "test", + effectiveMessageIds: [`${id}-message`], directMessageIds: [] }], + nextBlockId: 2, nextRunId: 2, messageRefs: { byRaw: {}, byRef: {} }, + }, + blockContents: new Map(), + lastMessages: [{ id: `${id}-message`, role: "user", contentType: "text", text: "original message" }], + lastMessagesFolded: false, + } as unknown as LegacySessionLike; +} + +const unrelated = 'null\n42\nfalse\n"text"\n[]\n{"truncated":\n'; + +test("SQLite session restore skips non-object JSONL before a valid archive", () => { + const dir = mkdtempSync(join(tmpdir(), "archive-scalar-session-")); + const store = UnifiedStore.open(join(dir, "index.db")); + try { + const archiveDir = join(dir, "archive"); + store.ingestSession(planIngest(session("wanted"))); + store.ingestSession(planIngest(session("other"))); + const before = store.getMessages("wanted"); + const blocks = store.getBlocks("wanted"); + assert.ok(store.archiveSession("wanted", { archiveDir }).ok); + assert.ok(store.archiveSession("other", { archiveDir }).ok); + const file = join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); + const valid = readFileSync(file, "utf8"); + const otherLine = valid.split("\n").find((line) => line.includes('"sessionId":"other"'))!; + writeFileSync(file, unrelated + valid); + assert.deepEqual(store.restoreArchived("wanted", { archiveDir }), { ok: true }); + assert.deepEqual(store.getMessages("wanted"), before); + assert.deepEqual(store.getBlocks("wanted"), blocks); + assert.ok(readFileSync(file, "utf8").split("\n").includes(otherLine)); + assert.deepEqual(store.restoreArchived("other", { archiveDir }), { ok: true }); + assert.deepEqual(store.restoreArchived("absent", { archiveDir }), { ok: false, reason: "export-missing" }); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("SQLite block restore skips null before the requested block without throwing", () => { + const dir = mkdtempSync(join(tmpdir(), "archive-scalar-block-")); + const store = UnifiedStore.open(join(dir, "index.db")); + try { + const archiveDir = join(dir, "archive"); + store.ingestSession(planIngest(session("wanted"))); + const before = store.getBlocks("wanted"); + assert.ok(store.archiveBlock("wanted", "b1", { archiveDir }).ok); + const file = join(archiveDir, UnifiedStore.BLOCK_ARCHIVE_LOG_NAME); + writeFileSync(file, unrelated + readFileSync(file, "utf8")); + assert.deepEqual(store.restoreBlock("wanted", "b1", { archiveDir }), { ok: true }); + assert.deepEqual(store.getBlocks("wanted"), before); + writeFileSync(file, unrelated); + assert.deepEqual(store.restoreBlock("absent", "b1", { archiveDir }), { ok: false, reason: "not-found" }); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("MySQL archive scanner reaches a valid later record and commits through the driver contract", async () => { + const dir = mkdtempSync(join(tmpdir(), "archive-scalar-mysql-")); + const source = UnifiedStore.open(join(dir, "source.db")); + try { + const archiveDir = join(dir, "archive"); + source.ingestSession(planIngest(session("wanted"))); + assert.ok(source.archiveSession("wanted", { archiveDir }).ok); + const file = join(archiveDir, MysqlBackend.ARCHIVE_LOG_NAME); + writeFileSync(file, unrelated + readFileSync(file, "utf8")); + const calls: string[] = []; + const conn = { + async beginTransaction() { calls.push("begin"); }, + async commit() { calls.push("commit"); }, + async rollback() { calls.push("rollback"); }, + release() { calls.push("release"); }, + async query(sql: string) { calls.push(sql); return [{ affectedRows: 1 }]; }, + }; + const driver = { createPool: () => ({ getConnection: async () => conn }) } as unknown as Mysql2Module; + const backend = new MysqlBackend("unused", driver); + assert.deepEqual(await backend.restoreArchived("wanted", { archiveDir }), { ok: true }); + assert.equal(calls[0], "begin"); + assert.ok(calls.some((sql) => sql.startsWith("INSERT INTO messages"))); + assert.deepEqual(calls.slice(-2), ["commit", "release"]); + assert.ok(!readFileSync(file, "utf8").includes('"sessionId":"wanted"')); + writeFileSync(file, unrelated); + const count = calls.length; + assert.deepEqual(await backend.restoreArchived("absent", { archiveDir }), { ok: false, reason: "export-missing" }); + assert.equal(calls.length, count); + } finally { + source.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); From 853cb9c8b65ebe40b36c521e63e0e34cde7cc723 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:59:09 +0800 Subject: [PATCH 64/67] fix(render): avoid argument limits when locating large fold anchors (cherry picked from commit 43a8954133df0f3530f1fa30322c034460091481) --- src/plugin-render.ts | 14 ++++++----- tests/plugin-render-endpoint.test.ts | 35 +++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/plugin-render.ts b/src/plugin-render.ts index 636aca2043..703207d39f 100644 --- a/src/plugin-render.ts +++ b/src/plugin-render.ts @@ -99,15 +99,17 @@ export async function handlePluginRender(req: http.IncomingMessage, res: http.Se const coveredSeqs = new Set(); const anchors: { pos: number; msg: BiliMessage }[] = []; for (const block of activeBlocks) { - const blockSeqs: number[] = []; + let maxCover: number | undefined; for (const id of block.effectiveMessageIds) { const seq = idToSeq.get(id); - if (seq !== undefined) blockSeqs.push(seq); + if (seq !== undefined) { + coveredSeqs.add(seq); + maxCover = maxCover === undefined ? seq : Math.max(maxCover, seq); + } } - if (blockSeqs.length === 0) continue; - for (const s of blockSeqs) coveredSeqs.add(s); - const maxCover = Math.max(...blockSeqs); - const pos = viewRows.findIndex((r) => r.seq > maxCover); + if (maxCover === undefined) continue; + const lastCoveredSeq = maxCover; + const pos = viewRows.findIndex((r) => r.seq > lastCoveredSeq); // #4732: pos === -1 means the fold reaches the END of the window — the // covered rows are suppressed below, so the anchor MUST still land // (at the tail) or the rendered prefix silently drops the whole fold diff --git a/tests/plugin-render-endpoint.test.ts b/tests/plugin-render-endpoint.test.ts index 7c3d696ecf..345803d0f7 100644 --- a/tests/plugin-render-endpoint.test.ts +++ b/tests/plugin-render-endpoint.test.ts @@ -17,7 +17,8 @@ import { startServer } from "../src/server.ts"; import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _resetPluginStateForTest } from "../src/plugin.ts"; -import { _resetSessionsForTest } from "../src/session.ts"; +import { getSession, _resetSessionsForTest } from "../src/session.ts"; +import type { MessageRow } from "../src/storage/store.ts"; import { _setForTest } from "../src/registry.ts"; import { resetToolRingForTest } from "../src/tool-ring.ts"; import { __resetIncrementalLaneForTests } from "../src/incremental-lane.js"; @@ -263,3 +264,35 @@ test("S5 guard rails: poison msgids hard-fail; count mismatch degrades silently" await h.close(); } }); + +test("render handles a fold covering 150000 rows without a function-argument limit", async (t) => { + const h = await harness(true); + try { + const count = 150_000; + const conv = "large-render"; + let rows: MessageRow[] = Array.from({ length: count + 1 }, (_, i) => ({ + seq: i * 2, msgId: `raw-${i}`, ref: null, refNum: null, role: "user", kind: "message", + content: JSON.stringify({ id: `raw-${i}`, role: "user", contentType: "text", text: i === count ? "uncovered tail" : `covered ${i}` }), + })); + t.mock.method(h.store, "unifiedTailViewRows", () => rows); + t.mock.method(h.store, "unifiedTailSeqs", () => rows.map((row) => row.seq)); + const session = getSession(conv); + (session.state.blocks as unknown[]).push({ + blockId: "large", active: true, + effectiveMessageIds: [...rows.slice(0, count).map((row) => row.msgId).reverse(), "outside-view"], + }); + const rendered = await h.render({ conversationId: conv, untilSeq: count + 1 }); + assert.equal(rendered.status, 200, JSON.stringify(rendered.body)); + const output = rendered.body as { appliedSeq: number; messages: { content: { type: string; text?: string }[] }[] }; + assert.equal(output.appliedSeq, count + 1); + assert.deepEqual(output.messages.flatMap((message) => message.content.map((part) => part.text)), ["", "uncovered tail"]); + + rows = [{ ...rows[count]!, seq: 0 }]; + const unresolved = await h.render({ conversationId: conv, untilSeq: 1 }); + assert.equal(unresolved.status, 200); + const unchanged = unresolved.body as { messages: { content: { text?: string }[] }[] }; + assert.deepEqual(unchanged.messages.flatMap((message) => message.content.map((part) => part.text)), ["uncovered tail"]); + } finally { + await h.close(); + } +}); From 702b5b013b68c984e9e735a8e7f6d0f0867a1cc5 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 19:11:34 +0800 Subject: [PATCH 65/67] perf(affinity): construct only sampled SimHash shingles (cherry picked from commit 140e970c1aacc10199196283ec3a20d28b9df863) --- scripts/bench-simhash-sampling.mjs | 26 ++++++++++++++++++++++++++ src/affinity-simhash.ts | 23 ++++++++--------------- tests/simhash-vote-range.test.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 15 deletions(-) create mode 100644 scripts/bench-simhash-sampling.mjs diff --git a/scripts/bench-simhash-sampling.mjs b/scripts/bench-simhash-sampling.mjs new file mode 100644 index 0000000000..ff61e22d0d --- /dev/null +++ b/scripts/bench-simhash-sampling.mjs @@ -0,0 +1,26 @@ +// Run with: node --max-old-space-size=512 --expose-gc --import tsx scripts/bench-simhash-sampling.mjs [tokens] +import { performance } from "node:perf_hooks"; +import { simhashHex } from "../src/affinity-simhash.ts"; + +const count = Number(process.argv[2] ?? 1_000_000); +if (!Number.isSafeInteger(count) || count < 1) throw new Error("tokens must be a positive safe integer"); +const canonical = Array.from({ length: count }, (_, i) => `word${i % 997}`).join(" "); +const timings = []; +let sketch; +for (let i = 0; i < 4; i++) { + global.gc?.(); + const start = performance.now(); + const result = simhashHex(canonical); + const elapsed = performance.now() - start; + if (sketch !== undefined && sketch !== result) throw new Error("unstable sketch"); + sketch = result; + if (i > 0) timings.push(elapsed); +} +console.log(JSON.stringify({ + count, + bytes: Buffer.byteLength(canonical), + sketch, + timings_ms: timings, + median_ms: [...timings].sort((a, b) => a - b)[1], + max_rss_kib: process.resourceUsage().maxRSS, +})); diff --git a/src/affinity-simhash.ts b/src/affinity-simhash.ts index f1ea3c6f60..5fc9695450 100644 --- a/src/affinity-simhash.ts +++ b/src/affinity-simhash.ts @@ -54,22 +54,15 @@ function fnv1a32(text: string, basis: number): number { * so a short message still yields a stable, comparable sketch. */ export function simhashHex(canonical: string): string { const tokens = canonical.match(/[\p{L}\p{N}_]+/gu) ?? []; - const features: string[] = []; - if (tokens.length < 2) { - for (const t of tokens) features.push(t); - } else { - for (let i = 0; i + 1 < tokens.length; i++) features.push(`${tokens[i]}\u0001${tokens[i + 1]}`); - } - if (features.length === 0) { - // No word tokens at all (pure punctuation/whitespace): hash the raw - // string so even that degenerate case is deterministic and comparable. - features.push(canonical); - } - const stride = Math.max(1, Math.ceil(features.length / MAX_FEATURES)); + const featureCount = Math.max(1, tokens.length - 1); + const stride = Math.max(1, Math.ceil(featureCount / MAX_FEATURES)); const v = new Int16Array(64); - for (let f = 0; f < features.length; f += stride) { - const h1 = fnv1a32(features[f]!, 2166136261); - const h2 = fnv1a32(features[f]!, 2166136261 ^ 0x9e3779b9); + for (let f = 0; f < featureCount; f += stride) { + // Construct only sampled shingles, not an additional full-size array + // for a long message. No-token inputs still hash the raw string. + const feature = tokens.length < 2 ? (tokens[0] ?? canonical) : `${tokens[f]}\u0001${tokens[f + 1]}`; + const h1 = fnv1a32(feature, 2166136261); + const h2 = fnv1a32(feature, 2166136261 ^ 0x9e3779b9); for (let b = 0; b < 64; b++) { const bit = b < 32 ? (h1 >>> b) & 1 : (h2 >>> (b - 32)) & 1; v[b] += bit === 1 ? 1 : -1; diff --git a/tests/simhash-vote-range.test.ts b/tests/simhash-vote-range.test.ts index 932d185826..2fb28a831e 100644 --- a/tests/simhash-vote-range.test.ts +++ b/tests/simhash-vote-range.test.ts @@ -18,3 +18,29 @@ test("unrelated repeated messages keep distinct sketches at the sampling cap", ( assert.notEqual(first, second); assert.equal(hammingHex(first, second), hammingHex(simhashHex("alpha alpha"), simhashHex("omega omega"))); }); + +test("SimHash sampling preserves persisted sketches across stride boundaries", () => { + // Captured from the original full-shingle implementation. Distinct tokens + // make a shifted sample observable, unlike a repeated-shingle fixture. + const sketches: [number, string][] = [ + [2, "896b965e996b0ce5"], + [128, "015bb052e72a4ca9"], + [255, "017b4012ee3a4d91"], + [256, "037b4012ee3a4d91"], + [257, "037b4012ee3a4d91"], + [258, "00486102fe224e09"], + [511, "00880302ffbac609"], + [512, "00880102ff3ac609"], + [513, "00880102ff3ac609"], + [100_000, "009213c27f3acc29"], + ]; + for (const [count, expected] of sketches) { + const canonical = Array.from({ length: count }, (_, i) => `word${i % 997}`).join(" "); + assert.equal(simhashHex(canonical), expected, `${count} tokens`); + } + const unicode = Array.from({ length: 1025 }, (_, i) => `中${i % 257}é`).join(" "); + assert.equal(simhashHex(unicode), "568a84089d1b5303"); + assert.equal(simhashHex(""), "1f2be47c811c9dc5"); + assert.equal(simhashHex("!!! 😀"), "74c4a446e2763713"); + assert.equal(simhashHex("唯一"), "50817b0bf9fa05fa"); +}); From 87611b385e05a6b9b307605257612915f136c3a6 Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 19:01:15 +0800 Subject: [PATCH 66/67] fix(storage): match KV key prefixes literally (cherry picked from commit 56242e3d65f016a91f65171056a4ebb9c5987e3f) --- src/storage/mysql.ts | 4 ++-- src/storage/store.ts | 4 ++-- tests/storage-backend-contract.test.ts | 25 +++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index e58d62b749..b3c2c8b18e 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -2208,8 +2208,8 @@ export class MysqlBackend implements StorageBackend { } if (keyPrefix !== undefined && keyPrefix !== "") { validateExtKey(keyPrefix, "key prefix"); - where.push("kv_key LIKE CONCAT(?, '%')"); - args.push(keyPrefix.replace(/[\\%_]/g, (ch) => "\\" + ch)); + where.push("kv_key LIKE CONCAT(?, '%') AND LOCATE(CAST(? AS BINARY), CAST(kv_key AS BINARY)) = 1"); + args.push(keyPrefix.replace(/[\\%_]/g, (ch) => "\\" + ch), keyPrefix); } const sql = "SELECT namespace, kv_key, value_json, updated_at FROM kv_store" + diff --git a/src/storage/store.ts b/src/storage/store.ts index 3414133b8b..5586982e34 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2613,8 +2613,8 @@ export class UnifiedStore { } if (keyPrefix !== undefined && keyPrefix !== "") { validateExtKey(keyPrefix, "key prefix"); - where.push("key LIKE ? ESCAPE '\\'"); - args.push(likeEscape(keyPrefix) + "%"); + where.push("key LIKE ? ESCAPE '\\' AND instr(key, ?) = 1"); + args.push(likeEscape(keyPrefix) + "%", keyPrefix); } const sql = "SELECT namespace, key, value_json, updated_at FROM kv_store" + diff --git a/tests/storage-backend-contract.test.ts b/tests/storage-backend-contract.test.ts index 9ce9c24e24..6fd514a0d0 100644 --- a/tests/storage-backend-contract.test.ts +++ b/tests/storage-backend-contract.test.ts @@ -78,6 +78,31 @@ test("registry: sqlite resolves through the registry; names are advertised", asy } }); +for (const lane of lanes) { + test(`${lane.name}: KV prefixes match literal keys without case or accent folding`, async () => { + const dir = scratch(); + const backend = await lane.open(dir); + const namespace = `kv-prefix-${lane.name}-${process.pid}-${Date.now()}`; + try { + for (const key of ["Prefix:one", "prefix:two", "café:one", "cafe:two", "a%_\\[literal]:one", "aXYZliteral:two"]) { + await backend.setKv(namespace, key, { key }); + } + const keys = async (prefix: string) => (await backend.listKv(namespace, prefix)).map((r) => r.key); + assert.deepEqual(await keys("Prefix:"), ["Prefix:one"]); + assert.deepEqual(await keys("prefix:"), ["prefix:two"]); + assert.deepEqual(await keys("café:"), ["café:one"]); + assert.deepEqual(await keys("cafe:"), ["cafe:two"]); + assert.deepEqual(await keys("a%_\\[literal]:"), ["a%_\\[literal]:one"]); + assert.deepEqual(await keys("missing:"), []); + assert.equal((await keys("")).length, 6); + } finally { + await backend.deleteKv(namespace); + await backend.close(); + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +} + test("contract: ingest → reads round-trip through the interface", async () => { const dir = scratch(); try { From 65f036c424174756d7bd97d8b3aeaea1b07114ee Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 17:15:07 +0800 Subject: [PATCH 67/67] fix(ci): preserve live fork PR runs during janitor cleanup (cherry picked from commit d992d5dc09eca67596266b7d9e0b818894341716) --- .github/workflows/ci-janitor.yml | 13 +-- scripts/live-files.mjs | 2 + tests/ci-janitor-fork-heads.test.ts | 123 ++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 tests/ci-janitor-fork-heads.test.ts diff --git a/.github/workflows/ci-janitor.yml b/.github/workflows/ci-janitor.yml index 9790a4aa07..a79c22490c 100644 --- a/.github/workflows/ci-janitor.yml +++ b/.github/workflows/ci-janitor.yml @@ -80,7 +80,7 @@ jobs: warned=0 collect() { - gh api --paginate --jq '.workflow_runs[] | [.id, .status, .created_at, .event, (.head_branch // "")] | @tsv' \ + gh api --paginate --jq '.workflow_runs[] | [.id, .status, .created_at, .event, (.head_branch // "-"), (.head_repository.full_name // "-")] | @tsv' \ "repos/$REPO/actions/runs?status=$1&per_page=100" } { collect queued; collect in_progress; } | sort -u > /tmp/janitor-runs.tsv @@ -111,7 +111,7 @@ jobs: # Actions job, so EVERY rule below (zombie reap, age limits, master # warnings) was dead code from day one (2026-10-30). Feed the collected # list explicitly. - while IFS=$'\t' read -r id status created event branch; do + while IFS=$'\t' read -r id status created event branch head_repo; do if [ "$id" = "$SELF_ID" ]; then continue; fi age_min=$(( (now - $(date -d "$created" +%s)) / 60 )) @@ -122,9 +122,12 @@ jobs: fi reason="" - if [ "$event" = "pull_request" ] && [ -n "$branch" ]; then - if ! gh api "repos/$REPO/branches/$branch" >/dev/null 2>&1; then - reason="head branch '$branch' deleted" + if [ "$event" = "pull_request" ] && [ "$branch" != "-" ] && [ "$head_repo" != "-" ]; then + branch_ref=$(jq -rn --arg ref "$branch" '$ref | @uri') + branch_response=$(gh api --include "repos/$head_repo/branches/$branch_ref" 2>/dev/null) || true + branch_status=$(printf '%s\n' "$branch_response" | awk 'NR == 1 { print $2 }') + if [ "$branch_status" = "404" ] && gh api "repos/$head_repo" >/dev/null 2>&1; then + reason="head branch '$head_repo:$branch' deleted" fi fi if [ -z "$reason" ] && [ "$status" = "queued" ] && [ "$starve_mode" = "1" ] && [ "$STARVED_QUEUE_MAX" -gt 0 ]; then diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index 91e5d49457..63adb1c646 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -137,6 +137,8 @@ export const LIVE_FILES = [ // Real npm pack/install children use mkdtemp package fixtures and the // preload's per-process XDG roots; they bind no listener ports. "tests/pack-install.test.ts", + // Runs the Linux janitor shell against per-case local API fixtures; no ports or credentials. + "tests/ci-janitor-fork-heads.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; diff --git a/tests/ci-janitor-fork-heads.test.ts b/tests/ci-janitor-fork-heads.test.ts new file mode 100644 index 0000000000..d95cd079dc --- /dev/null +++ b/tests/ci-janitor-fork-heads.test.ts @@ -0,0 +1,123 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +interface Run { + id: number; + status: string; + event: string; + head_branch: string | null; + head_repository: { full_name: string } | null; + created_at: string; +} + +const workflow = readFileSync(path.resolve(import.meta.dirname, "../.github/workflows/ci-janitor.yml"), "utf8"); +const script = workflow.slice(workflow.indexOf(" set -euo pipefail")).replace(/^ {10}/gm, ""); +const available = process.platform === "linux" + && spawnSync("bash", ["--version"]).status === 0 + && spawnSync("jq", ["--version"]).status === 0; + +function run(id: number, repo: string | null, branch: string | null, age = 0): Run { + return { + id, status: "in_progress", event: "pull_request", head_branch: branch, + head_repository: repo ? { full_name: repo } : null, + created_at: new Date(Date.now() - age * 60_000).toISOString(), + }; +} + +function sweep(runs: Run[], responses: Record, dryRun = false) { + const dir = mkdtempSync(path.join(tmpdir(), "ci-janitor-")); + try { + writeFileSync(path.join(dir, "data.json"), JSON.stringify({ runs, responses })); + writeFileSync(path.join(dir, "calls.jsonl"), ""); + writeFileSync(path.join(dir, "gh"), `#!${process.execPath} +const fs = require("node:fs"); +const cp = require("node:child_process"); +const path = require("node:path"); +const dir = process.env.JANITOR_FIXTURE; +const { runs, responses } = JSON.parse(fs.readFileSync(path.join(dir, "data.json"), "utf8")); +const args = process.argv.slice(2); +const endpoint = args.find(value => value.startsWith("repos/")); +fs.appendFileSync(path.join(dir, "calls.jsonl"), JSON.stringify(endpoint) + "\\n"); +if (endpoint.includes("/actions/runs?status=")) { + const status = new URL("https://fixture/" + endpoint).searchParams.get("status"); + const query = args[args.indexOf("--jq") + 1]; + const result = cp.spawnSync("jq", ["-r", query], { + input: JSON.stringify({ workflow_runs: runs.filter(run => run.status === status) }), encoding: "utf8", + }); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); + process.exit(result.status ?? 1); +} +const status = endpoint.endsWith("/cancel") ? 202 : (responses[endpoint] ?? 404); +if (args.includes("--include")) process.stdout.write("HTTP/2.0 " + status + "\\n\\n"); +process.stdout.write("{}\\n"); +process.exit(status >= 400 ? 1 : 0); +`); + chmodSync(path.join(dir, "gh"), 0o755); + // Keep the workflow's temporary inventory inside this fixture too. + const result = spawnSync("bash", ["-c", script.replaceAll("/tmp/janitor-runs.tsv", path.join(dir, "runs.tsv"))], { + encoding: "utf8", + env: { ...process.env, PATH: `${dir}${path.delimiter}${process.env.PATH}`, JANITOR_FIXTURE: dir, + REPO: "owner/project", SELF_ID: "999", QUEUE_MAX: "120", PROGRESS_MAX: "240", + STARVED_QUEUE: "30", STARVED_QUEUE_MAX: "60", DRY_RUN: dryRun ? "1" : "", GH_TOKEN: "" }, + }); + assert.equal(result.status, 0, result.stderr || result.stdout); + const calls = readFileSync(path.join(dir, "calls.jsonl"), "utf8").trim().split("\n").map(line => JSON.parse(line) as string); + return { calls, cancelled: calls.filter(call => call.endsWith("/cancel")), output: result.stdout }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +test("janitor keeps a live fork head and looks it up in its owning repository", { skip: !available }, () => { + const result = sweep([run(1, "contributor/project", "fix/a+b")], { + "repos/contributor/project/branches/fix%2Fa%2Bb": 200, + }); + assert.deepEqual(result.cancelled, []); + assert.ok(result.calls.includes("repos/contributor/project/branches/fix%2Fa%2Bb")); +}); + +test("janitor distinguishes equal branch names in different repositories", { skip: !available }, () => { + const result = sweep([run(1, "contributor/project", "same-name")], { + "repos/owner/project/branches/same-name": 200, + "repos/contributor/project": 200, + }); + assert.deepEqual(result.cancelled, ["repos/owner/project/actions/runs/1/cancel"]); +}); + +test("janitor still cancels a deleted same-repository branch", { skip: !available }, () => { + const result = sweep([run(2, "owner/project", "deleted")], { "repos/owner/project": 200 }); + assert.deepEqual(result.cancelled, ["repos/owner/project/actions/runs/2/cancel"]); +}); + +for (const status of [403, 429, 500]) { + test(`janitor does not interpret HTTP ${status} as a deleted branch`, { skip: !available }, () => { + const result = sweep([run(3, "owner/project", "live")], { + "repos/owner/project/branches/live": status, "repos/owner/project": 200, + }); + assert.deepEqual(result.cancelled, []); + }); +} + +test("janitor does not infer deletion from an inaccessible repository or missing head metadata", { skip: !available }, () => { + const result = sweep([run(4, "hidden/project", "live"), run(5, null, "live"), run(6, "owner/project", null)], {}); + assert.deepEqual(result.cancelled, []); +}); + +test("janitor retains age cleanup and protects scheduled master runs", { skip: !available }, () => { + const result = sweep([ + run(7, "contributor/project", "live", 300), + { ...run(8, "owner/project", "master", 300), event: "schedule" }, + ], { "repos/contributor/project/branches/live": 200 }); + assert.deepEqual(result.cancelled, ["repos/owner/project/actions/runs/7/cancel"]); +}); + +test("janitor dry-run reports deleted branches without cancelling", { skip: !available }, () => { + const result = sweep([run(9, "owner/project", "deleted")], { "repos/owner/project": 200 }, true); + assert.deepEqual(result.cancelled, []); + assert.match(result.output, /head branch .*deleted/); +});