diff --git a/src/agent/dsh-native.ts b/src/agent/dsh-native.ts index 19a4494cd8..5904fba40e 100644 --- a/src/agent/dsh-native.ts +++ b/src/agent/dsh-native.ts @@ -58,7 +58,8 @@ import { isVersionNewer } from "../update.js"; import { VERSION } from "../version.js"; import { APIG_RESIGN_SCHEME, unresolvedRefusals } from "../apig-resign.js"; import { adminRelayEnabled, resolveResignSettings } from "../config.js"; -import { dshContextTabEnabled, dshContextTabLabel } from "../knobs.js"; +import { decodeRequestBody } from "../content-encoding.js"; +import { dshContextTabEnabled, dshContextTabLabel, limitsDecodedBytes } from "../knobs.js"; import { MIN_ASSUMED_WINDOW_TOKENS } from "../util.js"; import { markNativeHost, nativeAttachOrigin, nativeBootstrapGate, nativeProxyScriptPath, proxyEnvOrigin, singleFlight } from "./native-bootstrap.js"; import { bodyProvablyOverWindow as sharedBodyProvablyOverWindow, installNativeFetchIntercept, liveStateSlotEffect, noteRoutedOrigin, observeRoutedOrigin, routedBiliModelUrl, type GateVerdict, type NativeInterceptState } from "./native-intercept.js"; @@ -1336,25 +1337,70 @@ function noteForkVerdict(verdict: Extract): voi } } -function bodyJsonOf(init: RequestInit | undefined): unknown { - const raw = init?.body; - if (typeof raw === "string") { - try { - return JSON.parse(raw); - } catch { - return undefined; +// #4829: the content-encoding header across every shape fetch accepts +// (Headers instance, [name, value] pairs, name → value record). Case- +// insensitive; malformed shapes yield undefined = "no encoding declared". +function contentEncodingOf(init: RequestInit | undefined): string | undefined { + const headers = init?.headers; + if (headers === undefined || headers === null) return undefined; + try { + if (typeof (headers as Headers).get === "function") { + const v = (headers as Headers).get("content-encoding"); + return v === null ? undefined : v; + } + } catch { + return undefined; + } + if (Array.isArray(headers)) { + for (const pair of headers) { + if (Array.isArray(pair) && pair.length >= 2 && String(pair[0]).toLowerCase() === "content-encoding" && typeof pair[1] === "string") return pair[1]; } + return undefined; } - if (raw instanceof Uint8Array || raw instanceof ArrayBuffer) { - try { - return JSON.parse(new TextDecoder().decode(raw)); - } catch { - return undefined; + if (typeof headers === "object") { + for (const [k, v] of Object.entries(headers)) { + if (k.toLowerCase() !== "content-encoding") continue; + if (typeof v === "string") return v; + if (Array.isArray(v)) return v.join(", "); } } return undefined; } +// #4829: hosts may ship the request body content-encoded — the Codex SSE +// transport (pi-ai) sends zstd-compressed Uint8Array bodies with a +// content-encoding header. Decode for INSPECTION ONLY (fork-adoption +// projection, #4082 recall fingerprints); the outgoing bytes are never +// touched. Decoding reuses the server-side codec table (#619) so lane and +// proxy agree on which codings are understood; every failure mode +// (undecodable bytes, unsupported coding, decompression-bomb cap) yields +// undefined = today's unparseable behavior. A JS string cannot be compressed, +// so it parses as-is even under an encoding header. +async function bodyJsonOf(init: RequestInit | undefined): Promise { + const raw = init?.body; + let text: string | undefined; + if (typeof raw === "string") { + text = raw; + } else if (raw instanceof Uint8Array || raw instanceof ArrayBuffer) { + const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw); + const encoding = contentEncodingOf(init); + if (encoding === undefined) { + text = new TextDecoder().decode(bytes); + } else { + try { + text = (await decodeRequestBody(encoding, Buffer.from(bytes), limitsDecodedBytes())).body.toString("utf8"); + } catch { + return undefined; + } + } + } + if (text === undefined) return undefined; + try { + return JSON.parse(text); + } catch { + return undefined; + } +} async function maybeForkAdoptBeforeSend(ctx: PluginContext, init: RequestInit | undefined): Promise { const child = forkChildVerdict(ctx); @@ -1367,10 +1413,10 @@ async function maybeForkAdoptBeforeSend(ctx: PluginContext, init: RequestInit | `bili-native-dsh: fork adoption for ${child.sid} skipped at the tools-ready gate — this request arrived before bili tool registration completed; later requests can still adopt (#4215)`); return; } - const body = bodyJsonOf(init); + const body = await bodyJsonOf(init); if (body === undefined) { noteForkGateOnce(`gate:body-parse:${child.sid}`, "warn", - `bili-native-dsh: fork adoption for ${child.sid} skipped at the body-parse gate — the request body was not parseable JSON, so the replay cannot be projected (#4215)`); + `bili-native-dsh: fork adoption for ${child.sid} skipped at the body-parse gate — the request body was not parseable JSON after content-encoding decode, so the replay cannot be projected (#4215/#4829)`); return; } if (sideShapedBody(body)) { @@ -1607,14 +1653,14 @@ const pendingNegotiatedStamps = new WeakMap(); * contains a remembered boundary for this lane — provably a known session's * continuation — claim the request for the plugin lane; beforeSend stamps the * negotiated sid. No evidence or ambiguity ⇒ undefined ⇒ today's refusal. */ -function maybeNegotiatedClaim(url: string, init: RequestInit | undefined): string | undefined { +async function maybeNegotiatedClaim(url: string, init: RequestInit | undefined): Promise { if (init === undefined) return undefined; const laneKey = recallLaneKey(url); // #4082 review: probe before parsing — agentless sidecars (title-gen etc.) // hit this path constantly, and multi-MB quota-retry bodies must not pay // full JSON.parse + per-message hashing when no session was ever learned. if (!recallLaneHasEntries(laneKey)) return undefined; - const body = bodyJsonOf(init); + const body = await bodyJsonOf(init); if (body === undefined || typeof body !== "object") return undefined; const messages = (body as { messages?: unknown }).messages; if (!Array.isArray(messages) || messages.length === 0) return undefined; @@ -1653,7 +1699,7 @@ async function maybeRecallBeforeSend(ctx: PluginContext, url: string, init: Requ if (!register.toolsReady || init === undefined) return; const attr = attributionOf(ctx); if (attr.state === "ok") { - const body = bodyJsonOf(init); + const body = await bodyJsonOf(init); if (body === undefined || typeof body !== "object") return; const messages = (body as { messages?: unknown }).messages; if (!Array.isArray(messages) || messages.length === 0) return; @@ -2107,12 +2153,30 @@ export function apply(ctx: PluginContext): void { // recall — the body itself may prove it is a known session's // continuation (retry / model switch shape). A claim here routes the // request onto the plugin lane with the negotiated sid; refusal below - // stays byte-identical when there is no evidence. - if (attr.state === "none" && maybeNegotiatedClaim(url, init) !== undefined) { - // Deliberately NOT noteAttrRecovery(): the host attribution is - // still broken — the health signal stays honest; the REQUEST is - // what got rescued, not the initiator scope (#4082). - return true; + // stays byte-identical when there is no evidence. #4829: the claim may + // now decode content-encoded bodies, so this arm settles async — the + // gate contract already tolerates Promise verdicts (threw arm). + if (attr.state === "none") { + // #4829 review: without init there is no body to decode and the + // claim can never succeed — settle synchronously so this shape + // keeps the OLD verdict class: native-ws-intercept.ts compares + // its gate result with === false WITHOUT awaiting. + if (init === undefined) { + threwStreak = 0; + threwGiveUpNoticed = false; + return refuseUnattributed(url, attr, body, init); + } + return maybeNegotiatedClaim(url, init).then((claimed) => { + if (claimed !== undefined) { + // Deliberately NOT noteAttrRecovery(): the host attribution + // is still broken — the health signal stays honest; the + // REQUEST is what got rescued, not the initiator scope (#4082). + return true; + } + threwStreak = 0; + threwGiveUpNoticed = false; + return refuseUnattributed(url, attr, body, init); + }); } if (attr.state === "threw") { return settleThrewAttribution(attr.message, url, body, init).then((verdict) => { diff --git a/tests/dsh-native.test.ts b/tests/dsh-native.test.ts index 4392cba37e..e8967f16d6 100644 --- a/tests/dsh-native.test.ts +++ b/tests/dsh-native.test.ts @@ -1879,7 +1879,11 @@ test("#1117 apply() installs takeoverGate keyed on currentInitiator attribution" const gate = _stateTakeoverGateForTest(); assert.ok(gate !== undefined, "takeoverGate installed"); // unattributed (background chain, third-party in-process plugin) → NOT claimed - assert.equal(gate("https://api.anthropic.com/v1/messages"), false); + const rawNoneGate = gate("https://api.anthropic.com/v1/messages"); + // #4829 review: the no-init (WS-lane-shaped) call must keep its SYNCHRONOUS + // verdict class — native-ws-intercept.ts compares with === false, no await. + assert.equal(rawNoneGate, false); + assert.equal(await rawNoneGate, false); // attributed (the host's own agent chain) → claimed ctx.setInitiator({ session: { id: "s1117" } }); assert.equal(gate("https://api.anthropic.com/v1/messages"), true); @@ -1914,8 +1918,9 @@ test("#1158 apply() gate refusal logs each endpoint once per process with attrib const refusedLines = (): string[] => (fs.existsSync(logFile) ? fs.readFileSync(logFile, "utf8") : "").split("\n").filter((l) => l.length > 0 && l.includes("takeover gate refused")); try { const url = "https://api.gate-probe.test/v1/chat/completions"; - assert.equal(gate(url), false, "unattributed → refused"); - assert.equal(gate(url), false, "second refusal of the same endpoint"); + // #4829: the none arm settles async — the recall claim may decode bodies + assert.equal(await gate(url), false, "unattributed → refused"); + assert.equal(await gate(url), false, "second refusal of the same endpoint"); let lines = refusedLines(); assert.equal(lines.length, 1, `expected one refusal line per endpoint, got: ${lines.join(" | ")}`); // #3651 paste-safe sink: non-public hosts are masked in the line; the dedup key @@ -1923,11 +1928,11 @@ test("#1158 apply() gate refusal logs each endpoint once per process with attrib assert.match(lines[0]!, /https:\/\/\/v1\/chat\/completions/); assert.match(lines[0]!, /no active initiator attribution/); // A query string neither spawns a new line nor leaks into it. - assert.equal(gate(`${url}?key=sk-do-not-log`), false); + assert.equal(await gate(`${url}?key=sk-do-not-log`), false); assert.equal(refusedLines().length, 1); assert.ok(!fs.readFileSync(logFile, "utf8").includes("do-not-log"), "query string must not reach the log"); // A different endpoint earns its own line. - assert.equal(gate("https://api.other-probe.test/v1/messages"), false); + assert.equal(await gate("https://api.other-probe.test/v1/messages"), false); assert.equal(refusedLines().length, 2); // Attributed traffic claims silently. ctx.setInitiator({ session: { id: "s1158" } }); @@ -1969,11 +1974,12 @@ test("#1158 L2 gate three-state: thrown attribution is a distinct state; counts const refusedLines = (): string[] => (fs.existsSync(logFile) ? fs.readFileSync(logFile, "utf8") : "").split("\n").filter((l) => l.length > 0 && l.includes("takeover gate refused")); try { const url = "https://api.gate-l2.test/v1/chat/completions"; - // four same-state (none) refusals → exactly one line; counting continues silently - assert.equal(gate(url), false); - assert.equal(gate(url), false); - assert.equal(gate(url), false); - assert.equal(gate(url), false); + // four same-state (none) refusals → exactly one line; counting continues + // silently (#4829: the none arm settles async — the recall claim may decode bodies) + assert.equal(await gate(url), false); + assert.equal(await gate(url), false); + assert.equal(await gate(url), false); + assert.equal(await gate(url), false); let lines = refusedLines(); assert.equal(lines.length, 1, `expected one line for repeated same-state refusals, got: ${lines.join(" | ")}`); assert.match(lines[0]!, /no active initiator attribution/); @@ -1997,7 +2003,7 @@ test("#1158 L2 gate three-state: thrown attribution is a distinct state; counts assert.equal(refusedLines().length, 2); // back to plain none → transition the other way, count keeps running ctx.agents.currentInitiator = () => undefined; - assert.equal(gate(url), false); + assert.equal(await gate(url), false); lines = refusedLines(); assert.equal(lines.length, 3); assert.match(lines[2]!, /refusals so far: 7 \(state threw→none\)$/); diff --git a/tests/issue4829-zstd-fork-gate.test.ts b/tests/issue4829-zstd-fork-gate.test.ts new file mode 100644 index 0000000000..34a2b470fd --- /dev/null +++ b/tests/issue4829-zstd-fork-gate.test.ts @@ -0,0 +1,147 @@ +// #4829: the fork-adoption body-parse gate must see through supported +// content-encodings. The Codex SSE transport (pi-ai) ships zstd-compressed +// Uint8Array request bodies with a content-encoding header; the old +// plain-text-only parser failed them all, so seeded fork children of long +// compressed conversations could NEVER adopt — resume-inheritance miss → +// multi-million-token preflight compression → client disconnects before the +// model request is forwarded. Pins both directions (#3617): encoded bodies +// reach the adopter protocol, undecodable or unsupported encodings still hit +// the body-parse gate exactly once, and plain-text behavior is unchanged. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs, { mkdtempSync, rmSync } from "node:fs"; +import http from "node:http"; +import { once } from "node:events"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import zlib from "node:zlib"; +import { _maybeForkAdoptBeforeSendForTest, _resetForkGateDiagnosticsForTest, _resetRegisterForTest, type PluginContext } from "../src/agent/dsh-native.ts"; +import { resetForkCapabilityCacheForTest } from "../src/agent/fork-adopt.ts"; + +process.env.NODE_ENV = "test"; +const testRoot = mkdtempSync(join(tmpdir(), "bili-issue4829-")); +test.after(() => rmSync(testRoot, { recursive: true, force: true })); +for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = testRoot; +const logFile = join(testRoot, "billion-context", "bili.log"); + +// Each positive test needs its OWN child sid: the adopter marks a child done +// after its first terminal outcome, and a later test reusing the sid would be +// silently dropped by the done-set (zero log lines). +function ctxSeeded(sid = "child-sid"): PluginContext { + return { + tools: { register: () => undefined }, + commands: { register: () => undefined }, + agents: { currentInitiator: () => ({ session: { id: sid, header: { isSeeded: true, parentSession: "parent-sid" } } }) }, + }; +} + +const GATE_LINE = /fork adoption for .* (skipped at the \S+ gate|deferred for a side-shaped request)/; + +// A main-shaped openai body (tools present, no tiny budget) so only the +// body-parse gate can block adoption. +const mainBody = JSON.stringify({ + model: "gpt-test", + messages: [ + { role: "system", content: "sys" }, + { role: "user", content: "hello" }, + ], + tools: [{ type: "function", function: { name: "bash", description: "d", parameters: { type: "object", properties: {} } } }], +}); + +function freshScenario(base: string | undefined, toolsReady: boolean): void { + _resetRegisterForTest(base, toolsReady); + _resetForkGateDiagnosticsForTest(); + resetForkCapabilityCacheForTest(); + fs.rmSync(logFile, { force: true }); +} + +function logLines(): string[] { + try { + return fs.readFileSync(logFile, "utf8").split("\n").filter((l) => l.length > 0); + } catch { + return []; + } +} + +// Drive one seeded-child dispatch against a mock proxy that answers every +// route with an empty-capabilities manifest — reaching that manifest probe is +// proof the body passed the parse gate into the adopter protocol (#4215's +// side-shape test uses the same sentinel: "does not advertise the fork +// capability"). +async function runAgainstMockProxy(sid: string, init: RequestInit): Promise { + const srv = http.createServer((_req, res) => { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ capabilities: {} })); }); + srv.listen(0, "127.0.0.1"); + await once(srv, "listening"); + const addr = srv.address(); + assert(addr && typeof addr === "object"); + try { + _resetRegisterForTest(`http://127.0.0.1:${addr.port}`, true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(sid), init); + } finally { + srv.close(); + srv.closeAllConnections?.(); + await once(srv, "close").catch(() => undefined); + } + return logLines(); +} + +test("zstd-compressed body reaches the adopter protocol (#4829)", async (t) => { + // The production shape: pi-ai hands fetch a Headers instance + a + // zstd-compressed Uint8Array body. + const zstdCompressSync = (zlib as unknown as { zstdCompressSync?: (buf: Buffer) => Buffer }).zstdCompressSync; + if (typeof zstdCompressSync !== "function") { + t.skip("native zlib zstd unavailable on this Node build"); + return; + } + freshScenario(undefined, true); + const lines = await runAgainstMockProxy("child-zstd", { + headers: new Headers({ "content-encoding": "zstd" }), + body: new Uint8Array(zstdCompressSync(Buffer.from(mainBody))), + }); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 0, JSON.stringify(lines)); + assert.equal(lines.filter((l) => /does not advertise the fork capability/.test(l)).length, 1, JSON.stringify(lines)); +}); + +test("gzip-compressed body reaches the adopter protocol (#4829)", async () => { + freshScenario(undefined, true); + const lines = await runAgainstMockProxy("child-gzip", { + headers: { "content-encoding": "gzip" }, + body: new Uint8Array(zlib.gzipSync(Buffer.from(mainBody))), + }); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 0, JSON.stringify(lines)); + assert.equal(lines.filter((l) => /does not advertise the fork capability/.test(l)).length, 1, JSON.stringify(lines)); +}); + +test("undecodable bytes under a supported coding still hit the body-parse gate (#4829 negative)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), { + headers: new Headers({ "content-encoding": "zstd" }), + body: new Uint8Array([0x28, 0xb5, 0x2f, 0xfd, 0xde, 0xad, 0xbe, 0xef]), + }); + const lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*skipped at the body-parse gate/); +}); + +test("unsupported coding hits the body-parse gate without guessing (#4829 negative)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), { + headers: [["Content-Encoding", "foo"]], + body: new Uint8Array(Buffer.from(mainBody)), + }); + const lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*skipped at the body-parse gate/); +}); + +test("plain-text failures keep hitting the body-parse gate with the updated message (#4829)", async () => { + freshScenario("http://proxy.invalid", true); + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), { body: "not-json" }); + let lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1, JSON.stringify(lines)); + assert.match(lines[0]!, /\[warn\].*skipped at the body-parse gate.*after content-encoding decode/); + // a missing body hits the same gate and does not re-log per shape + await _maybeForkAdoptBeforeSendForTest(ctxSeeded(), undefined); + lines = logLines(); + assert.equal(lines.filter((l) => GATE_LINE.test(l)).length, 1); +});