From e3542852119eea82f0de530c657ff72cbec58039 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 24 Oct 2026 13:01:37 +0800 Subject: [PATCH 1/3] fix: plaintext body dumps now lead with a credentials-warning banner (#4482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opt-in body dumps (ACP_DUMP_BODY / diagnostics.dumpBody, REREQUEST dumps, BILI_DUMP_4XX err/summary-err dumps) write the full plaintext conversation body to disk and may carry real credentials (tool arguments, host-injected content), yet nothing warned the user. #4397's reporter saw live credentials in such dumps; sharing them into an issue leaks them publicly. No automatic redaction — byte-exactness is the point of these dumps. Instead every plaintext-body dump file now starts with one standing warning line (PLAINTEXT DUMP ... scrub before sharing): - raw/*-REQ.txt, raw/*-INCOMING.txt, dumps/req-*.json, req-*-REREQUEST.json and err-*.json / summary-err-*.json all get the banner (DUMP_PLAINTEXT_BANNER in src/dump-banner.ts). - bili acp-cache diff strips the known leading banner line explicitly, so pre-banner (old-format) dumps keep parsing byte-identically — old and new files also mix fine in one dir (tested). The banner text must stay free of ": " (head lines parse as headers); pinned by test. - tools/analyze-dumps.mjs strips the banner before JSON.parse; its local copy of the literal is checked against src/dump-banner.ts by test. - CONFIGURATION.md en+zh warnings for dumpBody/dumpReq/dump4xx go through the gen-config-docs seed (website/config-reference/diagnostics.yaml), keeping the docs-sync drift guard green. Behavior change (intended, disclosed per owner policy): dump file FORMAT changes for all six writers — a one-line banner is prepended. Consumers that strictly parsed dumps/req-*.json or err-*.json as pure JSON from outside this repo will see a non-JSON first line; in-repo consumers are updated. Body bytes after line one are untouched (byte-diff forensics preserved). Tests: new tests/issue4482-dump-banner.test.ts (banner pinned + tool sync, stripDumpBanner unit, real-writer harness asserting INCOMING/REQ/JSON banners with intact bodies, runDiff old/new/mixed-dir equivalence) plus banner assertions added to error-dump / issue1993 / reasoning-echo suites. --- CONFIGURATION.md | 6 +- CONFIGURATION.zh-CN.md | 6 +- src/acp-cache-diff.ts | 8 + src/dump-banner.ts | 16 ++ src/error-dump.ts | 7 +- src/loop/core.ts | 3 +- src/server/handle.ts | 3 +- src/server/relay.ts | 10 +- tests/error-dump.test.ts | 13 +- tests/issue1993-preflight-400.test.ts | 5 +- tests/issue4482-dump-banner.test.ts | 224 ++++++++++++++++++++++ tests/reasoning-echo-400.test.ts | 5 +- tools/analyze-dumps.mjs | 8 +- website/config-reference/diagnostics.yaml | 12 +- 14 files changed, 299 insertions(+), 27 deletions(-) create mode 100644 src/dump-banner.ts create mode 100644 tests/issue4482-dump-banner.test.ts diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 0b36517f5f..a1b746c5e8 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -225,10 +225,10 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | Key | Type | Default | Env | Description | |-----|------|---------|-----|-------------| | `diagnostics` | object | {} (all defaults) | — | Local troubleshooting knobs (dumps, render/injection toggles, count-tokens passthrough). | -| `diagnostics.dumpBody` | boolean | false | ACP_DUMP_BODY | Dump full request/response bodies to the log for troubleshooting. | -| `diagnostics.dumpReq` | boolean | true | ACP_DUMP_REQ | Log outgoing upstream request metadata per call. | +| `diagnostics.dumpBody` | boolean | false | ACP_DUMP_BODY | Dump full request/response bodies to the log for troubleshooting. Dumps are PLAINTEXT and may contain credentials — scrub before sharing (#4482). | +| `diagnostics.dumpReq` | boolean | true | ACP_DUMP_REQ | Log outgoing upstream request metadata per call; with dumpBody on, also gates the plaintext body dumps (scrub before sharing, #4482). | | `diagnostics.rawDumpDir` | string | /raw | ACP_RAW_DUMP_DIR | Directory for raw wire dumps. | -| `diagnostics.dump4xx` | boolean | false | BILI_DUMP_4XX | Persist upstream 4xx responses to disk for post-mortem inspection. | +| `diagnostics.dump4xx` | boolean | false | BILI_DUMP_4XX | Persist upstream 4xx responses to disk for post-mortem inspection. Bodies are PLAINTEXT and may contain credentials — scrub before sharing (#4482). | | `diagnostics.dump4xxMaxBytes` | number | 2097152 (floor 1024) | BILI_DUMP_4XX_MAX_BYTES | Size cap for a single 4xx dump file. | | `diagnostics.maxTotalBytes` | number | unset (off) | BILI_DUMP_MAX_TOTAL_BYTES | Total-size cap for debug dump dirs (dumps/, raw/, SSE dump dir); oldest files deleted first when exceeded (floor 1 MiB). Dumps have no rotation by default and can grow tens of GB/day under active capture (#2412). | | `diagnostics.maxAgeDays` | number | unset (off) | BILI_DUMP_MAX_AGE_DAYS | Age cap in days for debug dump dirs; files older than this are deleted (floor 1 h). Off by default. | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 7acd1c5ae3..4efa171ac6 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -225,10 +225,10 @@ | Key | Type | Default | Env | Description | |-----|------|---------|-----|-------------| | `diagnostics` | object | {} (all defaults) | — | 本地排障旋钮(转储、渲染/注入开关、count-tokens 透传)。 | -| `diagnostics.dumpBody` | boolean | false | ACP_DUMP_BODY | 把完整请求/响应体转储到日志用于排障。 | -| `diagnostics.dumpReq` | boolean | true | ACP_DUMP_REQ | 逐次调用记录发往上游的请求元数据。 | +| `diagnostics.dumpBody` | boolean | false | ACP_DUMP_BODY | 把完整请求/响应体转储到日志用于排障。转储内容为明文,可能包含凭据——分享前必须清洗(#4482)。 | +| `diagnostics.dumpReq` | boolean | true | ACP_DUMP_REQ | 逐次调用记录发往上游的请求元数据;开启 dumpBody 时同时是明文请求体转储的闸门(分享前必须清洗,#4482)。 | | `diagnostics.rawDumpDir` | string | /raw | ACP_RAW_DUMP_DIR | 原始线路转储目录。 | -| `diagnostics.dump4xx` | boolean | false | BILI_DUMP_4XX | 把上游 4xx 响应落盘以便事后检查。 | +| `diagnostics.dump4xx` | boolean | false | BILI_DUMP_4XX | 把上游 4xx 响应落盘以便事后检查。转储内容为明文,可能包含凭据——分享前必须清洗(#4482)。 | | `diagnostics.dump4xxMaxBytes` | number | 2097152 (floor 1024) | BILI_DUMP_4XX_MAX_BYTES | 单个 4xx 转储文件的大小上限。 | | `diagnostics.maxTotalBytes` | number | unset (off) | BILI_DUMP_MAX_TOTAL_BYTES | 调试转储目录(dumps/、raw/、SSE dump 目录)的总大小上限;超限时按最旧优先删除(下限 1 MiB)。转储默认无任何轮转,持续捕获下每天可增长数十 GB(#2412)。 | | `diagnostics.maxAgeDays` | number | unset (off) | BILI_DUMP_MAX_AGE_DAYS | 调试转储目录的保留天数上限;超过该天数的文件被删除(下限 1 小时)。默认关闭。 | diff --git a/src/acp-cache-diff.ts b/src/acp-cache-diff.ts index ca141e4ad9..d01d191dce 100644 --- a/src/acp-cache-diff.ts +++ b/src/acp-cache-diff.ts @@ -4,6 +4,7 @@ // its producer instead of surfacing as an unattributed TTL bucket (#1254). import fs from "node:fs"; import path from "node:path"; +import { DUMP_PLAINTEXT_BANNER } from "./dump-banner.js"; import { defaultLogFile } from "./paths.js"; import { safeSessionId } from "./server/headers.js"; @@ -130,6 +131,13 @@ function collectDumps(root: string): DumpMeta[] { function parseDumpFile(file: string): DumpContent | null { let buf: Buffer; try { buf = fs.readFileSync(file); } catch { return null; } + // #4482: new-format dumps open with the plaintext-warning banner line; drop + // it explicitly (not by relying on the banner lacking ": ") so header + // parsing and body bytes stay identical to pre-banner files. + const nl = buf.indexOf(0x0a); + if (nl > 0 && buf.subarray(0, nl).toString("utf8").replace(/\r$/, "") === DUMP_PLAINTEXT_BANNER) { + buf = buf.subarray(nl + 1); + } // #3331: the request-line + headers block can be terminated by \r\n\r\n as // well as \n\n (dump files written on Windows / by CRLF-emitting clients); // a bare \n\n probe misses the header block entirely on such files. diff --git a/src/dump-banner.ts b/src/dump-banner.ts new file mode 100644 index 0000000000..8e3276088b --- /dev/null +++ b/src/dump-banner.ts @@ -0,0 +1,16 @@ +// #4482: body dumps exist for byte-exact forensics (prefix diffs, rejection +// post-mortems), so their content is NEVER redacted — the full plaintext +// conversation body lands on disk and may carry real credentials (tool +// arguments, host-injected content). Every plaintext-body dump file therefore +// starts with this standing warning so a dump that gets shared (issue +// attachments, pastes) announces what it carries before the reader scrolls +// past it. Keep the text free of ": " — dump parsers treat head lines as +// headers and the first line is skipped as the request line. +export const DUMP_PLAINTEXT_BANNER = "PLAINTEXT DUMP — full conversation body, may contain credentials; scrub before sharing"; + +/** Drop one leading banner line if `text` starts with it (LF or CRLF). */ +export function stripDumpBanner(text: string): string { + const nl = text.indexOf("\n"); + if (nl > 0 && text.slice(0, nl).replace(/\r$/, "") === DUMP_PLAINTEXT_BANNER) return text.slice(nl + 1); + return text; +} diff --git a/src/error-dump.ts b/src/error-dump.ts index e9f43e7e87..6620639cb3 100644 --- a/src/error-dump.ts +++ b/src/error-dump.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import { DUMP_PLAINTEXT_BANNER } from "./dump-banner.js"; import { dumpsDir } from "./paths.js"; import { log as loggerLog } from "./logger.js"; import { dump4xxEnabled as knobDump4xxEnabled, dump4xxMaxBytes as knobDump4xxMaxBytes } from "./knobs.js"; @@ -10,6 +11,8 @@ import { gcDumpDirIfConfigured } from "./state-gc.js"; // standing body dump (ACP_DUMP_BODY=1) must be armed BEFORE the incident; this // one fires on the failure itself. Still off by default — conversation bodies // leak to disk (#276) — enable with BILI_DUMP_4XX=1 or diagnostics.dump4xx. +// #4482: like every plaintext-body dump, each file starts with the standing +// credentials warning banner (DUMP_PLAINTEXT_BANNER). let failCount = 0; let lastFailLog = 0; @@ -48,7 +51,7 @@ export function dumpRejectedBody(status: number, sessionId: string, body: string fs.mkdirSync(dir, { recursive: true }); const sid = sessionId.replace(/[^a-zA-Z0-9_-]/g, "_"); const out = path.join(dir, `err-${Date.now()}-${sid}-${status}.json`); - fs.writeFileSync(out, `${text}${marker}`); + fs.writeFileSync(out, `${DUMP_PLAINTEXT_BANNER}\n${text}${marker}`); loggerLog("info", `[dump] upstream ${status} rejected body written to ${out}`); gcDumpDirIfConfigured(dir); return out; @@ -80,7 +83,7 @@ export function dumpSummaryRejection(status: number, sessionId: string, request: fs.mkdirSync(dir, { recursive: true }); const sid = sessionId.replace(/[^a-zA-Z0-9_-]/g, "_"); const out = path.join(dir, `summary-err-${Date.now()}-${sid}-${status}.json`); - fs.writeFileSync(out, JSON.stringify({ status, request: side(request), response: side(response) }, null, 2)); + fs.writeFileSync(out, `${DUMP_PLAINTEXT_BANNER}\n${JSON.stringify({ status, request: side(request), response: side(response) }, null, 2)}`); loggerLog("info", `[dump] upstream ${status} rejected summary exchange written to ${out}`); gcDumpDirIfConfigured(dir); return out; diff --git a/src/loop/core.ts b/src/loop/core.ts index d03120c6c5..e9c261578b 100644 --- a/src/loop/core.ts +++ b/src/loop/core.ts @@ -36,6 +36,7 @@ import { formatUpstreamError, proxyDispatcher } from "../upstream-proxy.js"; import { warnCacheCollapse } from "../cache-warn.js"; import { dumpRejectedBody } from "../error-dump.js"; import { shedEgressIfOverBudget } from "../egress-shed.js"; +import { DUMP_PLAINTEXT_BANNER } from "../dump-banner.js"; import { bodyDumpEnabled as knobBodyDumpEnabled, imageRescaleMaxDim } from "../knobs.js"; import { dumpsDir } from "../paths.js"; import { isStrictReasoningEcho, modelIdOf, normalizeStrictEchoBody } from "../strict-echo.js"; @@ -1488,7 +1489,7 @@ export async function* runCompressLoop( const dumpDir = dumpsDir(); fs.mkdirSync(dumpDir, { recursive: true }); const sid = (ctx.session.id ?? "unknown").replace(/[^a-zA-Z0-9_-]/g, "_"); - fs.writeFileSync(path.join(dumpDir, `req-${Date.now()}-${sid}-REREQUEST.json`), JSON.stringify(newBody, null, 2)); + fs.writeFileSync(path.join(dumpDir, `req-${Date.now()}-${sid}-REREQUEST.json`), `${DUMP_PLAINTEXT_BANNER}\n${JSON.stringify(newBody, null, 2)}`); } catch { /* best-effort */ } } let respResult: { response: Response; clearTimer: () => void }; diff --git a/src/server/handle.ts b/src/server/handle.ts index a1d5f4bcb6..29272d47d8 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -61,6 +61,7 @@ import { FORCE_TEXT_PROTOCOL } from "./inject.js"; import { prepareAnthropic } from "./prepare-anthropic.js"; import { prepareOpenai } from "./prepare-openai.js"; import { prepareGoogle, prepareGoogleCountTokens } from "./prepare-google.js"; +import { DUMP_PLAINTEXT_BANNER } from "../dump-banner.js"; import { prepareResponses, prepareResponsesCompact } from "./prepare-responses.js"; import { bodyDumpEnabled, isModelDiscoveryPath, isV1InternalControlPlanePath, logDumpFailure, logUnrecognizedPath } from "./observability.js"; import { adminGateDenial, handleAdminRoute, isManagementPath } from "./admin.js"; @@ -2550,7 +2551,7 @@ async function handleRequest( Object.fromEntries(Object.entries(req.headers).map(([k, v]) => [k, Array.isArray(v) ? v.join(",") : String(v)])), ); const hdrText = Object.entries(hdrs).map(([k, v]) => `${k}: ${v}`).join("\n"); - fs.writeFileSync(path.join(rawDir, `${Date.now()}-${safeSessionId(session.id)}-INCOMING.txt`), `${req.method} ${maskUrlsInText(req.url ?? "")}\n${hdrText}\n\n${bodyBuffer.toString("utf8")}`); + fs.writeFileSync(path.join(rawDir, `${Date.now()}-${safeSessionId(session.id)}-INCOMING.txt`), `${DUMP_PLAINTEXT_BANNER}\n${req.method} ${maskUrlsInText(req.url ?? "")}\n${hdrText}\n\n${bodyBuffer.toString("utf8")}`); gcDumpDirIfConfigured(rawDir); } catch (err) { logDumpFailure("INCOMING dump", err); } } diff --git a/src/server/relay.ts b/src/server/relay.ts index 8745f844c8..cabdfa597d 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -55,6 +55,7 @@ import { clearUpstreamAlertsForHost, recordUpstreamAlert, recordUpstreamCircuitF import { describeUpstreamFailure, formatUpstreamError, proxyDispatcher, recordUpstreamConnection } from "../upstream-proxy.js"; import { withUpstreamFailure, type UpstreamFailureKind } from "../upstream-fail.js"; import { applyEstimateCalibration, currentCalibrationFactor, inspectContextOverflow, MIN_ASSUMED_WINDOW_TOKENS, normalizeUpstreamOrigin, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "../util.js"; +import { DUMP_PLAINTEXT_BANNER } from "../dump-banner.js"; import { recordPckAblation } from "../pck-probe.js"; import { applyOutputSteering, applyOutputSteeringJson } from "../output-steering.js"; import { estimateWireOverhead } from "./budget.js"; @@ -490,9 +491,9 @@ export async function forward( const out = path.join(dumpDir, `req-${Date.now()}-${safeSessionId(sid)}.json`); try { const pretty = JSON.stringify(JSON.parse(wireText), null, 2); - fs.writeFileSync(out, pretty); + fs.writeFileSync(out, `${DUMP_PLAINTEXT_BANNER}\n${pretty}`); } catch { - fs.writeFileSync(out, wireText); + fs.writeFileSync(out, `${DUMP_PLAINTEXT_BANNER}\n${wireText}`); } log("info", `[debug] forwarded body written to ${out}`); gcDumpDirIfConfigured(dumpDir); @@ -513,7 +514,8 @@ export async function forward( // headers/exact body bytes; response status+headers) so two consecutive // requests can be byte-diffed to locate a cache-breaker that the JSON body // dump (which re-formats and omits headers) may hide. Enabled with - // ACP_DUMP_BODY=1 (credential header values + non-public hosts masked). + // ACP_DUMP_BODY=1 (credential header values + non-public hosts masked; + // #4482: body is plaintext by design — the leading banner warns). const rawBase = bodyDumpEnabled() ? (() => { @@ -538,7 +540,7 @@ export async function forward( ? wireBody : Buffer.from(wireBody).toString("utf8"); const reqPath = `${rawBase}-REQ.txt`; - fs.writeFileSync(reqPath, `${req.method ?? "POST"} ${maskUrlForLog(upstreamUrl)}\n${hdrText}\n\n${bodyText}`); + fs.writeFileSync(reqPath, `${DUMP_PLAINTEXT_BANNER}\n${req.method ?? "POST"} ${maskUrlForLog(upstreamUrl)}\n${hdrText}\n\n${bodyText}`); log("info", `[debug] RAW request dump: ${reqPath}`); gcDumpDirIfConfigured(path.dirname(rawBase)); } catch (err) { logDumpFailure("REQ dump", err); } diff --git a/tests/error-dump.test.ts b/tests/error-dump.test.ts index 1932c4b3f0..1773698c02 100644 --- a/tests/error-dump.test.ts +++ b/tests/error-dump.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { DUMP_PLAINTEXT_BANNER, stripDumpBanner } from "../src/dump-banner.js"; import { dumpRejectedBody, dumpSummaryRejection } from "../src/error-dump.js"; import { setLogCapture } from "../src/logger.js"; import { rmrf } from "./tmp-rm.ts"; @@ -57,7 +58,9 @@ test("on: JSON body is pretty-printed and parseable", () => { const files = errFiles(); assert.equal(files.length, 1); assert.match(files[0]!, /^err-\d+-s1-400\.json$/); - const parsed = JSON.parse(fs.readFileSync(out!, "utf8")) as { model: string; messages: unknown[] }; + const text = fs.readFileSync(out!, "utf8"); + assert.ok(text.startsWith(`${DUMP_PLAINTEXT_BANNER}\n`), "#4482: plaintext warning banner must lead the file"); + const parsed = JSON.parse(stripDumpBanner(text)) as { model: string; messages: unknown[] }; assert.equal(parsed.model, "m"); assert.equal(parsed.messages.length, 1); } finally { @@ -70,7 +73,7 @@ test("on: non-JSON body passes through raw", () => { try { const out = dumpRejectedBody(400, "s1", "bad gateway"); assert.ok(out); - assert.equal(fs.readFileSync(out!, "utf8"), "bad gateway"); + assert.equal(fs.readFileSync(out!, "utf8"), `${DUMP_PLAINTEXT_BANNER}\nbad gateway`); } finally { restoreDumpEnv(); } @@ -139,7 +142,9 @@ test("#1993 summary dump on: envelope carries status plus both sides as strings" const files = summaryErrFiles(); assert.equal(files.length, 1); assert.match(files[0]!, /^summary-err-\d+-s1-400\.json$/); - const parsed = JSON.parse(fs.readFileSync(out!, "utf8")) as { status: number; request: unknown; response: unknown }; + const text = fs.readFileSync(out!, "utf8"); + assert.ok(text.startsWith(`${DUMP_PLAINTEXT_BANNER}\n`), "#4482: plaintext warning banner must lead the file"); + const parsed = JSON.parse(stripDumpBanner(text)) as { status: number; request: unknown; response: unknown }; assert.equal(parsed.status, 400); assert.equal(typeof parsed.request, "string"); assert.equal(typeof parsed.response, "string"); @@ -159,7 +164,7 @@ test("#1993 summary dump on: non-JSON sides stay raw, oversized side is capped", const text = fs.readFileSync(out!, "utf8"); assert.match(text, /\[truncated: \d+ more character\(s\)\]/); assert.ok(text.length < 2048 * 3 + 512, `capped file should be far smaller than raw (${text.length})`); - const parsed = JSON.parse(text) as { request: unknown; response: unknown }; + const parsed = JSON.parse(stripDumpBanner(text)) as { request: unknown; response: unknown }; assert.equal(parsed.request, "req"); assert.ok(typeof parsed.response === "string" && (parsed.response as string).includes("[truncated:")); } finally { diff --git a/tests/issue1993-preflight-400.test.ts b/tests/issue1993-preflight-400.test.ts index 62f2af400b..2a65e46dd2 100644 --- a/tests/issue1993-preflight-400.test.ts +++ b/tests/issue1993-preflight-400.test.ts @@ -17,6 +17,7 @@ import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; import { listSessions } from "../src/session.ts"; +import { DUMP_PLAINTEXT_BANNER, stripDumpBanner } from "../src/dump-banner.ts"; import { setLogCapture } from "../src/logger.ts"; import { rmrf } from "./tmp-rm.ts"; @@ -319,7 +320,9 @@ test("#1993 BILI_DUMP_4XX persists the rejected summary exchange (request + resp const files = fs.readdirSync(dumpDir).filter((f) => f.startsWith("summary-err-") && f.endsWith("-400.json")); assert.ok(files.length >= 1, `expected at least one summary-err dump, dir has: ${JSON.stringify(fs.readdirSync(dumpDir))}`); - const parsed = JSON.parse(fs.readFileSync(path.join(dumpDir, files[0]!), "utf8")) as { status: number; request: unknown; response: unknown }; + const text = fs.readFileSync(path.join(dumpDir, files[0]!), "utf8"); + assert.ok(text.startsWith(`${DUMP_PLAINTEXT_BANNER}\n`), "#4482: plaintext warning banner must lead the dump"); + const parsed = JSON.parse(stripDumpBanner(text)) as { status: number; request: unknown; response: unknown }; assert.equal(parsed.status, 400); assert.ok(typeof parsed.request === "string" && parsed.request.includes("instructions"), "dump must carry the request bytes"); assert.ok(typeof parsed.response === "string" && parsed.response.includes("request too large: 523200"), "dump must carry the response bytes"); diff --git a/tests/issue4482-dump-banner.test.ts b/tests/issue4482-dump-banner.test.ts new file mode 100644 index 0000000000..996de2b203 --- /dev/null +++ b/tests/issue4482-dump-banner.test.ts @@ -0,0 +1,224 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { DUMP_PLAINTEXT_BANNER, stripDumpBanner } from "../src/dump-banner.js"; +import { runDiff } from "../src/acp-cache-diff.ts"; +import { rmrf } from "./tmp-rm.ts"; + +// #4482: opt-in body dumps land on disk as full plaintext conversation bodies +// (credentials included) with zero warning. These tests pin the standing +// warning banner on the real writer paths (raw/*-INCOMING.txt, raw/*-REQ.txt, +// dumps/req-*.json) and prove bili acp-cache diff still classifies pre-banner +// (old-format) dumps identically to bannered ones — including mixed dirs. + +const M1 = { role: "user", content: "hello there" }; +const M2 = { role: "assistant", content: "hi!" }; +const BODY_MARKER = "dump-banner-marker-QQQ42"; + +function chatBody(model: string, msgs: unknown[]): string { return JSON.stringify({ model, messages: msgs }); } +function tmpdir(prefix: string): string { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); } + +test("#4482: banner text is pinned and the analyze-dumps.mjs copy stays in sync", () => { + assert.equal(DUMP_PLAINTEXT_BANNER, "PLAINTEXT DUMP — full conversation body, may contain credentials; scrub before sharing"); + // parseDumpFile treats head lines as potential "k: v" headers; a ": " in + // the banner would silently pollute every parsed dump's header map. + assert.ok(!DUMP_PLAINTEXT_BANNER.includes(": "), "banner must stay free of ': '"); + const toolSrc = fs.readFileSync(fileURLToPath(new URL("../tools/analyze-dumps.mjs", import.meta.url)), "utf8"); + assert.ok(toolSrc.includes(`const PLAINTEXT_BANNER = "${DUMP_PLAINTEXT_BANNER}";`), "tools/analyze-dumps.mjs banner literal drifted from src/dump-banner.ts"); +}); + +test("#4482: stripDumpBanner drops LF/CRLF banner lines, leaves everything else intact", () => { + assert.equal(stripDumpBanner(`${DUMP_PLAINTEXT_BANNER}\n{"a":1}`), '{"a":1}'); + assert.equal(stripDumpBanner(`${DUMP_PLAINTEXT_BANNER}\r\n{"a":1}`), '{"a":1}'); + assert.equal(stripDumpBanner('{"a":1}'), '{"a":1}'); + assert.equal(stripDumpBanner(`${DUMP_PLAINTEXT_BANNER} extra\n{"a":1}`), `${DUMP_PLAINTEXT_BANNER} extra\n{"a":1}`); +}); + +function close(server: http.Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +function listFiles(dir: string): string[] { + if (!fs.existsSync(dir)) return []; + const out: string[] = []; + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, e.name); + if (e.isDirectory()) out.push(...listFiles(p)); + else out.push(p); + } + return out; +} + +function baseOpts(): ProxyOptions { + return { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: {}, + modelContextLimit: 400_000, + kernelConfig: defaultConfig(400_000), + compress: { injectTool: false, injectNudge: false }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + }; +} + +interface Env { + xdg: string | undefined; + body: string | undefined; +} + +function saveEnv(): Env { + return { xdg: process.env.XDG_STATE_HOME, body: process.env.ACP_DUMP_BODY }; +} + +function restoreEnv(prev: Env): void { + if (prev.xdg === undefined) delete process.env.XDG_STATE_HOME; + else process.env.XDG_STATE_HOME = prev.xdg; + if (prev.body === undefined) delete process.env.ACP_DUMP_BODY; + else process.env.ACP_DUMP_BODY = prev.body; +} + +async function startHarness(tmpRoot: string): Promise<{ proxy: http.Server; upstream: http.Server; proxyPort: number; upstreamPort: number }> { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const upstream = http.createServer((_req, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ id: "r1", object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }] })); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const opts = baseOpts(); + opts.routes = { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: 400_000 } } } }; + const proxy = await startServer(opts); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + return { proxy, upstream, proxyPort, upstreamPort }; +} + +test("#4482: real writers lead INCOMING/REQ/JSON dumps with the banner, body intact", async () => { + const tmpRoot = tmpdir("bili-4482-writers-"); + const prev = saveEnv(); + process.env.XDG_STATE_HOME = tmpRoot; + process.env.ACP_DUMP_BODY = "1"; + let h: Awaited> | undefined; + try { + h = await startHarness(tmpRoot); + const resp = await fetch(`http://127.0.0.1:${h.proxyPort}/bili/http://127.0.0.1:${h.upstreamPort}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-test", messages: [{ role: "user", content: BODY_MARKER }] }), + }); + assert.equal(resp.status, 200); + await resp.text(); + + const files = listFiles(path.join(tmpRoot, "billion-context")); + const incoming = files.filter((f) => f.endsWith("-INCOMING.txt")); + assert.equal(incoming.length, 1, `expected exactly one INCOMING dump, got: ${files.join(", ") || "(none)"}`); + const reqTxt = files.filter((f) => f.endsWith("-REQ.txt")); + assert.equal(reqTxt.length, 1, `expected exactly one REQ dump, got: ${files.join(", ") || "(none)"}`); + const jsonDumps = files.filter((f) => /req-\d+-.+\.json$/.test(f)); + assert.equal(jsonDumps.length, 1, `expected exactly one dumps/req-*.json, got: ${files.filter((f) => f.includes("dumps")).join(", ") || "(none)"}`); + + for (const f of [...incoming, ...reqTxt]) { + const text = fs.readFileSync(f, "utf8"); + assert.ok(text.startsWith(`${DUMP_PLAINTEXT_BANNER}\n`), `${path.basename(f)} must lead with the plaintext warning banner`); + assert.ok(text.includes(BODY_MARKER), `${path.basename(f)} must still carry the exact body bytes`); + } + const jsonText = fs.readFileSync(jsonDumps[0], "utf8"); + assert.ok(jsonText.startsWith(`${DUMP_PLAINTEXT_BANNER}\n`), "dumps/req-*.json must lead with the plaintext warning banner"); + const parsed = JSON.parse(stripDumpBanner(jsonText)) as { messages: Array<{ content?: unknown }> }; + assert.ok(parsed.messages.some((m) => String(m.content).includes(BODY_MARKER)), "body after the banner must parse and carry the marker"); + } finally { + restoreEnv(prev); + if (h) { await close(h.proxy); await close(h.upstream); } + rmrf(tmpRoot); + } +}); + +const T_A = Date.parse("2026-10-24T08:00:00.000Z"); +const T_B = Date.parse("2026-10-24T08:00:05.000Z"); + +function writeSession(dir: string, sid: string, banner: boolean): void { + const w = (ts: number, kind: "INCOMING" | "REQ", body: string): void => { + const head = ["POST /v1/chat/completions", "content-type: application/json"].join("\n"); + fs.writeFileSync(path.join(dir, `${ts}-${sid}-${kind}.txt`), `${banner ? `${DUMP_PLAINTEXT_BANNER}\n` : ""}${head}\n\n${body}`); + }; + w(T_A, "INCOMING", chatBody("gpt-x", [M1])); + w(T_A + 1, "REQ", chatBody("gpt-x", [M1])); + w(T_B, "INCOMING", chatBody("gpt-x", [M1, M2])); + w(T_B + 1, "REQ", chatBody("gpt-x", [M1, M2])); +} + +function pairShape(p: { category: string; attribution: string | null; outgoingAppend: { kind: string } | null; incomingAppend: { kind: string } | null; outgoingExtra: number; incomingExtra: number | null; modelA: string | null; modelB: string | null; divergence: { offset: number } | null }): Record { + return { + category: p.category, + attribution: p.attribution, + outgoingAppend: p.outgoingAppend?.kind ?? null, + incomingAppend: p.incomingAppend?.kind ?? null, + outgoingExtra: p.outgoingExtra, + incomingExtra: p.incomingExtra, + modelA: p.modelA, + modelB: p.modelB, + divergenceOffset: p.divergence?.offset ?? null, + }; +} + +test("#4482: runDiff classifies bannered dumps identically to pre-banner ones (old dumps stay readable)", () => { + const oldDir = tmpdir("bili-4482-oldfmt-"); + const newDir = tmpdir("bili-4482-newfmt-"); + try { + writeSession(oldDir, "ses_old", false); + writeSession(newDir, "ses_new", true); + const rOld = runDiff(oldDir, { noLog: true }); + const rNew = runDiff(newDir, { noLog: true }); + assert.equal(rOld.totalPairs, 1); + assert.equal(rNew.totalPairs, 1); + const shapeOld = pairShape(rOld.sessions[0]!.pairs[0]!); + const shapeNew = pairShape(rNew.sessions[0]!.pairs[0]!); + assert.deepEqual(shapeNew, shapeOld, "banner must not change classification, extras, models or divergence offsets"); + assert.equal(shapeOld.category, "pure-append"); + assert.equal(shapeOld.modelA, "gpt-x"); + assert.equal(shapeOld.modelB, "gpt-x"); + } finally { rmrf(oldDir); rmrf(newDir); } +}); + +test("#4482: runDiff handles a mixed dir (pre-banner + bannered dumps side by side)", () => { + const dir = tmpdir("bili-4482-mixed-"); + try { + writeSession(dir, "ses_old", false); + writeSession(dir, "ses_new", true); + const report = runDiff(dir, { noLog: true }); + assert.equal(report.totalPairs, 2); + for (const s of report.sessions) { + for (const p of s.pairs) { + assert.equal(p.category, "pure-append", `session ${s.sid} must classify pure-append regardless of banner presence`); + assert.equal(p.modelA, "gpt-x"); + assert.equal(p.modelB, "gpt-x"); + } + } + } finally { rmrf(dir); } +}); diff --git a/tests/reasoning-echo-400.test.ts b/tests/reasoning-echo-400.test.ts index 586c9bc5f4..7c5e4fbd10 100644 --- a/tests/reasoning-echo-400.test.ts +++ b/tests/reasoning-echo-400.test.ts @@ -10,6 +10,7 @@ import { startServer } from "../src/server.ts"; import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { DUMP_PLAINTEXT_BANNER, stripDumpBanner } from "../src/dump-banner.ts"; import { setLogCapture } from "../src/logger.ts"; import { rmrf } from "./tmp-rm.ts"; @@ -162,7 +163,9 @@ test("#762: main-path 400 learns strict-echo, next request is normalized, reject const files = fs.readdirSync(dumpDir).filter((f) => f.startsWith("err-")); assert.equal(files.length, 1, `expected exactly one 4xx dump, got: ${files.join(", ")}`); assert.match(files[0]!, /^err-\d+-re400-1-400\.json$/); - const dumped = JSON.parse(fs.readFileSync(path.join(dumpDir, files[0]!), "utf8")) as { model: string; messages: Array<{ role?: string; content?: unknown; tool_calls?: unknown }> }; + const rawText = fs.readFileSync(path.join(dumpDir, files[0]!), "utf8"); + assert.ok(rawText.startsWith(`${DUMP_PLAINTEXT_BANNER}\n`), "#4482: plaintext warning banner must lead the dump"); + const dumped = JSON.parse(stripDumpBanner(rawText)) as { model: string; messages: Array<{ role?: string; content?: unknown; tool_calls?: unknown }> }; assert.equal(dumped.model, "gpt-test"); // #1881: injectTool=false sessions still carry the ACP-TAGS prohibition — // the client's 6 messages + the injected head system. diff --git a/tools/analyze-dumps.mjs b/tools/analyze-dumps.mjs index 98b0df2e47..628bc08ac3 100644 --- a/tools/analyze-dumps.mjs +++ b/tools/analyze-dumps.mjs @@ -18,6 +18,9 @@ import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { homedir } from "node:os"; +// #4482: body dumps now open with this plaintext-warning line (src/dump-banner.ts). +const PLAINTEXT_BANNER = "PLAINTEXT DUMP — full conversation body, may contain credentials; scrub before sharing"; + const args = process.argv.slice(2); const dir = args[0] || join(homedir(), ".local", "state", "billion-context", "dumps"); const sid = args[1]; @@ -46,7 +49,10 @@ for (const f of files) { } function load(f) { - return JSON.parse(readFileSync(join(dir, f), "utf8")); + let text = readFileSync(join(dir, f), "utf8"); + const nl = text.indexOf("\n"); + if (nl > 0 && text.slice(0, nl).replace(/\r$/, "") === PLAINTEXT_BANNER) text = text.slice(nl + 1); + return JSON.parse(text); } function msgKey(body) { diff --git a/website/config-reference/diagnostics.yaml b/website/config-reference/diagnostics.yaml index 2ca8842960..b80a695a0e 100644 --- a/website/config-reference/diagnostics.yaml +++ b/website/config-reference/diagnostics.yaml @@ -17,15 +17,15 @@ entries: default: "false" env: ACP_DUMP_BODY group: diagnostics - desc_en: "Dump full request/response bodies to the log for troubleshooting." - desc_zh: "把完整请求/响应体转储到日志用于排障。" + desc_en: "Dump full request/response bodies to the log for troubleshooting. Dumps are PLAINTEXT and may contain credentials — scrub before sharing (#4482)." + desc_zh: "把完整请求/响应体转储到日志用于排障。转储内容为明文,可能包含凭据——分享前必须清洗(#4482)。" - key: diagnostics.dumpReq type: boolean default: "true" env: ACP_DUMP_REQ group: diagnostics - desc_en: "Log outgoing upstream request metadata per call." - desc_zh: "逐次调用记录发往上游的请求元数据。" + desc_en: "Log outgoing upstream request metadata per call; with dumpBody on, also gates the plaintext body dumps (scrub before sharing, #4482)." + desc_zh: "逐次调用记录发往上游的请求元数据;开启 dumpBody 时同时是明文请求体转储的闸门(分享前必须清洗,#4482)。" - key: diagnostics.rawDumpDir type: string default: "/raw" @@ -38,8 +38,8 @@ entries: default: "false" env: BILI_DUMP_4XX group: diagnostics - desc_en: "Persist upstream 4xx responses to disk for post-mortem inspection." - desc_zh: "把上游 4xx 响应落盘以便事后检查。" + desc_en: "Persist upstream 4xx responses to disk for post-mortem inspection. Bodies are PLAINTEXT and may contain credentials — scrub before sharing (#4482)." + desc_zh: "把上游 4xx 响应落盘以便事后检查。转储内容为明文,可能包含凭据——分享前必须清洗(#4482)。" - key: diagnostics.dump4xxMaxBytes type: number default: "2097152 (floor 1024)" From b9ba79b858d15f0fff3ee815d3a9b87ba82ae832 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 24 Oct 2026 13:07:06 +0800 Subject: [PATCH 2/3] docs: observability comment claimed raw/*-RES.txt carries the request body It only ever writes the response status + masked headers (src/server/relay.ts). Corrected the file list so the gate contract comment matches the writers. (review nit, unrelated to #4482 content) --- src/server/observability.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/server/observability.ts b/src/server/observability.ts index 8043b36244..7b2b827d58 100644 --- a/src/server/observability.ts +++ b/src/server/observability.ts @@ -2,9 +2,10 @@ import { log as loggerLog } from "../logger.js"; import { maskUrlsInText } from "../log-mask.js"; import { bodyDumpEnabled as knobBodyDumpEnabled } from "../knobs.js"; -// Body dumps (dumps/req-*.json, raw/*-REQ.txt, raw/*-RES.txt, raw/*-INCOMING.txt, -// req-*-REREQUEST.json) write the full plaintext request body and are off by -// default. They are decoupled from --debug (verbose logging) and enabled only +// Body dumps (dumps/req-*.json, raw/*-REQ.txt, raw/*-INCOMING.txt, +// req-*-REREQUEST.json) write the full plaintext request body; raw/*-RES.txt +// carries the response status + masked headers only. All are off by default. +// They are decoupled from --debug (verbose logging) and enabled only // with ACP_DUMP_BODY=1 / diagnostics.dumpBody so `bili ` users don't // leak conversation bodies to disk by default (#276). export function bodyDumpEnabled(): boolean { From 060948943e0ff0f62694288cffabad47ee2c7a70 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 24 Oct 2026 13:29:31 +0800 Subject: [PATCH 3/3] fix(ci): register codex-mcp-shell + codex-plugin-version-sync in LIVE_FILES Both files landed on master via af8e5dc80 (#4449) without LIVE_FILES registration, so the run-ci-tests process-spawn tripwire fails every shard of every PR branch (inherited red master, unrelated to this PR's change). Neither binds a port: codex-mcp-shell coordinates its spawned shell child via ENTRY_REPORT_FILE, codex-plugin-version-sync runs execFileSync synchronously. Appended at list end (slot = index). --- scripts/live-files.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index 720e339d2b..c3a8926eb3 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -57,6 +57,11 @@ export const LIVE_FILES = [ // #4434 (split-brain surface): spawns bili CLI children against a throwaway // origin; own probe server is listen(0) so only the zone band matters "tests/issue-split-brain-surface.test.ts", + // #4449 (af8e5dc80): both landed without registration and tripped the + // run-ci-tests guard on every PR branch; neither binds a port (shell + // child coordinated via ENTRY_REPORT_FILE; version-sync is execFileSync) + "tests/codex-mcp-shell.test.ts", + "tests/codex-plugin-version-sync.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry;