diff --git a/CONFIGURATION.md b/CONFIGURATION.md index f0cda90f8d..593985c3dd 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -222,6 +222,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `diagnostics.maxTotalBytes` | number | unset (off) | BILI_DUMP_MAX_TOTAL_BYTES | Total-size cap for debug dump dirs (dumps/, raw/, SSE dump dir); oldest files deleted first when exceeded (floor 1 MiB). Dumps have no rotation by default and can grow tens of GB/day under active capture (#2412). | | `diagnostics.maxAgeDays` | number | unset (off) | BILI_DUMP_MAX_AGE_DAYS | Age cap in days for debug dump dirs; files older than this are deleted (floor 1 h). Off by default. | | `diagnostics.renderNone` | boolean | false | ACP_RENDER_NONE | Disable all ACP tag rendering (raw wire study mode). | +| `diagnostics.invariantJudge` | boolean | true | BILI_INVARIANT_JUDGE | Keep the I1/I2 outbound-invariant judge armed as an L1 advisory (#3938): every request's outbound bytes are reconciled against the client's inbound history; violations log at warn, clean verdicts at debug, traffic is never blocked. Set false or BILI_INVARIANT_JUDGE=0 to silence it. | | `diagnostics.noInjectTool` | boolean | false | ACP_NO_INJECT_TOOL | Stop injecting the acp_compress tool definition into requests. | | `diagnostics.noCompressPrompt` | boolean | false | ACP_NO_COMPRESS_PROMPT | Stop appending the compression doctrine to system prompts. | | `diagnostics.countTokensPassthrough` | boolean | false | ACP_COUNT_TOKENS_PASSTHROUGH | Forward /v1/messages/count_tokens to the upstream instead of answering locally. | @@ -869,6 +870,7 @@ Since #2030 every pure-behavior knob has a config-file key alongside its env var "maxTotalBytes": null, // cap total dump-dir bytes; null = no cap (#2412) "maxAgeDays": null, // delete dump files older than N days; null = off "renderNone": false, // stop injecting mNNNNN render tags + "invariantJudge": true, // L1 advisory outbound-invariant judge (#3938) "noInjectTool": false, // suppress compress tool injection "noCompressPrompt": false, // suppress compress prompt text "countTokensPassthrough": false, // forward /count_tokens verbatim @@ -954,8 +956,8 @@ Security boundary: the file controls ONLY *when* to upgrade and *which published ### `diagnostics` -- **Type:** `{ dumpBody?: boolean; dumpReq?: boolean; rawDumpDir?: string; dump4xx?: boolean; dump4xxMaxBytes?: number; maxTotalBytes?: number; maxAgeDays?: number; renderNone?: boolean; noInjectTool?: boolean; noCompressPrompt?: boolean; countTokensPassthrough?: boolean; compressProtocol?: "tools" | "text" }` -- **Default:** `{ dumpBody: false, dumpReq: true, rawDumpDir: /raw, dump4xx: false, dump4xxMaxBytes: 2097152, maxTotalBytes: null, maxAgeDays: null, renderNone: false, noInjectTool: false, noCompressPrompt: false, countTokensPassthrough: false, compressProtocol: "tools" }` +- **Type:** `{ dumpBody?: boolean; dumpReq?: boolean; rawDumpDir?: string; dump4xx?: boolean; dump4xxMaxBytes?: number; maxTotalBytes?: number; maxAgeDays?: number; renderNone?: boolean; invariantJudge?: boolean; noInjectTool?: boolean; noCompressPrompt?: boolean; countTokensPassthrough?: boolean; compressProtocol?: "tools" | "text" }` +- **Default:** `{ dumpBody: false, dumpReq: true, rawDumpDir: /raw, dump4xx: false, dump4xxMaxBytes: 2097152, maxTotalBytes: null, maxAgeDays: null, renderNone: false, invariantJudge: true, noInjectTool: false, noCompressPrompt: false, countTokensPassthrough: false, compressProtocol: "tools" }` - **Status:** ACTIVE - **Description:** Debug/diagnostic toggles that used to be env-only (`ACP_DUMP_BODY`, `ACP_DUMP_REQ`, `ACP_RAW_DUMP_DIR`, `BILI_DUMP_4XX`, `BILI_DUMP_4XX_MAX_BYTES`, `ACP_RENDER_NONE`, `ACP_NO_INJECT_TOOL`, `ACP_NO_COMPRESS_PROMPT`, `ACP_COUNT_TOKENS_PASSTHROUGH`, `ACP_COMPRESS_PROTOCOL`). Mostly read live per request (no restart needed), except `compressProtocol` which resolves once at boot like the env var did. `renderNone` stops the `mNNNNN` render tags from being injected into outgoing history — disable only if your workflow doesn't need ref-based compression (#933). Retention bounds (#2412): dump dirs are write-only with no rotation by default — active body/SSE capture measured at ~63 GB/day can fill a disk in days. Set `maxTotalBytes` and/or `maxAgeDays` to bound them (oldest-first deletion, `err-*`/`summary-err-*` incident dumps protected, deletions logged at warn level); leave both unset for unbounded retention. Details in the [env table](#config-file-keys-for-environment-knobs-2030). @@ -1738,6 +1740,7 @@ File keys resolve only when the matching env var is unset. Defaults in parenthes | `BILI_IMAGE_BILLING` | `imageBilling` | auto (resolves to pixels) | | `BILI_IMAGE_RESCALE` | `imageRescale` | true | | `BILI_IMAGE_TOKEN_CAP` | `imageTokenCap` | unset (uncapped) | +| `BILI_INVARIANT_JUDGE` | `diagnostics.invariantJudge` | true | | `BILI_KEEP_ALIVE_TIMEOUT_MS` | `network.keepAliveTimeoutMs` | 5000 | | `BILI_LOG_MASK_HOSTS` | `maskHosts` | true | | `BILI_MAX_SESSIONS` | `sessions.max` | 256 | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 8115d9fe12..e18885c7f2 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -222,6 +222,7 @@ | `diagnostics.maxTotalBytes` | number | unset (off) | BILI_DUMP_MAX_TOTAL_BYTES | 调试转储目录(dumps/、raw/、SSE dump 目录)的总大小上限;超限时按最旧优先删除(下限 1 MiB)。转储默认无任何轮转,持续捕获下每天可增长数十 GB(#2412)。 | | `diagnostics.maxAgeDays` | number | unset (off) | BILI_DUMP_MAX_AGE_DAYS | 调试转储目录的保留天数上限;超过该天数的文件被删除(下限 1 小时)。默认关闭。 | | `diagnostics.renderNone` | boolean | false | ACP_RENDER_NONE | 禁用所有 ACP 标签渲染(原始线路研究模式)。 | +| `diagnostics.invariantJudge` | boolean | true | BILI_INVARIANT_JUDGE | 保持 I1/I2 出站不变量判官以 L1 advisory 形态开启(#3938):每笔请求的出站字节都与客户端入站历史对账,违规以 warn 记录、干净判定以 debug 记录,绝不阻塞流量。设为 false 或 BILI_INVARIANT_JUDGE=0 可静默。 | | `diagnostics.noInjectTool` | boolean | false | ACP_NO_INJECT_TOOL | 停止向请求注入 acp_compress 工具定义。 | | `diagnostics.noCompressPrompt` | boolean | false | ACP_NO_COMPRESS_PROMPT | 停止向系统提示词附加压缩教条文本。 | | `diagnostics.countTokensPassthrough` | boolean | false | ACP_COUNT_TOKENS_PASSTHROUGH | 把 /v1/messages/count_tokens 转发给上游,而不是本地应答。 | @@ -869,6 +870,7 @@ "maxTotalBytes": null, // 转储目录总字节上限;null = 不限制(#2412) "maxAgeDays": null, // 删除超过 N 天的转储文件;null = 关闭 "renderNone": false, // 停止向出站历史注入 mNNNNN 渲染标签 + "invariantJudge": true, // L1 advisory 出站不变量判官(#3938) "noInjectTool": false, // 抑制 compress 工具注入 "noCompressPrompt": false, // 抑制压缩提示文本 "countTokensPassthrough": false, // /count_tokens 原样转发 @@ -954,8 +956,8 @@ ### `diagnostics` -- **类型:** `{ dumpBody?: boolean; dumpReq?: boolean; rawDumpDir?: string; dump4xx?: boolean; dump4xxMaxBytes?: number; maxTotalBytes?: number; maxAgeDays?: number; renderNone?: boolean; noInjectTool?: boolean; noCompressPrompt?: boolean; countTokensPassthrough?: boolean; compressProtocol?: "tools" | "text" }` -- **默认:** `{ dumpBody: false, dumpReq: true, rawDumpDir: /raw, dump4xx: false, dump4xxMaxBytes: 2097152, maxTotalBytes: null, maxAgeDays: null, renderNone: false, noInjectTool: false, noCompressPrompt: false, countTokensPassthrough: false, compressProtocol: "tools" }` +- **类型:** `{ dumpBody?: boolean; dumpReq?: boolean; rawDumpDir?: string; dump4xx?: boolean; dump4xxMaxBytes?: number; maxTotalBytes?: number; maxAgeDays?: number; renderNone?: boolean; invariantJudge?: boolean; noInjectTool?: boolean; noCompressPrompt?: boolean; countTokensPassthrough?: boolean; compressProtocol?: "tools" | "text" }` +- **默认:** `{ dumpBody: false, dumpReq: true, rawDumpDir: /raw, dump4xx: false, dump4xxMaxBytes: 2097152, maxTotalBytes: null, maxAgeDays: null, renderNone: false, invariantJudge: true, noInjectTool: false, noCompressPrompt: false, countTokensPassthrough: false, compressProtocol: "tools" }` - **状态:** ACTIVE - **说明:** 此前仅有环境变量形态的调试/诊断开关(`ACP_DUMP_BODY`、`ACP_DUMP_REQ`、`ACP_RAW_DUMP_DIR`、`BILI_DUMP_4XX`、`BILI_DUMP_4XX_MAX_BYTES`、`ACP_RENDER_NONE`、`ACP_NO_INJECT_TOOL`、`ACP_NO_COMPRESS_PROMPT`、`ACP_COUNT_TOKENS_PASSTHROUGH`、`ACP_COMPRESS_PROTOCOL`)。除 `compressProtocol` 在启动时一次性解析(同旧环境变量行为)外,其余均按请求实时读取,无需重启。`renderNone` 停止向外发历史注入 `mNNNNN` 渲染标签 —— 仅当你的工作流不需要基于 ref 的压缩时才关闭(#933)。保留期限约束(#2412):转储目录只写不删、默认无轮转 —— 实测持续开启 body/SSE 捕获约 63 GB/天,数天内即可写满磁盘。设置 `maxTotalBytes` 和/或 `maxAgeDays` 可为其设限(最旧优先删除,`err-*`/`summary-err-*` 事故证据文件受保护不删,删除动作以 warn 级别记录日志);两者都不设置则保持无限期保留。详见[环境变量表](#环境变量的配置键对照-2030)。 @@ -1737,6 +1739,7 @@ ACP 原生 agent(当前为 `pi` 扩展)会在每个进程内向代理上报 | `BILI_IMAGE_BILLING` | `imageBilling` | auto (resolves to pixels) | | `BILI_IMAGE_RESCALE` | `imageRescale` | true | | `BILI_IMAGE_TOKEN_CAP` | `imageTokenCap` | unset (uncapped) | +| `BILI_INVARIANT_JUDGE` | `diagnostics.invariantJudge` | true | | `BILI_KEEP_ALIVE_TIMEOUT_MS` | `network.keepAliveTimeoutMs` | 5000 | | `BILI_LOG_MASK_HOSTS` | `maskHosts` | true | | `BILI_MAX_SESSIONS` | `sessions.max` | 256 | diff --git a/src/config.ts b/src/config.ts index 6d26b6e4e0..a7e5934643 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1949,6 +1949,10 @@ type FileConfig = { dump4xx?: boolean; dump4xxMaxBytes?: number; renderNone?: boolean; + /** #3938 decision ②: arm the I1/I2 outbound-invariant judge as an L1 + * advisory (warn-log violations, never blocks). Default true; + * BILI_INVARIANT_JUDGE=0/false disarms. */ + invariantJudge?: boolean; noInjectTool?: boolean; noCompressPrompt?: boolean; countTokensPassthrough?: boolean; diff --git a/src/invariant-judge.ts b/src/invariant-judge.ts new file mode 100644 index 0000000000..21d52cfeed --- /dev/null +++ b/src/invariant-judge.ts @@ -0,0 +1,362 @@ +// #3842 — Outbound-invariant judge (the "attribution layer"). +// +// Decides, for ONE request, whether bili rewrote the client's own history on +// the way out. It reconciles the INBOUND core (what the client sent) against +// the OUTBOUND core (what bili forwarded) at the kernel canonical-core level +// (#3658) — wire-neutral, so same-wire and cross-wire share one code path. +// +// Invariants (owner design, #3842): +// I1 (no fold): every inbound stable message appears at the outbound in +// order, normalized-equal. Only legal diffs: (a) tail appends, (b) the +// named, count-bounded exemption list — chain checkpoint / acp_summary +// carriers (outbound-injected, or inbound-ECHOED by stateful hosts, which +// match their own twin by identity), policy reasoning drops, crush +// envelopes. +// I2 (fold): all divergence is confined to [earliest-fold-start, end); the +// bytes before the fold point stay identical, and the fold carrier stays +// pinned (anchor drift is a violation too). +// +// Single-request independence: the inbound core IS the client ground truth, so +// the judge needs no cross-request state — structurally avoiding the process- +// local seam pairing's restart blindness (#2773). +// +// Division of labor (kept separate on purpose): this judge = attribution +// ("whose fault"); seamLcp (#2131, consecutive outbound-vs-previous-outbound) +// = miss detection ("money lost"). A client restart/id-change passes I1 +// correctly yet still breaks the provider prefix cache — that layer owns it. +// +// normalize-on-compare (renderTags=text-only, #2483/#3842): render tags are +// re-stamped onto historical assistant text EVERY request, so they are stripped +// from BOTH sides before comparison. This closes the exemption list WITHOUT +// changing a single outbound byte (zero behavior change). +// +// Pure: no I/O, no mutation. `judgeInboundOutbound` never throws on well-formed +// input; the runtime wrapper records events only and never blocks traffic. + +import type { CoreMessage } from "acp-kernel"; +import { openaiToCore, anthropicToCore, responsesToCore, googleToCore } from "acp-kernel/wire"; +import { log as loggerLog } from "./logger.js"; +import { invariantJudgeEnabled } from "./knobs.js"; + +// `ref`; angle brackets hex-escaped per KDD#2. +// An optional leading newline is consumed too — the renderer injects the tag on +// its own line, so leaving it behind would false-positive every re-tagged turn. +const RENDER_TAG_RE = /\n?\x3cacp\s[^>]*\x3e[\s\S]*?\x3c\/acp\x3e/g; +const CHAIN_TAG_PREFIX = "\x3cbili-chain "; +const SUMMARY_MARKER = "[Compressed conversation section]"; +// kernel/src/crush.ts CRUSH_KEY — a tool-result decoding to an object with this +// key was legitimately crushed (lossless, deterministic), not rewritten. +const CRUSH_KEY = "__acp_crush"; + +type ViolationKind = + | "mid-history-rewrite" + | "mid-history-drop" + | "mid-insertion" + | "fold-boundary-drift" + | "carrier-drift" + | "exemption-overflow"; + +interface Violation { + kind: ViolationKind; + /** Inbound index implicated (-1 when the finding is positional on the exit side). */ + index: number; + detail: string; +} + +export interface JudgeVerdict { + ok: boolean; + folded: boolean; + foldStart?: number; + matched: number; + total: number; + violations: Violation[]; + /** Named-exemption volume consumed this request (#3842 count-bounded list). */ + exempt?: { carrier: number; reasoning: number; crush: number }; + /** Egress lane the verdict came from — main first-send vs loop round-2+. */ + lane?: "main" | "loop"; +} + +interface JudgeOptions { + // Carriers are few per request; a wider gap than this before the next stable + // match reads as a rewrite, not an injection. Default 4. + maxCarrierGap?: number; + // #3842 named exemptions are COUNT-BOUNDED: carrier/reasoning/crush volume + // beyond this per request is itself a violation signal (a defect dressing up + // as exemptions), even when every stable message matched. Default 128 — a + // steady folded session stays far under it (≈ one carrier per live block + + // one reasoning item per reasoning turn). + maxExempt?: number; +} + +function normText(m: CoreMessage): string { + return (m.text ?? "").replace(RENDER_TAG_RE, "").trim(); +} + +function isReasoning(m: CoreMessage): boolean { + return m.contentType === "reasoning"; +} + +function isCarrier(m: CoreMessage): boolean { + if (m.summaryOfBlockId !== undefined) return true; + const t = m.text ?? ""; + return t.startsWith(CHAIN_TAG_PREFIX) || t.includes(SUMMARY_MARKER); +} + +function isSummaryCarrier(m: CoreMessage): boolean { + if (m.summaryOfBlockId !== undefined) return true; + const t = m.text ?? ""; + return t.includes(SUMMARY_MARKER) || m.id.startsWith("acp_summary"); +} + +function isCrushedToolResult(m: CoreMessage): boolean { + if (m.contentType !== "tool-result") return false; + const t = (m.text ?? "").trim(); + if (!t.startsWith("{")) return false; + try { + const o = JSON.parse(t) as unknown; + return !!o && typeof o === "object" && Object.hasOwn(o as Record, CRUSH_KEY); + } catch { + return false; + } +} + +function fp(m: CoreMessage): string { + const id = m.toolCallId ? `#${m.toolCallId}` : ""; + const name = m.contentType === "tool-call" && m.toolName ? `:${m.toolName}` : ""; + const t = normText(m).replace(/\s+/g, " ").trim().slice(0, 80); + return `${m.role}/${m.contentType}${name}${id}|${t}`; +} + +// Identity is content-primary: tool-call ids are re-minted on re-projection +// (#2821) while bytes stay identical, so a differing toolCallId never vetoes a +// content-equal pair (order-centric, KDD#11). toolName stays a hard gate. +function stableEqual(a: CoreMessage, b: CoreMessage): boolean { + if (a.role !== b.role || a.contentType !== b.contentType) return false; + if (a.contentType === "tool-call" && a.toolName && b.toolName && a.toolName !== b.toolName) return false; + if (isCrushedToolResult(b)) return true; + return normText(a) === normText(b); +} + +/** + * Reconcile one request's inbound core against its outbound core. Returns a + * structured verdict; never throws on well-formed input. On a corrupted stream + * it may report several related violations — `ok` is authoritative, details aid triage. + */ +export function judgeInboundOutbound( + entry: readonly CoreMessage[], + exit: readonly CoreMessage[], + opts: JudgeOptions = {}, +): JudgeVerdict { + const maxGap = opts.maxCarrierGap ?? 4; + const maxExempt = opts.maxExempt ?? 128; + const violations: Violation[] = []; + let matched = 0; + let total = 0; + + // The fold point is the first summary carrier; everything before it is the + // frozen prefix (must match), everything from it on is the fold/tail (free). + const sumIdx = exit.findIndex(isSummaryCarrier); + const folded = sumIdx !== -1; + const freezeEnd = folded ? sumIdx : exit.length; + let foldStart = -1; + + let j = 0; + for (let i = 0; i < entry.length; i++) { + const e = entry[i]; + if (isReasoning(e)) continue; + if (folded && j >= freezeEnd) { foldStart = i; break; } // remainder is the folded region + total++; + + // Stateful hosts re-send previously-injected carriers in their inbound + // history; such an entry must match its own twin by identity — skipping + // exit carriers for it would false-positive every steady-state post-fold + // turn. The gap budget counts NON-carrier elements only: a stack of + // injected carriers between stable messages is exempt volume, not drift. + const eCarrier = isCarrier(e); + let found = -1; + let nearTwin = false; + let scanned = 0; + for (let k = j; k < freezeEnd; k++) { + const x = exit[k]; + if (isReasoning(x)) continue; + if (!eCarrier && isCarrier(x)) continue; + scanned++; + if (scanned > maxGap) break; + if (stableEqual(e, x)) { found = k; break; } + if (x.role === e.role && x.contentType === e.contentType && (!e.toolCallId || !x.toolCallId || x.toolCallId === e.toolCallId)) nearTwin = true; + } + + if (found < 0) { + violations.push({ + kind: nearTwin ? "mid-history-rewrite" : "mid-history-drop", + index: i, + detail: `inbound[${i}] ${fp(e)} absent from outbound frozen prefix`, + }); + j++; // soft resync: one bad cell must not cascade into a total unfold (#2821) + continue; + } + + for (let k = j; k < found; k++) { + const x = exit[k]; + if (!isCarrier(x) && !isReasoning(x)) { + violations.push({ kind: "mid-insertion", index: i, detail: `unexplained outbound element before inbound[${i}]: ${fp(x)}` }); + } + } + matched++; + j = found + 1; + } + + // Carrier pinning (I2): a non-carrier element sitting before the fold carrier + // that exceeds the matched frozen prefix means the anchor drifted. + if (folded && sumIdx > matched) { + for (let k = matched; k < sumIdx; k++) { + const x = exit[k]; + if (!isCarrier(x) && !isReasoning(x)) { + violations.push({ kind: "carrier-drift", index: -1, detail: `element before fold carrier at outbound[${k}]: ${fp(x)}` }); + break; + } + } + } + + // The named exemptions are COUNT-BOUNDED (#3842): the ledger is the whole + // outbound exempt volume (carriers + policy reasoning drops + crush + // envelopes), independent of matching — a defect mass-producing exempt- + // looking elements surfaces even when every stable message matched. + const carrierExempt = exit.filter(isCarrier).length; + const reasoningExempt = exit.filter(isReasoning).length; + const crushExempt = exit.filter(isCrushedToolResult).length; + const exemptTotal = carrierExempt + reasoningExempt + crushExempt; + if (exemptTotal > maxExempt) { + violations.push({ + kind: "exemption-overflow", + index: -1, + detail: `named-exemption volume ${exemptTotal} exceeds cap ${maxExempt} (carrier=${carrierExempt} reasoning=${reasoningExempt} crush=${crushExempt})`, + }); + } + + return { + ok: violations.length === 0, + folded, + ...(folded && foldStart >= 0 ? { foldStart } : {}), + matched, + total, + violations, + exempt: { carrier: carrierExempt, reasoning: reasoningExempt, crush: crushExempt }, + }; +} + +// --- Runtime wiring (advisory, armed by default — #3938 decision ②) -------- +// L1 advisory: armed by default (invariantJudgeEnabled knob: env +// BILI_INVARIANT_JUDGE=0/false or config diagnostics.invariantJudge=false to +// disarm). It reconciles every request through the real path, reports verdicts +// to the sink (default: violations at warn, clean verdicts at debug), and +// never mutates the body or throws out. Owner design: #3842, enablement +// decision recorded in #3938. +interface InvariantJudgeInput { + /** Inbound cores (what the client sent) — already computed by prepare. */ + originalMessages?: readonly CoreMessage[]; + /** Outbound request body (the exact bytes forwarded) — re-parsed to core here. */ + body?: string | Buffer; + protocol?: string; + sessionId?: string; +} + +// Re-parse the ACTUAL outbound body through its own wire codec so the exit side +// reflects exactly what crossed the wire. prepare builds the body from +// rebuiltMessages, NOT from processedMessages — comparing those internal +// snapshots diverges for reasons unrelated to the invariant (token estimation, +// ref re-minting, ordering repairs). Same codec on both sides ⇒ directly +// comparable and wire-neutral, which is also what makes cross-wire work. +function parseExitCores(body: string | Buffer | undefined, protocol: string | undefined): CoreMessage[] | null { + if (!body || !protocol) return null; + const text = typeof body === "string" ? body : body.toString("utf8"); + let parsed: unknown; + try { parsed = JSON.parse(text); } catch { return null; } + try { + switch (protocol) { + case "openai": return openaiToCore(parsed as Parameters[0]).msgs; + case "anthropic": return anthropicToCore(parsed as Parameters[0]).msgs; + case "responses": return responsesToCore(parsed as Parameters[0]).msgs; + case "google": return googleToCore(parsed as Parameters[0]).msgs; + default: return null; + } + } catch { return null; } +} + +type Sink = (v: JudgeVerdict) => void; +// Test-only fault injector: simulates a bili defect in the exit core so the +// wired path can be proven to detect it without depending on an unfixed bug. +// It operates on a throwaway copy — the relayed body is never touched. +type Fault = (exitCores: CoreMessage[]) => CoreMessage[]; +let armedOverride: boolean | null = null; +let sink: Sink | null = null; +let fault: Fault | null = null; + +function defaultSink(sessionId: string | undefined, lane: "main" | "loop"): Sink { + return (v) => { + const where = `[${sessionId ?? "unknown"}] [invariant-judge${v.lane ? `/${v.lane}` : ``}]`; + if (!v.ok) { + const head = v.violations[0]; + loggerLog("warn", `${where} I${v.folded ? 2 : 1} violation: ${head.kind} @${head.index} — ${head.detail} (${v.violations.length} finding(s), matched ${v.matched}/${v.total}${v.exempt ? `, exempt carrier=${v.exempt.carrier} reasoning=${v.exempt.reasoning} crush=${v.exempt.crush}` : ``})`); + } else { + loggerLog("debug", `${where} clean ${v.folded ? "fold" : "growth"} verdict: matched ${v.matched}/${v.total}${v.exempt ? `, exempt carrier=${v.exempt.carrier} reasoning=${v.exempt.reasoning} crush=${v.exempt.crush}` : ``}`); + } + }; +} + +export function _setInvariantJudgeForTest(enabled: boolean, s?: Sink): void { + armedOverride = enabled ? true : false; + sink = enabled ? (s ?? null) : null; +} + +export function _setInvariantJudgeFaultForTest(fn?: Fault | null): void { + fault = fn ?? null; +} + +function armed(): boolean { + if (armedOverride !== null) return armedOverride; + return invariantJudgeEnabled(); +} + +export function maybeRunInvariantJudge(input: InvariantJudgeInput): void { + if (!armed()) return; + try { + const entry = input.originalMessages ?? []; + const exitCores0 = parseExitCores(input.body, input.protocol); + if (!exitCores0) return; // unparseable/unknown wire → skip, never false-positive + const exitCores = fault ? fault([...exitCores0]) : exitCores0; + const v = judgeInboundOutbound(entry, exitCores); + // Every judged request reports its verdict (ok or not): consumers must be + // able to see folded-but-clean traffic too, not just violations — that is + // how the live I2 path proves a real fold stayed green (#3842). + (sink ?? defaultSink(input.sessionId, "main"))({ ...v, lane: "main" }); + } catch { + // advisory layer: a judge failure must never disturb the request + } +} + +// Loop rounds 2+ rebuild the body from bili's OWN accumulated state, not from +// the client's inbound — the entry ground truth for them is the PREVIOUS +// outbound (I1: only tail appends + named exemptions may differ). `prev` is +// the last sent body's retained full copy (clipped/oversized bodies are +// absent and the check silently skips — no false positives from partial +// forensics). Same-wire by construction; a mid-session wire switch fails the +// parse/pair loudly, which is exactly the #3849 signal. +export function maybeJudgeContinuity( + prev: string | Buffer | null | undefined, + body: string | Buffer, + protocol: string | undefined, + sessionId?: string, +): void { + if (!armed() || !prev) return; + try { + const entry = parseExitCores(prev, protocol); + const exitCores0 = parseExitCores(body, protocol); + if (!entry || !exitCores0) return; + const exitCores = fault ? fault([...exitCores0]) : exitCores0; + const v = judgeInboundOutbound(entry, exitCores); + (sink ?? defaultSink(sessionId, "loop"))({ ...v, lane: "loop" }); + } catch { + // advisory layer: a judge failure must never disturb the request + } +} diff --git a/src/knobs.ts b/src/knobs.ts index 40c5f0b899..d9f7972aa7 100644 --- a/src/knobs.ts +++ b/src/knobs.ts @@ -631,6 +631,18 @@ export function renderNone(): boolean { return fileDiag().renderNone === true; } +/** BILI_INVARIANT_JUDGE > diagnostics.invariantJudge > true (#3938 decision ②). + * The I1/I2 outbound-invariant judge runs as an L1 advisory on every request — + * violations log at warn, clean verdicts at debug, traffic is never blocked. + * "0"/"false" disarms (env precedence follows the boolean-off convention so an + * operator can silence a noisy judge without a config edit). */ +export function invariantJudgeEnabled(): boolean { + const envRaw = process.env.BILI_INVARIANT_JUDGE; + if (envRaw !== undefined) return !(envRaw === "0" || envRaw === "false" || envRaw === "off"); + const f = fileDiag().invariantJudge; + return typeof f === "boolean" ? f : true; +} + /** ACP_NO_INJECT_TOOL > diagnostics.noInjectTool > false. */ export function noInjectTool(): boolean { const envRaw = process.env.ACP_NO_INJECT_TOOL; diff --git a/src/loop/core.ts b/src/loop/core.ts index bae08d95de..d03120c6c5 100644 --- a/src/loop/core.ts +++ b/src/loop/core.ts @@ -9,6 +9,7 @@ import { estimateWireOverhead } from "../server/budget.js"; import { handleAcpStatus } from "../acp-status.js"; import { STATUS_REPORT_CLOSE, STATUS_REPORT_OPEN } from "../acp-panel.js"; import { credentialFingerprint, generationParamFp, handleAcpCache, noteForwardedBody, noteForwardedImageFacts, settleUsageReport } from "../cache-ledger.js"; +import { maybeJudgeContinuity } from "../invariant-judge.js"; import { countImagesInRawBody } from "../image-tokens.js"; import { diagnoseSuccessWithoutUsage, lastCompressSuffix, withSessionLock, type Session } from "../session.js"; import type { BiliMessage } from "acp-kernel/wire"; @@ -500,9 +501,18 @@ export async function* runCompressLoop( // #2131 follow-up: the message array lives under `messages` (chat), // `input` (Responses) or `contents` (Google) depending on wire shape. const wireArr = [wireObj.messages, wireObj.input, wireObj.contents].find((v): v is unknown[] => Array.isArray(v)); + // #3938 (PR-A): loop rounds 2+ have no client inbound — their I1 entry + // ground truth is the PREVIOUS outbound. Grab it BEFORE noteForwardedBody + // overwrites the slot; clipped/oversized bodies retain no full copy and + // the continuity check silently skips (no false positives from partial + // forensics). Advisory; never blocks. + const prevSentFull = ctx.session.transient?.sentBody?.full ?? null; // #3803: generation-parameter fingerprint from the full wire object (never the capped head) — // a host reasoning-effort / thinking-config flip re-partitions the provider cache namespace. noteForwardedBody(ctx.session, wireBodyStr, wireArr !== undefined ? wireArr.length : null, credentialFingerprint(requestOptions.headers), generationParamFp(wireObj)); + if (prevSentFull) { + maybeJudgeContinuity(prevSentFull, wireBodyStr, ctx.protocol, ctx.session.id); + } // #1843 L1: capture the round's image facts for the learning layer — the // text side must mirror what outboundPayloadBreakdown bills (messages + // wire overhead) so observed image mass = billed total - textSide. diff --git a/src/server/relay.ts b/src/server/relay.ts index cfd3af7f7a..b31bc9cbd0 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1,4 +1,5 @@ import { checkOutboundBody, OutboundBodyTooLargeError } from "../request-body-budget.js"; +import { maybeRunInvariantJudge } from "../invariant-judge.js"; // #1440 P2 cut 4: upstream relay zone (forward) extracted verbatim from src/server.ts — pure move, zero behavior change. import http from "node:http"; @@ -670,6 +671,17 @@ export async function forward( // mid-history break to the client (host plugin insert / paste hoisting) // instead of blaming the rebuild. noteInboundMessages(prepared.session, prepared.originalMessages); + // #3938 (PR-A): judge the exact outbound bytes at the universal main-lane + // send chokepoint — the same string just paired, AFTER compat-role / + // steering / chain-stamp / shed rewrites, so I1/I2 see the wire truth. + // Advisory; never blocks. Replaces the pre-send hook (handle.ts) which + // judged a pre-rewrite snapshot. + maybeRunInvariantJudge({ + originalMessages: prepared.originalMessages, + body: sentBody, + protocol: prepared.protocol, + sessionId: prepared.session.id, + }); } // #3148: round-level breaker — while the endpoint is provably dead, fail // fast WITHOUT contacting it instead of burning another full cold send + diff --git a/tests/invariant-judge-e2e.test.ts b/tests/invariant-judge-e2e.test.ts new file mode 100644 index 0000000000..82c21b7fcc --- /dev/null +++ b/tests/invariant-judge-e2e.test.ts @@ -0,0 +1,254 @@ +// #3842 — I1/I2 judge on REAL traffic: host-behavior generator + defect fixtures. +// The judge is armed on EVERY request through the actual proxy path (handle.ts). +// Two end-to-end claims, deterministic (fixed seed, single wire): +// (A) ZERO false positives across a seeded stream of legitimate host behaviors — +// append / edit-prior / retry / model-switch / restart — including HOST-side +// history rewrites, which the judge attributes to the client (entry==exit => +// green), never to bili; +// (B) the wired path FIRES on each injected defect shape (rewrite / reorder / +// drop / mid-insertion) — proof the attribution layer has teeth live; +// (C) a REAL fold through preflight (tiny window) stays green end-to-end — +// including the fold-creating request, which is judged on the FINAL body +// (post-preflight) — and post-fold prefix corruption fires (I2 teeth live). +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { defaultConfig, type CoreMessage } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _resetPluginStateForTest } from "../src/plugin.ts"; +import { _resetSessionsForTest } from "../src/session.ts"; +import { _setForTest } from "../src/registry.ts"; +import { resetToolRingForTest } from "../src/tool-ring.ts"; +import { _setInvariantJudgeForTest, _setInvariantJudgeFaultForTest, type JudgeVerdict } from "../src/invariant-judge.ts"; + +const root = mkdtempSync(join(tmpdir(), "bili-invariant-judge-")); +test.after(() => rmSync(root, { recursive: true, force: true })); +for (const key of ["HOME", "APPHOME", "HERMES_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = root; +process.env.NODE_ENV = "test"; + +// Deterministic PRNG so CI runs are reproducible (tests must not rely on luck). +function mulberry32(seed: number): () => number { + return () => { + seed |= 0; + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +type Msg = { role: "user" | "assistant"; content: string }; + +async function harness(opts: { context?: number } = {}) { + const ctx = opts.context ?? 400000; + _resetSessionsForTest(); + _resetPluginStateForTest(); + resetToolRingForTest(); + _setForTest({}); + const store = new SessionStore({ dir: join(root, "sessions"), enabled: false }); + _setStoreForTest(store); + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + res.writeHead(200, { "content-type": "application/json" }); + // Substantive content: preflight compression asks THIS mock for summaries, + // and the kernel rejects sub-minimum-size summaries (#3842 I2 live path). + // Wire-aware: the fixture upstream answers in the lane's native shape so + // anthropic/google traffic exercises their own codecs end to end. + const url = req.url ?? ""; + const substantive = `ok ${"lorem ipsum dolor sit amet ".repeat(18)}`; + if (url.includes("/v1/messages")) { + res.end(JSON.stringify({ id: "msg_fixture", type: "message", role: "assistant", model: "model-test", content: [{ type: "text", text: substantive }], stop_reason: "end_turn", usage: { input_tokens: 10, output_tokens: 10 } })); + return; + } + if (url.includes(":generateContent")) { + res.end(JSON.stringify({ candidates: [{ content: { role: "model", parts: [{ text: substantive }] }, finishReason: "STOP" }], usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 10 } })); + return; + } + res.end(JSON.stringify({ id: "chat_fixture", choices: [{ message: { role: "assistant", content: substantive }, finish_reason: "stop" }] })); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const addr = upstream.address(); + assert(addr && typeof addr === "object"); + const upstreamUrl = `http://127.0.0.1:${addr.port}`; + const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: upstreamUrl, routes: { [upstreamUrl]: { models: { "model-test": { context: ctx }, "model-alt": { context: ctx } } } }, modelContextLimit: ctx, kernelConfig: defaultConfig(ctx), compress: { injectTool: false, injectNudge: false, preserveRecentMessages: 1, preserveRecentTokens: 0, minCompressRangeChars: 100 }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", log: false, debug: false, passthrough: false, autoUpdate: false, mitm: { enabled: false, domains: [] } } as ProxyOptions); + await once(proxy, "listening"); + const paddr = proxy.address(); + assert(paddr && typeof paddr === "object"); + const origin = `http://127.0.0.1:${paddr.port}`; + type Wire = "openai" | "anthropic" | "google"; + const send = async (messages: Msg[], opts: { model?: string; conv?: string; wire?: Wire } = {}) => { + const wire = opts.wire ?? "openai"; + const headers = { "content-type": "application/json", "x-bili-plugin": "fixture", "x-bili-plugin-conversation": opts.conv ?? "invariant" }; + const model = opts.model ?? "model-test"; + let url: string, body: string; + if (wire === "anthropic") { + url = `${origin}/bili/${upstreamUrl}/v1/messages`; + body = JSON.stringify({ model, max_tokens: 1024, stream: false, messages: messages.map((m) => ({ role: m.role, content: m.content })) }); + } else if (wire === "google") { + url = `${origin}/bili/${upstreamUrl}/v1beta/models/${model}:generateContent`; + body = JSON.stringify({ contents: messages.map((m) => ({ role: m.role === "assistant" ? "model" : "user", parts: [{ text: m.content }] })) }); + } else { + url = `${origin}/bili/${upstreamUrl}/v1/chat/completions`; + body = JSON.stringify({ model, messages, max_tokens: 1024, stream: false }); + } + const r = await fetch(url, { method: "POST", headers, body }); + assert.equal(r.status, 200, await r.text()); + }; + return { send, close: async () => { store.cancelAll(); proxy.close(); proxy.closeIdleConnections?.(); upstream.close(); upstream.closeIdleConnections?.(); await Promise.all([once(proxy, "close"), once(upstream, "close")]); } }; +} + +test("generator: zero false positives across seeded legitimate host behaviors", async () => { + const h = await harness(); + const send = h.send; + const seen: JudgeVerdict[] = []; + _setInvariantJudgeForTest(true, (v) => { if (!v.ok) seen.push(v); }); + try { + const rand = mulberry32(0xc0ffee); + const pool = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet"]; + const pick = (): string => pool[Math.floor(rand() * pool.length)] ?? "word"; + const history: Msg[] = []; + const pushTurn = () => { + const last = history[history.length - 1]; + const role: Msg["role"] = !last || last.role === "user" ? "assistant" : "user"; + history.push({ role, content: `${pick()} ${pick()} ${pick()}` }); + }; + for (let step = 0; step < 60; step++) { + const r = rand(); + if (history.length < 2 || r < 0.45) { + pushTurn(); + } else if (r < 0.65) { + const i = Math.floor(rand() * history.length); + const cur = history[i]; + if (cur) history[i] = { ...cur, content: `${pick()} EDIT` }; + } else if (r >= 0.8) { + await send(history.map((m) => ({ ...m })), { model: rand() < 0.5 ? "model-alt" : "model-test" }); + continue; + } + await send(history.map((m) => ({ ...m }))); + } + for (let k = 0; k < 3; k++) { + await send([{ role: "user", content: `${pick()} ${pick()}` }], { conv: `restart-${k}` }); + } + assert.equal(seen.length, 0, JSON.stringify(seen.slice(0, 3))); + } finally { + _setInvariantJudgeForTest(false); + await h.close(); + } +}); + +test("wired path fires on each injected defect shape", async () => { + const h = await harness(); + const seen: JudgeVerdict[] = []; + _setInvariantJudgeForTest(true, (v) => { if (!v.ok) seen.push(v); }); + const base: Msg[] = [ + { role: "user", content: "one one one one" }, + { role: "assistant", content: "two two two two" }, + { role: "user", content: "three three three three" }, + ]; + const faults: Array<[string, (c: CoreMessage[]) => CoreMessage[]]> = [ + ["rewrite", (c) => c.map((m, i) => (i === 1 ? { ...m, text: `${m.text ?? ""} MUTATED` } : m))], + ["reorder", (c) => { if (c.length < 3) return c; const a = [...c]; const t = a[1]!; a[1] = a[2]!; a[2] = t; return a; }], + ["drop", (c) => c.filter((_, i) => i !== 1)], + ["mid-insertion", (c) => { if (c.length < 2) return c; const a = [...c]; a.splice(1, 0, { ...(a[1] as CoreMessage), text: "INJECTED INJECTED" }); return a; }], + ]; + try { + await h.send(base.map((m) => ({ ...m }))); + assert.equal(seen.length, 0, JSON.stringify(seen)); + for (const [name, fn] of faults) { + const before = seen.length; + _setInvariantJudgeFaultForTest(fn); + await h.send(base.map((m) => ({ ...m }))); + assert.ok(seen.length > before, `judge must fire on ${name}: ${JSON.stringify(seen.slice(before))}`); + _setInvariantJudgeFaultForTest(null); + } + } finally { + _setInvariantJudgeFaultForTest(null); + _setInvariantJudgeForTest(false); + await h.close(); + } +}); + +test("I2 live path: real preflight fold stays green; post-fold corruption fires", async () => { + const h = await harness({ context: 8000 }); + const seen: JudgeVerdict[] = []; + _setInvariantJudgeForTest(true, (v) => seen.push(v)); + try { + const history: Msg[] = []; + let folded = false; + for (let step = 0; step < 60 && !folded; step++) { + const role: Msg["role"] = history.length % 2 === 0 ? "user" : "assistant"; + history.push({ role, content: `turn ${step} ${"lorem ipsum dolor sit amet ".repeat(30)}` }); + await h.send(history.map((m) => ({ ...m }))); + folded = seen.some((v) => v.folded); + } + assert.ok(folded, "preflight must fold within 60 turns on an 8K window"); + assert.equal(seen.filter((v) => !v.ok).length, 0, JSON.stringify(seen.filter((v) => !v.ok).slice(0, 3))); + for (let k = 0; k < 3; k++) await h.send(history.map((m) => ({ ...m }))); + assert.equal(seen.filter((v) => !v.ok).length, 0, JSON.stringify(seen.filter((v) => !v.ok).slice(0, 3))); + // Teeth: corrupt the frozen prefix (or plant an unexplained element when the + // whole prefix was folded away) -> the armed live judge must fire. + const isSum = (m: CoreMessage) => m.summaryOfBlockId !== undefined || (m.text ?? "").includes("[Compressed conversation section]") || m.id.startsWith("acp_summary"); + _setInvariantJudgeFaultForTest((c) => { + const a = [...c]; + const s = a.findIndex(isSum); + if (s <= 0) { + const donor = a[a.length - 1]!; + a.unshift({ ...donor, text: "UNEXPLAINED PLANTED ELEMENT" }); + return a; + } + let k = s - 1; + while (k >= 0 && isSum(a[k]!)) k--; + if (k < 0) { + const donor = a[a.length - 1]!; + a.unshift({ ...donor, text: "UNEXPLAINED PLANTED ELEMENT" }); + return a; + } + return a.map((m, i) => (i === k ? { ...m, text: `${m.text ?? ""} MUTATED` } : m)); + }); + const before = seen.filter((v) => !v.ok).length; + await h.send(history.map((m) => ({ ...m }))); + assert.ok(seen.filter((v) => !v.ok).length > before, "judge must fire on post-fold corruption"); + } finally { + _setInvariantJudgeFaultForTest(null); + _setInvariantJudgeForTest(false); + await h.close(); + } +}); + +// #3938 (PR-A): the judge speaks every wire — the relay chokepoint re-parses the +// OUTBOUND bytes through the lane's own codec, so each wire needs its own live +// proof: zero false positives on legitimate growth + teeth on a defect. +for (const wire of ["anthropic", "google"] as const) { + test(`${wire} lane: legitimate growth stays green, injected defect fires`, async () => { + const h = await harness(); + const seen: JudgeVerdict[] = []; + _setInvariantJudgeForTest(true, (v) => { if (!v.ok) seen.push(v); }); + try { + const history: Msg[] = []; + for (let step = 0; step < 8; step++) { + const role: Msg["role"] = history.length % 2 === 0 ? "user" : "assistant"; + history.push({ role, content: `turn ${step} ${"lorem ipsum dolor sit amet ".repeat(6)}` }); + await h.send(history.map((m) => ({ ...m })), { wire }); + } + assert.equal(seen.length, 0, JSON.stringify(seen.slice(0, 3))); + _setInvariantJudgeFaultForTest((c) => c.map((m, i) => (i === 1 ? { ...m, text: `${m.text ?? ""} MUTATED` } : m))); + const before = seen.length; + await h.send(history.map((m) => ({ ...m })), { wire }); + assert.ok(seen.length > before, `judge must fire on ${wire} mid-history rewrite: ${JSON.stringify(seen.slice(before))}`); + } finally { + _setInvariantJudgeFaultForTest(null); + _setInvariantJudgeForTest(false); + await h.close(); + } + }); +} diff --git a/tests/invariant-judge.test.ts b/tests/invariant-judge.test.ts new file mode 100644 index 0000000000..bc88f68335 --- /dev/null +++ b/tests/invariant-judge.test.ts @@ -0,0 +1,272 @@ +// #3842 — I1/I2 outbound-invariant judge: unit-level proof of the decision logic. +// Positive fixtures pin the exemption list at zero false positives; negative +// controls prove the judge goes red on the within-request rewrite/drop/reorder/ +// insert shapes (#3676/#3769 family) and on fold-boundary drift (I2). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { CoreMessage } from "acp-kernel"; +import { judgeInboundOutbound, maybeRunInvariantJudge, maybeJudgeContinuity, _setInvariantJudgeForTest, type JudgeVerdict } from "../src/invariant-judge.js"; +import { setLogCapture } from "../src/logger.js"; + +let n = 0; +function mk(role: CoreMessage["role"], contentType: CoreMessage["contentType"], text?: string, extra: Partial = {}): CoreMessage { + n++; + return { id: `m${String(n).padStart(5, "0")}`, role, contentType, ...(text !== undefined ? { text } : {}), ...extra }; +} +const TAG = "\x3cacp tokens=\"2\" type=\"text\"\x3em00001\x3c/acp\x3e"; +function tagged(t: string): string { return `${t}\n${TAG}`; } + +test("I1 identity: entry == exit passes", () => { + const a = [mk("user", "text", "hello"), mk("assistant", "text", "hi there")]; + const v = judgeInboundOutbound(a, a); + assert.equal(v.ok, true); + assert.equal(v.matched, 2); +}); + +test("I1 render tags added on exit are normalized away (no false positive)", () => { + const entry = [mk("user", "text", "hello"), mk("assistant", "text", "hi there")]; + const exit = [mk("user", "text", "hello"), mk("assistant", "text", tagged("hi there"))]; + assert.equal(judgeInboundOutbound(entry, exit).ok, true); +}); + +test("I1 tail append (this turn's new message) is allowed", () => { + const entry = [mk("user", "text", "hello"), mk("assistant", "text", "hi")]; + const exit = [...entry, mk("user", "text", "follow-up question")]; + assert.equal(judgeInboundOutbound(entry, exit).ok, true); +}); + +test("I1 trailing chain-checkpoint carrier is exempt", () => { + const entry = [mk("user", "text", "hello"), mk("assistant", "text", "hi")]; + const exit = [...entry, mk("user", "text", "\x3cbili-chain \"abc\"/>\x3c/bili-chain>")]; + assert.equal(judgeInboundOutbound(entry, exit).ok, true); +}); + +test("I1 policy reasoning drop is exempt", () => { + const entry = [mk("user", "text", "q"), mk("assistant", "reasoning", "thinking..."), mk("assistant", "text", "a")]; + const exit = [mk("user", "text", "q"), mk("assistant", "text", "a")]; + assert.equal(judgeInboundOutbound(entry, exit).ok, true); +}); + +test("I1 deterministic crush transform of a tool-result is exempt", () => { + const entry = [mk("user", "text", "run"), mk("tool", "tool-result", "[{\"a\":1},{\"a\":2},{\"a\":3}]"), mk("assistant", "text", "done")]; + const crushed = JSON.stringify({ __acp_crush: "rows", rows: 3, const: { a: 1 }, items: [] }); + const exit = [mk("user", "text", "run"), mk("tool", "tool-result", crushed), mk("assistant", "text", "done")]; + assert.equal(judgeInboundOutbound(entry, exit).ok, true); +}); + +test("I1 tool-call id re-mint with identical content does not false-positive (#2821)", () => { + const entry = [mk("assistant", "tool-call", "{\"cmd\":\"ls\"}", { toolName: "bash", toolCallId: "call_1" })]; + const exit = [mk("assistant", "tool-call", "{\"cmd\":\"ls\"}", { toolName: "bash", toolCallId: "call_REMINTED" })]; + assert.equal(judgeInboundOutbound(entry, exit).ok, true); +}); + +test("NEG mid-history rewrite (stable assistant text altered) -> red", () => { + const entry = [mk("user", "text", "hello"), mk("assistant", "text", "the answer is 42"), mk("user", "text", "thanks")]; + const exit = [mk("user", "text", "hello"), mk("assistant", "text", "the answer is 43"), mk("user", "text", "thanks")]; + const v = judgeInboundOutbound(entry, exit); + assert.equal(v.ok, false); + assert.ok(v.violations.some((x) => x.kind === "mid-history-rewrite")); +}); + +test("NEG mid-history drop (stable message vanished, no fold) -> red", () => { + const entry = [mk("user", "text", "one"), mk("assistant", "text", "two"), mk("user", "text", "three")]; + const exit = [mk("user", "text", "one"), mk("user", "text", "three")]; + const v = judgeInboundOutbound(entry, exit); + assert.equal(v.ok, false); + assert.ok(v.violations.some((x) => x.kind === "mid-history-drop" || x.kind === "mid-history-rewrite")); +}); + +test("NEG reorder (stable messages out of order) -> red", () => { + const entry = [mk("user", "text", "A"), mk("assistant", "text", "B"), mk("user", "text", "C")]; + const exit = [mk("user", "text", "A"), mk("user", "text", "C"), mk("assistant", "text", "B")]; + assert.equal(judgeInboundOutbound(entry, exit).ok, false); +}); + +test("NEG mid-insertion (non-carrier element lands before a stable match) -> red", () => { + const entry = [mk("user", "text", "A"), mk("assistant", "text", "B")]; + const exit = [mk("user", "text", "A"), mk("system", "text", "INJECTED MIDDLE"), mk("assistant", "text", "B")]; + const v = judgeInboundOutbound(entry, exit); + assert.equal(v.ok, false); + assert.ok(v.violations.some((x) => x.kind === "mid-insertion")); +}); + +test("I2 fold: frozen prefix intact, folded region replaced by summary carrier -> green", () => { + const entry = [ + mk("user", "text", "preamble"), + mk("assistant", "text", "old-1"), + mk("user", "text", "old-2"), + mk("assistant", "text", "old-3"), + ]; + const exit = [ + mk("user", "text", "preamble"), + mk("user", "text", "[Compressed conversation section]\nold stuff summarized", { summaryOfBlockId: "b5" }), + mk("user", "text", "new question"), + ]; + const v = judgeInboundOutbound(entry, exit); + assert.equal(v.ok, true); + assert.equal(v.folded, true); + assert.equal(v.foldStart, 1); +}); + +test("I2 fold boundary drift: frozen-prefix element altered before the summary -> red", () => { + const entry = [ + mk("user", "text", "preamble"), + mk("assistant", "text", "old-1"), + mk("user", "text", "old-2"), + ]; + const exit = [ + mk("user", "text", "PREAMBLE MUTATED"), + mk("user", "text", "[Compressed conversation section]", { summaryOfBlockId: "b5" }), + ]; + const v = judgeInboundOutbound(entry, exit); + assert.equal(v.ok, false); + assert.equal(v.folded, true); +}); + +test("I1 stateful-host echo: inbound acp_summary carrier matches its twin (entry==exit) -> green", () => { + const a = [ + mk("user", "text", "hello"), + mk("user", "text", "[Compressed conversation section]\nold stuff", { summaryOfBlockId: "b3" }), + mk("assistant", "text", "still here"), + ]; + const v = judgeInboundOutbound(a, a); + assert.equal(v.ok, true, JSON.stringify(v.violations)); +}); + +test("I1 stateful-host echo: inbound chain-checkpoint carrier matches its twin -> green", () => { + const a = [ + mk("user", "text", "hello"), + mk("user", "text", "\x3cbili-chain \"abc\"/>\x3c/bili-chain>"), + mk("assistant", "text", "hi"), + ]; + const v = judgeInboundOutbound(a, a); + assert.equal(v.ok, true, JSON.stringify(v.violations)); +}); + +test("I1 five consecutive injected carriers before the next stable match stay exempt (gap counts non-carriers)", () => { + const ck = (t: string) => mk("user", "text", t); + const entry = [mk("user", "text", "A"), mk("assistant", "text", "B")]; + const exit = [ + entry[0]!, + ck("\x3cbili-chain \"c1\"/>\x3c/bili-chain>"), + ck("\x3cbili-chain \"c2\"/>\x3c/bili-chain>"), + ck("\x3cbili-chain \"c3\"/>\x3c/bili-chain>"), + ck("\x3cbili-chain \"c4\"/>\x3c/bili-chain>"), + ck("\x3cbili-chain \"c5\"/>\x3c/bili-chain>"), + entry[1]!, + ]; + const v = judgeInboundOutbound(entry, exit); + assert.equal(v.ok, true, JSON.stringify(v.violations)); +}); + +test("carrier text drift in place stays green: inbound elements at/after the fold point live in bili's owned render region (host edits to persisted summaries are normalized by the canonical re-render, so they are not attributable to the host)", () => { + const entry = [ + mk("user", "text", "hello"), + mk("user", "text", "[Compressed conversation section]\nold stuff", { summaryOfBlockId: "b3" }), + ]; + const exit = [ + mk("user", "text", "hello"), + mk("user", "text", "[Compressed conversation section]\nDIFFERENT CONTENT", { summaryOfBlockId: "b3" }), + ]; + const v = judgeInboundOutbound(entry, exit); + assert.equal(v.ok, true, JSON.stringify(v.violations)); + assert.equal(v.folded, true); + assert.equal(v.matched, 1); +}); + +// --- #3938 (PR-A): exemption ledger, default-arm wiring, loop continuity ---- + +test("exemption ledger rides every verdict (carrier/reasoning/crush volumes)", () => { + const entry = [mk("user", "text", "q"), mk("assistant", "reasoning", "thinking..."), mk("assistant", "text", "a")]; + const exit = [entry[0]!, entry[2]!, mk("user", "text", "\x3cbili-chain \"x\"/\x3e/bili-chain\x3e")]; + const v = judgeInboundOutbound(entry, exit); + assert.equal(v.ok, true); + assert.equal(v.exempt?.carrier, 1); + assert.equal(v.exempt?.reasoning, 0); + assert.equal(v.exempt?.crush, 0); +}); + +test("exemption-overflow: exempt volume beyond the cap is itself a violation (#3842 count-bounded list)", () => { + const carriers = Array.from({ length: 130 }, () => mk("user", "text", "\x3cbili-chain \"x\"/\x3e/bili-chain\x3e")); + const v = judgeInboundOutbound([], carriers); + assert.equal(v.ok, false); + assert.ok(v.violations.some((x) => x.kind === "exemption-overflow")); + assert.equal(v.exempt?.carrier, 130); + const raised = judgeInboundOutbound([], carriers, { maxExempt: 200 }); + assert.equal(raised.ok, true); +}); + +test("steady folded traffic stays far under the default cap (128)", () => { + const history = Array.from({ length: 30 }, (_, i) => mk(i % 2 === 0 ? "user" : "assistant", "text", `turn ${i}`)); + const exit = [mk("user", "text", "[Compressed conversation section] folded"), ...history.slice(-4)]; + const v = judgeInboundOutbound(history, exit); + assert.ok(v.folded); + assert.ok((v.exempt?.carrier ?? 0) <= 2, JSON.stringify(v.exempt)); +}); + +test("default-armed wiring: violations log at warn via the tee logger; BILI_INVARIANT_JUDGE=0 disarms", () => { + const lines: Array<[string, string]> = []; + setLogCapture((level, msg) => lines.push([level, msg])); + const entry = [mk("user", "text", "hello"), mk("assistant", "text", "hi"), mk("user", "text", "q")]; + const body = JSON.stringify({ model: "m", messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "hi REWRITTEN BY DEFECT" }, + { role: "user", content: "q" }, + ] }); + try { + maybeRunInvariantJudge({ originalMessages: entry, body, protocol: "openai", sessionId: "s-judge" }); + const hit = lines.find(([, m]) => m.includes("[invariant-judge")); + assert.ok(hit, JSON.stringify(lines.slice(0, 3))); + assert.equal(hit![0], "warn"); + assert.ok(hit![1].includes("s-judge")); + assert.ok(hit![1].includes("mid-history-rewrite")); + lines.length = 0; + process.env.BILI_INVARIANT_JUDGE = "0"; + maybeRunInvariantJudge({ originalMessages: entry, body, protocol: "openai", sessionId: "s-judge" }); + assert.equal(lines.length, 0); + } finally { + delete process.env.BILI_INVARIANT_JUDGE; + setLogCapture(null); + } +}); + +test("loop continuity judge: prev-outbound -> next-outbound append is green, mid edit is red", () => { + const seen: JudgeVerdict[] = []; + _setInvariantJudgeForTest(true, (v) => seen.push(v)); + try { + const prev = JSON.stringify({ model: "m", messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "hi" }, + ] }); + const grow = JSON.stringify({ model: "m", messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "hi" }, + { role: "user", content: "tool result + follow-up" }, + ] }); + maybeJudgeContinuity(prev, grow, "openai", "s-loop"); + assert.equal(seen.at(-1)?.ok, true); + assert.equal(seen.at(-1)?.lane, "loop"); + const corrupt = JSON.stringify({ model: "m", messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "hi MUTATED MID-LOOP" }, + { role: "user", content: "tool result + follow-up" }, + ] }); + maybeJudgeContinuity(prev, corrupt, "openai", "s-loop"); + assert.equal(seen.at(-1)?.ok, false); + assert.ok(seen.at(-1)!.violations.some((x) => x.kind === "mid-history-rewrite")); + } finally { + _setInvariantJudgeForTest(false); + } +}); + +test("loop continuity judge: absent prev (clipped/oversized forensics) silently skips", () => { + const seen: JudgeVerdict[] = []; + _setInvariantJudgeForTest(true, (v) => seen.push(v)); + try { + maybeJudgeContinuity(null, JSON.stringify({ messages: [] }), "openai", "s-loop"); + maybeJudgeContinuity(undefined, JSON.stringify({ messages: [] }), "openai", "s-loop"); + assert.equal(seen.length, 0); + } finally { + _setInvariantJudgeForTest(false); + } +}); diff --git a/website/config-reference/diagnostics.yaml b/website/config-reference/diagnostics.yaml index 18dc6f3486..2ca8842960 100644 --- a/website/config-reference/diagnostics.yaml +++ b/website/config-reference/diagnostics.yaml @@ -68,6 +68,13 @@ entries: group: diagnostics desc_en: "Disable all ACP tag rendering (raw wire study mode)." desc_zh: "禁用所有 ACP 标签渲染(原始线路研究模式)。" + - key: diagnostics.invariantJudge + type: boolean + default: "true" + env: BILI_INVARIANT_JUDGE + group: diagnostics + desc_en: "Keep the I1/I2 outbound-invariant judge armed as an L1 advisory (#3938): every request's outbound bytes are reconciled against the client's inbound history; violations log at warn, clean verdicts at debug, traffic is never blocked. Set false or BILI_INVARIANT_JUDGE=0 to silence it." + desc_zh: "保持 I1/I2 出站不变量判官以 L1 advisory 形态开启(#3938):每笔请求的出站字节都与客户端入站历史对账,违规以 warn 记录、干净判定以 debug 记录,绝不阻塞流量。设为 false 或 BILI_INVARIANT_JUDGE=0 可静默。" - key: diagnostics.noInjectTool type: boolean default: "false"