From 3896863d9cfceb0691aa353425d3113eab8311fe Mon Sep 17 00:00:00 2001 From: mz Date: Sun, 1 Nov 2026 17:08:57 +0300 Subject: [PATCH 1/4] fix(diagnostics): distinguish size rejections and retain bounded causes --- CHANGELOG.md | 2 + reference/architecture.md | 1 + src/server/rejection-diagnostics.ts | 120 ++++++++++++++++++ src/server/relay.ts | 25 +++- src/util.ts | 32 +++-- tests/issue4452-markerless-reject.test.ts | 47 +++++-- tests/rejection-diagnostics.test.ts | 147 ++++++++++++++++++++++ 7 files changed, 348 insertions(+), 26 deletions(-) create mode 100644 src/server/rejection-diagnostics.ts create mode 100644 tests/rejection-diagnostics.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b28c8d421..313d0bb89f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Fixed +* **普通格式/参数错误的上游 400 不再被误诊为「上下文溢出」(#4452)**:`isMarkerlessOverflowReject` 原判据是 `(400 || 413) && 无任何溢出标记`,于是畸形请求、schema 校验失败、role 顺序不对的 400(各家中转站返回的正文几乎一模一样)全部命中,日志打出「无溢出标记 → 自愈无法启动 → 该路由校准饥饿」的警告——而这类请求根本没有「太大」的证据,纯属制造假警报。现在只有 **413** 才成立:413 本身就是载荷过大这一语义的判据,正文丢了标记才是真正的中转站重写问题;400 交给既有的溢出标记解析器(未命中即普通错误,不告警)。已知溢出的解析与 refold / emergency-shrink / window-learn 全部决策一字未动。 +* **中转站拒绝原因不再被截断在「All targets failed」之后(#4452)**:`← upstream : ` 只保留正文前 600 字节,而扇出型中转站在全部目标失败时返回的是聚合体,真正的死因在最后一个尝试的 `details.error.message` 里——永远落在截断之外,拒绝原因因此无法从日志诊断。现在该行追加一个有界、脱敏的终局原因(`; terminal cause [attempt N/M]: …`):只读 `attempts[last].details.error.message` 一条、经共享 `redactSecretsInText` 洗凭证、折叠全部空白(杜绝换行伪造日志行)、上限 300 字符、正文超过 64 KiB 直接不解析;正文里没有该结构时后缀为空,日志行与原实现逐字节一致。纯诊断,不改重试判定与响应字节。 * **#5412: 异步存储后端的「压缩冻结会话」现在也能自愈。** #5408 事故(一个会话连续 17 次摘要漂移后被保护机制冻住、8 小时无人知晓)的主修复 #5409 只教了同步存储路径(SQLite 直写)自救:漂移挂起期间保存会自动升级为全量重导入并清掉索引里的残留行。异步路径(opt-in 网络后端,如 mysql)那份孪生代码没搬——同样被冻住的会话在那上面永远救不回来,每次保存都重写不完整视图、下轮采样继续漂移。本 PR 在 `src/storage/shadow.ts` 的异步入队路径补上同一行对齐(`completeLog = hasRawTail || driftedSessions.has(id)`),行为与同步路径逐字一致;回归测试同时钉住正门(latch 挂起的折叠保存清掉外来行)与负门(未挂起保存不清扫)。 - **中断频繁的会话压缩提醒几乎不触发,积到几十万 token 才压一次(#5415,0.1.210 引入)**:#5293 的修复把每次用户中断后的增长参照点直接抬到当前读数,连「中断前已实测的真实增长」也一起清零;高频中断的客户端(实测一份日志 4.5 小时 85 次中断)永远攒不到增长台阶,30 万可压内容 idle 数小时,最后一次性压 33 万。现在吸收时把被清零的实测增长记入「欠账」(`nudge.absorbedDebt`)并累计吸收次数(`nudge.absorbCount`),两者都只在**第 2 次起**的中断后生效:单次中断字节级保持 #5293 的安静(不打扰用户刚改的主意——欠账再多也不当场弹提醒);第二次中断才证明会话是「高频中断型」而非「改主意」,欠账从此与新增量一起计入增长台阶,恢复 209 时代的节奏。另加「大积压排水」兑底:欠账>0 且可压量≥2×阈值且当前无未消化提醒时不再等增长台阶(与 #194 同哲学);无欠账/单次中断的会话决策字节不变。压缩成功时欠账与计数随其它基线一起清零;idle 日志同步显示 `growth X + absorbed Y across N aborts < floor Z`(单次中断显示 `held (single abort, #5293)`)便于排查。tests:kernel/tests/issue5415-absorb-debt-nudge.test.ts(欠账付台阶/单次中断钉死不弹/排水兑底/忽略提醒不重发/无欠账字节不变/压缩后清账+清计数);tests/issue5415-absorb-debt.test.ts(吸收记账算术、欠账+计数、累次累计、参照点已越过 anchor 不记账也计数、陈旧 flag 丢弃)。 - **长会话每回合把 Node 主线程卡死十几秒、网页面板冻结(#5337)**:每回合的压缩筛查把历史切成若干可折叠单元,逐个跑「能否折叠」判定——而每次判定都从零重扫全量消息数组(id 查找无索引,五处线性 `.find()`/`includes`),还重复重算全套整史统计(受保护/过期工具调用集合、工具配对表、末尾锚点、完整性预扫)。总工作量≈单元数×全史长度=历史规模平方级,且同步跑在主线程:2.3万条核心消息实测单回合约17秒(事件循环被占死,请求排队;#5304 同签名报告)。修复=把这批 O(n) 推导提升为每轮筛查**一次**构建(`buildProjectionShared`)、共享给同一轮所有单元;id→序号一次性建表替五处线性查找;「已丢弃」成员判断从 includes-in-some 改为 Set。判定逻辑本身一字未动(#3236 单一判定纪律),只是输入到达判定的方式变了。同机同语料实测:2万条消息完整推荐链 9028ms→约572ms(≈16×),回落到线性量级。单次调用路径(apply/report)保持原有内联计算,输出逐字节不变。kernel v0.0.175。tests/issue5337-shared-screening.test.ts(共享/非共享投影逐字段等价+降级边界一致+n=20000 耗时上界钉);既有 issue3406 曲线钉与 #3236 一致性回归不变。 diff --git a/reference/architecture.md b/reference/architecture.md index baa641963f..bb1522412e 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -28,6 +28,7 @@ billion-context/ │ │ ├── prepare-openai.ts # OpenAI chat request preparation (#1440) │ │ ├── prepare-responses.ts # Responses request preparation + codex compact/prompt-cache helpers (#1440) │ │ ├── relay.ts # Upstream relay zone: forward() + fake-completion recovery (#1440 P2 cut 4) +│ │ ├── rejection-diagnostics.ts # Bounded terminal provider cause from direct or nested aggregate rejections │ │ ├── side-request.ts # Tool-surface check for auxiliary side requests │ │ └── stream-io.ts # Small response-body reader for non-2xx inspection │ ├── knobs.ts # Single knob resolver: env > config file > default (#2030) diff --git a/src/server/rejection-diagnostics.ts b/src/server/rejection-diagnostics.ts new file mode 100644 index 0000000000..0aa5e76576 --- /dev/null +++ b/src/server/rejection-diagnostics.ts @@ -0,0 +1,120 @@ +/** + * #4452: bounded terminal-cause diagnostics for a relayed upstream REJECTION. + * + * A relay/gateway that fans a request out across candidate targets may answer a + * total failure with ONE aggregate body carrying every per-target attempt, e.g.: + * + * { "error": { "message": "All targets failed", "type": "upstream_error" }, + * "attempts": [ { "target": "…", "details": { "error": { "message": "…" } } }, … ] } + * + * bili's own log line (`← upstream : `) keeps only the first + * 600 bytes of that body, so the operator sees the OUTER narrative — "all + * targets failed" — while the per-target cause sits further in. This module + * reads exactly ONE such nested cause (the last usable attempt's) when a body + * has that shape, reporting its attempt position rather than guessing causality. + * + * Hard bounds, deliberately: + * - ONE message, never a recursive walk: no `JSON.stringify` of the payload, no + * traversal of `cause`/`errors`/`attempts` chains, no echoing of the request + * body. Only one `attempts[].details.error.message`, with the aggregate + * accepted directly or beneath its standard `error` envelope. + * - Credential scrubbing goes through the shared log-mask sanitizer + * (`redactSecretsInText`), the SAME choke point the logger applies, so a + * provider that echoes the rejected key cannot smuggle it into the log. It is + * a high-signal credential scrubber, NOT a general privacy filter: it does not + * mask arbitrary URLs or paths, and the far stronger guarantee here is + * structural — only this one nested string is ever read, never a body dump. + * - Bounded to `MAX_DIAGNOSTIC_CHARS`; ALL whitespace collapsed so a newline + * (including the `\r\n` frame-injection shape) can never forge a second log + * field. + * - Absent, non-string, empty, or structurally-unrelated payloads return + * `undefined`: the caller then logs exactly what it logged before, so a + * rejection that carries no attempt detail is unchanged byte-for-byte. + * + * Recovery policy is NOT touched here — this module only reads a rejection + * body; every retry/refold/arm decision stays in the relay. + */ + +import { redactSecretsInText } from "../log-mask.js"; + +/** Upper bound on the retained cause text, independent of the outer snippet. */ +const MAX_DIAGNOSTIC_CHARS = 300; + +/** Do not even attempt to parse a body larger than this: an aggregate error + * reply is small (a few KB); a multi-megabyte body is a stream (or an HTML + * error page), not a JSON rejection envelope, and parsing it would cost the + * request more than the diagnostic is worth. */ +const MAX_PARSED_BYTES = 64 * 1024; + +interface DiagnosticResult { + /** The last usable attempt's nested cause, sanitized + bounded. */ + message: string; + /** 0-based index of the attempt the cause came from (for the log line). */ + attemptIndex: number; + /** How many attempts the aggregate body carried. */ + attempts: number; +} + +/** Collapse ALL whitespace (newlines included) and clamp to the bound. */ +function bound(text: string): string { + const collapsed = text.replace(/[\u0000-\u001f\u007f]/g, " ").replace(/\s+/g, " ").trim(); + if (collapsed.length <= MAX_DIAGNOSTIC_CHARS) return collapsed; + return `${collapsed.slice(0, MAX_DIAGNOSTIC_CHARS)} …`; +} + +/** Read `value[name]` only when `value` is a non-array object, so a nested + * field is reached without a generic record guard that would prove nothing + * about the field actually consumed here. */ +function child(value: unknown, name: string): unknown { + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + return (value as Record)[name]; +} + +/** + * Read the single last-reported nested cause out of a relay rejection body. + * + * `payload` is the raw upstream error-body text (the same string the relay + * already buffers to detect an overflow). Returns `undefined` whenever there is + * no usable `attempts[].details.error.message`, leaving the caller's existing + * logging untouched. + * + * Selection rule: the LAST attempt that carries a non-empty + * `details.error.message`, scanning from the end. If the terminal attempt is + * silent, an earlier cause is reported with its actual position and is NOT + * labeled terminal: its existence does not establish why the last stage failed. + */ +export function rejectionDiagnostics(payload: string | null | undefined): DiagnosticResult | undefined { + if (typeof payload !== "string" || payload.length === 0) return undefined; + if (Buffer.byteLength(payload, "utf8") > MAX_PARSED_BYTES) return undefined; + const trimmed = payload.trim(); + // A rejection body must be a JSON object envelope — anything else (an HTML + // error page, a bare text message, an SSE frame) carries no attempts[]. + if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return undefined; + } + const attempts = child(parsed, "attempts") ?? child(child(parsed, "error"), "attempts"); + if (!Array.isArray(attempts) || attempts.length === 0) return undefined; + for (let i = attempts.length - 1; i >= 0; i--) { + const raw = child(child(child(attempts[i], "details"), "error"), "message"); + if (typeof raw !== "string") continue; + const sanitized = bound(redactSecretsInText(raw)); + if (sanitized.length === 0) continue; + return { message: sanitized, attemptIndex: i, attempts: attempts.length }; + } + return undefined; +} + +/** Render the diagnostic as the compact suffix the relay appends to its + * `← upstream ` line. Returns "" when there is no usable cause, so the + * caller can concatenate unconditionally and stay byte-identical without one. + * Pure numbers and the sanitized cause only — never the payload. */ +export function rejectionDiagnosticSuffix(payload: string | null | undefined): string { + const diag = rejectionDiagnostics(payload); + if (diag === undefined) return ""; + const position = diag.attempts > 1 ? ` attempt ${diag.attemptIndex + 1}/${diag.attempts}` : ""; + return `; ${diag.attemptIndex === diag.attempts - 1 ? "terminal" : "reported"} cause${position}: ${diag.message}`; +} diff --git a/src/server/relay.ts b/src/server/relay.ts index 1b7642d5f3..248b4075d0 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -66,6 +66,7 @@ import { applyOutputSteering, applyOutputSteeringJson } from "../output-steering import { estimateWireOverhead } from "./budget.js"; import { buildForwardHeaders, connectionNamedHeaders, RESPONSE_ONLY_STRIP_HEADERS, safeSessionId, UPSTREAM_HOP_HEADERS } from "./headers.js"; import { bodyDumpEnabled, logDumpFailure } from "./observability.js"; +import { rejectionDiagnosticSuffix } from "./rejection-diagnostics.js"; import { effectiveAbsorbBlock } from "./prepare-responses.js"; import { awaitDrain, bufferToStream, dumpStreamToFile, pipeThrough, readStreamToBuffer } from "./stream-io.js"; import { buildForwardTarget, outboundPayloadBreakdown, registerRequestAbort, beginStreamKeepalive, effectiveTokenCount, googlePathKind, imageBillingFor, imageTokenCapFor, inboundResignCredential, outboundContextEstimates, repairResponsesAssistantOrdering, resolveUpstream, stripKernelSummaries, usageGradeInputBaseline, DOOMED_PROBE_ABSOLUTE_BYTES, DOOMED_PROBE_BYTES_PER_TOKEN, type Prepared } from "../server.js"; @@ -1557,14 +1558,18 @@ async function forwardCaptured( } log("warn", `[${s.id}] upstream image-count wall (cap ${info.imageCountCap}, request carried ${info.imageCountSeen ?? "?"}) — learned for pre-send convergence on this route; no token-space arm (#4124)`); } else if (isMarkerlessOverflowReject(upstream.status, errText)) { - // #4452: the gateway rewrote this overflow-shaped reject into a body with no - // parseable marker, so bili's self-heal cannot engage from it — name that here - // (+ the last meter reading) so a body-rewriting gateway's silent starvation is - // diagnosable from the log rather than the opaque line further down. + // #4452: a 413 IS the payload-size verdict, yet this body carries no + // parseable marker, so bili's self-heal cannot engage from it — name that + // here (+ the last meter reading) so a body-rewriting gateway's silent + // starvation is diagnosable from the log rather than the opaque line + // further down. Only 413 reaches this branch: an unmarked 400 is the + // generic malformed/schema-rejected bucket across relays and says nothing + // about size, so it is never diagnosed as overflow here (see + // isMarkerlessOverflowReject). const est = s.stats.lastInputTokens > 0 ? s.stats.lastInputTokens : 0; const win = typeof s.metadata.effectiveContextLimit === "number" && s.metadata.effectiveContextLimit > 0 ? s.metadata.effectiveContextLimit : 0; const meterNote = est > 0 && win > 0 ? `; last settled input ~${Math.round(est)} vs effective window ${win}` : ""; - log("warn", `[${s.id}] ← upstream ${upstream.status} carries no parseable overflow marker (an intermediate gateway rewrote the rejection body?) — bili's overflow self-heal (refold / emergency-shrink / window-learn) CANNOT engage from this rejection${meterNote}; repeated at similar size = calibration-starved route (#4452)`); + log("warn", `[${s.id}] ← upstream ${upstream.status} carries no parseable overflow marker (an intermediate gateway rewrote the payload-size rejection body?) — bili's overflow self-heal (refold / emergency-shrink / window-learn) CANNOT engage from this rejection${meterNote}; repeated at similar size = calibration-starved route (#4452)`); } // #762: learn strict-echo on the MAIN request path too. The loop-only // learner (src/loop/core.ts) never sees client-originated 400s, so a @@ -1600,7 +1605,15 @@ async function forwardCaptured( let snippet = bodyText.slice(0, 600).replace(/\s+/g, " ").trim(); if (bodyText.length > 600) snippet += " …"; if (!snippet) snippet = "(no body)"; - loggerLog("warn", `[${errSid}] ← upstream ${upstream.status}${reqIdText}: ${snippet}`); + // #4452: a relay that fans out to several targets answers a total failure + // with ONE aggregate body whose real cause sits in the LAST attempt's + // `details.error.message` — past the 600-byte snippet above, so the log + // showed only "all targets failed". Append that single terminal cause + // (bounded + credential-scrubbed by the helper; empty when absent, leaving + // this line byte-identical to before). Read-only: no recovery decision + // here depends on it. + const terminalCause = rejectionDiagnosticSuffix(bodyText); + loggerLog("warn", `[${errSid}] ← upstream ${upstream.status}${reqIdText}: ${snippet}${terminalCause}`); // #4204: feed the egress throttle governor with this send's FINAL verdict // (one observation per request — in-request refolds/resends above all // converge into this same response handling). A 429 arms/extends the diff --git a/src/util.ts b/src/util.ts index c960263f5a..05181dfc06 100644 --- a/src/util.ts +++ b/src/util.ts @@ -527,18 +527,30 @@ export function inspectContextOverflow(status: number, bodyText: string): Contex }; } -// #4452: an intermediate gateway that rewrites EVERY rejection into one generic -// body (Cherry Studio et al. strip the token counts / window from each overflow -// 400) makes inspectContextOverflow return isOverflow:false even when the payload -// really overflows — so bili's whole overflow self-heal (#1195 same-request -// refold, armOverflowShrink, window learn) keys off a marker that is never there -// and silently never engages, starving the route of calibration. This predicate -// names exactly that shape — a 4xx overflow-SHAPED reject whose body matches no -// pattern — so the relay can log it loudly instead of the opaque `← upstream 400` -// line. Deliberately EXCLUDES 5xx (those already self-heal via the local estimate, +// #4452: an intermediate gateway that rewrites a payload-size rejection into one +// generic body (Cherry Studio et al. strip the token counts / window) makes +// inspectContextOverflow return isOverflow:false even when the payload really +// was too large — so bili's whole overflow self-heal (#1195 same-request refold, +// armOverflowShrink, window learn) keys off a marker that is never there and +// silently never engages, starving the route of calibration. This predicate +// names exactly that shape — an unmarked size-wall reject whose body matches no +// pattern — so the relay can log it loudly instead of the opaque +// `← upstream 413` line. +// +// ONLY 413 carries that meaning on its own. A 400 is the generic "malformed / +// schema-invalid / role-rejected / auth-shaped" bucket across relays: the SAME +// opaque body (`{"error":{"message":"Internal server error", +// "code":"bad_request"}}`) is emitted for a truncated JSON body, an unsupported +// field and a genuinely oversized payload alike, so treating every unmarked 400 +// as evidence of overflow manufactured a false overflow diagnosis for ordinary +// malformed requests (#4452 follow-up). 413 Request Entity Too Large is the +// status whose SEMANTICS are already the payload-size verdict, which is why it +// can be diagnosed without a body marker. +// +// Deliberately EXCLUDES 5xx (those already self-heal via the local estimate, // #604) and any body that DOES carry a marker (self-heal engages normally then). export function isMarkerlessOverflowReject(status: number, bodyText: string): boolean { - return (status === 400 || status === 413) && !CONTEXT_OVERFLOW_PATTERNS.some((p) => p.test(bodyText)); + return status === 413 && !CONTEXT_OVERFLOW_PATTERNS.some((p) => p.test(bodyText)); } /** Default cap on the output-headroom reservation, as a fraction of the diff --git a/tests/issue4452-markerless-reject.test.ts b/tests/issue4452-markerless-reject.test.ts index 817cbda1bb..d3bd1c9749 100644 --- a/tests/issue4452-markerless-reject.test.ts +++ b/tests/issue4452-markerless-reject.test.ts @@ -2,18 +2,27 @@ import test from "node:test"; import assert from "node:assert/strict"; import { inspectContextOverflow, isMarkerlessOverflowReject } from "../src/util.ts"; -// #4452: an intermediate gateway that rewrites every overflow rejection into one +// #4452: an intermediate gateway that rewrites a payload-size rejection into one // generic body (Cherry Studio strips the token counts / window) defeats the // overflow MARKER gate — inspectContextOverflow returns isOverflow:false, so the // whole self-heal never engages and the route starves of calibration. The relay // must surface that shape explicitly (isMarkerlessOverflowReject) instead of the -// opaque `← upstream 400` line. Gate discipline (#3617): assert BOTH directions — -// the marker-less shape IS flagged, and anything self-heal already handles (a real -// marker, a 5xx, a non-overflow 4xx) is NOT. +// opaque `← upstream 413` line. Gate discipline (#3617): assert BOTH directions — +// the marker-less SIZE verdict (413) IS flagged, and anything self-heal already +// handles (a real marker, a 5xx, a non-size 4xx) is NOT. +// +// #4452 follow-up: the generic 400 is NOT a size verdict. Every ordinary +// malformed / schema-invalid / role-rejected request across relays answers with +// the SAME opaque body, so diagnosing an unmarked 400 as markerless overflow +// manufactured a false overflow warning for requests that carry no evidence of +// being too large at all. // Verbatim from the #4438/#4452 field report — identical for every request id. const GENERIC_BODY = '{"error":{"message":"Internal server error","type":"invalid_request_error","code":"bad_request"}}'; +// A schema/malformed 400 body in the shape a strict backend emits. +const MALFORMED_BODY = '{"error":{"message":"Invalid request: messages[3].content must be a string or an array","type":"invalid_request_error","code":"invalid_value"}}'; + const OPENAI_OVERFLOW_BODY = JSON.stringify({ error: { message: @@ -25,35 +34,53 @@ const OPENAI_OVERFLOW_BODY = JSON.stringify({ const CJK_OVERFLOW_BODY = JSON.stringify({ error: { message: "上下文长度超出限制", type: "invalid_request_error" } }); -test("generic gateway body on 400 is overflow-shaped but carries no parseable marker", () => { +test("generic gateway body on 413 is the markerless SIZE verdict and is flagged", () => { // The first-order failure: the marker gate itself says "not overflow". + assert.equal(inspectContextOverflow(413, GENERIC_BODY).isOverflow, false); + // …yet a 413 whose body dropped the marker is exactly the shape the relay + // must flag as self-heal-starved. + assert.equal(isMarkerlessOverflowReject(413, GENERIC_BODY), true); +}); + +test("an unmarked 400 is NOT diagnosed as overflow (generic malformed/schema rejection)", () => { + // Neither body is overflow evidence: no status semantics and no marker. assert.equal(inspectContextOverflow(400, GENERIC_BODY).isOverflow, false); - // …yet it is exactly the shape the relay must now flag as self-heal-starved. - assert.equal(isMarkerlessOverflowReject(400, GENERIC_BODY), true); + assert.equal(isMarkerlessOverflowReject(400, GENERIC_BODY), false); + // The same must hold for a clearly schema-shaped 400 body. + assert.equal(inspectContextOverflow(400, MALFORMED_BODY).isOverflow, false); + assert.equal(isMarkerlessOverflowReject(400, MALFORMED_BODY), false); }); -test("generic gateway body on 413 is also flagged", () => { - assert.equal(inspectContextOverflow(413, GENERIC_BODY).isOverflow, false); - assert.equal(isMarkerlessOverflowReject(413, GENERIC_BODY), true); +test("an unmarked 413 is not double-counted as a marker overflow", () => { + // 413 without a marker: the self-heal gate is closed, which is exactly why + // the warning exists; it must never claim isOverflow. + assert.equal(inspectContextOverflow(413, MALFORMED_BODY).isOverflow, false); + assert.equal(isMarkerlessOverflowReject(413, MALFORMED_BODY), true); }); test("a body WITH an overflow marker is NOT markerless (self-heal engages normally)", () => { assert.equal(inspectContextOverflow(400, OPENAI_OVERFLOW_BODY).isOverflow, true); assert.equal(isMarkerlessOverflowReject(400, OPENAI_OVERFLOW_BODY), false); + assert.equal(inspectContextOverflow(413, OPENAI_OVERFLOW_BODY).isOverflow, true); + assert.equal(isMarkerlessOverflowReject(413, OPENAI_OVERFLOW_BODY), false); }); test("a CJK-marked overflow body is NOT markerless", () => { assert.equal(inspectContextOverflow(400, CJK_OVERFLOW_BODY).isOverflow, true); assert.equal(isMarkerlessOverflowReject(400, CJK_OVERFLOW_BODY), false); + assert.equal(isMarkerlessOverflowReject(413, CJK_OVERFLOW_BODY), false); }); test("5xx is excluded — those already self-heal via the local estimate (#604)", () => { assert.equal(isMarkerlessOverflowReject(502, GENERIC_BODY), false); assert.equal(isMarkerlessOverflowReject(503, GENERIC_BODY), false); + assert.equal(isMarkerlessOverflowReject(507, GENERIC_BODY), false); }); test("non-overflow 4xx and success codes are excluded (no false alarm on auth/malformed)", () => { assert.equal(isMarkerlessOverflowReject(200, GENERIC_BODY), false); assert.equal(isMarkerlessOverflowReject(401, GENERIC_BODY), false); + assert.equal(isMarkerlessOverflowReject(404, GENERIC_BODY), false); + assert.equal(isMarkerlessOverflowReject(422, GENERIC_BODY), false); assert.equal(isMarkerlessOverflowReject(429, GENERIC_BODY), false); }); diff --git a/tests/rejection-diagnostics.test.ts b/tests/rejection-diagnostics.test.ts new file mode 100644 index 0000000000..7c1733c348 --- /dev/null +++ b/tests/rejection-diagnostics.test.ts @@ -0,0 +1,147 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { rejectionDiagnostics, rejectionDiagnosticSuffix } from "../src/server/rejection-diagnostics.ts"; + +// #4452: a fan-out relay answers a total failure with ONE aggregate body whose +// real cause sits nested in the last attempt. bili's log line keeps only the +// first 600 bytes, i.e. "all targets failed" — the cause was undiagnosable. +// These pin the extractor's contract: read ONE terminal nested message, bounded +// and credential-scrubbed, changing nothing about a body that carries none. + +/** Build the aggregate shape a fan-out relay returns on total failure. */ +function aggregate(attempts: unknown[], outer = "All targets failed"): string { + return JSON.stringify({ error: { message: outer, type: "upstream_error" }, attempts }); +} + +function attempt(message: unknown): Record { + return { target: "t", details: { error: { message } } }; +} + +test("reads the terminal attempt's nested cause", () => { + const body = aggregate([attempt("connection reset by peer"), attempt("upstream returned 529 overloaded")]); + const diag = rejectionDiagnostics(body); + assert.ok(diag); + assert.equal(diag.message, "upstream returned 529 overloaded"); + assert.equal(diag.attemptIndex, 1); + assert.equal(diag.attempts, 2); +}); + +test("a single attempt is indexed 0 and the suffix omits the position", () => { + const diag = rejectionDiagnostics(aggregate([attempt("model not found")])); + assert.ok(diag); + assert.equal(diag.attemptIndex, 0); + assert.equal(diag.attempts, 1); +}); + +test("later attempts win over earlier retry noise", () => { + const body = aggregate([attempt("noise early"), attempt("noise mid"), attempt("densely informative final cause")]); + assert.equal(rejectionDiagnostics(body)?.message, "densely informative final cause"); +}); + +test("a terminal attempt with no usable message falls back to the nearest informative earlier one", () => { + const body = aggregate([attempt("real cause"), { target: "t", details: {} }, attempt(undefined)]); + const diag = rejectionDiagnostics(body); + assert.ok(diag); + assert.equal(diag.message, "real cause"); +}); + +test("an empty nested message is skipped", () => { + const body = aggregate([attempt("")]); + assert.equal(rejectionDiagnostics(body), undefined); + assert.equal(rejectionDiagnosticSuffix(body), ""); +}); + +test("empty / non-string message at every attempt yields no diagnostic", () => { + assert.equal(rejectionDiagnostics(aggregate([attempt(12), attempt(null), attempt("")])), undefined); + assert.equal(rejectionDiagnosticSuffix(aggregate([attempt(12), attempt(null)])), ""); +}); + +test("a payload with no attempts[] is untouched (no diagnostic)", () => { + const bodies = [ + '{"error":{"message":"Internal server error"}}', + '{"error":{"message":"context_length_exceeded: 200000 tokens"}}', + '{"attempts":[]}', + '{"attempts":"not-an-array"}', + "{not json", + "Internal Server Error", + "502 Bad Gateway", + "", + ]; + for (const body of bodies) { + assert.equal(rejectionDiagnostics(body), undefined, `expected none for: ${body}`); + assert.equal(rejectionDiagnosticSuffix(body), "", `expected empty suffix for: ${body}`); + } +}); + +test("non-object / non-array attempt entries are skipped, not thrown on", () => { + const body = aggregate([null, "string-attempt", 4, [], { details: null }, { details: { error: null } }, attempt("surviving cause")]); + assert.equal(rejectionDiagnostics(body)?.message, "surviving cause"); +}); + +test("null / undefined payload is tolerated and yields no suffix", () => { + assert.equal(rejectionDiagnostics(null), undefined); + assert.equal(rejectionDiagnostics(undefined), undefined); + assert.equal(rejectionDiagnosticSuffix(null), ""); + assert.equal(rejectionDiagnosticSuffix(undefined), ""); +}); + +test("whitespace, newlines and control shapes are collapsed into ONE line", () => { + const hostile = "boom\r\n[attacker] ← upstream 200: forged line\n\tsecond"; + const suffix = rejectionDiagnosticSuffix(aggregate([attempt(hostile)])); + assert.ok(!suffix.includes("\n"), "no newline may survive into the log line"); + assert.ok(!suffix.includes("\r"), "no carriage return may survive"); +}); + +test("credential-shaped text is scrubbed by the shared sanitizer", () => { + const body = aggregate([attempt("rejected: authorization: Bearer abcdefghijklmnopqrstuvwxyz012345")]); + const suffix = rejectionDiagnosticSuffix(body); + assert.ok(!suffix.includes("abcdefghijklmnopqrstuvwxyz012345"), "bearer token must not reach the log"); +}); + +test("an sk- API key echoed by a provider is masked", () => { + const suffix = rejectionDiagnosticSuffix(aggregate([attempt("incorrect key sk-proj-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA supplied")])); + assert.ok(!suffix.includes("sk-proj-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); +}); + +test("a long cause is clamped, and the clamp is bounded", () => { + const suffix = rejectionDiagnosticSuffix(aggregate([attempt("x".repeat(4000))])); + // 300 retained chars + the ellipsis marker + the fixed prefix. + assert.ok(suffix.length < 340, `suffix too long: ${suffix.length}`); + assert.ok(suffix.endsWith(" …")); +}); + +test("an oversized body is not parsed at all", () => { + const huge = aggregate([attempt("y".repeat(64 * 1024 + 10))]); + assert.equal(rejectionDiagnostics(huge), undefined); + assert.equal(rejectionDiagnosticSuffix(huge), ""); +}); + +test("the suffix never echoes the outer narrative or the raw payload", () => { + const body = aggregate([attempt("inner cause")], "All targets failed: secret-route-label"); + const suffix = rejectionDiagnosticSuffix(body); + assert.ok(!suffix.includes("All targets failed")); + assert.ok(!suffix.includes("secret-route-label")); + assert.ok(!suffix.includes("{"), "no JSON structure may leak into the log"); +}); + +test("reads the terminal provider cause from CCR's nested error envelope", () => { + const body = JSON.stringify({ + error: { + message: "All target providers failed.", + attempts: [attempt("earlier transient error"), attempt("messages.32: tool_use has no following tool_result")], + }, + }); + assert.equal(rejectionDiagnostics(body)?.message, "messages.32: tool_use has no following tool_result"); + assert.equal(rejectionDiagnostics(body)?.attemptIndex, 1); +}); + +test("terminal control bytes cannot reach the diagnostic line", () => { + const body = aggregate([attempt("refused\u001b[2J\u0000\u007f by provider")]); + assert.equal(rejectionDiagnostics(body)?.message, "refused [2J by provider"); +}); + +test("the parsing budget counts UTF-8 bytes rather than UTF-16 characters", () => { + const body = aggregate([attempt("错".repeat(24 * 1024))]); + assert.ok(body.length < 64 * 1024); + assert.equal(rejectionDiagnostics(body), undefined); +}); From c749eb82e585b3e6c80ed5c619b3d1296f9ab59b Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 2 Nov 2026 00:29:03 +0800 Subject: [PATCH 2/4] fix(diagnostics): strip C1 control chars (incl. NEL) from the bounded rejection cause MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on #5482. bound() collapsed only C0 + DEL before clamping, so C1 control characters survived into the appended log suffix — in particular U+0085 (NEXT LINE) and U+009C (file separator), which render as line/field breaks. A relay that echoed such a byte into attempts[].details.error.message could forge a second log field, contradicting the module's own "no newline may forge a second log field" invariant (pinned by the \r\n test). Widen the strip to C1 (U+007f-U+009f) and extend the control-shape regression test to pin NEL and file-separator. No behavior change beyond the log-line sanitization. --- src/server/rejection-diagnostics.ts | 12 +++++++----- tests/rejection-diagnostics.test.ts | 6 ++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/server/rejection-diagnostics.ts b/src/server/rejection-diagnostics.ts index 0aa5e76576..729238c548 100644 --- a/src/server/rejection-diagnostics.ts +++ b/src/server/rejection-diagnostics.ts @@ -24,9 +24,9 @@ * a high-signal credential scrubber, NOT a general privacy filter: it does not * mask arbitrary URLs or paths, and the far stronger guarantee here is * structural — only this one nested string is ever read, never a body dump. - * - Bounded to `MAX_DIAGNOSTIC_CHARS`; ALL whitespace collapsed so a newline - * (including the `\r\n` frame-injection shape) can never forge a second log - * field. + * - Bounded to `MAX_DIAGNOSTIC_CHARS`; ALL control characters (C0+C1, incl. + * NEL U+0085) and whitespace collapsed so a newline (the `\r\n` shape or any + * C1 line-break) can never forge a second log field. * - Absent, non-string, empty, or structurally-unrelated payloads return * `undefined`: the caller then logs exactly what it logged before, so a * rejection that carries no attempt detail is unchanged byte-for-byte. @@ -55,9 +55,11 @@ interface DiagnosticResult { attempts: number; } -/** Collapse ALL whitespace (newlines included) and clamp to the bound. */ +/** Collapse control chars (C0+C1 — incl. NEL U+0085) and all whitespace to + * single spaces, then clamp, so no control byte can survive as a forged field + * break in the log line. */ function bound(text: string): string { - const collapsed = text.replace(/[\u0000-\u001f\u007f]/g, " ").replace(/\s+/g, " ").trim(); + const collapsed = text.replace(/[\u0000-\u001f\u007f-\u009f]/g, " ").replace(/\s+/g, " ").trim(); if (collapsed.length <= MAX_DIAGNOSTIC_CHARS) return collapsed; return `${collapsed.slice(0, MAX_DIAGNOSTIC_CHARS)} …`; } diff --git a/tests/rejection-diagnostics.test.ts b/tests/rejection-diagnostics.test.ts index 7c1733c348..b6802fdb7a 100644 --- a/tests/rejection-diagnostics.test.ts +++ b/tests/rejection-diagnostics.test.ts @@ -85,11 +85,13 @@ test("null / undefined payload is tolerated and yields no suffix", () => { assert.equal(rejectionDiagnosticSuffix(undefined), ""); }); -test("whitespace, newlines and control shapes are collapsed into ONE line", () => { - const hostile = "boom\r\n[attacker] ← upstream 200: forged line\n\tsecond"; +test("whitespace, newlines and control shapes (incl. C1 NEL) are collapsed into ONE line", () => { + const hostile = "boom\r\n[attacker] ← upstream 200: forged line\u0085forged2\n\tsecond\u009c"; const suffix = rejectionDiagnosticSuffix(aggregate([attempt(hostile)])); assert.ok(!suffix.includes("\n"), "no newline may survive into the log line"); assert.ok(!suffix.includes("\r"), "no carriage return may survive"); + assert.ok(!suffix.includes("\u0085"), "C1 next-line (U+0085) may not survive as a field break"); + assert.ok(!suffix.includes("\u009c"), "C1 file-separator (U+009C) may not survive as a field break"); }); test("credential-shaped text is scrubbed by the shared sanitizer", () => { From 8d283b9e55e111c858d388888d835282c566256b Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 19:36:44 +0800 Subject: [PATCH 3/4] test(ci): issue5152 startReceiver waits for a valid port value, not file existence (#4599) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The receiver fixture creates its .port file before the write lands; on a loaded runner the existsSync poll can win that race and read back an empty file, so Number("") === 0 tripped the assertion (CI: ubuntu-24 shard 2, run 42202673910 — second occurrence of this signature after PR #5327's one-off, recorded in #4599 as record-only). Poll until the content parses as a positive integer instead; the 15s deadline and the diagnostic message are unchanged. --- ...ue5152-owner-beat-survives-host-stall.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/issue5152-owner-beat-survives-host-stall.test.ts b/tests/issue5152-owner-beat-survives-host-stall.test.ts index 671977001c..900a53e120 100644 --- a/tests/issue5152-owner-beat-survives-host-stall.test.ts +++ b/tests/issue5152-owner-beat-survives-host-stall.test.ts @@ -53,13 +53,21 @@ async function startReceiver(dir: string): Promise { const portFile = `${log}.port`; const child = spawn(process.execPath, [RECEIVER, log], { stdio: "ignore", detached: true }); child.unref(); + // Poll for a VALID port, not mere file existence: the receiver creates the + // file before its write lands, so existsSync can win that race on a loaded + // runner and read back an empty file (Number("") === 0). + let port = 0; + let portText = ""; const deadline = Date.now() + 15_000; - while (!existsSync(portFile) && Date.now() < deadline) { + while (port === 0 && Date.now() < deadline) { + if (existsSync(portFile)) { + portText = readFileSync(portFile, "utf8").trim(); + const parsed = Number(portText); + if (Number.isInteger(parsed) && parsed > 0) port = parsed; + } Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20); } - assert.ok(existsSync(portFile), "receiver must publish its port"); - const port = Number(readFileSync(portFile, "utf8").trim()); - assert.ok(Number.isInteger(port) && port > 0, `receiver port must be a number, got ${port}`); + assert.ok(port > 0, `receiver must publish its port (file held "${portText}")`); return { port, beats: () => (existsSync(log) ? readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map(Number) : []), From 47d07897392881ae72a6a1f8d1df830089bba11b Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 20:59:40 +0800 Subject: [PATCH 4/4] docs: place rejection-diagnostics.ts in its alphabetical slot in the server module map (#5482 review) --- reference/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference/architecture.md b/reference/architecture.md index c3ad47688e..2819562c7f 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -27,8 +27,8 @@ billion-context/ │ │ ├── prepare-google.ts # Gemini request preparation incl. :countTokens twin (#1440) │ │ ├── prepare-openai.ts # OpenAI chat request preparation (#1440) │ │ ├── prepare-responses.ts # Responses request preparation + codex compact/prompt-cache helpers (#1440) -│ │ ├── relay.ts # Upstream relay zone: forward() + fake-completion recovery (#1440 P2 cut 4) │ │ ├── rejection-diagnostics.ts # Bounded terminal provider cause from direct or nested aggregate rejections +│ │ ├── relay.ts # Upstream relay zone: forward() + fake-completion recovery (#1440 P2 cut 4) │ │ ├── side-request.ts # Tool-surface check for auxiliary side requests │ │ └── stream-io.ts # Small response-body reader for non-2xx inspection │ ├── knobs.ts # Single knob resolver: env > config file > default (#2030)