From 6906dfa13850752a424f1c208673f9c441271cbe Mon Sep 17 00:00:00 2001 From: ework-daemon Date: Sat, 24 Oct 2026 03:33:52 +0800 Subject: [PATCH] fix: name body-rewriting-gateway overflow starvation in the relay log (#4452) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An intermediate gateway that rewrites every reject into one generic body (Cherry Studio et al.) strips the token counts / window from each overflow 400, so inspectContextOverflow returns isOverflow:false and bili's whole overflow self-heal silently never engages — the route starves of calibration and re-sends the same oversized payload forever. Decision-free observability slice of #4452 (behavioral recovery awaits owner sign-off on thresholds): - add isMarkerlessOverflowReject() (util.ts): a 4xx overflow-shaped reject whose body matches no CONTEXT_OVERFLOW_PATTERNS (excludes 5xx, which already self-heal via the local estimate, and any body that carries a marker). - the relay logs a loud warn naming the starvation + the last meter reading (settled input vs effective window) instead of only the opaque `<- upstream 400` line. Purely additive logging — no behavior change when isOverflow is true, no new config surface. --- src/server/relay.ts | 16 ++++-- src/util.ts | 14 ++++++ tests/issue4452-markerless-reject.test.ts | 59 +++++++++++++++++++++++ 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tests/issue4452-markerless-reject.test.ts diff --git a/src/server/relay.ts b/src/server/relay.ts index b7ecbdfea3..cfb0ed0030 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -54,7 +54,7 @@ import { safePrefix, safeSuffix, scrubLoneSurrogatesOnWire } from "../text-safe. import { clearUpstreamAlertsForHost, recordUpstreamAlert, recordUpstreamCircuitFailure, recordUpstreamCircuitSuccess, upstreamCircuitBlockReason, upstreamCircuitLastFailureKind } from "../upstream-alerts.js"; import { describeUpstreamFailure, formatUpstreamError, proxyDispatcher, recordUpstreamConnection } from "../upstream-proxy.js"; import { withUpstreamFailure, type UpstreamFailureKind } from "../upstream-fail.js"; -import { applyEstimateCalibration, currentCalibrationFactor, inspectContextOverflow, MIN_ASSUMED_WINDOW_TOKENS, normalizeUpstreamOrigin, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "../util.js"; +import { applyEstimateCalibration, currentCalibrationFactor, inspectContextOverflow, isMarkerlessOverflowReject, MIN_ASSUMED_WINDOW_TOKENS, normalizeUpstreamOrigin, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "../util.js"; import { recordPckAblation } from "../pck-probe.js"; import { applyOutputSteering, applyOutputSteeringJson } from "../output-steering.js"; import { estimateWireOverhead } from "./budget.js"; @@ -1347,9 +1347,19 @@ export async function forward( } if (prepared?.session && errBody) { const s = prepared.session; - const info = inspectContextOverflow(upstream.status, errBody.toString("utf8")); + const errText = errBody.toString("utf8"); + const info = inspectContextOverflow(upstream.status, errText); if (info.isOverflow) { armOverflowShrink(info); + } else if (isMarkerlessOverflowReject(upstream.status, errText)) { + // #4452: the gateway rewrote this overflow-shaped reject into a body with no + // parseable marker, so bili's self-heal cannot engage from it — name that here + // (+ the last meter reading) so a body-rewriting gateway's silent starvation is + // diagnosable from the log rather than the opaque line further down. + const est = s.stats.lastInputTokens > 0 ? s.stats.lastInputTokens : 0; + const win = typeof s.metadata.effectiveContextLimit === "number" && s.metadata.effectiveContextLimit > 0 ? s.metadata.effectiveContextLimit : 0; + const meterNote = est > 0 && win > 0 ? `; last settled input ~${Math.round(est)} vs effective window ${win}` : ""; + log("warn", `[${s.id}] ← upstream ${upstream.status} carries no parseable overflow marker (an intermediate gateway rewrote the rejection body?) — bili's overflow self-heal (refold / emergency-shrink / window-learn) CANNOT engage from this rejection${meterNote}; repeated at similar size = calibration-starved route (#4452)`); } // #762: learn strict-echo on the MAIN request path too. The loop-only // learner (src/loop/core.ts) never sees client-originated 400s, so a @@ -1357,7 +1367,7 @@ export async function forward( // dropping reasoning and every following turn split again. // #2169: match BOTH field spellings — the chat wire says // reasoning_content, DeepSeek's Responses wire says reasoning_text. - if (upstream.status === 400 && /reasoning_(?:content|text)/i.test(errBody.toString("utf8"))) { + if (upstream.status === 400 && /reasoning_(?:content|text)/i.test(errText)) { if (s.metadata.strictReasoningEcho !== true) { s.metadata.strictReasoningEcho = true; markDirty(s); diff --git a/src/util.ts b/src/util.ts index 95f8aba135..895d24ffc7 100644 --- a/src/util.ts +++ b/src/util.ts @@ -428,6 +428,20 @@ export function inspectContextOverflow(status: number, bodyText: string): Contex }; } +// #4452: an intermediate gateway that rewrites EVERY rejection into one generic +// body (Cherry Studio et al. strip the token counts / window from each overflow +// 400) makes inspectContextOverflow return isOverflow:false even when the payload +// really overflows — so bili's whole overflow self-heal (#1195 same-request +// refold, armOverflowShrink, window learn) keys off a marker that is never there +// and silently never engages, starving the route of calibration. This predicate +// names exactly that shape — a 4xx overflow-SHAPED reject whose body matches no +// pattern — so the relay can log it loudly instead of the opaque `← upstream 400` +// line. Deliberately EXCLUDES 5xx (those already self-heal via the local estimate, +// #604) and any body that DOES carry a marker (self-heal engages normally then). +export function isMarkerlessOverflowReject(status: number, bodyText: string): boolean { + return (status === 400 || status === 413) && !CONTEXT_OVERFLOW_PATTERNS.some((p) => p.test(bodyText)); +} + /** Default cap on the output-headroom reservation, as a fraction of the * context window (#896, aligned with billion-context-pi #207). Reserving the * FULL registered max output halves the effective input budget on models whose diff --git a/tests/issue4452-markerless-reject.test.ts b/tests/issue4452-markerless-reject.test.ts new file mode 100644 index 0000000000..817cbda1bb --- /dev/null +++ b/tests/issue4452-markerless-reject.test.ts @@ -0,0 +1,59 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { inspectContextOverflow, isMarkerlessOverflowReject } from "../src/util.ts"; + +// #4452: an intermediate gateway that rewrites every overflow rejection into one +// generic body (Cherry Studio strips the token counts / window) defeats the +// overflow MARKER gate — inspectContextOverflow returns isOverflow:false, so the +// whole self-heal never engages and the route starves of calibration. The relay +// must surface that shape explicitly (isMarkerlessOverflowReject) instead of the +// opaque `← upstream 400` line. Gate discipline (#3617): assert BOTH directions — +// the marker-less shape IS flagged, and anything self-heal already handles (a real +// marker, a 5xx, a non-overflow 4xx) is NOT. + +// Verbatim from the #4438/#4452 field report — identical for every request id. +const GENERIC_BODY = '{"error":{"message":"Internal server error","type":"invalid_request_error","code":"bad_request"}}'; + +const OPENAI_OVERFLOW_BODY = JSON.stringify({ + error: { + message: + "This model's maximum context length is 128000 tokens. However, you requested 132096 tokens (your request used 20 inputs and 4096 output tokens). Please reduce the length of the inputs or output.", + type: "invalid_request_error", + code: "context_length_exceeded", + }, +}); + +const CJK_OVERFLOW_BODY = JSON.stringify({ error: { message: "上下文长度超出限制", type: "invalid_request_error" } }); + +test("generic gateway body on 400 is overflow-shaped but carries no parseable marker", () => { + // The first-order failure: the marker gate itself says "not overflow". + assert.equal(inspectContextOverflow(400, GENERIC_BODY).isOverflow, false); + // …yet it is exactly the shape the relay must now flag as self-heal-starved. + assert.equal(isMarkerlessOverflowReject(400, GENERIC_BODY), true); +}); + +test("generic gateway body on 413 is also flagged", () => { + assert.equal(inspectContextOverflow(413, GENERIC_BODY).isOverflow, false); + assert.equal(isMarkerlessOverflowReject(413, GENERIC_BODY), true); +}); + +test("a body WITH an overflow marker is NOT markerless (self-heal engages normally)", () => { + assert.equal(inspectContextOverflow(400, OPENAI_OVERFLOW_BODY).isOverflow, true); + assert.equal(isMarkerlessOverflowReject(400, OPENAI_OVERFLOW_BODY), false); +}); + +test("a CJK-marked overflow body is NOT markerless", () => { + assert.equal(inspectContextOverflow(400, CJK_OVERFLOW_BODY).isOverflow, true); + assert.equal(isMarkerlessOverflowReject(400, CJK_OVERFLOW_BODY), false); +}); + +test("5xx is excluded — those already self-heal via the local estimate (#604)", () => { + assert.equal(isMarkerlessOverflowReject(502, GENERIC_BODY), false); + assert.equal(isMarkerlessOverflowReject(503, GENERIC_BODY), false); +}); + +test("non-overflow 4xx and success codes are excluded (no false alarm on auth/malformed)", () => { + assert.equal(isMarkerlessOverflowReject(200, GENERIC_BODY), false); + assert.equal(isMarkerlessOverflowReject(401, GENERIC_BODY), false); + assert.equal(isMarkerlessOverflowReject(429, GENERIC_BODY), false); +});