From 67406c0cfc5c30d9412c39fda154aa75fe8cbb47 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 19:03:29 +0800 Subject: [PATCH 1/5] fix: answer 429-hold locally before ingesting large request bodies (#5795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #4204 egress hold refused requests only AFTER the full ingest (JSON.parse + openaiToCore + snapshot pinning + processTurn), so every host retry of a held send still paid ~5.5-7.2s of proxy CPU and a hundreds-of-MB allocation spike per attempt on a 51 MiB session. A read-only peek of the governor (upstreamThrottlePeekReject — rejects only, never mints probe tokens) now answers >=1 MiB POSTs up front in the dispatch path; sub-MiB traffic, forge/classifier local intercepts, passthrough-mark lanes, chained-bili hop markers and model-mismatched model-scoped holds all keep their exact pre-change paths, with the post-ingest gate unchanged as backstop. The single reject-response renderer (answerLocalThrottleReject) keeps both sites byte-identical. --- src/cache-ledger.ts | 7 +++- src/server/handle.ts | 33 +++++++++++++++++++ src/server/relay.ts | 70 ++++++++++++++++++++++++++++++++++------ src/upstream-throttle.ts | 34 ++++++++++++++++++- 4 files changed, 133 insertions(+), 11 deletions(-) diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index 62d11c6044..c3d3a77b0e 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -642,6 +642,11 @@ export function noteClientAbort(session: Session): void { lastClientAbort.set(session, Date.now()); } +// #5795: single source of truth for the credential header names — the 429 +// governor's hold key (upstream-throttle.ts) and the pre-ingest fast path +// (server/handle.ts) must read exactly these headers or holds go unmatched. +export const CREDENTIAL_HEADER_NAMES = ["authorization", "x-api-key", "api-key", "x-goog-api-key"] as const; + /** #2131: stable fingerprint of the outbound credential — the ONLY form the * key is ever persisted in (raw secrets never reach the ledger, logs or the * web UI). Checks the standard bearer/api-key headers, case-insensitively; @@ -651,7 +656,7 @@ export function credentialFingerprint(headers: Record | undefine let cred: string | undefined; for (const [name, value] of Object.entries(headers)) { const n = name.toLowerCase(); - if (n === "authorization" || n === "x-api-key" || n === "api-key" || n === "x-goog-api-key") { + if ((CREDENTIAL_HEADER_NAMES as readonly string[]).includes(n)) { // Strip the auth-scheme envelope so "Bearer X" (authorization) and // a bare "X" (x-api-key) hash to the SAME credential identity. const v = value.trim().replace(/^Bearer\s+/i, "").trim(); diff --git a/src/server/handle.ts b/src/server/handle.ts index 229ad3ca34..a574085a36 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -87,6 +87,21 @@ import { ccrPluginWireOk, storeEffectiveCcr, storeEffectiveImageRescale, type Cc import { DEFAULT_DECIDE_MAX_TOKENS } from "../nudge-decide.js"; import { BodyTooLargeError, buildForwardTarget, forward, forwardUpstreamUrl, googleModelFromPath, googlePathKind, headroomEffectiveLogged, headroomFallbackLogged, headerValue, imageBillingFor, imageReserveFor, imageTokenCapFor, isCountTokensRequest, isPreflightFailFast, logRequestCost, NON_CONVERSATION_RELAY_WARN_CAP, nonConversationRelayWarned, prepareCountTokens, preflightCompressIfNeeded, readBody, resolveKnownOutputCeiling, resolveUpstream, resignSettingsFor, selfAddressedLocalPath, scrubAnthropicPck, scrubCompatDrop, vertexAnthropicModelFromPath, warnModelNameMissIfNew, warnRouteMissIfNew, type Prepared } from "../server.js"; import { normalizeVertexModelId } from "../vertex.js"; +import { CREDENTIAL_HEADER_NAMES } from "../cache-ledger.js"; +import { requestModelFromBody } from "../upstream-throttle.js"; +import { tryAnswerEarlyHold } from "./relay.js"; + +// #5795: the credential subset of the inbound headers — exactly the names +// credentialFingerprint reads (CREDENTIAL_HEADER_NAMES), or the fast path's +// hold key would diverge from the governor's and account holds go invisible. +function earlyHoldCredHeaders(req: http.IncomingMessage): Record { + const out: Record = {}; + for (const n of CREDENTIAL_HEADER_NAMES) { + const v = headerValue(req, n); + if (v !== undefined && v !== "") out[n] = v; + } + return out; +} // #5243: warn-once per model when the live codex model table is unavailable and // window sizing falls back to config.toml model_context_window. @@ -487,6 +502,24 @@ async function handleRequest( await forward(req, res, opts, scrubCompatDrop(scrubAnthropicPck(protocol, bodyBuffer, log), compatDropPaths, log), null, core, config, log, route, instanceId, undefined, undefined, undefined, googleModel ?? vertexModel); return; } + // #5795: pre-ingest 429-hold fast path — while the egress governor holds + // this origin+credential, refuse BEFORE prepare pays the O(history) + // parse/convert/pin cost of a send that will not egress anyway (~6s per + // refused turn on the field-reported multi-MiB histories). Sits above the + // hop-marker branch so chained-bili relays get the identical local refusal + // one stage earlier; sub-threshold bodies and unknown-wire requests fall + // through untouched (see tryAnswerEarlyHold). + if (tryAnswerEarlyHold({ + method: req.method, + upstreamUrl: forwardUpstreamUrl(req, opts, route), + credHeaders: earlyHoldCredHeaders(req), + protocol, + bodyBytes: bodyBuffer.length, + requestModel: googleModel ?? vertexModel ?? requestModelFromBody(bodyBuffer), + res, + streamErrorShape: opts.streamErrorShape, + log, + })) return; // #300: bili→bili chain detection. If the inbound request already carries // the x-bili-hop marker, an upstream bili instance already ran the // compression pipeline on it. Processing it again would double-compress diff --git a/src/server/relay.ts b/src/server/relay.ts index 88bf9da3ea..0569024a14 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -58,7 +58,7 @@ import { safePrefix, safeSuffix, scrubLoneSurrogatesOnWire } from "../text-safe. import { clearUpstreamAlertsForHost, noteCircuitFailFastLog, recordUpstreamAlert, recordUpstreamCircuitFailure, recordUpstreamCircuitSuccess, upstreamCircuitBlockReason, upstreamCircuitLastFailureKind, upstreamCircuitRetryAfterSeconds } from "../upstream-alerts.js"; import { describeUpstreamFailure, formatUpstreamError, proxyDispatcher, recordUpstreamConnection } from "../upstream-proxy.js"; import { markExpectedFailFast, withUpstreamFailure, type UpstreamFailureKind } from "../upstream-fail.js"; -import { keyringModel429Scope, noteUpstreamThrottle429, noteUpstreamThrottleResolved, releaseUpstreamThrottleProbe, requestModelFromBody, upstreamThrottleGate } from "../upstream-throttle.js"; +import { keyringModel429Scope, noteUpstreamThrottle429, noteUpstreamThrottleResolved, releaseUpstreamThrottleProbe, requestModelFromBody, type ThrottleLocalReject, upstreamThrottleGate, upstreamThrottlePeekReject } from "../upstream-throttle.js"; import { applyEstimateCalibration, currentCalibrationFactor, inspectContextOverflow, isMarkerlessOverflowReject, MIN_ASSUMED_WINDOW_TOKENS, normalizeUpstreamOrigin, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "../util.js"; import { billOutputUsage } from "../billing.js"; import { DUMP_PLAINTEXT_BANNER } from "../dump-banner.js"; @@ -251,6 +251,65 @@ export function armFirstByteWatchdog(o: { return timer; } +// #5795: single source of truth for the local 429/424 hold response — shared by +// the post-ingest gate below and the pre-ingest fast path in handle.ts so the +// two sites cannot drift on status/headers/body shape. +export function answerLocalThrottleReject( + res: http.ServerResponse, + reject: ThrottleLocalReject, + protocol: WireProtocol | null | undefined, + streamErrorShape: "protocol" | "completion" | undefined, + log: (level: string, msg: string) => void, + label?: string, +): void { + log(reject.status === 424 ? "warn" : "debug", `[${label ?? "passthrough"}] [acp-proxy] 429-hold: answered locally with ${reject.status} without egress (#4204)`); + if (res.headersSent) { + emitStreamError(res, protocol ?? "openai", reject.message, undefined, streamErrorShape); + return; + } + res.writeHead(reject.status, reject.retryAfterSec !== null + ? { "content-type": "application/json", "retry-after": String(reject.retryAfterSec) } + : { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { type: "rate_limit_error", code: reject.code, stage: "forward", message: reject.message, retryable: reject.status === 429, ...(reject.model === undefined ? {} : { scope: "model", model: reject.model }) } })); +} + +// #5795: pre-ingest 429-hold fast path. While a hold is active for this +// origin+credential the send will be answered locally anyway — answering +// BEFORE prepare skips the O(history) parse/convert/pin cost of a request that +// will not egress regardless (the held retry storms in the field report cost +// ~6s of ingest per refused turn on multi-MiB histories). Completes #4204's +// contract: zero egress while held → also zero ingest while held. +// Read-only peek: never mints probe tokens or mutates governor state, so the +// real gate in forwardCaptured keeps owning admission/probe accounting. The +// conservative edges are deliberate: sub-threshold bodies keep their exact +// pre-#5795 path (forge/classifier/compact-trigger shapes intercept locally +// during prepare and must not be pre-empted), and lanes whose credential bili +// itself overrides (#2336) key holds under the overridden fingerprint which +// this raw-header peek cannot see — those fall through to the post-ingest gate +// and are refused there exactly as before. +export const EARLY_HOLD_GATE_MIN_BYTES = 1024 * 1024; + +export function tryAnswerEarlyHold(o: { + method?: string; + upstreamUrl: string; + credHeaders: Record; + protocol: WireProtocol | null; + bodyBytes: number; + requestModel?: string; + res: http.ServerResponse; + streamErrorShape?: "protocol" | "completion"; + log: (level: string, msg: string) => void; + /** Test seam: mirrors the governor's own explicit-now convention. */ + now?: number; +}): boolean { + if (o.method === "GET" || o.method === "HEAD") return false; + if (o.protocol === null || o.bodyBytes < EARLY_HOLD_GATE_MIN_BYTES) return false; + const reject = upstreamThrottlePeekReject(o.upstreamUrl, o.credHeaders, o.now ?? Date.now(), o.requestModel); + if (reject === undefined) return false; + answerLocalThrottleReject(o.res, reject, o.protocol, o.streamErrorShape, o.log, "pre-ingest"); + return true; +} + export function forward(...args: Parameters): Promise { return withCacheCapture(args[4]?.session, () => forwardCaptured(...args)); } @@ -857,14 +916,7 @@ async function forwardCaptured( throttleProbeToken = throttleGate.probeToken; const throttleReject = throttleGate.reject; if (throttleReject !== undefined) { - log(throttleReject.status === 424 ? "warn" : "debug", `[${prepared?.session.id ?? "passthrough"}] [acp-proxy] 429-hold: answered locally with ${throttleReject.status} without egress (#4204)`); - if (res.headersSent) emitStreamError(res, prepared?.protocol ?? "openai", throttleReject.message, undefined, opts.streamErrorShape); - else { - res.writeHead(throttleReject.status, throttleReject.retryAfterSec !== null - ? { "content-type": "application/json", "retry-after": String(throttleReject.retryAfterSec) } - : { "content-type": "application/json" }); - res.end(JSON.stringify({ error: { type: "rate_limit_error", code: throttleReject.code, stage: "forward", message: throttleReject.message, retryable: throttleReject.status === 429, ...(throttleReject.model === undefined ? {} : { scope: "model", model: throttleReject.model }) } })); - } + answerLocalThrottleReject(res, throttleReject, prepared?.protocol, opts.streamErrorShape, log, prepared?.session.id); return; } } diff --git a/src/upstream-throttle.ts b/src/upstream-throttle.ts index bd54dc2b14..197e3d2a61 100644 --- a/src/upstream-throttle.ts +++ b/src/upstream-throttle.ts @@ -292,7 +292,7 @@ const NOOP_CLASSIFICATION_RESERVATION: ThrottleClassificationReservation = { release(): void {}, }; -interface ThrottleLocalReject { +export interface ThrottleLocalReject { /** 429 while under budget (Retry-After attached); 424 terminal. */ status: 429 | 424; code: "bili_throttle_hold" | "bili_throttle_terminal"; @@ -651,6 +651,38 @@ export function upstreamThrottleGate( return { probeToken }; } +/** + * #5795: read-only counterpart of upstreamThrottleGate's REJECT branch. + * Returns the local rejection a send to this url+headers would receive RIGHT + * NOW, without minting probe tokens or mutating any hold state — the real + * gate in forward() still owns admission and probe accounting. Used by the + * pre-ingest fast path in server/handle.ts to refuse held sends BEFORE they + * pay the O(history) parse/convert/pin cost of a request that will not + * egress anyway. The reject loop is deliberately duplicated rather than + * shared: the gate's loop and its mutation are atomic, while this one must + * never mutate — a flag-based shared loop would couple the two. + */ +export function upstreamThrottlePeekReject( + url: string, + headers: Record, + now: number = Date.now(), + requestModel?: string, +): ThrottleLocalReject | undefined { + if (!throttle429Enabled()) return undefined; + const { key } = holdKey(url, headers); + let rejectState: HoldState | undefined; + for (const state of holds.values()) { + if (state.accountKey !== key) continue; + if (!state.provisional && state.model !== undefined && state.model !== requestModel) continue; + if (slotBlocked(state, now)) rejectState = strongerReject(rejectState, state); + } + for (const group of provisionalGroups.values()) { + const state = group.state; + if (state.accountKey !== key) continue; + if (slotBlocked(state, now)) rejectState = strongerReject(rejectState, state); + } + return rejectState === undefined ? undefined : makeReject(rejectState, now); +} /** Observe a classified upstream 429 verdict for this send. */ export function noteUpstreamThrottle429( From 3bb2d8728f82dd71f832c0cc036165ab6a7823fa Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 19:03:29 +0800 Subject: [PATCH 2/5] test: regression rows for the pre-ingest 429-hold fastpath (#5795) Unit: peek/gate parity incl. terminal 424, peek read-only probe-slot guarantee, model-scoped conservatism, credential isolation, provisional groups, disabled governor, threshold boundary, exact 429/424 response shape. E2E hermetic proxy: held big turn = zero egress + local 429 with Retry-After, clean big turn egresses, small turns during a hold keep the post-ingest refusal shape, keyring model-scoped hold does not block other models' big turns. --- tests/issue5795-early-hold-fastpath.test.ts | 449 ++++++++++++++++++++ 1 file changed, 449 insertions(+) create mode 100644 tests/issue5795-early-hold-fastpath.test.ts diff --git a/tests/issue5795-early-hold-fastpath.test.ts b/tests/issue5795-early-hold-fastpath.test.ts new file mode 100644 index 0000000000..8ebacc258f --- /dev/null +++ b/tests/issue5795-early-hold-fastpath.test.ts @@ -0,0 +1,449 @@ +// #5795: pre-ingest 429-hold fast path — while the egress governor (#4204) +// holds origin+credential, sends ≥ EARLY_HOLD_GATE_MIN_BYTES are refused +// BEFORE prepare pays the O(history) parse/convert/pin cost. Invariants: +// 1. upstreamThrottlePeekReject is byte-for-byte the gate's REJECT branch +// (parity rows) but NEVER mutates — above all it must not consume the +// single-flight probe slot the real gate owns. +// 2. tryAnswerEarlyHold gates both directions: refuses when a hold applies, +// falls through untouched when it does not (method/protocol/size/model). +// 3. Hermetic E2E: a held multi-MiB send egresses ZERO bytes (the field bug: +// every host retry re-paid ~6s of ingest for a send that could not leave), +// while sub-threshold and model-mismatched sends keep their exact +// pre-#5795 behavior. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import http from "node:http"; +import { once } from "node:events"; +import os from "node:os"; +import path from "node:path"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest, _resetStoreForTest } from "../src/persist.ts"; +import { _resetPluginStateForTest } from "../src/plugin.ts"; +import { _resetSessionsForTest } from "../src/session.ts"; +import { closeServers } from "./server-close-guard.ts"; +import { + activeThrottleHolds, + beginProvisionalUpstreamThrottle429, + keyringModel429Scope, + noteUpstreamThrottle429, + upstreamThrottleGate, + upstreamThrottlePeekReject, + _resetUpstreamThrottleForTest, +} from "../src/upstream-throttle.ts"; +import { EARLY_HOLD_GATE_MIN_BYTES, tryAnswerEarlyHold } from "../src/server/relay.ts"; + +const URL_A = "http://z.example/api/coding/paas/v4/chat/completions"; +const H_EMPTY: Record = {}; +const H_CRED_A: Record = { authorization: "Bearer AAA" }; +const H_CRED_B: Record = { authorization: "Bearer BBB" }; +// Integer-second timestamps: keeps HTTP-date Retry-After fixtures exact. +const T0 = 1_000_000; + +const KNOB_ENVS = [ + "BILI_THROTTLE_429", + "BILI_THROTTLE_429_MAX_CONSECUTIVE", + "BILI_THROTTLE_429_BASE_DELAY_MS", + "BILI_THROTTLE_429_MAX_DELAY_MS", +] as const; + +function withKnobs(vars: Partial>, fn: () => void): void { + const saved: Record = {}; + for (const k of KNOB_ENVS) saved[k] = process.env[k]; + try { + for (const [k, v] of Object.entries(vars)) process.env[k] = v; + fn(); + } finally { + for (const k of KNOB_ENVS) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + } +} + +async function withKnobsAsync(vars: Partial>, fn: () => Promise): Promise { + const saved: Record = {}; + for (const k of KNOB_ENVS) saved[k] = process.env[k]; + try { + for (const [k, v] of Object.entries(vars)) process.env[k] = v; + await fn(); + } finally { + for (const k of KNOB_ENVS) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + } +} + +const KNOWN_KNOBS: Partial> = { + BILI_THROTTLE_429: "1", + BILI_THROTTLE_429_MAX_CONSECUTIVE: "5", + BILI_THROTTLE_429_BASE_DELAY_MS: "2000", + BILI_THROTTLE_429_MAX_DELAY_MS: "30000", +}; + +// --------------------------------------------------------------------------- +// upstreamThrottlePeekReject — parity with the gate's REJECT branch, zero mutation +// --------------------------------------------------------------------------- + +test("#5795: peek is byte-parity with the gate's reject branch (active hold)", () => { + withKnobs(KNOWN_KNOBS, () => { + _resetUpstreamThrottleForTest(); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + const gateReject = upstreamThrottleGate(URL_A, H_EMPTY, T0 + 100).reject; + assert.ok(gateReject, "gate rejects mid-window"); + assert.deepEqual(upstreamThrottlePeekReject(URL_A, H_EMPTY, T0 + 100), gateReject); + // Repeated peeks never drift the hold either. + assert.ok(upstreamThrottlePeekReject(URL_A, H_EMPTY, T0 + 200)); + assert.equal(activeThrottleHolds()[0].streak, 1); + }); +}); + +test("#5795: peek parity on a TERMINAL hold (424, no Retry-After)", () => { + withKnobs({ ...KNOWN_KNOBS, BILI_THROTTLE_429_MAX_CONSECUTIVE: "2" }, () => { + _resetUpstreamThrottleForTest(); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + const p = upstreamThrottlePeekReject(URL_A, H_EMPTY, T0 + 100); + assert.ok(p); + assert.equal(p.status, 424); + assert.equal(p.code, "bili_throttle_terminal"); + assert.equal(p.retryAfterSec, null); + assert.deepEqual(p, upstreamThrottleGate(URL_A, H_EMPTY, T0 + 100).reject); + }); +}); + +test("#5795: peek is read-only — it must not consume the single-flight probe slot", () => { + withKnobs(KNOWN_KNOBS, () => { + _resetUpstreamThrottleForTest(); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + const edge = activeThrottleHolds()[0].nextAttemptAt; + assert.equal(upstreamThrottlePeekReject(URL_A, H_EMPTY, edge), undefined, "window open: nothing to refuse"); + const admitted = upstreamThrottleGate(URL_A, H_EMPTY, edge); + assert.equal(admitted.reject, undefined, "the real gate still admits the probe"); + assert.equal(typeof admitted.probeToken, "symbol", "peek must not have minted/consumed the probe token"); + // Mid-window, peek and gate agree AND the gate's reject carries no token. + const dup = upstreamThrottleGate(URL_A, H_EMPTY, T0 + 100); + assert.ok(dup.reject); + assert.equal(dup.probeToken, undefined); + assert.deepEqual(upstreamThrottlePeekReject(URL_A, H_EMPTY, T0 + 100), dup.reject); + }); +}); + +test("#5795: peek honors model-scoped holds conservatively (exact match only)", () => { + withKnobs(KNOWN_KNOBS, () => { + _resetUpstreamThrottleForTest(); + const model = "deepseek-v4.1-flash"; + const modelScope = keyringModel429Scope(429, model, JSON.stringify({ error: { code: "keyring_model_temporarily_unavailable", model } })); + assert.equal(modelScope, model); + noteUpstreamThrottle429(URL_A, H_CRED_A, null, T0, modelScope); + assert.ok(upstreamThrottlePeekReject(URL_A, H_CRED_A, T0 + 100, model), "exact model match is refused"); + assert.equal(upstreamThrottlePeekReject(URL_A, H_CRED_A, T0 + 100, "other-model"), undefined, "another model passes"); + assert.equal(upstreamThrottlePeekReject(URL_A, H_CRED_A, T0 + 100), undefined, "unknown model passes (cannot prove mismatch)"); + }); +}); + +test("#5795: peek keeps credential pools isolated", () => { + withKnobs(KNOWN_KNOBS, () => { + _resetUpstreamThrottleForTest(); + noteUpstreamThrottle429(URL_A, H_CRED_A, null, T0); + assert.ok(upstreamThrottlePeekReject(URL_A, H_CRED_A, T0 + 100)); + assert.equal(upstreamThrottlePeekReject(URL_A, H_CRED_B, T0 + 100), undefined, "other credential unaffected"); + }); +}); + +test("#5795: peek sees provisional groups (account-scoped while classifying)", () => { + withKnobs(KNOWN_KNOBS, () => { + _resetUpstreamThrottleForTest(); + const settle = beginProvisionalUpstreamThrottle429(URL_A, H_EMPTY, "10", T0, undefined); + const p = upstreamThrottlePeekReject(URL_A, H_EMPTY, T0 + 100); + assert.ok(p, "provisional hold refuses like a confirmed one"); + assert.equal(p?.status, 429); + settle(); + }); +}); + +test("#5795: disabled governor — peek is inert even with an armed hold", () => { + withKnobs(KNOWN_KNOBS, () => { + _resetUpstreamThrottleForTest(); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + assert.ok(upstreamThrottlePeekReject(URL_A, H_EMPTY, T0 + 100), "enabled: refuses"); + }); + withKnobs({ ...KNOWN_KNOBS, BILI_THROTTLE_429: "0" }, () => { + assert.equal(upstreamThrottlePeekReject(URL_A, H_EMPTY, T0 + 100), undefined, "disabled: inert"); + }); +}); + +// --------------------------------------------------------------------------- +// tryAnswerEarlyHold — gate directions (refuse when it should, pass when not) +// --------------------------------------------------------------------------- + +interface FakeRes { + headersSent: boolean; + headStatus?: number; + headHeaders?: Record; + endBody?: string; +} + +function makeFakeRes(): FakeRes & http.ServerResponse { + const f: FakeRes = { headersSent: false }; + (f as { writeHead?: unknown }).writeHead = (status: number, headers?: Record) => { + f.headStatus = status; + f.headHeaders = headers; + }; + (f as { end?: unknown }).end = (body?: string) => { + f.endBody = body ?? ""; + }; + return f as unknown as FakeRes & http.ServerResponse; +} + +function earlyHoldBase(logs: string[]) { + return { + upstreamUrl: URL_A, + credHeaders: H_EMPTY, + protocol: "openai" as const, + log: (level: string, msg: string) => logs.push(`${level} ${msg}`), + }; +} + +test("#5795: tryAnswerEarlyHold pass-through rows (no refusal, response untouched)", () => { + withKnobs(KNOWN_KNOBS, () => { + _resetUpstreamThrottleForTest(); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + const big = EARLY_HOLD_GATE_MIN_BYTES + 1; + const logs: string[] = []; + + for (const method of ["GET", "HEAD"]) { + const res = makeFakeRes(); + assert.equal(tryAnswerEarlyHold({ ...earlyHoldBase(logs), method, bodyBytes: big, res }), false, `${method} never consults the hold`); + assert.equal(res.headStatus, undefined); + } + const pnull = makeFakeRes(); + assert.equal(tryAnswerEarlyHold({ ...earlyHoldBase(logs), method: "POST", protocol: null, bodyBytes: big, res: pnull }), false, "unknown wire stays verbatim"); + assert.equal(pnull.headStatus, undefined); + const small = makeFakeRes(); + assert.equal(tryAnswerEarlyHold({ ...earlyHoldBase(logs), method: "POST", bodyBytes: EARLY_HOLD_GATE_MIN_BYTES - 1, res: small }), false, "sub-threshold keeps its exact pre-#5795 path"); + assert.equal(small.headStatus, undefined); + assert.equal(logs.length, 0, "pass-through rows log nothing"); + + _resetUpstreamThrottleForTest(); + const admit = makeFakeRes(); + assert.equal(tryAnswerEarlyHold({ ...earlyHoldBase(logs), method: "POST", bodyBytes: big, res: admit }), false, "no hold: admitted"); + assert.equal(admit.headStatus, undefined); + }); +}); + +test("#5795: tryAnswerEarlyHold answers a held send with the exact local 429 shape", () => { + withKnobs(KNOWN_KNOBS, () => { + _resetUpstreamThrottleForTest(); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + const logs: string[] = []; + const res = makeFakeRes(); + assert.equal(tryAnswerEarlyHold({ ...earlyHoldBase(logs), method: "POST", bodyBytes: EARLY_HOLD_GATE_MIN_BYTES, now: T0 + 100, res }), true, "threshold boundary consults the hold"); + assert.equal(res.headStatus, 429); + assert.equal(res.headHeaders?.["content-type"], "application/json"); + assert.ok(res.headHeaders?.["retry-after"], "Retry-After rides the local 429"); + const body = JSON.parse(res.endBody ?? "") as { error: { type: string; code: string; stage: string; message: string; retryable: boolean } }; + assert.equal(body.error.type, "rate_limit_error"); + assert.equal(body.error.code, "bili_throttle_hold"); + assert.equal(body.error.stage, "forward"); + assert.equal(body.error.retryable, true); + assert.ok(logs.some((l) => l.includes("[pre-ingest]") && l.includes("answered locally with 429")), "fast-path refusal is logged distinctly"); + }); +}); + +test("#5795: tryAnswerEarlyHold answers a TERMINAL hold with 424 (no Retry-After, non-retryable)", () => { + withKnobs({ ...KNOWN_KNOBS, BILI_THROTTLE_429_MAX_CONSECUTIVE: "2" }, () => { + _resetUpstreamThrottleForTest(); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + noteUpstreamThrottle429(URL_A, H_EMPTY, null, T0); + const logs: string[] = []; + const res = makeFakeRes(); + assert.equal(tryAnswerEarlyHold({ ...earlyHoldBase(logs), method: "POST", bodyBytes: EARLY_HOLD_GATE_MIN_BYTES, now: T0 + 100, res }), true); + assert.equal(res.headStatus, 424); + assert.equal(res.headHeaders?.["retry-after"], undefined, "terminal has no Retry-After"); + const body = JSON.parse(res.endBody ?? "") as { error: { code: string; retryable: boolean } }; + assert.equal(body.error.code, "bili_throttle_terminal"); + assert.equal(body.error.retryable, false); + assert.ok(logs.some((l) => l.startsWith("warn")), "terminal refusals log at warn"); + }); +}); + +// --------------------------------------------------------------------------- +// Hermetic E2E — held multi-MiB send egresses zero bytes +// --------------------------------------------------------------------------- + +interface Capture { raw: string; headerKeys: string[] } + +function completionBody(): string { + return JSON.stringify({ + id: "chatcmpl-test", object: "chat.completion", created: 1, model: "model-test", + choices: [{ index: 0, message: { role: "assistant", content: "answer" }, finish_reason: "stop" }], + usage: { prompt_tokens: 100, completion_tokens: 4, total_tokens: 104 }, + }); +} + +function makeScriptedUpstream(sink: Capture[], first: { status: number; body: string } | null) { + let n = 0; + return http.createServer((req, res) => { + let data = ""; + req.on("data", (c) => (data += c)); + req.on("end", () => { + sink.push({ raw: data, headerKeys: Object.keys(req.headers).map((k) => k.toLowerCase()) }); + if (n === 0 && first !== null) { + res.writeHead(first.status, { "content-type": "application/json" }); + res.end(first.body); + return; + } + n++; + res.writeHead(200, { "content-type": "application/json" }); + res.end(completionBody()); + }); + }); +} + +interface Harness2 { root: string; store: SessionStore; captured: Capture[]; upstream: http.Server } + +function makeHarness(tag: string): Harness2 { + _resetStoreForTest(); + _resetPluginStateForTest(); + _resetSessionsForTest(); + const root = mkdtempSync(path.join(os.tmpdir(), `e5795-${tag}-`)); + for (const k of ["HOME", "APPHOME", "HERMES_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[k] = root; + const store = new SessionStore({ dir: path.join(root, "sessions"), enabled: false }); + _setStoreForTest(store); + const captured: Capture[] = []; + return { root, store, captured, upstream: makeScriptedUpstream(captured, null) }; +} + +async function bringUp(h: Harness2) { + h.upstream.listen(0, "127.0.0.1"); + await once(h.upstream, "listening"); + const uaddr = h.upstream.address(); + assert(uaddr && typeof uaddr === "object"); + const upstreamUrl = `http://127.0.0.1:${uaddr.port}`; + const proxy = await startServer({ + port: 0, host: "127.0.0.1", upstream: upstreamUrl, + routes: { [upstreamUrl]: { models: { "model-test": { context: 2_000_000 } } } }, + modelContextLimit: 2_000_000, + kernelConfig: defaultConfig(2_000_000), + compress: { injectTool: false, injectNudge: false }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, debug: true, passthrough: false, autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + await once(proxy, "listening"); + const paddr = proxy.address(); + assert(paddr && typeof paddr === "object"); + return { proxy, origin: `http://127.0.0.1:${paddr.port}`, upstreamUrl }; +} + +function bigTurn(model: string, targetBytes: number): unknown { + const overhead = JSON.stringify({ model, messages: [{ role: "user", content: "" }], max_tokens: 64, stream: false }).length; + return { model, messages: [{ role: "user", content: "x".repeat(targetBytes - overhead) }], max_tokens: 64, stream: false }; +} + +async function post(origin: string, upstreamUrl: string, body: unknown): Promise<{ status: number; retryAfter: string | null; error?: { type?: string; code?: string; stage?: string; retryable?: boolean } }> { + const res = await fetch(`${origin}/bili/${upstreamUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const text = await res.text(); + let error: { type?: string; code?: string; stage?: string; retryable?: boolean } | undefined; + try { + const parsed = JSON.parse(text) as { error?: typeof error }; + error = parsed.error; + } catch { /* non-JSON success body */ } + return { status: res.status, retryAfter: res.headers.get("retry-after"), error }; +} + +test("#5795 e2e: held send ≥1MiB is refused pre-ingest — zero egress, local shape intact", async () => { + await withKnobsAsync(KNOWN_KNOBS, async () => { + const roots: string[] = []; + try { + const h0 = makeHarness("hold"); + roots.push(h0.root); + h0.upstream.close(); + h0.upstream = makeScriptedUpstream(h0.captured, { + status: 429, + body: JSON.stringify({ error: { type: "server_error", message: "[overloaded] the server is overloaded; retry shortly" } }), + }); + const h = await bringUp(h0); + // Turn 1 (small): egresses, upstream 429 arms the account hold. + const r1 = await post(h.origin, h.upstreamUrl, { model: "model-test", messages: [{ role: "user", content: "hi" }], max_tokens: 64, stream: false }); + assert.equal(r1.status, 429, "turn 1 relays the upstream 429"); + assert.equal(h0.captured.length, 1); + // Turn 2 (≥1MiB): held LOCALLY before prepare — the core invariant. + const r2 = await post(h.origin, h.upstreamUrl, bigTurn("model-test", EARLY_HOLD_GATE_MIN_BYTES + 64 * 1024)); + assert.equal(r2.status, 429, "held turn answered locally"); + assert.equal(h0.captured.length, 1, "ZERO egress for the held turn — no re-upload cost paid"); + assert.ok(r2.retryAfter !== null, "local refusal carries Retry-After"); + assert.equal(r2.error?.code, "bili_throttle_hold"); + assert.equal(r2.error?.stage, "forward"); + assert.equal(r2.error?.retryable, true); + // Turn 3 (small, still held): the post-ingest gate still catches it too. + const r3 = await post(h.origin, h.upstreamUrl, { model: "model-test", messages: [{ role: "user", content: "again" }], max_tokens: 64, stream: false }); + assert.equal(r3.status, 429); + assert.equal(h0.captured.length, 1, "small held turn still refused by the original gate"); + assert.equal(r3.error?.code, "bili_throttle_hold"); + h0.store.cancelAll(); + await closeServers([h.proxy, h0.upstream], "#5795 hold"); + } finally { + for (const root of roots) rmSync(root, { recursive: true, force: true }); + } + }); +}); + +test("#5795 e2e negative: a ≥1MiB send with NO hold egresses normally", async () => { + await withKnobsAsync(KNOWN_KNOBS, async () => { + const roots: string[] = []; + try { + const h0 = makeHarness("clean"); + roots.push(h0.root); + const h = await bringUp(h0); + const r = await post(h.origin, h.upstreamUrl, bigTurn("model-test", EARLY_HOLD_GATE_MIN_BYTES + 64 * 1024)); + assert.equal(r.status, 200, "no hold: the fast path must not over-refuse"); + assert.equal(h0.captured.length, 1, "egressed once"); + assert.ok(h0.captured[0].raw.length >= EARLY_HOLD_GATE_MIN_BYTES, "full multi-MiB body reached upstream"); + h0.store.cancelAll(); + await closeServers([h.proxy, h0.upstream], "#5795 clean"); + } finally { + for (const root of roots) rmSync(root, { recursive: true, force: true }); + } + }); +}); + +test("#5795 e2e: model-scoped hold — other models egress, matching model held", async () => { + await withKnobsAsync(KNOWN_KNOBS, async () => { + const roots: string[] = []; + try { + const h0 = makeHarness("modelscope"); + roots.push(h0.root); + h0.upstream.close(); + h0.upstream = makeScriptedUpstream(h0.captured, { + status: 429, + body: JSON.stringify({ error: { code: "keyring_model_temporarily_unavailable", model: "model-a", message: "model temporarily unavailable" } }), + }); + const h = await bringUp(h0); + const r1 = await post(h.origin, h.upstreamUrl, { model: "model-a", messages: [{ role: "user", content: "hi" }], max_tokens: 64, stream: false }); + assert.equal(r1.status, 429, "turn 1 relays the KeyRing model 429"); + assert.equal(h0.captured.length, 1); + const r2 = await post(h.origin, h.upstreamUrl, bigTurn("model-b", EARLY_HOLD_GATE_MIN_BYTES + 64 * 1024)); + assert.equal(r2.status, 200, "model-b is outside the model-scoped hold — must egress"); + assert.equal(h0.captured.length, 2); + const r3 = await post(h.origin, h.upstreamUrl, bigTurn("model-a", EARLY_HOLD_GATE_MIN_BYTES + 64 * 1024)); + assert.equal(r3.status, 429, "model-a is held"); + assert.equal(h0.captured.length, 2, "zero egress for the held model"); + assert.equal(r3.error?.code, "bili_throttle_hold"); + h0.store.cancelAll(); + await closeServers([h.proxy, h0.upstream], "#5795 modelscope"); + } finally { + for (const root of roots) rmSync(root, { recursive: true, force: true }); + } + }); +}); From c26dbea1917a8feccbb9d63fe1a4abcdd9a61cd8 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 20:41:32 +0800 Subject: [PATCH 3/5] fix: drop unused export from answerLocalThrottleReject (unused-exports gate) (#5795) --- src/server/relay.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/relay.ts b/src/server/relay.ts index 0569024a14..2783b4741a 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -254,7 +254,7 @@ export function armFirstByteWatchdog(o: { // #5795: single source of truth for the local 429/424 hold response — shared by // the post-ingest gate below and the pre-ingest fast path in handle.ts so the // two sites cannot drift on status/headers/body shape. -export function answerLocalThrottleReject( +function answerLocalThrottleReject( res: http.ServerResponse, reject: ThrottleLocalReject, protocol: WireProtocol | null | undefined, From bf482e9eb2caab20ccf4c35f489f091cd1f11c61 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 21:14:41 +0800 Subject: [PATCH 4/5] test(#5795): scripted upstream advances on every request, not just 200s With a first-response fixture set, n++ sat only on the 200 path, so the fixture replayed its first response forever: the model-scope row's egressing turn received another 429 and its 200 assertion could never pass (deterministic fail, reproduced locally). Counter now advances on every request; fixture behavior for the hold/clean rows is unchanged. --- tests/issue5795-early-hold-fastpath.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/issue5795-early-hold-fastpath.test.ts b/tests/issue5795-early-hold-fastpath.test.ts index 8ebacc258f..ca3ed0c7a4 100644 --- a/tests/issue5795-early-hold-fastpath.test.ts +++ b/tests/issue5795-early-hold-fastpath.test.ts @@ -293,12 +293,16 @@ function makeScriptedUpstream(sink: Capture[], first: { status: number; body: st req.on("data", (c) => (data += c)); req.on("end", () => { sink.push({ raw: data, headerKeys: Object.keys(req.headers).map((k) => k.toLowerCase()) }); - if (n === 0 && first !== null) { + // Advance on EVERY request: with `first` set, incrementing only on + // the 200 path replayed the first response forever, so the + // model-scope row's "other model egresses" turn could never see a 200. + const isFirst = n === 0 && first !== null; + n++; + if (isFirst) { res.writeHead(first.status, { "content-type": "application/json" }); res.end(first.body); return; } - n++; res.writeHead(200, { "content-type": "application/json" }); res.end(completionBody()); }); From feb8be9e19f91a3b5fed942da3d96b5dd1b7d688 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 21:14:41 +0800 Subject: [PATCH 5/5] test(#5795): word-separated big-turn content (homogeneous runs spin maskUrlsInText) These e2e requests mint fresh ANONYMOUS sessions, whose provenance path masks the ENTIRE first message through maskUrlsInText before truncating to the preview cap. The #4191 any-scheme regex ([a-z][a-z0-9+.-]*://) backtracks O(n^2) over long scheme-char runs (measured: 7.4s @ 100k chars, exact 4x per doubling), so a megabyte of 'x' froze the event loop for minutes and hung this suite in CI. Whitespace-separated words keep the byte size identical while keeping the scan linear. The underlying masker defect is pre-existing and tracked separately. --- tests/issue5795-early-hold-fastpath.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/issue5795-early-hold-fastpath.test.ts b/tests/issue5795-early-hold-fastpath.test.ts index ca3ed0c7a4..dee9135d70 100644 --- a/tests/issue5795-early-hold-fastpath.test.ts +++ b/tests/issue5795-early-hold-fastpath.test.ts @@ -346,9 +346,19 @@ async function bringUp(h: Harness2) { return { proxy, origin: `http://127.0.0.1:${paddr.port}`, upstreamUrl }; } +// The fill must NOT be one homogeneous character run: these requests mint a +// FRESH ANONYMOUS session (no session header), whose provenance path masks the +// ENTIRE first message through maskUrlsInText BEFORE truncating to the preview +// cap. That masker's scheme-prefix scan ([a-z][a-z0-9+.-]*://) backtracks +// O(n²) over long scheme-char runs — a megabyte of 'x' spins the event loop +// for minutes (pre-existing since the #4191 any-scheme regex; tracked +// separately). Whitespace-separated words keep the byte size identical while +// keeping the scan linear. +const BIG_TURN_WORDS = "lorem ipsum dolor sit amet "; function bigTurn(model: string, targetBytes: number): unknown { const overhead = JSON.stringify({ model, messages: [{ role: "user", content: "" }], max_tokens: 64, stream: false }).length; - return { model, messages: [{ role: "user", content: "x".repeat(targetBytes - overhead) }], max_tokens: 64, stream: false }; + const fill = BIG_TURN_WORDS.repeat(Math.ceil((targetBytes - overhead) / BIG_TURN_WORDS.length)).slice(0, targetBytes - overhead); + return { model, messages: [{ role: "user", content: fill }], max_tokens: 64, stream: false }; } async function post(origin: string, upstreamUrl: string, body: unknown): Promise<{ status: number; retryAfter: string | null; error?: { type?: string; code?: string; stage?: string; retryable?: boolean } }> {