Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/cache-ledger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,11 @@ export function noteClientAbort(session: Session): void {
lastClientAbort.set(session, Date.now());
}

// #5795: single source of truth for the credential header names — the 429
// governor's hold key (upstream-throttle.ts) and the pre-ingest fast path
// (server/handle.ts) must read exactly these headers or holds go unmatched.
export const CREDENTIAL_HEADER_NAMES = ["authorization", "x-api-key", "api-key", "x-goog-api-key"] as const;

/** #2131: stable fingerprint of the outbound credential — the ONLY form the
* key is ever persisted in (raw secrets never reach the ledger, logs or the
* web UI). Checks the standard bearer/api-key headers, case-insensitively;
Expand All @@ -651,7 +656,7 @@ export function credentialFingerprint(headers: Record<string, string> | undefine
let cred: string | undefined;
for (const [name, value] of Object.entries(headers)) {
const n = name.toLowerCase();
if (n === "authorization" || n === "x-api-key" || n === "api-key" || n === "x-goog-api-key") {
if ((CREDENTIAL_HEADER_NAMES as readonly string[]).includes(n)) {
// Strip the auth-scheme envelope so "Bearer X" (authorization) and
// a bare "X" (x-api-key) hash to the SAME credential identity.
const v = value.trim().replace(/^Bearer\s+/i, "").trim();
Expand Down
33 changes: 33 additions & 0 deletions src/server/handle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,21 @@ import { ccrPluginWireOk, storeEffectiveCcr, storeEffectiveImageRescale, type Cc
import { DEFAULT_DECIDE_MAX_TOKENS } from "../nudge-decide.js";
import { BodyTooLargeError, buildForwardTarget, forward, forwardUpstreamUrl, googleModelFromPath, googlePathKind, headroomEffectiveLogged, headroomFallbackLogged, headerValue, imageBillingFor, imageReserveFor, imageTokenCapFor, isCountTokensRequest, isPreflightFailFast, logRequestCost, NON_CONVERSATION_RELAY_WARN_CAP, nonConversationRelayWarned, prepareCountTokens, preflightCompressIfNeeded, readBody, resolveKnownOutputCeiling, resolveUpstream, resignSettingsFor, selfAddressedLocalPath, scrubAnthropicPck, scrubCompatDrop, vertexAnthropicModelFromPath, warnModelNameMissIfNew, warnRouteMissIfNew, type Prepared } from "../server.js";
import { normalizeVertexModelId } from "../vertex.js";
import { CREDENTIAL_HEADER_NAMES } from "../cache-ledger.js";
import { requestModelFromBody } from "../upstream-throttle.js";
import { tryAnswerEarlyHold } from "./relay.js";

// #5795: the credential subset of the inbound headers — exactly the names
// credentialFingerprint reads (CREDENTIAL_HEADER_NAMES), or the fast path's
// hold key would diverge from the governor's and account holds go invisible.
function earlyHoldCredHeaders(req: http.IncomingMessage): Record<string, string> {
const out: Record<string, string> = {};
for (const n of CREDENTIAL_HEADER_NAMES) {
const v = headerValue(req, n);
if (v !== undefined && v !== "") out[n] = v;
}
return out;
}

// #5243: warn-once per model when the live codex model table is unavailable and
// window sizing falls back to config.toml model_context_window.
Expand Down Expand Up @@ -487,6 +502,24 @@ async function handleRequest(
await forward(req, res, opts, scrubCompatDrop(scrubAnthropicPck(protocol, bodyBuffer, log), compatDropPaths, log), null, core, config, log, route, instanceId, undefined, undefined, undefined, googleModel ?? vertexModel);
return;
}
// #5795: pre-ingest 429-hold fast path — while the egress governor holds
// this origin+credential, refuse BEFORE prepare pays the O(history)
// parse/convert/pin cost of a send that will not egress anyway (~6s per
// refused turn on the field-reported multi-MiB histories). Sits above the
// hop-marker branch so chained-bili relays get the identical local refusal
// one stage earlier; sub-threshold bodies and unknown-wire requests fall
// through untouched (see tryAnswerEarlyHold).
if (tryAnswerEarlyHold({
method: req.method,
upstreamUrl: forwardUpstreamUrl(req, opts, route),
credHeaders: earlyHoldCredHeaders(req),
protocol,
bodyBytes: bodyBuffer.length,
requestModel: googleModel ?? vertexModel ?? requestModelFromBody(bodyBuffer),
res,
streamErrorShape: opts.streamErrorShape,
log,
})) return;
// #300: bili→bili chain detection. If the inbound request already carries
// the x-bili-hop marker, an upstream bili instance already ran the
// compression pipeline on it. Processing it again would double-compress
Expand Down
70 changes: 61 additions & 9 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ import { safePrefix, safeSuffix, scrubLoneSurrogatesOnWire } from "../text-safe.
import { clearUpstreamAlertsForHost, noteCircuitFailFastLog, recordUpstreamAlert, recordUpstreamCircuitFailure, recordUpstreamCircuitSuccess, upstreamCircuitBlockReason, upstreamCircuitLastFailureKind, upstreamCircuitRetryAfterSeconds } from "../upstream-alerts.js";
import { describeUpstreamFailure, formatUpstreamError, proxyDispatcher, recordUpstreamConnection } from "../upstream-proxy.js";
import { markExpectedFailFast, withUpstreamFailure, type UpstreamFailureKind } from "../upstream-fail.js";
import { keyringModel429Scope, noteUpstreamThrottle429, noteUpstreamThrottleResolved, releaseUpstreamThrottleProbe, requestModelFromBody, upstreamThrottleGate } from "../upstream-throttle.js";
import { keyringModel429Scope, noteUpstreamThrottle429, noteUpstreamThrottleResolved, releaseUpstreamThrottleProbe, requestModelFromBody, type ThrottleLocalReject, upstreamThrottleGate, upstreamThrottlePeekReject } from "../upstream-throttle.js";
import { applyEstimateCalibration, currentCalibrationFactor, inspectContextOverflow, isMarkerlessOverflowReject, MIN_ASSUMED_WINDOW_TOKENS, normalizeUpstreamOrigin, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "../util.js";
import { billOutputUsage } from "../billing.js";
import { DUMP_PLAINTEXT_BANNER } from "../dump-banner.js";
Expand Down Expand Up @@ -251,6 +251,65 @@ export function armFirstByteWatchdog(o: {
return timer;
}

// #5795: single source of truth for the local 429/424 hold response — shared by
// the post-ingest gate below and the pre-ingest fast path in handle.ts so the
// two sites cannot drift on status/headers/body shape.
function answerLocalThrottleReject(
res: http.ServerResponse,
reject: ThrottleLocalReject,
protocol: WireProtocol | null | undefined,
streamErrorShape: "protocol" | "completion" | undefined,
log: (level: string, msg: string) => void,
label?: string,
): void {
log(reject.status === 424 ? "warn" : "debug", `[${label ?? "passthrough"}] [acp-proxy] 429-hold: answered locally with ${reject.status} without egress (#4204)`);
if (res.headersSent) {
emitStreamError(res, protocol ?? "openai", reject.message, undefined, streamErrorShape);
return;
}
res.writeHead(reject.status, reject.retryAfterSec !== null
? { "content-type": "application/json", "retry-after": String(reject.retryAfterSec) }
: { "content-type": "application/json" });
res.end(JSON.stringify({ error: { type: "rate_limit_error", code: reject.code, stage: "forward", message: reject.message, retryable: reject.status === 429, ...(reject.model === undefined ? {} : { scope: "model", model: reject.model }) } }));
}

// #5795: pre-ingest 429-hold fast path. While a hold is active for this
// origin+credential the send will be answered locally anyway — answering
// BEFORE prepare skips the O(history) parse/convert/pin cost of a request that
// will not egress regardless (the held retry storms in the field report cost
// ~6s of ingest per refused turn on multi-MiB histories). Completes #4204's
// contract: zero egress while held → also zero ingest while held.
// Read-only peek: never mints probe tokens or mutates governor state, so the
// real gate in forwardCaptured keeps owning admission/probe accounting. The
// conservative edges are deliberate: sub-threshold bodies keep their exact
// pre-#5795 path (forge/classifier/compact-trigger shapes intercept locally
// during prepare and must not be pre-empted), and lanes whose credential bili
// itself overrides (#2336) key holds under the overridden fingerprint which
// this raw-header peek cannot see — those fall through to the post-ingest gate
// and are refused there exactly as before.
export const EARLY_HOLD_GATE_MIN_BYTES = 1024 * 1024;

export function tryAnswerEarlyHold(o: {
method?: string;
upstreamUrl: string;
credHeaders: Record<string, string>;
protocol: WireProtocol | null;
bodyBytes: number;
requestModel?: string;
res: http.ServerResponse;
streamErrorShape?: "protocol" | "completion";
log: (level: string, msg: string) => void;
/** Test seam: mirrors the governor's own explicit-now convention. */
now?: number;
}): boolean {
if (o.method === "GET" || o.method === "HEAD") return false;
if (o.protocol === null || o.bodyBytes < EARLY_HOLD_GATE_MIN_BYTES) return false;
const reject = upstreamThrottlePeekReject(o.upstreamUrl, o.credHeaders, o.now ?? Date.now(), o.requestModel);
if (reject === undefined) return false;
answerLocalThrottleReject(o.res, reject, o.protocol, o.streamErrorShape, o.log, "pre-ingest");
return true;
}

export function forward(...args: Parameters<typeof forwardCaptured>): Promise<void> {
return withCacheCapture(args[4]?.session, () => forwardCaptured(...args));
}
Expand Down Expand Up @@ -857,14 +916,7 @@ async function forwardCaptured(
throttleProbeToken = throttleGate.probeToken;
const throttleReject = throttleGate.reject;
if (throttleReject !== undefined) {
log(throttleReject.status === 424 ? "warn" : "debug", `[${prepared?.session.id ?? "passthrough"}] [acp-proxy] 429-hold: answered locally with ${throttleReject.status} without egress (#4204)`);
if (res.headersSent) emitStreamError(res, prepared?.protocol ?? "openai", throttleReject.message, undefined, opts.streamErrorShape);
else {
res.writeHead(throttleReject.status, throttleReject.retryAfterSec !== null
? { "content-type": "application/json", "retry-after": String(throttleReject.retryAfterSec) }
: { "content-type": "application/json" });
res.end(JSON.stringify({ error: { type: "rate_limit_error", code: throttleReject.code, stage: "forward", message: throttleReject.message, retryable: throttleReject.status === 429, ...(throttleReject.model === undefined ? {} : { scope: "model", model: throttleReject.model }) } }));
}
answerLocalThrottleReject(res, throttleReject, prepared?.protocol, opts.streamErrorShape, log, prepared?.session.id);
return;
}
}
Expand Down
34 changes: 33 additions & 1 deletion src/upstream-throttle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ const NOOP_CLASSIFICATION_RESERVATION: ThrottleClassificationReservation = {
release(): void {},
};

interface ThrottleLocalReject {
export interface ThrottleLocalReject {
/** 429 while under budget (Retry-After attached); 424 terminal. */
status: 429 | 424;
code: "bili_throttle_hold" | "bili_throttle_terminal";
Expand Down Expand Up @@ -651,6 +651,38 @@ export function upstreamThrottleGate(
return { probeToken };
}

/**
* #5795: read-only counterpart of upstreamThrottleGate's REJECT branch.
* Returns the local rejection a send to this url+headers would receive RIGHT
* NOW, without minting probe tokens or mutating any hold state — the real
* gate in forward() still owns admission and probe accounting. Used by the
* pre-ingest fast path in server/handle.ts to refuse held sends BEFORE they
* pay the O(history) parse/convert/pin cost of a request that will not
* egress anyway. The reject loop is deliberately duplicated rather than
* shared: the gate's loop and its mutation are atomic, while this one must
* never mutate — a flag-based shared loop would couple the two.
*/
export function upstreamThrottlePeekReject(
url: string,
headers: Record<string, string>,
now: number = Date.now(),
requestModel?: string,
): ThrottleLocalReject | undefined {
if (!throttle429Enabled()) return undefined;
const { key } = holdKey(url, headers);
let rejectState: HoldState | undefined;
for (const state of holds.values()) {
if (state.accountKey !== key) continue;
if (!state.provisional && state.model !== undefined && state.model !== requestModel) continue;
if (slotBlocked(state, now)) rejectState = strongerReject(rejectState, state);
}
for (const group of provisionalGroups.values()) {
const state = group.state;
if (state.accountKey !== key) continue;
if (slotBlocked(state, now)) rejectState = strongerReject(rejectState, state);
}
return rejectState === undefined ? undefined : makeReject(rejectState, now);
}

/** Observe a classified upstream 429 verdict for this send. */
export function noteUpstreamThrottle429(
Expand Down
Loading
Loading