Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1434,7 +1434,7 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr
child.contentStore = cloneStoreForRefs(parentStore, new Set(prefix.filter((m) => m.ref !== null).map((m) => m.ref!))) ?? undefined;
child.metadata.publicSnapshotTextComparable = true;
child.metadata.publicSnapshotStoredRefs = Object.keys(child.contentStore?.byRef ?? {});
for (const key of ["pluginAgent", "lastModel", "effectiveConfig", "effectiveConfigEpoch", "effectiveCcr", "effectiveContextLimit", "lastWindowSource", "systemPromptTokens"]) if (parent.metadata[key] !== undefined) child.metadata[key] = structuredClone(parent.metadata[key]);
for (const key of ["pluginAgent", "lastModel", "effectiveConfig", "effectiveConfigEpoch", "effectiveCcr", "effectiveContextLimit", "reservedContextLimit", "lastWindowSource", "systemPromptTokens"]) if (parent.metadata[key] !== undefined) child.metadata[key] = structuredClone(parent.metadata[key]);
child.metadata.parentConversationId = request.parentConversationId;
child.metadata.parentRevision = request.parentRevision;
child.stats.contextTokens = prune(child.pluginSnapshot, child.state).reduce((sum, m) => sum + countMessageTokens(m), 0) + (typeof child.metadata.systemPromptTokens === "number" ? child.metadata.systemPromptTokens : 0);
Expand Down
7 changes: 4 additions & 3 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3421,9 +3421,10 @@ export async function preflightCompressIfNeeded(
// windowShrinkReason (set ONLY by the operator-override and codex-align
// paths) — NOT just on limit < resolvedNativeWindow: the per-request
// output-headroom reservation (reserveOutputHeadroom) also lowers
// reqConfig.modelContextLimit below native for every non-Anthropic turn
// with a max_tokens, so comparing alone would emit this note for a
// setting the operator never touched (#737 review).
// reqConfig.modelContextLimit below native for every turn whose route
// reserves headroom (non-anthropic wires plus anthropic routes lacking
// separate-output-budget evidence, #4205), so comparing alone would emit
// this note for a setting the operator never touched (#737 review).
const ffEmbeddedUrl = route?.rewrittenUrl;
const ffProviderKeys = Object.keys(opts.routes);
const shrinkNote = buildWindowShrinkNote({
Expand Down
13 changes: 9 additions & 4 deletions src/server/budget.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,15 @@ export { readOutputBudget };

// #453 hard backstop: cap the forwarded output budget so input+output can never
// exceed the window on request-rebuilding upstreams (vLLM rejects an oversized
// total instead of clamping). Non-Anthropic only — Anthropic enforces its input
// limit independently of max_tokens (see shouldReserveOutputHeadroom). The proxy
// owns both sides of the sum, so capping output to (window - input - margin)
// makes the overflow impossible even when the agent ignores the compress nudge.
// total instead of clamping). Wired into the openai/responses/google prepare
// lanes only — canonical Anthropic routes enforce their input limit
// independently of max_tokens, so they get no clamp call site. #4205 boundary:
// wrapper providers on the anthropic wire (completion billed against the same
// window) DO reserve headroom at the band layer but keep no egress clamp here;
// their residual over-requests ride the overflow self-heal (#4093) instead.
// The proxy owns both sides of the sum, so capping output to (window - input -
// margin) makes the overflow impossible even when the agent ignores the
// compress nudge.
const OUTPUT_CLAMP_MARGIN_PCT = 0.05;
const OUTPUT_CLAMP_MIN_MARGIN = 2048;
const OUTPUT_CLAMP_FLOOR = 1024;
Expand Down
37 changes: 23 additions & 14 deletions src/server/handle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ import { resolveClaudeTranscriptLineage } from "../claude-transcript-lineage.js"
import { consumePluginRegisterFor, pluginAgentHeader, pluginConversationHeader, pluginHeadersMatchModel, pluginReportedContextWindow, pluginReportedMaxOutput, pluginReportedModel, pluginRequestAgentHeader, pluginRuntimeInfoFor, pluginRuntimeInfoForConversation, publicForkInputCheck, recordChainVerdict, recordPluginSession, rememberPluginMessages, resolveConversation, runtimeConversationId, takePendingPluginRegister } from "../plugin.js";
import { scheduleAffinityPersist } from "../affinity-persist.js";
import { evaluateChain, extractChainCarriers } from "../chain-checkpoint.js";
import { BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, isLoopbackAddress, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, type WireProtocol } from "../util.js";
import { BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, isLoopbackAddress, normalizeUpstreamOrigin, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, type WireProtocol } from "../util.js";
import { checkTunnelDestination, tunnelAllowlistFromEnv } from "../tunnel-guard.js";
import { resolveCompatDropFields } from "../compat-drop.js";
import { DecompressedTooLargeError, decodeRequestBody } from "../content-encoding.js";
Expand Down Expand Up @@ -1996,7 +1996,7 @@ async function handleRequest(
// model's real window. overflowArmTokens is set ONLY by an upstream
// context-overflow 400 and cleared by the next real usage report.
const armedForGuard = typeof session.stats.overflowArmTokens === "number" && session.stats.overflowArmTokens > 0 ? session.stats.overflowArmTokens : 0;
const guard = sideRequestGuard(parsed, protocol, reqConfig.modelContextLimit, imageBillingFor(opts, route?.rewrittenUrl ?? upstreamOrigin), imageTokenCapFor(opts, route?.rewrittenUrl ?? upstreamOrigin), headroomCap, armedForGuard, imageReserveFor(session, protocol, parsed, opts, route?.rewrittenUrl ?? upstreamOrigin));
const guard = sideRequestGuard(parsed, protocol, reqConfig.modelContextLimit, imageBillingFor(opts, route?.rewrittenUrl ?? upstreamOrigin), imageTokenCapFor(opts, route?.rewrittenUrl ?? upstreamOrigin), headroomCap, armedForGuard, imageReserveFor(session, protocol, parsed, opts, route?.rewrittenUrl ?? upstreamOrigin), shouldReserveOutputHeadroom(protocol, normalizeUpstreamOrigin(route?.rewrittenUrl ?? upstreamOrigin), requestModel));
if (guard.blocked) {
log("warn", `[${session.id}] side request (~${guard.estimate} tokens) ≥ effective window ${guard.limit} (model=${reqModel ?? "?"}) — NOT forwarded: guaranteed upstream 400 (side requests bypass preflight by design, #388)`);
if (!res.headersSent && !res.writableEnded && !res.destroyed) {
Expand Down Expand Up @@ -2062,11 +2062,14 @@ async function handleRequest(
// kernel's nudge/truncate bands sit below (window - reserved) and a
// context+output overflow can't happen on a small window (e.g. 100k with a
// large max_tokens — the most common "context blew up" cause; none of the
// three layers reserved room for the output before this). Anthropic is
// exempt: its input limit is enforced independently of max_tokens
// (separate output budget), so reserving would shift every band down by
// maxOutput on every session for no safety gain — see
// shouldReserveOutputHeadroom. The request's own budget field is the exact
// three layers reserved room for the output before this). #4205: EVERY
// wire reserves unless the route carries positive evidence of a separate
// output budget (real-Claude routing — api.anthropic.com or a claude-*
// model id; see shouldReserveOutputHeadroom / hasSeparateOutputBudgetEvidence).
// The old blanket anthropic exemption is what let wrapper providers on
// that wire (DeepSeek /anthropic bills completion against the same
// window) 400 at the wall (#3900); proven-share learning (#3973) composes
// here later. The request's own budget field is the exact
// output budget requested for THIS turn, so it is precise and per-request;
// when the harness omits every budget field (#924 fallback below) the
// model's declared max output stands in for it.
Expand All @@ -2079,7 +2082,8 @@ async function handleRequest(
// self-heal above handles the resulting overflow. Feeds reqConfig (→
// processTurn `config`), so diagNudge shows the reserved window (no extra log).
const nativeWindow = reqConfig.modelContextLimit;
if (shouldReserveOutputHeadroom(protocol)) {
const headroomOrigin = normalizeUpstreamOrigin(route?.rewrittenUrl ?? upstreamOrigin);
if (shouldReserveOutputHeadroom(protocol, headroomOrigin, requestModel)) {
const p = parsed as Record<string, unknown>;
const rawMax = p.max_tokens ?? p.max_completion_tokens ?? p.max_output_tokens;
let maxOutput = typeof rawMax === "number" ? rawMax : 0;
Expand Down Expand Up @@ -2188,12 +2192,17 @@ async function handleRequest(
}
}
}
// Record the FINAL effective window (post self-heal + output-headroom)
// so the status panel / acp_status show the window the kernel is actually
// using, in every mode (plugin AND wire). #393: previously this was set
// pre-self-heal and only for plugin sessions, so wire-mode panels fell
// back to a hardcoded 200K.
session.metadata.effectiveContextLimit = reqConfig.modelContextLimit;
// #4205 caliber: status surfaces record the UNRESERVED window (post
// self-heal, pre output-headroom) — the wall the client/upstream
// declares, stable across turns regardless of per-request max_tokens.
// The reserved judging window (what nudge/truncate/preflight actually
// judge against) rides reqConfig into the kernel AND the per-request
// twin below: request-scoped consumers read the twin instead (#3738's
// arm clamp must clamp to the limit the kernel judges, not display N).
// #393: both were previously one value set post-self-heal only for
// plugin sessions, so wire-mode panels fell back to a hardcoded 200K.
session.metadata.effectiveContextLimit = nativeWindow;
session.metadata.reservedContextLimit = reqConfig.modelContextLimit;
// #955 runtime-info: record the model id + window source for this
// session so /__bili/plugin/status can show them pre-first-request and
// post-hoc forensics can tell which source sized the window.
Expand Down
39 changes: 26 additions & 13 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -593,6 +593,10 @@ export async function forward(
// No LocalRefusalRecord here: the doomed body itself re-trips this fuse on
// every retry, so a record would only outlive the turn and lock the route
// the way #3974 suffered.
// #4205: the window below is the UNRESERVED wall (effectiveContextLimit
// records N) — the bar sizes against the actual upstream limit; bodies
// between the reserved band and the wall ride forward→400→refold rescue
// instead of an instant 413 (slightly looser than the pre-#4205 R-sized bar).
if (prepared && !prepared.sidePassthrough && req.method !== "GET" && req.method !== "HEAD") {
const declaredWindow = typeof prepared.session.metadata.effectiveContextLimit === "number" && prepared.session.metadata.effectiveContextLimit > 0
? prepared.session.metadata.effectiveContextLimit
Expand Down Expand Up @@ -1069,11 +1073,12 @@ export async function forward(
//
// #3738: the armed value must be ≤ the limit the kernel actually
// JUDGES against, or the arm can never trigger the fold it
// exists to cause. The kernel compares the payload to
// `session.metadata.effectiveContextLimit` — the
// output-headroom-RESERVED window (handle.ts:2137
// `reserveOutputHeadroom`), e.g. 1,048,576 - 256,000 = 792,576 —
// while `info.window` is the RAW stated window (1,048,576). A
// exists to cause. The kernel judges against reqConfig's
// output-headroom-RESERVED window, recorded per-request as
// `session.metadata.reservedContextLimit` (#4205 split the display
// off into the unreserved `effectiveContextLimit`), e.g. 1,048,576
// - 256,000 = 792,576 — while `info.window` is the RAW stated
// window (1,048,576). A
// live limb hit exactly this: upstream 400 "maximum context
// length is 1048576 ... you requested 1097304 (841304 in the
// messages, 256000 in the completion)", the arm sat at 1,048,576
Expand All @@ -1092,9 +1097,9 @@ export async function forward(
// (a stated count exceeds the reserved limit exactly when the
// completion share pushed the total over the window).
const measured = typeof info.requestedMessages === "number" && info.requestedMessages > 0 ? info.requestedMessages : undefined;
const judgingLimit = typeof s.metadata.effectiveContextLimit === "number" && s.metadata.effectiveContextLimit > 0
? s.metadata.effectiveContextLimit
: 0;
const judgingLimit = typeof s.metadata.reservedContextLimit === "number" && s.metadata.reservedContextLimit > 0
? s.metadata.reservedContextLimit
: typeof s.metadata.effectiveContextLimit === "number" && s.metadata.effectiveContextLimit > 0 ? s.metadata.effectiveContextLimit : 0;
const unclampedBaseline = measured ?? info.window;
const armedBaseline = judgingLimit > 0 && judgingLimit < unclampedBaseline ? judgingLimit : unclampedBaseline;
s.stats.lastInputTokens = armedBaseline;
Expand All @@ -1118,11 +1123,13 @@ export async function forward(
const reserve = typeof info.requestedCompletion === "number" && info.requestedCompletion > 0 ? info.requestedCompletion : undefined;
if (reserve !== undefined) s.stats.overflowArmCompletionReserve = reserve;
// #3450: one-time mismatch warning — the upstream-stated window
// differs from the window bili actually judges on.
// differs from the window bili actually judges on. #4205: compare
// against the UNRESERVED declaration, so routes that reserve output
// headroom no longer warn on every arm just because R < N.
const declaredWindow = typeof s.metadata.effectiveContextLimit === "number" ? s.metadata.effectiveContextLimit : 0;
if (declaredWindow > 0 && declaredWindow !== info.window && !s.stats.overflowArmMismatchWarned) {
s.stats.overflowArmMismatchWarned = true;
log("warn", `[${s.id}] upstream-stated window ${info.window} differs from the window in use (${declaredWindow}, model=${reqModel ?? "unknown"}) — folding targets min of the two; set compress.modelContextLimit if the declaration is wrong (#3450)`);
log("warn", `[${s.id}] upstream-stated window ${info.window} differs from the declared window in use (${declaredWindow}, model=${reqModel ?? "unknown"}) — folding targets the smallest of the stated window, the declared window and the per-route output reservation; set compress.modelContextLimit if the declaration is wrong (#3450)`);
}
log("warn", `[${s.id}] upstream context overflow (model=${reqModel ?? "unknown"}) — window ${info.window} stated upstream${measured !== undefined ? `, rejected payload ~${measured} tokens stated` : ""}${reserve !== undefined ? `, completion reservation ${reserve} tokens` : ""}; armed emergency shrink at ${armedBaseline}${armedBaseline < unclampedBaseline ? ` (clamped to the judging limit ${judgingLimit}, #3738)` : ""}, declared window unchanged (#987)`);
// #3900/#4079: the stated count is a usage-grade measurement of
Expand All @@ -1146,13 +1153,19 @@ export async function forward(
// declared window arms at the declaration — which is what
// the #496 image-relay forward-once gate needs to break the
// #488 400 loop after exactly one rejected forward.
const declared = typeof s.metadata.effectiveContextLimit === "number" ? s.metadata.effectiveContextLimit : 0;
// #4205: arm against the JUDGING (reserved) window so the #3738
// invariant holds here too; pre-#4205 persisted sessions carry that
// value in effectiveContextLimit (the field then held R), hence the
// fallback.
const judging = typeof s.metadata.reservedContextLimit === "number" && s.metadata.reservedContextLimit > 0
? s.metadata.reservedContextLimit
: typeof s.metadata.effectiveContextLimit === "number" ? s.metadata.effectiveContextLimit : 0;
let est = 0;
try {
// #1492: same CJK-aware lower bound as armFailureShrink.
est = defaultCountTokens(rawBody ?? "");
} catch { est = 0; }
const arm = Math.max(0, Math.min(declared, Number.isFinite(est) ? est : declared));
const arm = Math.max(0, Math.min(judging, Number.isFinite(est) ? est : judging));
if (arm > 0) {
s.stats.lastInputTokens = arm;
// #1839: an estimate promoted to "usage" would enter the nudge
Expand All @@ -1162,7 +1175,7 @@ export async function forward(
s.stats.overflowArmTokens = arm; // #1110: guard reads this, not the baseline
s.stats.lastInputTokensOrigin = normalizeUpstreamOrigin(upstreamUrl); // #2313: same-route evidence, see the window branch above
}
log("warn", `[${s.id}] upstream context overflow (window not parseable, model=${reqModel ?? "unknown"}) — armed emergency shrink at ~${arm} tokens (min of declared ${declared} and payload estimate), nothing learned (#987): ${info.message}`);
log("warn", `[${s.id}] upstream context overflow (window not parseable, model=${reqModel ?? "unknown"}) — armed emergency shrink at ~${arm} tokens (min of judging window ${judging} and payload estimate), nothing learned (#987): ${info.message}`);
}
// The armed emergency (lastInputTokens) lives in memory only
// until scheduled — the error path returns before forward()'s
Expand Down
Loading
Loading