Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,10 @@ interface PersistedSession {
overflowArmTokens?: number;
overflowArmCompletionReserve?: number;
overflowArmMismatchWarned?: boolean;
// #3450 follow-up: provider-lifetime completion proof + its scope.
provenCompletionWall?: number;
provenCompletionWallOrigin?: string;
provenCompletionWallModel?: string;
contextTokens?: number;
contextTokensSource?: string;
// #2129: measurement-state fields buildRecord persists via spread —
Expand Down Expand Up @@ -2787,6 +2791,10 @@ function buildSession(parsed: PersistedSession): Session {
// #3450: proven completion reservation + one-time warn flag — legacy files lack both.
overflowArmCompletionReserve: typeof stats.overflowArmCompletionReserve === "number" && Number.isFinite(stats.overflowArmCompletionReserve) && stats.overflowArmCompletionReserve > 0 ? stats.overflowArmCompletionReserve : undefined,
overflowArmMismatchWarned: stats.overflowArmMismatchWarned === true ? true : undefined,
// #3450 follow-up: the proof outlives the arm, so a restart must not forget it; absent = no proof.
provenCompletionWall: typeof stats.provenCompletionWall === "number" && Number.isFinite(stats.provenCompletionWall) && stats.provenCompletionWall > 0 ? stats.provenCompletionWall : undefined,
provenCompletionWallOrigin: typeof stats.provenCompletionWallOrigin === "string" && stats.provenCompletionWallOrigin.length > 0 ? stats.provenCompletionWallOrigin : undefined,
provenCompletionWallModel: typeof stats.provenCompletionWallModel === "string" && stats.provenCompletionWallModel.length > 0 ? stats.provenCompletionWallModel : undefined,
// In-memory only — a fresh process has no pending compress fold.
compressCreditTokens: 0,
contextTokens: Math.max(0, stats.contextTokens ?? parsed.contextTokens ?? 0),
Expand Down
34 changes: 32 additions & 2 deletions src/server/handle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ import { resolveClaudeTranscriptLineage } from "../claude-transcript-lineage.js"
import { consumePluginRegisterFor, pluginAgentHeader, pluginConversationHeader, pluginHeadersMatchModel, pluginReportedContextWindow, pluginReportedMaxOutput, pluginReportedModel, pluginRequestAgentHeader, pluginRuntimeInfoFor, pluginRuntimeInfoForConversation, publicForkInputCheck, recordChainVerdict, recordPluginSession, rememberPluginMessages, resolveConversation, runtimeConversationId, takePendingPluginRegister } from "../plugin.js";
import { scheduleAffinityPersist } from "../affinity-persist.js";
import { evaluateChain, extractChainCarriers } from "../chain-checkpoint.js";
import { BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, isLoopbackAddress, normalizeUpstreamOrigin, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, type WireProtocol } from "../util.js";
import { BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, isLoopbackAddress, normalizeUpstreamOrigin, provenCompletionReserve, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, type WireProtocol } from "../util.js";
import { checkTunnelDestination, tunnelAllowlistFromEnv } from "../tunnel-guard.js";
import { resolveCompatDropFields } from "../compat-drop.js";
import { DecompressedTooLargeError, decodeRequestBody } from "../content-encoding.js";
Expand Down Expand Up @@ -2083,7 +2083,19 @@ async function handleRequest(
// processTurn `config`), so diagNudge shows the reserved window (no extra log).
const nativeWindow = reqConfig.modelContextLimit;
const headroomOrigin = normalizeUpstreamOrigin(route?.rewrittenUrl ?? upstreamOrigin);
if (shouldReserveOutputHeadroom(protocol, headroomOrigin, requestModel)) {
// #3450 follow-up (#3973): a 400 that stated a completion share disproves
// any separate-wall evidence for this origin+model — judging input against
// the full window would re-earn the same 400 every turn (#3738: nudge and
// preflight must judge on a number the upstream will accept). #4205 made
// the reservation the default for every wire without separate-budget
// evidence, so the proven share now forces the reservation where that
// evidence exists and clamps the resulting window below (see provenWindow).
const provenReserve = provenCompletionReserve(
session.stats,
headroomOrigin,
(parsed as { model?: string }).model,
);
if (shouldReserveOutputHeadroom(protocol, headroomOrigin, requestModel) || provenReserve > 0) {
const p = parsed as Record<string, unknown>;
const rawMax = p.max_tokens ?? p.max_completion_tokens ?? p.max_output_tokens;
let maxOutput = typeof rawMax === "number" ? rawMax : 0;
Expand Down Expand Up @@ -2157,6 +2169,17 @@ async function handleRequest(
}
}
}
if (provenReserve > 0 && !shouldReserveOutputHeadroom(protocol, headroomOrigin, requestModel)) {
// Proven share outranks the client-side fallbacks: it is what
// the upstream already counted into a rejection on this exact
// origin+model, so window − proven share is the input it accepts.
if (provenReserve > maxOutput) maxOutput = provenReserve;
const proofModel = (parsed as { model?: string }).model ?? "?";
if (!headroomFallbackLogged.has(`${proofModel}|proven-completion`)) {
headroomFallbackLogged.add(`${proofModel}|proven-completion`);
log("info", `[headroom] model=${proofModel}: reserving the PROVEN completion share ${provenReserve} although protocol ${protocol ?? "unknown"} is exempt (#3450) — this origin+model billed completion inside the wall in an earlier rejection; judging input against ${nativeWindow} - ${provenReserve}`);
}
}
// #2283 sub-defect 1: emit [window-change] when the per-session resolution
// triple (model/source/base) moves mid-session. Placed pre output-headroom so
// #2096's per-request max_tokens reservation can't read as a base change.
Expand All @@ -2178,6 +2201,13 @@ async function handleRequest(
log("info", `[${session.id}] fallback context window floored: ${reserved} → ${FALLBACK_EFFECTIVE_WINDOW_FLOOR} (model=${String(requestModel ?? "?")} not authoritatively identified; self-heal corrects it if the real window is smaller)`);
reserved = FALLBACK_EFFECTIVE_WINDOW_FLOOR;
}
// The #896 cap bounds GUESSED budgets; a proven share is measured —
// the upstream already counted it in a rejection on this origin+model —
// so the window may never stay above (declared window − proven share).
if (provenReserve > 0 && nativeWindow > provenReserve) {
const provenWindow = nativeWindow - provenReserve;
if (provenWindow < reserved) reserved = provenWindow;
}
if (reserved !== reqConfig.modelContextLimit) {
reqConfig = { ...reqConfig, modelContextLimit: reserved };
// #2096: the [window] line above already went out with the
Expand Down
13 changes: 12 additions & 1 deletion src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1121,7 +1121,18 @@ export async function forward(
// against the same wall — remember it so preflight folds against
// window − completion (the foldable quantity is MESSAGES).
const reserve = typeof info.requestedCompletion === "number" && info.requestedCompletion > 0 ? info.requestedCompletion : undefined;
if (reserve !== undefined) s.stats.overflowArmCompletionReserve = reserve;
if (reserve !== undefined) {
s.stats.overflowArmCompletionReserve = reserve;
// #3450 follow-up: same evidence on the provider's lifetime —
// the arm retires on the next usage report (#1110), this must
// not (see provenCompletionReserve). Scoped to the origin+model.
const provenOrigin = normalizeUpstreamOrigin(upstreamUrl);
if (provenOrigin !== undefined && reqModel !== undefined) {
s.stats.provenCompletionWall = reserve;
s.stats.provenCompletionWallOrigin = provenOrigin;
s.stats.provenCompletionWallModel = reqModel;
}
}
// #3450: one-time mismatch warning — the upstream-stated window
// differs from the window bili actually judges on. #4205: compare
// against the UNRESERVED declaration, so routes that reserve output
Expand Down
11 changes: 11 additions & 0 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,17 @@ export type Session = {
/** #3450: one-time mismatch warning already emitted for this session
* (upstream-stated window ≠ window in use). Provenance only. */
overflowArmMismatchWarned?: boolean;
/** #3450 follow-up: same completion-share proof as
* overflowArmCompletionReserve, on a PROVIDER lifetime — the arm retires
* on the next usage report (#1110), "this endpoint bills max_tokens
* inside the window" does not stop being true when one turn recovers.
* Scoped to the origin+model that stated it; read via
* provenCompletionReserve(). Absent/zero → no proof. */
provenCompletionWall?: number;
/** normalizeUpstreamOrigin() of the upstream that stated the proof. */
provenCompletionWallOrigin?: string;
/** Model on the rejected request that stated the proof. */
provenCompletionWallModel?: string;
/** Tokens compressed THIS turn whose fold has not yet materialized in
* an upstream usage report (the post-compress re-request re-sends the
* UNFOLDED history for prefix-cache reasons, so its usage report
Expand Down
28 changes: 28 additions & 0 deletions src/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,34 @@ export function shouldReserveOutputHeadroom(protocol: string | undefined, origin
return true;
}

/**
* The completion share an upstream PROVED it bills inside the context wall for
* one (origin, model) pair, or 0 when this request cannot claim that proof.
*
* #3450 learns the share from a rejected request but books it on the one-shot
* overflow arm, which retires on the next usage report (#1110) — the wrong
* lifetime for "this endpoint counts max_tokens inside the window", a PROVIDER
* fact that survives the fold which recovered the turn. Dropping it re-judges
* every later turn on the declared window, so an anthropic-wire host that bills
* completion inside the wall (DeepSeek's /anthropic: 1048576 stated,
* "requested 1370540 (1114540 in the messages, 256000 in the completion)")
* over-fills, 400s, and pays a new emergency fold each time.
*
* Scoped: a route or model change returns 0 and falls back to the plain
* protocol rule, never silently reserving on an unrelated upstream.
*/
export function provenCompletionReserve(
stats: { provenCompletionWall?: number; provenCompletionWallOrigin?: string; provenCompletionWallModel?: string } | undefined,
origin: string | undefined,
model: string | undefined,
): number {
const wall = stats?.provenCompletionWall;
if (typeof wall !== "number" || !Number.isFinite(wall) || wall <= 0) return 0;
if (origin === undefined || origin === "" || stats?.provenCompletionWallOrigin !== origin) return 0;
if (model === undefined || model === "" || stats?.provenCompletionWallModel !== model) return 0;
return wall;
}

/** System-carrier texts of a parsed request body, covering all four wire shapes:
* anthropic/openai-hoisted top-level `system` (string or text-block array),
* responses `instructions`, google `systemInstruction.parts`, and system/developer
Expand Down
31 changes: 30 additions & 1 deletion tests/context-overflow.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import test from "node:test";
import assert from "node:assert/strict";
import { inspectContextOverflow, reserveOutputHeadroom, shouldReserveOutputHeadroom } from "../src/util.ts";
import { inspectContextOverflow, provenCompletionReserve, reserveOutputHeadroom, shouldReserveOutputHeadroom } from "../src/util.ts";

// A context-overflow error is the only reliable signal that the configured
// window is wrong (e.g. the 200k fallback for an unknown model on a relay).
Expand Down Expand Up @@ -346,3 +346,32 @@ test("shouldReserveOutputHeadroom: reserve-by-default, real-Claude routing exemp
assert.equal(shouldReserveOutputHeadroom("anthropic", "http://127.0.0.1:8787", "DeepSeek-V3.2-Exp"), true, "#3900 shape: deepseek model on the anthropic wire → reserve");
assert.equal(shouldReserveOutputHeadroom("anthropic", "https://api.anthropic.com.evil.example"), true, "host match is exact — no suffix tricks");
});

// #3450 follow-up: a stated completion share is a PROVIDER fact, so it outlives
// the one-shot arm — but only for the origin+model that stated it.
const PROVEN = { provenCompletionWall: 7_000, provenCompletionWallOrigin: "http://127.0.0.1:9", provenCompletionWallModel: "deepseek-flash" };

test("provenCompletionReserve: matching origin+model returns the proven share", () => {
assert.equal(provenCompletionReserve(PROVEN, "http://127.0.0.1:9", "deepseek-flash"), 7_000);
});

test("provenCompletionReserve: no proof / incomplete proof → 0 (protocol rule unchanged)", () => {
assert.equal(provenCompletionReserve(undefined, "http://127.0.0.1:9", "deepseek-flash"), 0);
assert.equal(provenCompletionReserve({}, "http://127.0.0.1:9", "deepseek-flash"), 0);
assert.equal(provenCompletionReserve({ provenCompletionWallOrigin: PROVEN.provenCompletionWallOrigin, provenCompletionWallModel: PROVEN.provenCompletionWallModel }, "http://127.0.0.1:9", "deepseek-flash"), 0);
});

test("provenCompletionReserve: a different origin or model cannot claim the proof", () => {
assert.equal(provenCompletionReserve(PROVEN, "http://127.0.0.1:10", "deepseek-flash"), 0, "different upstream");
assert.equal(provenCompletionReserve(PROVEN, "http://127.0.0.1:9", "deepseek-chat"), 0, "different model");
assert.equal(provenCompletionReserve(PROVEN, undefined, "deepseek-flash"), 0, "unknown origin");
assert.equal(provenCompletionReserve(PROVEN, "http://127.0.0.1:9", undefined), 0, "unknown model");
assert.equal(provenCompletionReserve(PROVEN, "", "deepseek-flash"), 0, "empty origin");
assert.equal(provenCompletionReserve(PROVEN, "http://127.0.0.1:9", ""), 0, "empty model");
});

test("provenCompletionReserve: corrupt walls are rejected outright", () => {
for (const wall of [0, -1, Number.NaN, Number.POSITIVE_INFINITY, "7000"]) {
assert.equal(provenCompletionReserve({ ...PROVEN, provenCompletionWall: wall as number }, "http://127.0.0.1:9", "deepseek-flash"), 0, `wall=${String(wall)}`);
}
});
11 changes: 9 additions & 2 deletions tests/e2e-issue3900-overflow-teaches.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,16 @@ test("e2e #3900: two overflow 400s with stated counts publish k̂ — the estima
// pinned directly in the unit test; here the refold hook's re-prepare
// legitimately notes a fresh pair afterwards, so lastLocalTextEstimate
// may hold a new value by the time we look.)
// The arm keeps its #1110/#3738/#3450 semantics untouched.
// The arm keeps its #1110/#3738/#3450 semantics untouched: the
// baseline is still min(stated message tokens, the judging limit).
// Only the judging limit moves: this 400 proved the upstream bills a
// 5_000-token completion share against the 30_000 wall (#3900), so
// handle.ts rewrites effectiveContextLimit to 25_000 and the clamp
// lands there - the window the session can actually use. The clamp
// RULE is what this assertion pins; the stated ceiling is unchanged
// by it (asserted right below).
assert.equal(st.lastInputTokensSource, "overflow-arm");
assert.equal(st.lastInputTokens, 30_000, "armed at min(stated messages 40k, judging limit 30k) — #3738 clamp");
assert.equal(st.lastInputTokens, 25_000, "armed at min(stated messages 40k, proven judging limit 25k = 30k - 5k completion share) - #3738 clamp");
assert.equal(st.overflowArmTokens, 30_000, "guard keeps the stated ceiling");
assert.equal(st.overflowArmCompletionReserve, 5_000, "the completion share stays reserved (#3450)");
} finally {
Expand Down
Loading
Loading