Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions reference/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,7 @@ billion-context/
│ │ ├── media-bytes.ts # Shared inline-media byte accounting for native direct-send refusal gates
│ │ ├── native-bootstrap.ts # dist/agent/<entry>.js → dist/index.js resolution
│ │ ├── native-intercept.ts # Model-API fetch routing, attribution and proxy recovery
│ │ ├── prompt-mass.ts # Protocol-shaped prompt-mass pricing for the client-local doom gate (#7106)
│ │ ├── fetch-chain/
│ │ │ └── index.ts # Fetch guard, re-entry cutoff and pi-web-access coexistence
│ │ ├── native-ws-intercept.ts # WebSocket-companion intercept: Codex Responses upgrades → /bili/ lane (#2111)
Expand Down
45 changes: 37 additions & 8 deletions src/agent/native-intercept.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { logHostEvent } from "./host-log.js";
import { envMillis } from "./native-bootstrap.js";
import { matchModelEndpoint, type ModelEndpointPattern } from "../model-endpoints.js";
import { mediaBytesOfWireBody } from "./media-bytes.js";
import { promptMassOfWireBody } from "./prompt-mass.js";
import * as pfChannel from "./pf-channel.js";
import { probeLiveness, type LivenessVerdict } from "../liveness.js";
import { BILI_PASSTHROUGH_HEADER, CALIBRATED_TEXT_TOKEN_RATIO, MIN_ASSUMED_WINDOW_TOKENS } from "../util.js";
Expand Down Expand Up @@ -829,7 +830,7 @@ async function streamBytesCapped(stream: ReadableStream<Uint8Array>, cap: number
* restart (session state is persisted on disk), not another attempt (#3589). */
function oversizeRefusal(url: string): Response {
const message =
"billion-context refused this request locally: the host's attribution lookup threw mid-request (its agent scope is disposed/closing), and this body is priced over the model's context window under the calibrated size check (conservative text pricing — every declared byte counts as text, and the bar is the larger of the client-declared window and 1,000,000 tokens) — forwarding it uncompressed would be rejected upstream (#2924). " +
"billion-context refused this request locally: the host's attribution lookup threw mid-request (its agent scope is disposed/closing), and this body is priced over the model's context window under the calibrated size check (conservative text pricing — every declared byte of prompt mass counts as text (transport-only fields are excluded when the body parses, #7106), and the bar is the larger of the client-declared window and 1,000,000 tokens) — forwarding it uncompressed would be rejected upstream (#2924). " +
"While that scope stays dead, retrying this same call will keep failing; restart the client host once (its session state is persisted on disk, so nothing is lost) so the oversized history can be folded back under the window. If you cannot restart, start a fresh session instead. If the model's real window is larger than the client host declared, fix the host's model declaration and restart — the direct path reopens.";
const headers = { "content-type": "application/json", "x-bili-gate": "oversize-refusal" };
if (url.endsWith("/messages")) {
Expand All @@ -849,7 +850,7 @@ function oversizeRefusal(url: string): Response {
* (there is no broken scope to restart). */
function unattributedOversizeRefusal(url: string): Response {
const message =
"billion-context refused this request locally: it is not attributed to any bili-managed session, and its core text is priced over the model's context window (conservative text pricing — every declared byte counts as text, and the bar is never assumed below the larger of the client-declared window and 1,000,000 tokens) — sending it directly to the provider can only be rejected (every retry re-sends the same oversized history, #4068). " +
"billion-context refused this request locally: it is not attributed to any bili-managed session, and its core text is priced over the model's context window (conservative text pricing — every declared byte of prompt mass counts as text (transport-only fields are excluded when the body parses, #7106), and the bar is never assumed below the larger of the client-declared window and 1,000,000 tokens) — sending it directly to the provider can only be rejected (every retry re-sends the same oversized history, #4068). " +
"If the model's real window is larger than the client host declared, fix the host's model declaration and restart — the direct path reopens. Otherwise, route the conversation through a bili-managed session so it can be compressed (check the client's bili plugin attribution), or trim/remove the oversized content (e.g. a giant paste or tool result), or start a fresh session.";
const headers = { "content-type": "application/json", "x-bili-gate": "oversize-refusal-unattributed" };
if (url.endsWith("/messages")) {
Expand Down Expand Up @@ -905,7 +906,9 @@ function emptySlotOversizeRefusal(
: `against the ${doom.window.toLocaleString("en-US")} token local bar (${doom.declaredWindow.toLocaleString("en-US")} resolved by the active lane; the bar never assumes below ${MIN_ASSUMED_WINDOW_TOKENS.toLocaleString("en-US")}, #4079)`;
const basis = doom === undefined
? "this model request is larger than the silent-direct byte ceiling"
: `this model request's core text prices at ~${doom.estTokens.toLocaleString("en-US")} tokens (optimistic floor ${doom.floorTokens.toLocaleString("en-US")}) ${barNote}`;
: doom.promptTokens === undefined
? `this model request's core text prices at ~${doom.estTokens.toLocaleString("en-US")} tokens (optimistic floor ${doom.floorTokens.toLocaleString("en-US")}) ${barNote}`
: `this model request's prompt prices at ~${doom.estTokens.toLocaleString("en-US")} tokens (optimistic floor ${doom.floorTokens.toLocaleString("en-US")}; transport-only fields excluded on the ${doom.protocol ?? "detected"} lane, #7106) ${barNote}`;
const message =
"billion-context refused this request locally: the native interceptor's live-state slot is empty (the plugin is between activations or was deactivated), and " + basis + " — forwarding it would send the prompt to the provider uncompressed (#5405/#5996). " +
"The slot normally rebinds on the next plugin activation; retry shortly. If it persists, restart the host so the interceptor can rebind.";
Expand Down Expand Up @@ -958,18 +961,44 @@ export function bodyOverWindowDetail(
bodyBytes: number | undefined,
rawBody: string | undefined,
declaredWindow: number | undefined,
): { coreBytes: number; mediaBytes: number; floorTokens: number; estTokens: number; provablyOver: boolean; window: number; declaredWindow: number } | undefined {
): { coreBytes: number; mediaBytes: number; floorTokens: number; estTokens: number; provablyOver: boolean; window: number; declaredWindow: number; promptTokens?: number; protocol?: string } | undefined {
if (bodyBytes === undefined || declaredWindow === undefined) return undefined;
const window = Math.max(declaredWindow, MIN_ASSUMED_WINDOW_TOKENS);
// Worst case: zero media credits → core = full mass. If even THAT fits
// the calibrated bar, no raw body can refuse — skip the JSON walk.
if (Math.ceil(Math.ceil(bodyBytes / 4) * CALIBRATED_TEXT_TOKEN_RATIO) <= window) return undefined;
// #7106: bytes/3 bounds CJK-dense text (1 token per 3-byte char) from above;
// bytes/4 is only the ASCII bound and would skip this check for CJK bodies.
if (Math.ceil((bodyBytes / 3) * CALIBRATED_TEXT_TOKEN_RATIO) <= window) return undefined;
const mediaBytes = rawBody === undefined ? 0 : mediaBytesOfWireBody(rawBody);
const coreBytes = Math.max(bodyBytes - mediaBytes, 0);
const floorTokens = Math.ceil(coreBytes / 4);
const estTokens = Math.ceil(floorTokens * CALIBRATED_TEXT_TOKEN_RATIO);
// #7106: when the body parses, price the PROMPT mass instead of the whole
// wire body. DSH stamps transport-only top-level fields (dsh_session_log up
// to 8 MiB, dsh_plugin_packages) that never reach the model, so whole-body
// byte pricing refuses healthy in-window requests before the provider ever
// sees them. Byte pricing stays the fallback for bodies we cannot price by
// shape (non-JSON, or no recognizable prompt key).
const promptMass = promptMassOfWireBody(rawBody);
const floorTokens = promptMass === undefined
? Math.ceil(coreBytes / 4)
: promptMass.promptTokens; // media stays excluded, as the byte math has since #4111
// A shape-priced prompt is a direct token count (CJK-aware, same walker as the
// server gates), not a byte heuristic: compare it to the window as-is. Only the
// byte fallback keeps the 1.14 calibration bias.
// #4111 two-tier contract unchanged: the calibrated tier is floor x 1.14 and
// the bare floor is the provably-over tier — #7106 only changes what the
// floor PRICES (prompt mass when the body parses, whole-body bytes otherwise).
// #7106 review: apply the 1.14 calibration ONLY to the byte heuristic. A
// shape-priced prompt is already a token count, so multiplying it would
// double-calibrate and refuse at ~0.877 x window (the wrong direction for
// the false-refusal this issue fixes).
const estTokens = promptMass === undefined
? Math.ceil(floorTokens * CALIBRATED_TEXT_TOKEN_RATIO)
: floorTokens;
if (estTokens <= window) return undefined;
return { coreBytes, mediaBytes, floorTokens, estTokens, provablyOver: floorTokens > window, window, declaredWindow };
return {
coreBytes, mediaBytes, floorTokens, estTokens, provablyOver: floorTokens > window, window, declaredWindow,
...(promptMass === undefined ? {} : { promptTokens: promptMass.promptTokens, protocol: promptMass.protocol }),
};
}

/** The model context window every plugin lane stamps onto its model requests
Expand Down
76 changes: 76 additions & 0 deletions src/agent/prompt-mass.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/** #7106: price the PROMPT mass of a wire body, not the whole body.
*
* The client-local doom gate (#2924/#4068/#4111/#5405/#5996) estimates tokens
* from raw body BYTES: `ceil(bytes/4) * 1.14`, minus image bytes. That is a
* sound upper bound for a plain model request, but DSH requests carry
* transport-only top-level fields that never reach the model:
*
* dsh_session_log incremental session-log contribution (up to 8 MiB)
* dsh_plugin_packages plugin inventory blob
*
* Pricing them as prompt mass makes a healthy in-window request look like a
* 2.4M-token body on a 1M-token window, so the local gate refuses it before
* the provider ever sees it (#7090 family, client-side face; #7106).
*
* The client gate has no route config and no protocol header, but the body
* shape is unambiguous: google carries `contents`, responses `input`,
* anthropic `system`, openai `messages`. Once the lane is known, the prompt
* mass is exactly the text under that lane's prompt keys — counted with the
* same CJK-aware walker the server-side preflight uses, so both sides price
* the same bytes. Non-JSON / unrecognized bodies keep the byte estimate.
*/

import { estimateRawBodyTokens } from "../preflight.js";
import type { WireProtocol } from "../util.js";

/** Top-level keys that carry prompt mass, per wire protocol.
* (Same split as the server-side `PROMPT_BODY_KEYS` added by #7092; kept
* local so this client fix applies without depending on that PR landing.) */
const PROMPT_KEYS: Record<WireProtocol, readonly string[]> = {
openai: ["messages", "tools"],
anthropic: ["messages", "tools", "system"],
google: ["contents", "systemInstruction", "tools"],
responses: ["input", "instructions", "tools"],
};

interface WirePromptMass {
/** Lane the body shape was recognized as; only used for reporting. */
protocol: WireProtocol;
/** CJK-aware token mass of the prompt fields (turns + system + tools).
* Inline images are deliberately excluded: the text walk skips image
* keys (media stays uncounted, #4111). */
promptTokens: number;
}

/** Recognize the lane from the body shape. */
function sniffWireProtocolOfBody(body: Record<string, unknown>): WireProtocol | undefined {
if (body.contents !== undefined) return "google";
if (body.input !== undefined) return "responses";
if (body.system !== undefined) return "anthropic";
if (body.messages !== undefined) return "openai";
return undefined;
}

/** Prompt mass of one raw request body, or undefined when it cannot be priced
* from shape (non-JSON, non-object, or no recognizable prompt key). */
export function promptMassOfWireBody(raw: string | undefined): WirePromptMass | undefined {
if (raw === undefined) return undefined;
let body: unknown;
try {
body = JSON.parse(raw);
} catch {
return undefined;
}
if (body === null || typeof body !== "object" || Array.isArray(body)) return undefined;
const record = body as Record<string, unknown>;
const protocol = sniffWireProtocolOfBody(record);
if (protocol === undefined) return undefined;
const promptOnly: Record<string, unknown> = {};
for (const key of PROMPT_KEYS[protocol]) {
if (record[key] !== undefined) promptOnly[key] = record[key];
}
return {
protocol,
promptTokens: estimateRawBodyTokens(promptOnly),
};
}
2 changes: 1 addition & 1 deletion tests/dsh-native.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2417,7 +2417,7 @@ test("#4111 mediaBytesOfWireBody/bodyOverWindowDetail: prices the inline-media s
const mixedDoom = bodyOverWindowDetail({ bytes: mixedBytes }, mixedInit, 1000);
assert.ok(mixedDoom !== undefined, "same media PLUS >4MB of text stays over");
assert.equal(mixedDoom?.provablyOver, true, ">4MB of text: even the optimistic floor alone exceeds the window");
assert.equal(mixedDoom?.estTokens, Math.ceil((mixedDoom?.floorTokens ?? 0) * 1.14), "calibrated tier = floor × CALIBRATED_TEXT_TOKEN_RATIO (src/util.ts)");
assert.equal(mixedDoom?.estTokens, mixedDoom?.floorTokens, "#7106: a shape-priced prompt mass is compared to the window as-is (no 1.14 re-calibration); only the byte fallback keeps the calibration bias");
assert.equal(mixedDoom?.mediaBytes, Buffer.byteLength(bigUrl), "doom numbers carry the excluded media mass");
assert.equal(mixedDoom?.coreBytes, mixedBytes - Buffer.byteLength(bigUrl), "doom numbers carry media-excluded core bytes");
assert.equal(mixedDoom?.window, 1_000_000, "#4079 floor never assumes below the 1M common window");
Expand Down
Loading
Loading