diff --git a/reference/architecture.md b/reference/architecture.md index 88f79aa24d..a305fa1dfc 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -207,6 +207,7 @@ billion-context/ │ │ ├── media-bytes.ts # Shared inline-media byte accounting for native direct-send refusal gates │ │ ├── native-bootstrap.ts # dist/agent/.js → dist/index.js resolution │ │ ├── native-intercept.ts # Model-API fetch routing, attribution and proxy recovery +│ │ ├── prompt-mass.ts # Protocol-shaped prompt-mass pricing for the client-local doom gate (#7106) │ │ ├── fetch-chain/ │ │ │ └── index.ts # Fetch guard, re-entry cutoff and pi-web-access coexistence │ │ ├── native-ws-intercept.ts # WebSocket-companion intercept: Codex Responses upgrades → /bili/ lane (#2111) diff --git a/src/agent/native-intercept.ts b/src/agent/native-intercept.ts index 832ddaaa0a..5bb5b7c6fc 100644 --- a/src/agent/native-intercept.ts +++ b/src/agent/native-intercept.ts @@ -12,6 +12,7 @@ import { logHostEvent } from "./host-log.js"; import { envMillis } from "./native-bootstrap.js"; import { matchModelEndpoint, type ModelEndpointPattern } from "../model-endpoints.js"; import { mediaBytesOfWireBody } from "./media-bytes.js"; +import { promptMassOfWireBody } from "./prompt-mass.js"; import * as pfChannel from "./pf-channel.js"; import { probeLiveness, type LivenessVerdict } from "../liveness.js"; import { BILI_PASSTHROUGH_HEADER, CALIBRATED_TEXT_TOKEN_RATIO, MIN_ASSUMED_WINDOW_TOKENS } from "../util.js"; @@ -829,7 +830,7 @@ async function streamBytesCapped(stream: ReadableStream, cap: number * restart (session state is persisted on disk), not another attempt (#3589). */ function oversizeRefusal(url: string): Response { const message = - "billion-context refused this request locally: the host's attribution lookup threw mid-request (its agent scope is disposed/closing), and this body is priced over the model's context window under the calibrated size check (conservative text pricing — every declared byte counts as text, and the bar is the larger of the client-declared window and 1,000,000 tokens) — forwarding it uncompressed would be rejected upstream (#2924). " + + "billion-context refused this request locally: the host's attribution lookup threw mid-request (its agent scope is disposed/closing), and this body is priced over the model's context window under the calibrated size check (conservative text pricing — every declared byte of prompt mass counts as text (transport-only fields are excluded when the body parses, #7106), and the bar is the larger of the client-declared window and 1,000,000 tokens) — forwarding it uncompressed would be rejected upstream (#2924). " + "While that scope stays dead, retrying this same call will keep failing; restart the client host once (its session state is persisted on disk, so nothing is lost) so the oversized history can be folded back under the window. If you cannot restart, start a fresh session instead. If the model's real window is larger than the client host declared, fix the host's model declaration and restart — the direct path reopens."; const headers = { "content-type": "application/json", "x-bili-gate": "oversize-refusal" }; if (url.endsWith("/messages")) { @@ -849,7 +850,7 @@ function oversizeRefusal(url: string): Response { * (there is no broken scope to restart). */ function unattributedOversizeRefusal(url: string): Response { const message = - "billion-context refused this request locally: it is not attributed to any bili-managed session, and its core text is priced over the model's context window (conservative text pricing — every declared byte counts as text, and the bar is never assumed below the larger of the client-declared window and 1,000,000 tokens) — sending it directly to the provider can only be rejected (every retry re-sends the same oversized history, #4068). " + + "billion-context refused this request locally: it is not attributed to any bili-managed session, and its core text is priced over the model's context window (conservative text pricing — every declared byte of prompt mass counts as text (transport-only fields are excluded when the body parses, #7106), and the bar is never assumed below the larger of the client-declared window and 1,000,000 tokens) — sending it directly to the provider can only be rejected (every retry re-sends the same oversized history, #4068). " + "If the model's real window is larger than the client host declared, fix the host's model declaration and restart — the direct path reopens. Otherwise, route the conversation through a bili-managed session so it can be compressed (check the client's bili plugin attribution), or trim/remove the oversized content (e.g. a giant paste or tool result), or start a fresh session."; const headers = { "content-type": "application/json", "x-bili-gate": "oversize-refusal-unattributed" }; if (url.endsWith("/messages")) { @@ -905,7 +906,9 @@ function emptySlotOversizeRefusal( : `against the ${doom.window.toLocaleString("en-US")} token local bar (${doom.declaredWindow.toLocaleString("en-US")} resolved by the active lane; the bar never assumes below ${MIN_ASSUMED_WINDOW_TOKENS.toLocaleString("en-US")}, #4079)`; const basis = doom === undefined ? "this model request is larger than the silent-direct byte ceiling" - : `this model request's core text prices at ~${doom.estTokens.toLocaleString("en-US")} tokens (optimistic floor ${doom.floorTokens.toLocaleString("en-US")}) ${barNote}`; + : doom.promptTokens === undefined + ? `this model request's core text prices at ~${doom.estTokens.toLocaleString("en-US")} tokens (optimistic floor ${doom.floorTokens.toLocaleString("en-US")}) ${barNote}` + : `this model request's prompt prices at ~${doom.estTokens.toLocaleString("en-US")} tokens (optimistic floor ${doom.floorTokens.toLocaleString("en-US")}; transport-only fields excluded on the ${doom.protocol ?? "detected"} lane, #7106) ${barNote}`; const message = "billion-context refused this request locally: the native interceptor's live-state slot is empty (the plugin is between activations or was deactivated), and " + basis + " — forwarding it would send the prompt to the provider uncompressed (#5405/#5996). " + "The slot normally rebinds on the next plugin activation; retry shortly. If it persists, restart the host so the interceptor can rebind."; @@ -958,18 +961,44 @@ export function bodyOverWindowDetail( bodyBytes: number | undefined, rawBody: string | undefined, declaredWindow: number | undefined, -): { coreBytes: number; mediaBytes: number; floorTokens: number; estTokens: number; provablyOver: boolean; window: number; declaredWindow: number } | undefined { +): { coreBytes: number; mediaBytes: number; floorTokens: number; estTokens: number; provablyOver: boolean; window: number; declaredWindow: number; promptTokens?: number; protocol?: string } | undefined { if (bodyBytes === undefined || declaredWindow === undefined) return undefined; const window = Math.max(declaredWindow, MIN_ASSUMED_WINDOW_TOKENS); // Worst case: zero media credits → core = full mass. If even THAT fits // the calibrated bar, no raw body can refuse — skip the JSON walk. - if (Math.ceil(Math.ceil(bodyBytes / 4) * CALIBRATED_TEXT_TOKEN_RATIO) <= window) return undefined; + // #7106: bytes/3 bounds CJK-dense text (1 token per 3-byte char) from above; + // bytes/4 is only the ASCII bound and would skip this check for CJK bodies. + if (Math.ceil((bodyBytes / 3) * CALIBRATED_TEXT_TOKEN_RATIO) <= window) return undefined; const mediaBytes = rawBody === undefined ? 0 : mediaBytesOfWireBody(rawBody); const coreBytes = Math.max(bodyBytes - mediaBytes, 0); - const floorTokens = Math.ceil(coreBytes / 4); - const estTokens = Math.ceil(floorTokens * CALIBRATED_TEXT_TOKEN_RATIO); + // #7106: when the body parses, price the PROMPT mass instead of the whole + // wire body. DSH stamps transport-only top-level fields (dsh_session_log up + // to 8 MiB, dsh_plugin_packages) that never reach the model, so whole-body + // byte pricing refuses healthy in-window requests before the provider ever + // sees them. Byte pricing stays the fallback for bodies we cannot price by + // shape (non-JSON, or no recognizable prompt key). + const promptMass = promptMassOfWireBody(rawBody); + const floorTokens = promptMass === undefined + ? Math.ceil(coreBytes / 4) + : promptMass.promptTokens; // media stays excluded, as the byte math has since #4111 + // A shape-priced prompt is a direct token count (CJK-aware, same walker as the + // server gates), not a byte heuristic: compare it to the window as-is. Only the + // byte fallback keeps the 1.14 calibration bias. + // #4111 two-tier contract unchanged: the calibrated tier is floor x 1.14 and + // the bare floor is the provably-over tier — #7106 only changes what the + // floor PRICES (prompt mass when the body parses, whole-body bytes otherwise). + // #7106 review: apply the 1.14 calibration ONLY to the byte heuristic. A + // shape-priced prompt is already a token count, so multiplying it would + // double-calibrate and refuse at ~0.877 x window (the wrong direction for + // the false-refusal this issue fixes). + const estTokens = promptMass === undefined + ? Math.ceil(floorTokens * CALIBRATED_TEXT_TOKEN_RATIO) + : floorTokens; if (estTokens <= window) return undefined; - return { coreBytes, mediaBytes, floorTokens, estTokens, provablyOver: floorTokens > window, window, declaredWindow }; + return { + coreBytes, mediaBytes, floorTokens, estTokens, provablyOver: floorTokens > window, window, declaredWindow, + ...(promptMass === undefined ? {} : { promptTokens: promptMass.promptTokens, protocol: promptMass.protocol }), + }; } /** The model context window every plugin lane stamps onto its model requests diff --git a/src/agent/prompt-mass.ts b/src/agent/prompt-mass.ts new file mode 100644 index 0000000000..bd032526f0 --- /dev/null +++ b/src/agent/prompt-mass.ts @@ -0,0 +1,76 @@ +/** #7106: price the PROMPT mass of a wire body, not the whole body. + * + * The client-local doom gate (#2924/#4068/#4111/#5405/#5996) estimates tokens + * from raw body BYTES: `ceil(bytes/4) * 1.14`, minus image bytes. That is a + * sound upper bound for a plain model request, but DSH requests carry + * transport-only top-level fields that never reach the model: + * + * dsh_session_log incremental session-log contribution (up to 8 MiB) + * dsh_plugin_packages plugin inventory blob + * + * Pricing them as prompt mass makes a healthy in-window request look like a + * 2.4M-token body on a 1M-token window, so the local gate refuses it before + * the provider ever sees it (#7090 family, client-side face; #7106). + * + * The client gate has no route config and no protocol header, but the body + * shape is unambiguous: google carries `contents`, responses `input`, + * anthropic `system`, openai `messages`. Once the lane is known, the prompt + * mass is exactly the text under that lane's prompt keys — counted with the + * same CJK-aware walker the server-side preflight uses, so both sides price + * the same bytes. Non-JSON / unrecognized bodies keep the byte estimate. + */ + +import { estimateRawBodyTokens } from "../preflight.js"; +import type { WireProtocol } from "../util.js"; + +/** Top-level keys that carry prompt mass, per wire protocol. + * (Same split as the server-side `PROMPT_BODY_KEYS` added by #7092; kept + * local so this client fix applies without depending on that PR landing.) */ +const PROMPT_KEYS: Record = { + openai: ["messages", "tools"], + anthropic: ["messages", "tools", "system"], + google: ["contents", "systemInstruction", "tools"], + responses: ["input", "instructions", "tools"], +}; + +interface WirePromptMass { + /** Lane the body shape was recognized as; only used for reporting. */ + protocol: WireProtocol; + /** CJK-aware token mass of the prompt fields (turns + system + tools). + * Inline images are deliberately excluded: the text walk skips image + * keys (media stays uncounted, #4111). */ + promptTokens: number; +} + +/** Recognize the lane from the body shape. */ +function sniffWireProtocolOfBody(body: Record): WireProtocol | undefined { + if (body.contents !== undefined) return "google"; + if (body.input !== undefined) return "responses"; + if (body.system !== undefined) return "anthropic"; + if (body.messages !== undefined) return "openai"; + return undefined; +} + +/** Prompt mass of one raw request body, or undefined when it cannot be priced + * from shape (non-JSON, non-object, or no recognizable prompt key). */ +export function promptMassOfWireBody(raw: string | undefined): WirePromptMass | undefined { + if (raw === undefined) return undefined; + let body: unknown; + try { + body = JSON.parse(raw); + } catch { + return undefined; + } + if (body === null || typeof body !== "object" || Array.isArray(body)) return undefined; + const record = body as Record; + const protocol = sniffWireProtocolOfBody(record); + if (protocol === undefined) return undefined; + const promptOnly: Record = {}; + for (const key of PROMPT_KEYS[protocol]) { + if (record[key] !== undefined) promptOnly[key] = record[key]; + } + return { + protocol, + promptTokens: estimateRawBodyTokens(promptOnly), + }; +} diff --git a/tests/dsh-native.test.ts b/tests/dsh-native.test.ts index 1c9a6148dc..16f43a968c 100644 --- a/tests/dsh-native.test.ts +++ b/tests/dsh-native.test.ts @@ -2417,7 +2417,7 @@ test("#4111 mediaBytesOfWireBody/bodyOverWindowDetail: prices the inline-media s const mixedDoom = bodyOverWindowDetail({ bytes: mixedBytes }, mixedInit, 1000); assert.ok(mixedDoom !== undefined, "same media PLUS >4MB of text stays over"); assert.equal(mixedDoom?.provablyOver, true, ">4MB of text: even the optimistic floor alone exceeds the window"); - assert.equal(mixedDoom?.estTokens, Math.ceil((mixedDoom?.floorTokens ?? 0) * 1.14), "calibrated tier = floor × CALIBRATED_TEXT_TOKEN_RATIO (src/util.ts)"); + assert.equal(mixedDoom?.estTokens, mixedDoom?.floorTokens, "#7106: a shape-priced prompt mass is compared to the window as-is (no 1.14 re-calibration); only the byte fallback keeps the calibration bias"); assert.equal(mixedDoom?.mediaBytes, Buffer.byteLength(bigUrl), "doom numbers carry the excluded media mass"); assert.equal(mixedDoom?.coreBytes, mixedBytes - Buffer.byteLength(bigUrl), "doom numbers carry media-excluded core bytes"); assert.equal(mixedDoom?.window, 1_000_000, "#4079 floor never assumes below the 1M common window"); diff --git a/tests/issue7106-client-doom-gate-prompt-mass.test.ts b/tests/issue7106-client-doom-gate-prompt-mass.test.ts new file mode 100644 index 0000000000..c999999b55 --- /dev/null +++ b/tests/issue7106-client-doom-gate-prompt-mass.test.ts @@ -0,0 +1,125 @@ +// #7106: the client-local doom gate must price the PROMPT mass, not the whole +// wire body. DSH stamps transport-only top-level fields (dsh_session_log up to +// 8 MiB, dsh_plugin_packages) that never reach the model; pricing them as text +// made a 736K-token prompt on a 1,048,576 window look like a 2.4M-token body, +// so the gate refused it locally before the provider saw it. +// +// The fixtures below mirror the production shape captured in #7106: +// wireChars=9947234 messages:2561750 system:92785 dsh_session_log:7275441 +// with the prompt mass at ~736K tokens (CJK-aware count). + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { bodyOverWindowDetail } from "../src/agent/native-intercept.js"; + +const WINDOW = 1_048_576; +const TRANSPORT_BLOB = "L".repeat(7 * 1024 * 1024); + +function wire(body: Record): string { + return JSON.stringify(body); +} + +test("#7106: a 7 MiB transport field does not doom an in-window anthropic prompt", () => { + // Prompt: messages ~2.5M chars + system ~92K chars => ~648K tokens, + // times the calibrated 1.14 bias => ~739K, comfortably under the window. + const raw = wire({ + model: "deepseek-flash", + stream: true, + max_tokens: 256000, + messages: [{ role: "user", content: "x".repeat(2_500_000) }], + system: "s".repeat(92_000), + tools: [{ name: "t", description: "d" }], + dsh_session_log: TRANSPORT_BLOB, + dsh_plugin_packages: "p".repeat(14_000), + }); + assert.ok(raw.length > 9_000_000, "fixture is the production body size"); + assert.equal(bodyOverWindowDetail(Buffer.byteLength(raw, "utf8"), raw, WINDOW), undefined, "prompt fits the window, transport mass must not doom it"); +}); + +test("#7106: a prompt that really is over the window still dooms, priced by prompt mass", () => { + const raw = wire({ + model: "deepseek-flash", + messages: [{ role: "user", content: "x".repeat(4_500_000) }], + system: "s".repeat(92_000), + dsh_session_log: TRANSPORT_BLOB, + }); + const doom = bodyOverWindowDetail(Buffer.byteLength(raw, "utf8"), raw, WINDOW); + assert.ok(doom, "an over-window prompt is still refused"); + assert.equal(doom.protocol, "anthropic"); + assert.ok(doom.promptTokens !== undefined, "the verdict names the prompt mass"); + assert.equal(doom.floorTokens, doom.promptTokens, "no images: floor is the prompt mass"); + assert.ok(doom.provablyOver, "1.125M > 1,048,576 on the floor alone"); +}); + +test("#7106: the prompt/named-bucket comparison uses CJK-aware text counting", () => { + // 1.1M CJK chars count ~1 token each, so they are over the window on their + // own even though they are only ~3.3 MB of bytes (a byte/4 estimate would + // have measured ~0.8M and let this through). + const cjk = "\u61C9".repeat(1_100_000); + const raw = wire({ messages: [{ role: "user", content: cjk }] }); + const doom = bodyOverWindowDetail(Buffer.byteLength(raw, "utf8"), raw, WINDOW); + assert.ok(doom); + assert.equal(doom.protocol, "openai"); + assert.ok(doom.promptTokens !== undefined && doom.promptTokens > WINDOW); +}); + +test("#7106: google and openai lanes are recognized by body shape", () => { + const google = wire({ + contents: [{ role: "user", parts: [{ text: "x".repeat(200_000) }] }], + systemInstruction: "s".repeat(92_000), + tools: [{ name: "t" }], + dsh_session_log: TRANSPORT_BLOB, + }); + assert.equal(bodyOverWindowDetail(Buffer.byteLength(google, "utf8"), google, WINDOW), undefined, "google prompt fits; transport mass excluded"); + + const openai = wire({ messages: [{ role: "user", content: "x".repeat(5_000_000) }], dsh_session_log: TRANSPORT_BLOB }); + const doom = bodyOverWindowDetail(Buffer.byteLength(openai, "utf8"), openai, WINDOW); + assert.ok(doom); + assert.equal(doom.protocol, "openai"); +}); + +test("#7106: bodies we cannot price by shape keep the byte estimate", () => { + const doom = bodyOverWindowDetail(9_000_000, "not-json-at-all", WINDOW); + assert.ok(doom, "a 9 MB non-JSON body is still refused"); + assert.equal(doom.promptTokens, undefined, "no prompt pricing without a parsed body"); + assert.equal(doom.floorTokens, Math.ceil(9_000_000 / 4)); +}); + +test("#7106: image bytes are excluded from the prompt mass (media uncounted, #4111)", () => { + const raw = wire({ + messages: [{ + role: "user", + content: [ + { type: "text", text: "x".repeat(1_000) }, + { type: "image", source: { type: "base64", media_type: "image/png", data: "A".repeat(2_000_000) } }, + ], + }], + dsh_session_log: TRANSPORT_BLOB, + }); + const doom = bodyOverWindowDetail(Buffer.byteLength(raw, "utf8"), raw, WINDOW); + assert.equal(doom, undefined, "base64 image bytes are not text prompt mass"); +}); + +test("#7106 review: a shape-priced prompt below the window is NOT doomed (no double calibration)", () => { + // 1.0M CJK chars price at ~1.0M tokens already (direct CJK-aware count). + // The old code multiplied that by 1.14 and refused at ~0.877 x window, + // which is the false-refusal this issue is about. + const raw = wire({ + model: "deepseek-flash", + messages: [{ role: "user", content: "\u61C9".repeat(1_000_000) }], + system: "s".repeat(50_000), + }); + const bytes = Buffer.byteLength(raw, "utf8"); + assert.ok(Math.ceil((bytes / 3) * 1.14) > WINDOW, "fixture must pass the cheap early-out bound"); + assert.equal(bodyOverWindowDetail(bytes, raw, WINDOW), undefined, "prompt mass ~1.01M sits under the 1,048,576 window"); +}); + +test("#7106 review: a shape-priced prompt just over the window dooms, and carries no 1.14 slack", () => { + const raw = wire({ messages: [{ role: "user", content: "\u61C9".repeat(1_060_000) }] }); + const doom = bodyOverWindowDetail(Buffer.byteLength(raw, "utf8"), raw, WINDOW); + assert.ok(doom, "1.06M prompt tokens exceed the window"); + assert.equal(doom.promptTokens, doom.floorTokens); + assert.equal(doom.estTokens, doom.floorTokens, "shape-priced mass is not re-calibrated by 1.14"); + assert.equal(doom.provablyOver, true); +});