From dae34d4229bf96b0d0a9052d4e15aac330ac5a6e Mon Sep 17 00:00:00 2001 From: zhang-ming-hui Date: Thu, 19 Nov 2026 15:40:52 +0800 Subject: [PATCH] fix: price host transport fields as hop mass, not prompt mass (#7013) dsh_session_log / dsh_plugin_packages ride the model request body but never reach a tokenizer; the raw walk counted them into other, so the egress window fuse refused sends whose prompt fit the window (#7013/#7189/#6844/#7099/#7106). Adds an evidence-permitlist HOP_ONLY_BODY_KEYS and reports hop/promptTotal on RawBodyBreakdown (total unchanged, buckets still exhaustive); measuredOutbound now prices promptTotal. --- src/preflight.ts | 31 ++++++++++++++++++++- src/server.ts | 8 +++++- src/server/relay.ts | 2 ++ tests/egress-fuse-hop-mass.test.ts | 43 ++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 tests/egress-fuse-hop-mass.test.ts diff --git a/src/preflight.ts b/src/preflight.ts index 44c053354a..07e33f0558 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -465,6 +465,17 @@ export function clearEstimateCoreMessagesMemoForTests(): void { // `source.data` and Gemini `inline_data.data` were already excluded. const NON_TEXT_BODY_KEYS = new Set(["data", "url", "b64_json", "file_data", "image_url"]); +// #7013: some top-level fields are HOST->proxy transport, never model input. DSH +// ships its session-log envelope (`dsh_session_log`, observed up to 8 MiB on a +// forked session's first turn) and its plugin ledger (`dsh_plugin_packages`) in +// the same HTTP body the model request rides, but no upstream tokenizer ever +// sees them; priced by the raw walk they inflated `other` to ~1.6M tokens and +// the egress fuse refused sends whose real prompt was ~240K (#7013/#7189/#7099/ +// #7114). This is an EVIDENCE-PERMITLIST (same discipline as KDD #9): a key +// joins only WITH traffic evidence naming the host field — never wholesale, so a +// future prompt-bearing field cannot take cover here. +const HOP_ONLY_BODY_KEYS = new Set(["dsh_session_log", "dsh_plugin_packages"]); + export function estimateRawBodyTokens(parsed: unknown): number { let tokens = 0; const pending: [unknown, string | undefined][] = [[parsed, undefined]]; @@ -498,6 +509,12 @@ export interface RawBodyBreakdown { tools: number; system: number; other: number; + /** Hop-only transport mass INSIDE `other` (#7013). `total` keeps counting it + * — the raw meter stays raw — while a gate that asks "will an upstream + * accept this" prices `promptTotal` instead. */ + hop: number; + /** What the model is actually asked to read: total − hop. */ + promptTotal: number; } export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown { @@ -525,8 +542,20 @@ export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown { for (const [k, v] of Object.entries(value as Record)) pending.push([v, k, bucket]); } } + // #7013: hop-only mass is measured apart from prompt mass. The walk above + // still counts it (so `total` keeps meaning "everything in the body" and the + // four buckets stay exhaustive); the egress fuse prices `promptTotal`. + let hop = 0; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + for (const [k, v] of Object.entries(parsed as Record)) { + if (HOP_ONLY_BODY_KEYS.has(k)) hop += estimateRawBodyTokens(v); + } + } + const total = buckets[0] + buckets[1] + buckets[2] + buckets[3]; return { - total: buckets[0] + buckets[1] + buckets[2] + buckets[3], + total, + promptTotal: total - hop, + hop, messages: buckets[0], tools: buckets[1], system: buckets[2], diff --git a/src/server.ts b/src/server.ts index 3fb954b5ff..866acd0362 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3801,7 +3801,13 @@ export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?: } } const breakdown = estimateRawBodyBreakdown(parsed); - return { tokens: breakdown.total, breakdown }; + // #7013: the callers that read `tokens` (the egress window fuse and the + // estimate-only probe gate) ask "will the upstream accept this", so they are + // owed what the MODEL reads. Host transport fields (dsh_session_log …) ride + // the body but reach no tokenizer; counting them refused sends whose prompt + // fit the window. `breakdown.total` stays the raw meter for every other + // reader (logs, breakdown clause, tests). + return { tokens: breakdown.promptTotal, breakdown }; } export async function preflightCompressIfNeeded( diff --git a/src/server/relay.ts b/src/server/relay.ts index 7ce093b91b..49d43f50ed 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1040,6 +1040,8 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare const fuseRawTokens = fuseMeasured?.tokens; if (fuseMeasured !== undefined && fuseRawTokens !== undefined && fuseRawTokens > fuseWindow) { log("warn", `413: [${prepared.session.id}] egress window fuse measured-body profile: ${describeCoreTextDuplication(fuseMsgs)}`); + if (fuseMeasured.breakdown.hop > 0) + log("warn", `413: [${prepared.session.id}] egress window fuse hop-only profile: ~${fuseMeasured.breakdown.hop.toLocaleString("en-US")} of the measured ${fuseMeasured.breakdown.total.toLocaleString("en-US")} tokens are host transport metadata (HOP_ONLY_BODY_KEYS, never sent to a model) and were not priced against the window (#7013)`); const fuseMessage = `outbound body measures ~${fuseRawTokens.toLocaleString("en-US")} tokens (${measuredBreakdownClause(fuseMeasured.breakdown)}; CJK-aware count, the #554 guard meter) vs the effective model window ${declaredWindow.toLocaleString("en-US")} (bars never assume below ${MIN_ASSUMED_WINDOW_TOKENS} tokens, #4079) (model=${prepared.model ?? "unknown"}); the body ITSELF exceeds the window, so the send was stopped at the egress fuse instead of being forwarded (#5000/#4543/#4068). Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`; log("warn", `413: [${prepared.session.id}] egress window fuse: ${fuseMessage}`); if (res.headersSent) emitStreamError(res, prepared.protocol, fuseMessage, undefined, opts.streamErrorShape); diff --git a/tests/egress-fuse-hop-mass.test.ts b/tests/egress-fuse-hop-mass.test.ts new file mode 100644 index 0000000000..051fd6f380 --- /dev/null +++ b/tests/egress-fuse-hop-mass.test.ts @@ -0,0 +1,43 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { estimateRawBodyBreakdown, estimateRawBodyTokens } from "../src/preflight.ts"; +import { measuredOutbound } from "../src/server.ts"; + +// #7013: DSH ships hop-only transport fields (its session-log envelope and plugin +// ledger) inside the model request body. The raw walk counted them as prompt +// mass, so the egress window fuse refused sends whose real prompt fit the window +// (`other ~1,597,576` against a 1,048,576 window while messages[] was ~212K). +const ledger = { version: 1, packages: [{ name: "dsh", version: "0.2.2-rc.3" }] }; +const envelope = { version: 1, sessionFormatVersion: 4, afterSeq: 0, throughSeq: 5152, events: [{ text: "L".repeat(2_000_000) }] }; + +test("estimateRawBodyBreakdown: host transport fields are hop mass, not prompt mass (#7013)", () => { + const body = { + model: "deepseek-flash", + system: "s".repeat(1_000), + messages: [{ role: "user", content: "m".repeat(1_000) }], + dsh_session_log: envelope, + dsh_plugin_packages: ledger, + }; + const b = estimateRawBodyBreakdown(body); + assert.equal(b.hop, estimateRawBodyTokens(envelope) + estimateRawBodyTokens(ledger), "hop mass is exactly the transport fields"); + assert.equal(b.total, estimateRawBodyTokens(body), "raw meter unchanged: total still counts every leaf"); + assert.equal(b.total, b.messages + b.tools + b.system + b.other, "buckets stay exhaustive — hop lives inside other"); + assert.ok(b.hop > 400_000, "the 2 MB envelope is real mass"); + assert.equal(b.promptTotal, b.total - b.hop, "promptTotal removes exactly the hop mass"); + assert.ok(b.promptTotal < 1_000, "what the model reads stays tiny"); + + // NEGATIVE: a permitlist must not swallow prompt text. The same key NESTED + // inside messages[] is content the model reads, not hop mass. + const nested = estimateRawBodyBreakdown({ messages: [{ role: "user", content: "c", dsh_session_log: "y".repeat(2_000) }] }); + assert.equal(nested.hop, 0, "only TOP-LEVEL transport fields are hop mass"); + assert.ok(nested.messages > 0, "nested text still lands in the messages bucket"); + assert.equal(nested.promptTotal, nested.total, "nothing is removed when the key is not top-level"); +}); + +test("measuredOutbound: the gate prices prompt mass, so a hop-heavy body forwards (#7013)", () => { + const body = JSON.stringify({ model: "deepseek-flash", messages: [{ role: "user", content: "hi" }], dsh_session_log: envelope }); + const measured = measuredOutbound({ body }); + assert.ok(measured, "JSON body measures"); + assert.ok(measured!.tokens < 1_000, "gate sees prompt mass"); + assert.ok(measured!.breakdown.total > 400_000, "raw meter still sees the envelope"); +});