Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,17 @@ export function clearEstimateCoreMessagesMemoForTests(): void {
// `source.data` and Gemini `inline_data.data` were already excluded.
const NON_TEXT_BODY_KEYS = new Set(["data", "url", "b64_json", "file_data", "image_url"]);

// #7013: some top-level fields are HOST->proxy transport, never model input. DSH
// ships its session-log envelope (`dsh_session_log`, observed up to 8 MiB on a
// forked session's first turn) and its plugin ledger (`dsh_plugin_packages`) in
// the same HTTP body the model request rides, but no upstream tokenizer ever
// sees them; priced by the raw walk they inflated `other` to ~1.6M tokens and
// the egress fuse refused sends whose real prompt was ~240K (#7013/#7189/#7099/
// #7114). This is an EVIDENCE-PERMITLIST (same discipline as KDD #9): a key
// joins only WITH traffic evidence naming the host field — never wholesale, so a
// future prompt-bearing field cannot take cover here.
const HOP_ONLY_BODY_KEYS = new Set(["dsh_session_log", "dsh_plugin_packages"]);

export function estimateRawBodyTokens(parsed: unknown): number {
let tokens = 0;
const pending: [unknown, string | undefined][] = [[parsed, undefined]];
Expand Down Expand Up @@ -498,6 +509,12 @@ export interface RawBodyBreakdown {
tools: number;
system: number;
other: number;
/** Hop-only transport mass INSIDE `other` (#7013). `total` keeps counting it
* — the raw meter stays raw — while a gate that asks "will an upstream
* accept this" prices `promptTotal` instead. */
hop: number;
/** What the model is actually asked to read: total − hop. */
promptTotal: number;
}

export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown {
Expand Down Expand Up @@ -525,8 +542,20 @@ export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown {
for (const [k, v] of Object.entries(value as Record<string, unknown>)) pending.push([v, k, bucket]);
}
}
// #7013: hop-only mass is measured apart from prompt mass. The walk above
// still counts it (so `total` keeps meaning "everything in the body" and the
// four buckets stay exhaustive); the egress fuse prices `promptTotal`.
let hop = 0;
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (HOP_ONLY_BODY_KEYS.has(k)) hop += estimateRawBodyTokens(v);
}
}
const total = buckets[0] + buckets[1] + buckets[2] + buckets[3];
return {
total: buckets[0] + buckets[1] + buckets[2] + buckets[3],
total,
promptTotal: total - hop,
hop,
messages: buckets[0],
tools: buckets[1],
system: buckets[2],
Expand Down
8 changes: 7 additions & 1 deletion src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3801,7 +3801,13 @@ export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?:
}
}
const breakdown = estimateRawBodyBreakdown(parsed);
return { tokens: breakdown.total, breakdown };
// #7013: the callers that read `tokens` (the egress window fuse and the
// estimate-only probe gate) ask "will the upstream accept this", so they are
// owed what the MODEL reads. Host transport fields (dsh_session_log …) ride
// the body but reach no tokenizer; counting them refused sends whose prompt
// fit the window. `breakdown.total` stays the raw meter for every other
// reader (logs, breakdown clause, tests).
return { tokens: breakdown.promptTotal, breakdown };
}

export async function preflightCompressIfNeeded(
Expand Down
2 changes: 2 additions & 0 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1040,6 +1040,8 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
const fuseRawTokens = fuseMeasured?.tokens;
if (fuseMeasured !== undefined && fuseRawTokens !== undefined && fuseRawTokens > fuseWindow) {
log("warn", `413: [${prepared.session.id}] egress window fuse measured-body profile: ${describeCoreTextDuplication(fuseMsgs)}`);
if (fuseMeasured.breakdown.hop > 0)
log("warn", `413: [${prepared.session.id}] egress window fuse hop-only profile: ~${fuseMeasured.breakdown.hop.toLocaleString("en-US")} of the measured ${fuseMeasured.breakdown.total.toLocaleString("en-US")} tokens are host transport metadata (HOP_ONLY_BODY_KEYS, never sent to a model) and were not priced against the window (#7013)`);
const fuseMessage = `outbound body measures ~${fuseRawTokens.toLocaleString("en-US")} tokens (${measuredBreakdownClause(fuseMeasured.breakdown)}; CJK-aware count, the #554 guard meter) vs the effective model window ${declaredWindow.toLocaleString("en-US")} (bars never assume below ${MIN_ASSUMED_WINDOW_TOKENS} tokens, #4079) (model=${prepared.model ?? "unknown"}); the body ITSELF exceeds the window, so the send was stopped at the egress fuse instead of being forwarded (#5000/#4543/#4068). Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`;
log("warn", `413: [${prepared.session.id}] egress window fuse: ${fuseMessage}`);
if (res.headersSent) emitStreamError(res, prepared.protocol, fuseMessage, undefined, opts.streamErrorShape);
Expand Down
43 changes: 43 additions & 0 deletions tests/egress-fuse-hop-mass.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { estimateRawBodyBreakdown, estimateRawBodyTokens } from "../src/preflight.ts";
import { measuredOutbound } from "../src/server.ts";

// #7013: DSH ships hop-only transport fields (its session-log envelope and plugin
// ledger) inside the model request body. The raw walk counted them as prompt
// mass, so the egress window fuse refused sends whose real prompt fit the window
// (`other ~1,597,576` against a 1,048,576 window while messages[] was ~212K).
const ledger = { version: 1, packages: [{ name: "dsh", version: "0.2.2-rc.3" }] };
const envelope = { version: 1, sessionFormatVersion: 4, afterSeq: 0, throughSeq: 5152, events: [{ text: "L".repeat(2_000_000) }] };

test("estimateRawBodyBreakdown: host transport fields are hop mass, not prompt mass (#7013)", () => {
const body = {
model: "deepseek-flash",
system: "s".repeat(1_000),
messages: [{ role: "user", content: "m".repeat(1_000) }],
dsh_session_log: envelope,
dsh_plugin_packages: ledger,
};
const b = estimateRawBodyBreakdown(body);
assert.equal(b.hop, estimateRawBodyTokens(envelope) + estimateRawBodyTokens(ledger), "hop mass is exactly the transport fields");
assert.equal(b.total, estimateRawBodyTokens(body), "raw meter unchanged: total still counts every leaf");
assert.equal(b.total, b.messages + b.tools + b.system + b.other, "buckets stay exhaustive — hop lives inside other");
assert.ok(b.hop > 400_000, "the 2 MB envelope is real mass");
assert.equal(b.promptTotal, b.total - b.hop, "promptTotal removes exactly the hop mass");
assert.ok(b.promptTotal < 1_000, "what the model reads stays tiny");

// NEGATIVE: a permitlist must not swallow prompt text. The same key NESTED
// inside messages[] is content the model reads, not hop mass.
const nested = estimateRawBodyBreakdown({ messages: [{ role: "user", content: "c", dsh_session_log: "y".repeat(2_000) }] });
assert.equal(nested.hop, 0, "only TOP-LEVEL transport fields are hop mass");
assert.ok(nested.messages > 0, "nested text still lands in the messages bucket");
assert.equal(nested.promptTotal, nested.total, "nothing is removed when the key is not top-level");
});

test("measuredOutbound: the gate prices prompt mass, so a hop-heavy body forwards (#7013)", () => {
const body = JSON.stringify({ model: "deepseek-flash", messages: [{ role: "user", content: "hi" }], dsh_session_log: envelope });
const measured = measuredOutbound({ body });
assert.ok(measured, "JSON body measures");
assert.ok(measured!.tokens < 1_000, "gate sees prompt mass");
assert.ok(measured!.breakdown.total > 400_000, "raw meter still sees the envelope");
});
Loading