diff --git a/src/preflight.ts b/src/preflight.ts index ebe8d88b87..752adec01c 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -27,7 +27,7 @@ import { epochConfirmPending } from "./fold-reconcile.js"; import { deadSkipKeys, filterDeadRanges, parseStructuralEmptyVerdict, recordDeadRange } from "./dead-ranges.js"; import { peekRegistryOutputLimit } from "./registry.js"; import { safePrefix } from "./text-safe.js"; -import { applyEstimateCalibration, currentCalibrationFactor } from "./util.js"; +import { applyEstimateCalibration, currentCalibrationFactor, type WireProtocol } from "./util.js"; import { configuredSummaryPlan, type ConfiguredSummaryPlan } from "./external-summary-runtime.js"; import { messageTimestampSections } from "./external-summary-surface.js"; import { isCodexClient } from "./codex-compact.js"; @@ -502,45 +502,74 @@ export function estimateRawBodyTokens(parsed: unknown): number { return tokens; } +// #7013: the wire-format's PROMPT keys — the top-level fields the upstream's +// tokenizer actually counts against the model window. Everything else (telemetry +// uploads like DSH's `dsh_session_log`, config scalars, cache directives) rides +// the wire but consumes NO window, so a window-fit judgment must price only +// these — NOT a hardcoded exclusion of one plugin's field (the next plugin that +// registers a different upload field would recur the #7013 bug). The keys mirror +// what each prepare lane feeds its core converter: openai (messages — system +// rides inside it — + tools), anthropic (messages + tools + top-level system), +// google (contents + systemInstruction + tools), responses (input + instructions +// + tools). +export const PROMPT_BODY_KEYS: Record> = { + openai: new Set(["messages", "tools"]), + anthropic: new Set(["messages", "tools", "system"]), + google: new Set(["contents", "systemInstruction", "tools"]), + responses: new Set(["input", "instructions", "tools"]), +}; + /** #5000 item 3: which DIMENSION carries the outbound mass. The doom messages * must say whether it is messages[] (a giant paste / tool result), tools[] * (84 schemas in #5000), the system prompt, or structural other — "trim the * oversized content" is actively misleading for a 55.2 MiB body whose * messages[] is only 1.44 MiB. Same walk + same exclusions as * estimateRawBodyTokens, split on the TOP-LEVEL field, so `total` always - * equals estimateRawBodyTokens(parsed) exactly. */ + * equals estimateRawBodyTokens(parsed) exactly. #7013: `prompt` is the mass + * under the wire-format's prompt keys — what the window-fit gates price. */ export interface RawBodyBreakdown { total: number; messages: number; tools: number; system: number; other: number; + /** #7013: mass under the wire-format's prompt keys (PROMPT_BODY_KEYS) — + * what the upstream tokenizer counts against the model window. Non-prompt + * fields (telemetry uploads, config scalars) sit in `other` but NOT here. + * 0 when no promptKeys were supplied to the walk. */ + prompt: number; } -export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown { +export function estimateRawBodyBreakdown(parsed: unknown, promptKeys?: ReadonlySet): RawBodyBreakdown { const buckets: [number, number, number, number] = [0, 0, 0, 0]; // messages, tools, system, other - const pending: [unknown, string | undefined, number][] = []; + let prompt = 0; + const pending: [unknown, string | undefined, number, boolean][] = []; if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { for (const [k, v] of Object.entries(parsed as Record)) { if (isNonModelTopLevelKey(k)) continue; const bucket = k === "messages" ? 0 : k === "tools" ? 1 : k === "system" ? 2 : 3; - pending.push([v, k, bucket]); + const isPrompt = promptKeys ? promptKeys.has(k) : false; + pending.push([v, k, bucket, isPrompt]); } } else { - pending.push([parsed, undefined, 3]); + pending.push([parsed, undefined, 3, false]); } while (pending.length > 0) { - const [value, key, bucket] = pending.pop()!; + const [value, key, bucket, isPrompt] = pending.pop()!; if (typeof value === "string") { - if (!key || !NON_TEXT_BODY_KEYS.has(key)) buckets[bucket] += defaultCountTokens(value); + if (!key || !NON_TEXT_BODY_KEYS.has(key)) { + const t = defaultCountTokens(value); + buckets[bucket] += t; + if (isPrompt) prompt += t; + } continue; } if (Array.isArray(value)) { - for (const item of value) pending.push([item, key, bucket]); + for (const item of value) pending.push([item, key, bucket, isPrompt]); continue; } if (value && typeof value === "object") { - for (const [k, v] of Object.entries(value as Record)) pending.push([v, k, bucket]); + for (const [k, v] of Object.entries(value as Record)) pending.push([v, k, bucket, isPrompt]); } } return { @@ -549,6 +578,7 @@ export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown { tools: buckets[1], system: buckets[2], other: buckets[3], + prompt, }; } diff --git a/src/server.ts b/src/server.ts index 82cf750c8a..ac8181e233 100644 --- a/src/server.ts +++ b/src/server.ts @@ -86,7 +86,7 @@ import { auditStartupSummaryCredentials } from "./external-summary-credentials.j import { applyRanges } from "./stream.js"; import { attachSubagentSessions } from "./subagent-sessions.js"; import { buildSessionCacheReport, handleAcpCache, learnedImageReserve, noteClientAbort, readKeySwitchStats, readModelSwitchStats, settleUsageReport } from "./cache-ledger.js"; -import { extractBillingAttributionBlock, extractSummaryFromSse, preflightCompress, estimateCoreMessages, estimateCoreMessagesUpper, estimateCoreMessagesUpperBytes, describeCoreTextDuplication, estimateRawBodyTokens, estimateRawBodyBreakdown, type PreflightDeps, type PreflightResult, type RawBodyBreakdown } from "./preflight.js"; +import { extractBillingAttributionBlock, extractSummaryFromSse, preflightCompress, estimateCoreMessages, estimateCoreMessagesUpper, estimateCoreMessagesUpperBytes, describeCoreTextDuplication, estimateRawBodyTokens, estimateRawBodyBreakdown, PROMPT_BODY_KEYS, type PreflightDeps, type PreflightResult, type RawBodyBreakdown } from "./preflight.js"; import { buildDecisionPrompt, buildDirectiveText, consumeFallback, DEFAULT_DECIDE_MAX_TOKENS, DECIDE_TIMEOUT_MS, extractDecisionText, ladderMode, parseDecision, recordDecision, resolveDecisionRange, type DecideConfig, type DecisionOutcome } from "./nudge-decide.js"; import { gcConfigFromEnv, gcSessionFiles } from "./session-gc.js"; import { countImagesInParsedBody, countImagesInRawBody, imageTokensInRawBody, imageTokensInParsedBody, resolveImageBilling, upstreamHost, type ResolvedImageBilling } from "./image-tokens.js"; @@ -3777,7 +3777,15 @@ export function buildWindowShrinkNote(args: { // tokens are charged separately by the caller (they sit in binary-carrying // keys the walk excludes by design). Unparseable bodies return undefined — // callers keep the byte-floor fallback for them. +// #7013: the meter prices only the wire-format's PROMPT mass (PROMPT_BODY_KEYS) +// — what the upstream tokenizer counts against the model window. Non-prompt +// fields (telemetry uploads like DSH's dsh_session_log, config scalars) ride the +// wire but consume no window; pricing them made a ~100K-token prompt with an +// 8 MiB upload field 413. The full-body `total` is still reported in the +// breakdown for the 413 message, so the operator sees where the mass is. interface MeasuredOutbound { + /** #7013: the wire-format's prompt mass (PROMPT_BODY_KEYS) — the value the + * window-fit gates compare against the model window. */ tokens: number; breakdown: RawBodyBreakdown; } @@ -3785,12 +3793,13 @@ interface MeasuredOutbound { // #5000 item 3: dimension clause for the measured-doom messages — which part // of the body carries the mass, so "trim the oversized content" tells the // operator WHERE (a 55.2 MiB #5000 body with 1.44 MiB of messages[] means the -// mass is tools[], not a giant paste). +// mass is tools[], not a giant paste). #7013: `other` is non-prompt mass (the +// 413 gate prices messages/tools/system only) — the clause says so. export function measuredBreakdownClause(b: RawBodyBreakdown): string { - return `messages ~${b.messages.toLocaleString("en-US")} · tools[] ~${b.tools.toLocaleString("en-US")} · system ~${b.system.toLocaleString("en-US")} · other ~${b.other.toLocaleString("en-US")}`; + return `messages ~${b.messages.toLocaleString("en-US")} · tools[] ~${b.tools.toLocaleString("en-US")} · system ~${b.system.toLocaleString("en-US")} · other ~${b.other.toLocaleString("en-US")} (non-prompt, excluded from the window budget)`; } -export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?: unknown }): MeasuredOutbound | undefined { +export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?: unknown; protocol: WireProtocol }): MeasuredOutbound | undefined { let parsed: unknown = prepared.parsedBody; if (parsed === undefined) { const raw = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8"); @@ -3800,8 +3809,8 @@ export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?: return undefined; } } - const breakdown = estimateRawBodyBreakdown(parsed); - return { tokens: breakdown.total, breakdown }; + const breakdown = estimateRawBodyBreakdown(parsed, PROMPT_BODY_KEYS[prepared.protocol]); + return { tokens: breakdown.prompt, breakdown }; } export async function preflightCompressIfNeeded( @@ -4525,15 +4534,18 @@ export async function preflightCompressIfNeeded( // the #554 side-request guard's own: it walks every string field of the // serialized body (system + tools + arguments + messages, CJK-aware), so // unlike the m.text byte floor it sees the mass that actually rides the - // wire. Parse failure leaves both readings undefined — the byte-floor - // fallbacks stay binding for those bodies. + // wire. #7013: it prices only the wire-format's PROMPT mass (PROMPT_BODY_KEYS + // for outbound.protocol) — non-prompt fields (telemetry uploads like DSH's + // dsh_session_log, config scalars) ride the wire but consume no window, so + // they must not doom a fitting prompt. Parse failure leaves both readings + // undefined — the byte-floor fallbacks stay binding for those bodies. let outboundBudgetTokens: number | undefined; let outboundRawTokens: number | undefined; let outboundBreakdown: RawBodyBreakdown | undefined; try { const outBody: unknown = JSON.parse(typeof outbound.body === "string" ? outbound.body : outbound.body.toString("utf8")); - outboundBreakdown = estimateRawBodyBreakdown(outBody); - outboundRawTokens = outboundBreakdown.total; + outboundBreakdown = estimateRawBodyBreakdown(outBody, PROMPT_BODY_KEYS[outbound.protocol]); + outboundRawTokens = outboundBreakdown.prompt; const outField = outputBudgetField(outBody); const rawBudget = outField !== null ? readOutputBudget(outBody as Record, outField) : undefined; if (typeof rawBudget === "number" && rawBudget > 0) outboundBudgetTokens = rawBudget; @@ -4563,13 +4575,15 @@ export async function preflightCompressIfNeeded( // Images are NOT part of this comparison: an image-dominated overage is // the #496/#1800 billing-arbitration domain (forward, let upstream bill // or 400 for real evidence), not a doom — #857's screenshot turn pins - // that. The #554 meter excludes image payloads (image_url/data/url/...), - // so `outboundRawTokens` is text-only by construction. + // that. The #554 meter excludes image payloads (image_url/data/url/...) + // and, #7013, prices only the wire-format's prompt keys — so + // `outboundRawTokens` is prompt-text-only by construction (non-prompt + // telemetry uploads like DSH's dsh_session_log never doom here). const armMeasuredBar = typeof resolvedNativeWindow === "number" && resolvedNativeWindow > 0 ? resolvedNativeWindow : limit; if (outboundRawTokens !== undefined && outboundRawTokens > armMeasuredBar) { const outMsgs = outbound.processedMessages.length > 0 ? outbound.processedMessages : outbound.originalMessages; log("error", `[${session.id}] preflight 413 measured-body profile (post-fold): ${describeCoreTextDuplication(outMsgs)}`); - const measuredDoomMessage = `rebuilt outbound body measures ~${outboundRawTokens.toLocaleString("en-US")} TEXT tokens (${outboundBreakdown ? measuredBreakdownClause(outboundBreakdown) : "by field: n/a"}; CJK-aware count, the #554 guard meter; images billed separately, never doom here — #496/#1800/#857) vs the model window ${armMeasuredBar.toLocaleString("en-US")} (model=${model}); the body ITSELF exceeds the window, so it was NOT forwarded for upstream arbitration. Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`; + const measuredDoomMessage = `rebuilt outbound prompt measures ~${outboundRawTokens.toLocaleString("en-US")} tokens (${outboundBreakdown ? measuredBreakdownClause(outboundBreakdown) : "by field: n/a"}; CJK-aware count, the #554 guard meter; images billed separately, never doom here — #496/#1800/#857; non-prompt fields in the other bucket are excluded from the window budget — #7013) vs the model window ${armMeasuredBar.toLocaleString("en-US")} (model=${model}); the prompt ITSELF exceeds the window, so it was NOT forwarded for upstream arbitration. Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`; log("error", `[${session.id}] preflight fail-fast 413 (retryable=false): ${measuredDoomMessage}`); noteLocalRefusal(session, currentOrigin, model, limit); return { failFast: true, status: 413, message: measuredDoomMessage, retryable: false, respond: !res.writableEnded }; @@ -4666,12 +4680,14 @@ export async function preflightCompressIfNeeded( // false-refuse a natively-fitting body), the headroom limit as // fallback. Measured tier first, byte floor below stays as the // fallback for unparseable bodies. - // Same text-only rule as the #1839 arm above: image-dominated overage - // belongs to the #496/#1800 billing arbitration, never to a local doom. + // Same prompt-text-only rule as the #1839 arm above (#7013): image- + // dominated overage belongs to the #496/#1800 billing arbitration, and + // non-prompt telemetry uploads (DSH's dsh_session_log) never doom a + // fitting prompt, never to a local doom. const gateMeasuredBar = typeof resolvedNativeWindow === "number" && resolvedNativeWindow > 0 ? resolvedNativeWindow : limit; if (outboundRawTokens !== undefined && outboundRawTokens > gateMeasuredBar) { log("error", `[${session.id}] preflight 413 measured-body profile: ${describeCoreTextDuplication(outMsgs)}`); - const measuredDoomMessage = `outbound body measures ~${outboundRawTokens.toLocaleString("en-US")} TEXT tokens even after preflight compression (${outboundBreakdown ? measuredBreakdownClause(outboundBreakdown) : "by field: n/a"}; CJK-aware count, the #554 guard meter; images billed separately, never doom here — #496/#1800/#857) vs the model window ${gateMeasuredBar.toLocaleString("en-US")} (model=${model}); the body ITSELF exceeds the window, so it was NOT forwarded. Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`; + const measuredDoomMessage = `outbound prompt measures ~${outboundRawTokens.toLocaleString("en-US")} tokens even after preflight compression (${outboundBreakdown ? measuredBreakdownClause(outboundBreakdown) : "by field: n/a"}; CJK-aware count, the #554 guard meter; images billed separately, never doom here — #496/#1800/#857; non-prompt fields in the other bucket are excluded from the window budget — #7013) vs the model window ${gateMeasuredBar.toLocaleString("en-US")} (model=${model}); the prompt ITSELF exceeds the window, so it was NOT forwarded. Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`; log("error", `[${session.id}] preflight fail-fast 413 (retryable=false): ${measuredDoomMessage}`); noteLocalRefusal(session, currentOrigin, model, limit); return { failFast: true, status: 413, message: measuredDoomMessage, retryable: false, respond: !res.writableEnded }; diff --git a/src/server/relay.ts b/src/server/relay.ts index 29fa2581a4..4e493ecc5e 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1036,11 +1036,15 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare } return; } + // #7013: the fuse prices the wire-format's PROMPT mass (PROMPT_BODY_KEYS + // for prepared.protocol) — non-prompt fields (telemetry uploads like + // DSH's dsh_session_log, config scalars) ride the wire but consume no + // window, so they must not 413 a fitting prompt. const fuseMeasured = measuredOutbound(prepared); const fuseRawTokens = fuseMeasured?.tokens; if (fuseMeasured !== undefined && fuseRawTokens !== undefined && fuseRawTokens > fuseWindow) { log("warn", `413: [${prepared.session.id}] egress window fuse measured-body profile: ${describeCoreTextDuplication(fuseMsgs)}`); - const fuseMessage = `outbound body measures ~${fuseRawTokens.toLocaleString("en-US")} tokens (${measuredBreakdownClause(fuseMeasured.breakdown)}; CJK-aware count, the #554 guard meter) vs the effective model window ${declaredWindow.toLocaleString("en-US")} (bars never assume below ${MIN_ASSUMED_WINDOW_TOKENS} tokens, #4079) (model=${prepared.model ?? "unknown"}); the body ITSELF exceeds the window, so the send was stopped at the egress fuse instead of being forwarded (#5000/#4543/#4068). Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`; + const fuseMessage = `outbound prompt measures ~${fuseRawTokens.toLocaleString("en-US")} tokens (${measuredBreakdownClause(fuseMeasured.breakdown)}; CJK-aware count, the #554 guard meter; non-prompt fields in the other bucket are excluded from the window budget — #7013) vs the effective model window ${declaredWindow.toLocaleString("en-US")} (bars never assume below ${MIN_ASSUMED_WINDOW_TOKENS} tokens, #4079) (model=${prepared.model ?? "unknown"}); the prompt ITSELF exceeds the window, so the send was stopped at the egress fuse instead of being forwarded (#5000/#4543/#4068). Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`; log("warn", `413: [${prepared.session.id}] egress window fuse: ${fuseMessage}`); if (res.headersSent) emitStreamError(res, prepared.protocol, fuseMessage, undefined, opts.streamErrorShape); else { diff --git a/src/server/side-request.ts b/src/server/side-request.ts index 83b2c40993..154d67e8b6 100644 --- a/src/server/side-request.ts +++ b/src/server/side-request.ts @@ -1,5 +1,5 @@ import type { WireProtocol } from "../util.js"; -import { estimateRawBodyTokens } from "../preflight.js"; +import { estimateRawBodyBreakdown, PROMPT_BODY_KEYS } from "../preflight.js"; import { imageTokensInParsedBody } from "../image-tokens.js"; import { reserveOutputHeadroom, systemCarrierTexts } from "../util.js"; import { type ResolvedImageBilling } from "../image-tokens.js"; @@ -1012,9 +1012,12 @@ const SIDE_REQUEST_GUARD_TOLERANCE = 1.15; * reservation on routes that carry one (#4205: decided by the caller via * shouldReserveOutputHeadroom with route+model evidence — the guard stays * pure mechanics and self-reserves once from the UNRESERVED limit it is - * handed). blocked=false with limit<=0 means "window unknown — forward as - * before". blocked requires estimate >= limit x SIDE_REQUEST_GUARD_TOLERANCE - * (estimator bias). */ + * handed). #7013: the estimate prices only the wire-format's PROMPT mass + * (PROMPT_BODY_KEYS[protocol]) — non-prompt fields (telemetry uploads like + * DSH's dsh_session_log, config scalars) ride the wire but consume no window, + * so they must not block a fitting prompt. blocked=false with limit<=0 means + * "window unknown — forward as before". blocked requires estimate >= limit x + * SIDE_REQUEST_GUARD_TOLERANCE (estimator bias). */ export function sideRequestGuard( parsed: unknown, protocol: WireProtocol, @@ -1044,6 +1047,8 @@ export function sideRequestGuard( const maxOut = (field ? readOutputBudget(parsed as Record, field) : undefined) ?? 0; if (reserveHeadroom && limit > 0) limit = reserveOutputHeadroom(limit, maxOut, headroomCap); const imageMass = imageReserve ?? imageTokensInParsedBody(protocol, parsed, imageBilling, configuredCap); - const estimate = estimateRawBodyTokens(parsed) + imageMass; + // #7013: price the wire-format's prompt mass (PROMPT_BODY_KEYS[protocol]) — + // non-prompt fields (telemetry uploads, config scalars) consume no window. + const estimate = estimateRawBodyBreakdown(parsed, PROMPT_BODY_KEYS[protocol]).prompt + imageMass; return { blocked: limit > 0 && estimate >= limit * SIDE_REQUEST_GUARD_TOLERANCE, estimate, limit }; } diff --git a/tests/issue4079-egress-window-fuse.test.ts b/tests/issue4079-egress-window-fuse.test.ts index 6a4d423bb0..bfba552507 100644 --- a/tests/issue4079-egress-window-fuse.test.ts +++ b/tests/issue4079-egress-window-fuse.test.ts @@ -164,7 +164,7 @@ test("e2e #4079: a body that provably cannot fit any window is stopped at the eg assert.equal(parsed.error?.type, "request_too_large", "same structured too-large family as the byte budget"); assert.equal(parsed.error?.code, "egress_window_fuse", "the fuse names itself for log greppability"); assert.equal(parsed.error?.retryable, false, "4xx non-retryable class — retrying re-sends the same oversized history (#2847 semantics)"); - assert.match(parsed.error?.message ?? "", /outbound body measures ~[\d,]+ tokens/, "the measured token math is in the operator-facing message"); + assert.match(parsed.error?.message ?? "", /outbound prompt measures ~[\d,]+ tokens/, "the measured prompt-mass math is in the operator-facing message (#7013)"); assert.match(parsed.error?.message ?? "", /bars never assume below 1000000 tokens/, "the assumed-window floor is stated for the operator (#4079)"); assert.match(parsed.error?.message ?? "", /stopped at the egress fuse instead of being forwarded/, "the remedy wording is present"); // THE behavior under test: the doomed payload NEVER reaches the upstream. @@ -201,7 +201,7 @@ test("e2e #5000: a body whose mass rides tools[] — core text clearing the byte assert.equal(r.status, 413, `tools-mass body must fail fast (got ${r.status}): ${body.slice(0, 300)}`); assert.equal(parsed.error?.code, "preflight_compress_failed", "the preflight fail-fast house shape (the measured gate answers before the relay)"); assert.equal(parsed.error?.retryable, false, "4xx non-retryable class — retrying re-sends the same oversized history (#2847 semantics)"); - assert.match(parsed.error?.message ?? "", /outbound body measures ~[\d,]+ TEXT tokens/, "the measured tier names itself (text-only; image payloads are #496 arbitration, never doom)"); + assert.match(parsed.error?.message ?? "", /outbound prompt measures ~[\d,]+ tokens/, "the measured tier names itself (prompt-mass; non-prompt fields excluded — #7013; image payloads are #496 arbitration, never doom)"); assert.match(parsed.error?.message ?? "", /NOT forwarded/, "the remedy wording is present"); // THE behavior under test: the doomed payload NEVER reaches the upstream // (before the measured tier it cleared the byte floor and rode forward @@ -234,7 +234,7 @@ test("e2e #4543: a CJK body whose CORE TEXT alone measures over the floored fuse assert.equal(r.status, 413, `the CJK-measured body must stop at the fuse (got ${r.status}): ${body.slice(0, 300)}`); assert.equal(parsed.error?.code, "egress_window_fuse", "the fuse names itself"); assert.equal(parsed.error?.retryable, false, "non-retryable — retrying re-sends the same oversized history (#2847 semantics)"); - assert.match(parsed.error?.message ?? "", /outbound body measures ~[\d,]+ tokens/, "the measured token math is in the operator-facing message"); + assert.match(parsed.error?.message ?? "", /outbound prompt measures ~[\d,]+ tokens/, "the measured prompt-mass math is in the operator-facing message (#7013)"); assert.match(parsed.error?.message ?? "", /stopped at the egress fuse instead of being forwarded/, "the remedy wording is present"); // THE behavior under test: the doomed payload NEVER reaches the upstream. assert.equal(calls.length, 0, `upstream must see zero calls (got ${JSON.stringify(calls)})`); @@ -269,3 +269,68 @@ test("e2e #4079 both-ways: bodies under the doom bars still forward — includin rmrf(root); } }); + +// #7013: the fuse prices the wire-format's PROMPT mass, not the whole body. +// A non-prompt upload field (DSH's dsh_session_log — telemetry that rides the +// wire but consumes no window) must NOT 413 a fitting prompt. This reproduces +// the issue's exact repro: ~100K-token prompt + an 8 MiB dsh_session_log that +// used to measure ~2M tokens and doom the send. +test("e2e #7013: a giant non-prompt upload field does not doom a fitting prompt at the egress fuse", async () => { + const root = mkdtempSync(path.join(tmpdir(), "bili-7013-")); + const { ctx, calls, close } = await setup(); + try { + // No `model` in the body → preflight's `!model` early return skips its + // doom math; forward()'s egress fuse is the last line (#4079). The + // prompt is small (~1K tokens); the dsh_session_log upload is 8 MiB of + // non-prompt telemetry (~2M tokens if it were priced). Before #7013 the + // fuse's meter counted the upload and answered 413; now it prices only + // the prompt mass and forwards. + const r = await fetch(ctx.url, { + method: "POST", + headers: { ...ctx.headers }, + body: JSON.stringify({ + stream: true, + messages: [{ role: "user", content: "hello, a small fitting prompt" }], + dsh_session_log: "t".repeat(8_000_000), + }), + }); + const body = await r.text(); + assert.equal(r.status, 200, `the fitting prompt must forward despite the 8 MiB non-prompt upload (got ${r.status}): ${body.slice(0, 300)}`); + assert.equal(calls.length, 1, `the request reached the upstream exactly once (got ${calls.length})`); + assert.ok(calls[0]!.bytes > 8_000_000, "the upload rode the wire (the upstream ignores it; bili only mis-priced it)"); + } finally { + await close(); + rmrf(root); + } +}); + +// #7013 both-ways: a prompt that GENUINELY exceeds the window still dooms, even +// with a large non-prompt upload field present — the fix must not over-suppress +// the fuse (#3617). The prompt mass alone clears the floored 1M bar. +test("e2e #7013 both-ways: a genuinely over-window prompt still dooms at the egress fuse", async () => { + const root = mkdtempSync(path.join(tmpdir(), "bili-7013b-")); + const { ctx, calls, close } = await setup(); + try { + // ~1.2M CJK chars ≈ ~1.2M tokens of PROMPT mass — over the floored 1M + // bar on its own. Plus a 2 MiB non-prompt upload (which the fix ignores + // for the window budget). The prompt mass alone dooms the send. + const r = await fetch(ctx.url, { + method: "POST", + headers: { ...ctx.headers }, + body: JSON.stringify({ + stream: true, + messages: [{ role: "user", content: "数".repeat(1_200_000) }], + dsh_session_log: "t".repeat(2_000_000), + }), + }); + const body = await r.text(); + const parsed = JSON.parse(body) as { error?: { code?: string; message?: string } }; + assert.equal(r.status, 413, `the over-window prompt must still doom (got ${r.status}): ${body.slice(0, 300)}`); + assert.equal(parsed.error?.code, "egress_window_fuse", "the fuse names itself"); + assert.match(parsed.error?.message ?? "", /outbound prompt measures ~[\d,]+ tokens/, "the prompt-mass math is in the message"); + assert.equal(calls.length, 0, "the doomed payload never reached the upstream"); + } finally { + await close(); + rmrf(root); + } +}); diff --git a/tests/issue4185-identified-fresh-forward.test.ts b/tests/issue4185-identified-fresh-forward.test.ts index 2aeab5a083..c9a14f6126 100644 --- a/tests/issue4185-identified-fresh-forward.test.ts +++ b/tests/issue4185-identified-fresh-forward.test.ts @@ -217,8 +217,8 @@ test("identified fresh session whose MEASURED body exceeds the native window but const r = await post(proxyPort, upstreamPort, "s-4185-measured", incidentConversation(300_000)); const body = await r.text(); assert.equal(r.status, 413, `measured-over body must fail fast: ${body.slice(0, 300)}`); - assert.match(body, /outbound body measures ~[\d,]+ TEXT tokens even after preflight compression/, `measured diagnostic present: ${body.slice(0, 300)}`); - assert.ok(body.includes("the body ITSELF exceeds the window"), `doom wording present: ${body.slice(0, 300)}`); + assert.match(body, /outbound prompt measures ~[\d,]+ tokens even after preflight compression/, `measured diagnostic present: ${body.slice(0, 300)}`); + assert.ok(body.includes("the prompt ITSELF exceeds the window"), `doom wording present: ${body.slice(0, 300)}`); assert.equal(calls.filter((c) => c.body.includes("TOOLMARKER_4185_")).length, 0, "the giant payload was never forwarded"); }); }); @@ -228,7 +228,7 @@ test("identified fresh session with mathematically doomed bytes (>2MB core) → const r = await post(proxyPort, upstreamPort, "s-4185-doomed", incidentConversation(2_100_000)); const body = await r.text(); assert.equal(r.status, 413, "doomed bytes must still fail fast"); - assert.ok(body.includes("the body ITSELF exceeds the window"), `doom diagnostic present: ${body.slice(0, 300)}`); + assert.ok(body.includes("the prompt ITSELF exceeds the window"), `doom diagnostic present: ${body.slice(0, 300)}`); assert.equal(calls.length, 0, "nothing forwarded and no summarization spent"); }); }); diff --git a/tests/preflight-zero-fold-calibration.test.ts b/tests/preflight-zero-fold-calibration.test.ts index 7837ff72ba..972bc03213 100644 --- a/tests/preflight-zero-fold-calibration.test.ts +++ b/tests/preflight-zero-fold-calibration.test.ts @@ -171,14 +171,14 @@ test("zero folds: anonymous unknown baseline — measured over-window body is do const f = await run({ windowRatio: 0.6, anonymous: true }); assert.ok("failFast" in f.result, JSON.stringify(f.logs)); assert.equal(f.result.status, 413); - assert.ok(f.logs.some(line => line.includes("body ITSELF exceeds the window"))); + assert.ok(f.logs.some(line => line.includes("prompt ITSELF exceeds the window"))); }); test("zero folds: a prior scoped refusal keeps the anonymous body locally doomed", { timeout: 15_000 }, async () => { const f = await run({ windowRatio: 0.6, anonymous: true, localRefusal: true }); assert.ok("failFast" in f.result, JSON.stringify(f.logs)); assert.equal(f.result.status, 413); - assert.ok(f.logs.some(line => line.includes("body ITSELF exceeds the window"))); + assert.ok(f.logs.some(line => line.includes("prompt ITSELF exceeds the window"))); }); test("zero folds: image reserve is not deflated by text calibration", { timeout: 15_000 }, async () => { diff --git a/tests/raw-body-walk-depth.test.ts b/tests/raw-body-walk-depth.test.ts index 3357d5277d..fcff9e927f 100644 --- a/tests/raw-body-walk-depth.test.ts +++ b/tests/raw-body-walk-depth.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { defaultCountTokens } from "acp-kernel"; -import { estimateRawBodyBreakdown, estimateRawBodyTokens } from "../src/preflight.ts"; +import { estimateRawBodyBreakdown, estimateRawBodyTokens, PROMPT_BODY_KEYS } from "../src/preflight.ts"; // #6038: client-controlled request bodies may nest far beyond safe recursion // depth; the raw-body token walks must survive them without changing any @@ -81,6 +81,113 @@ test("shallow plumbing: exclusion set, key inheritance and bucket split stay int test("non-object top-level bodies land whole in the other bucket", () => { const bd = estimateRawBodyBreakdown("plain string body"); - assert.deepEqual(bd, { total: t("plain string body"), messages: 0, tools: 0, system: 0, other: t("plain string body") }); + assert.deepEqual(bd, { total: t("plain string body"), messages: 0, tools: 0, system: 0, other: t("plain string body"), prompt: 0 }); assert.equal(estimateRawBodyTokens("plain string body"), t("plain string body")); }); + +// #7013: the wire-format's PROMPT mass is what a window-fit gate prices. +// Non-prompt fields (telemetry uploads, config scalars) ride the wire but +// consume NO window — they must sit in `other` (and `total`) but NOT in +// `prompt`. The fix keys off PROMPT_BODY_KEYS per wire format, never a +// hardcoded exclusion of one plugin's field in the window-fit pricing. +// (Top-level DSH transport fields `dsh_*` go further: excluded from `total` +// entirely by the NON_MODEL_TOP_LEVEL_KEYS rule — see +// raw-body-dsh-extensions.test.ts — so they never reach `other` either.) + +test("#7013: openai prompt mass = messages + tools (system rides inside messages)", () => { + const body = { + system: "in-body system rides inside messages for openai", + messages: [{ role: "user", content: "hello message" }], + tools: [{ name: "search", description: "find things fast" }], + dsh_session_log: { entries: ["telemetry one", "telemetry two"] }, + metadata: { note: "structural note" }, + }; + const bd = estimateRawBodyBreakdown(body, PROMPT_BODY_KEYS.openai); + // messages bucket: "user" + "hello message"; tools bucket: "search" + "find things fast". + const expectedPrompt = t("user") + t("hello message") + t("search") + t("find things fast"); + assert.equal(bd.prompt, expectedPrompt); + // The non-prompt top-level system + metadata land in other/total but NOT prompt + // (dsh_session_log is excluded from total by the dsh_* rule — see the block note). + assert.equal(bd.prompt, bd.messages + bd.tools); + assert.ok(bd.other > 0, "metadata is non-prompt mass (system lands in the system bucket)"); + assert.equal(bd.total, bd.messages + bd.tools + bd.system + bd.other); + assert.ok(bd.total > bd.prompt, "total includes non-prompt mass the prompt mass excludes"); + assert.equal(bd.total, estimateRawBodyTokens(body), "total invariant holds with promptKeys"); +}); + +test("#7013: anthropic prompt mass includes top-level system", () => { + const body = { + system: "top-level anthropic system", + messages: [{ role: "user", content: "hello" }], + tools: [{ name: "n", description: "d" }], + metadata: "big telemetry blob", + }; + const bd = estimateRawBodyBreakdown(body, PROMPT_BODY_KEYS.anthropic); + assert.equal(bd.prompt, bd.messages + bd.tools + bd.system); + assert.ok(bd.other > 0 && bd.total > bd.prompt); +}); + +test("#7013: google prompt mass = contents + systemInstruction + tools", () => { + const body = { + contents: [{ role: "user", parts: [{ text: "hello" }] }], + systemInstruction: "google system", + tools: [{ name: "n", description: "d" }], + metadata: "big telemetry blob", + }; + const bd = estimateRawBodyBreakdown(body, PROMPT_BODY_KEYS.google); + // google's prompt keys (contents/systemInstruction) land in the `other` + // bucket (the split is on the openai/anthropic key names), so assert the + // prompt mass against the direct leaf count, not the bucket sum. + const expectedPrompt = t("user") + t("hello") + t("google system") + t("n") + t("d"); + assert.equal(bd.prompt, expectedPrompt); + assert.ok(bd.other > 0 && bd.total > bd.prompt, "metadata is non-prompt mass"); +}); + +test("#7013: responses prompt mass = input + instructions + tools", () => { + const body = { + input: [{ role: "user", content: "hello" }], + instructions: "responses instructions", + tools: [{ name: "n", description: "d" }], + metadata: "big telemetry blob", + }; + const bd = estimateRawBodyBreakdown(body, PROMPT_BODY_KEYS.responses); + const expectedPrompt = t("user") + t("hello") + t("responses instructions") + t("n") + t("d"); + assert.equal(bd.prompt, expectedPrompt); + assert.ok(bd.other > 0 && bd.total > bd.prompt, "metadata is non-prompt mass"); +}); + +test("#7013: a giant dsh_session_log upload does not doom a fitting prompt", () => { + // The issue's repro: ~100K-token prompt + an 8 MiB dsh_session_log upload. + // Top-level dsh_* fields are excluded from `total` by the NON_MODEL_TOP_LEVEL_KEYS + // rule (see raw-body-dsh-extensions.test.ts) and are never in the prompt mass, so + // the window-fit gate (which prices prompt) sees only the fitting prompt. + const promptText = "x".repeat(400_000); // ~100K tokens + const telemetry = "t".repeat(8_000_000); // ~2M tokens of dsh_session_log upload + const body = { + messages: [{ role: "user", content: promptText }], + dsh_session_log: telemetry, + }; + const bd = estimateRawBodyBreakdown(body, PROMPT_BODY_KEYS.openai); + assert.equal(bd.prompt, t("user") + t(promptText)); + assert.equal(bd.total, bd.prompt, "dsh_session_log contributes zero to total"); + assert.ok(bd.prompt < 200_000, "prompt mass stays within a realistic window"); +}); + +test("#7013: a giant non-prompt telemetry upload does not inflate the prompt mass", () => { + // The general mechanism (non-DSH non-prompt fields): the upload lands in + // other/total but NOT in prompt, so the window-fit gate (which prices prompt) + // does not 413 a fitting prompt on a large non-prompt upload. + const promptText = "x".repeat(400_000); // ~100K tokens + const telemetry = "t".repeat(8_000_000); // ~2M tokens of non-prompt upload + const body = { + messages: [{ role: "user", content: promptText }], + telemetry_upload: telemetry, + }; + const bd = estimateRawBodyBreakdown(body, PROMPT_BODY_KEYS.openai); + assert.equal(bd.prompt, t("user") + t(promptText)); + assert.equal(bd.other, t(telemetry)); + assert.ok(bd.total > 2 * bd.prompt, "total dwarfs the prompt mass by the upload"); + // The window-fit gate compares bd.prompt (not bd.total) against the window, + // so a fitting prompt no longer 413s on the upload. + assert.ok(bd.prompt < 200_000, "prompt mass stays within a realistic window"); +}); diff --git a/tests/side-request-isolation.test.ts b/tests/side-request-isolation.test.ts index e14cdc10bd..1ba110ed1d 100644 --- a/tests/side-request-isolation.test.ts +++ b/tests/side-request-isolation.test.ts @@ -7,7 +7,7 @@ import { startServer, isSideRequest, isStructuredOutputCall, outputBudgetField, import { resolveSideLane, SIDE_REQUEST_MAX_TOKENS } from "../src/server/side-request.ts"; import type { ProxyOptions } from "../src/config.ts"; import { recordPluginRuntimeInfo, _resetPluginStateForTest } from "../src/plugin.ts"; -import { estimateRawBodyTokens, estimateRawBodyBreakdown } from "../src/preflight.ts"; +import { estimateRawBodyTokens, estimateRawBodyBreakdown, PROMPT_BODY_KEYS } from "../src/preflight.ts"; import { inspectContextOverflow } from "../src/util.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { mkdtempSync } from "node:fs"; @@ -635,7 +635,10 @@ test("inspectContextOverflow: exceed_context_size_error pattern + (A / B > W) wi test("sideRequestGuard: raw-body fit against declared ∩ armed window minus output headroom (#554)", () => { const txt = "z".repeat(8000); const body = { model: MODEL, max_tokens: 100, stream: true, messages: [{ role: "user", content: txt }] }; - const est = estimateRawBodyTokens(body); + // #7013: the guard prices the wire-format's prompt mass (PROMPT_BODY_KEYS) — + // model/max_tokens/stream are non-prompt config scalars, so the reference is + // the prompt mass, not the whole-body total. + const est = estimateRawBodyBreakdown(body, PROMPT_BODY_KEYS.anthropic).prompt; assert.ok(est > 0); assert.equal(sideRequestGuard(body, "anthropic", 0, undefined, undefined, 1).blocked, false, "unknown window → forward as before"); assert.equal(sideRequestGuard(body, "anthropic", est + 1, undefined, undefined, 1).blocked, false, "fits"); @@ -647,7 +650,7 @@ test("sideRequestGuard: raw-body fit against declared ∩ armed window minus out // reserved. #4205: the reservation policy is passed from the call site — // these rows pin the reserving direction explicitly. const oa = { model: MODEL, max_completion_tokens: 2_000, stream: true, messages: [{ role: "user", content: txt }] }; - const oaEst = estimateRawBodyTokens(oa); + const oaEst = estimateRawBodyBreakdown(oa, PROMPT_BODY_KEYS.openai).prompt; const oaLimit = Math.floor(oaEst / 1.15); const g = sideRequestGuard(oa, "openai", oaLimit + 2_000, undefined, undefined, 1, 0, undefined, true); assert.equal(g.limit, oaLimit, "limit reduced by max_completion_tokens"); @@ -666,13 +669,13 @@ test("sideRequestGuard: raw-body fit against declared ∩ armed window minus out { type: "text", text: "z".repeat(4000) }, { type: "image", source: { type: "base64", media_type: "image/png", data: "A".repeat(8000) } }, ] }] }; - const imgEst = estimateRawBodyTokens(imgBody) + Math.ceil(8000 / 4); + const imgEst = estimateRawBodyBreakdown(imgBody, PROMPT_BODY_KEYS.anthropic).prompt + Math.ceil(8000 / 4); assert.equal(sideRequestGuard(imgBody, "anthropic", Math.floor(imgEst / 1.15), undefined, undefined, 1).blocked, true, "image cost included at boundary"); // CJK estimator bias: defaultCountTokens counts CJK per-char (~1.6x real), // so a CJK-heavy payload estimated at ~110% of the window must forward — // the upstream's real overflow 400 arms the evidence that blocks re-sends. const cjkBody = { model: MODEL, max_tokens: 100, stream: true, messages: [{ role: "user", content: "汉".repeat(4000) }] }; - const cjkEst = estimateRawBodyTokens(cjkBody); + const cjkEst = estimateRawBodyBreakdown(cjkBody, PROMPT_BODY_KEYS.anthropic).prompt; assert.ok(cjkEst >= 4000, "CJK counted per-char"); assert.equal(sideRequestGuard(cjkBody, "anthropic", Math.floor(cjkEst / 1.10), undefined, undefined, 1).blocked, false, "CJK over-estimation absorbed by tolerance"); assert.equal(sideRequestGuard(cjkBody, "anthropic", Math.floor(cjkEst / 1.20), undefined, undefined, 1).blocked, true, "genuinely oversized CJK still blocks");