Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 40 additions & 10 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import { epochConfirmPending } from "./fold-reconcile.js";
import { deadSkipKeys, filterDeadRanges, parseStructuralEmptyVerdict, recordDeadRange } from "./dead-ranges.js";
import { peekRegistryOutputLimit } from "./registry.js";
import { safePrefix } from "./text-safe.js";
import { applyEstimateCalibration, currentCalibrationFactor } from "./util.js";
import { applyEstimateCalibration, currentCalibrationFactor, type WireProtocol } from "./util.js";
import { configuredSummaryPlan, type ConfiguredSummaryPlan } from "./external-summary-runtime.js";
import { messageTimestampSections } from "./external-summary-surface.js";
import { isCodexClient } from "./codex-compact.js";
Expand Down Expand Up @@ -502,45 +502,74 @@ export function estimateRawBodyTokens(parsed: unknown): number {
return tokens;
}

// #7013: the wire-format's PROMPT keys — the top-level fields the upstream's
// tokenizer actually counts against the model window. Everything else (telemetry
// uploads like DSH's `dsh_session_log`, config scalars, cache directives) rides
// the wire but consumes NO window, so a window-fit judgment must price only
// these — NOT a hardcoded exclusion of one plugin's field (the next plugin that
// registers a different upload field would recur the #7013 bug). The keys mirror
// what each prepare lane feeds its core converter: openai (messages — system
// rides inside it — + tools), anthropic (messages + tools + top-level system),
// google (contents + systemInstruction + tools), responses (input + instructions
// + tools).
export const PROMPT_BODY_KEYS: Record<WireProtocol, ReadonlySet<string>> = {
openai: new Set(["messages", "tools"]),
anthropic: new Set(["messages", "tools", "system"]),
google: new Set(["contents", "systemInstruction", "tools"]),
responses: new Set(["input", "instructions", "tools"]),
};

/** #5000 item 3: which DIMENSION carries the outbound mass. The doom messages
* must say whether it is messages[] (a giant paste / tool result), tools[]
* (84 schemas in #5000), the system prompt, or structural other — "trim the
* oversized content" is actively misleading for a 55.2 MiB body whose
* messages[] is only 1.44 MiB. Same walk + same exclusions as
* estimateRawBodyTokens, split on the TOP-LEVEL field, so `total` always
* equals estimateRawBodyTokens(parsed) exactly. */
* equals estimateRawBodyTokens(parsed) exactly. #7013: `prompt` is the mass
* under the wire-format's prompt keys — what the window-fit gates price. */
export interface RawBodyBreakdown {
total: number;
messages: number;
tools: number;
system: number;
other: number;
/** #7013: mass under the wire-format's prompt keys (PROMPT_BODY_KEYS) —
* what the upstream tokenizer counts against the model window. Non-prompt
* fields (telemetry uploads, config scalars) sit in `other` but NOT here.
* 0 when no promptKeys were supplied to the walk. */
prompt: number;
}

export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown {
export function estimateRawBodyBreakdown(parsed: unknown, promptKeys?: ReadonlySet<string>): RawBodyBreakdown {
const buckets: [number, number, number, number] = [0, 0, 0, 0]; // messages, tools, system, other
const pending: [unknown, string | undefined, number][] = [];
let prompt = 0;
const pending: [unknown, string | undefined, number, boolean][] = [];
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (isNonModelTopLevelKey(k)) continue;
const bucket = k === "messages" ? 0 : k === "tools" ? 1 : k === "system" ? 2 : 3;
pending.push([v, k, bucket]);
const isPrompt = promptKeys ? promptKeys.has(k) : false;
pending.push([v, k, bucket, isPrompt]);
}
} else {
pending.push([parsed, undefined, 3]);
pending.push([parsed, undefined, 3, false]);
}
while (pending.length > 0) {
const [value, key, bucket] = pending.pop()!;
const [value, key, bucket, isPrompt] = pending.pop()!;
if (typeof value === "string") {
if (!key || !NON_TEXT_BODY_KEYS.has(key)) buckets[bucket] += defaultCountTokens(value);
if (!key || !NON_TEXT_BODY_KEYS.has(key)) {
const t = defaultCountTokens(value);
buckets[bucket] += t;
if (isPrompt) prompt += t;
}
continue;
}
if (Array.isArray(value)) {
for (const item of value) pending.push([item, key, bucket]);
for (const item of value) pending.push([item, key, bucket, isPrompt]);
continue;
}
if (value && typeof value === "object") {
for (const [k, v] of Object.entries(value as Record<string, unknown>)) pending.push([v, k, bucket]);
for (const [k, v] of Object.entries(value as Record<string, unknown>)) pending.push([v, k, bucket, isPrompt]);
}
}
return {
Expand All @@ -549,6 +578,7 @@ export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown {
tools: buckets[1],
system: buckets[2],
other: buckets[3],
prompt,
};
}

Expand Down
48 changes: 32 additions & 16 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ import { auditStartupSummaryCredentials } from "./external-summary-credentials.j
import { applyRanges } from "./stream.js";
import { attachSubagentSessions } from "./subagent-sessions.js";
import { buildSessionCacheReport, handleAcpCache, learnedImageReserve, noteClientAbort, readKeySwitchStats, readModelSwitchStats, settleUsageReport } from "./cache-ledger.js";
import { extractBillingAttributionBlock, extractSummaryFromSse, preflightCompress, estimateCoreMessages, estimateCoreMessagesUpper, estimateCoreMessagesUpperBytes, describeCoreTextDuplication, estimateRawBodyTokens, estimateRawBodyBreakdown, type PreflightDeps, type PreflightResult, type RawBodyBreakdown } from "./preflight.js";
import { extractBillingAttributionBlock, extractSummaryFromSse, preflightCompress, estimateCoreMessages, estimateCoreMessagesUpper, estimateCoreMessagesUpperBytes, describeCoreTextDuplication, estimateRawBodyTokens, estimateRawBodyBreakdown, PROMPT_BODY_KEYS, type PreflightDeps, type PreflightResult, type RawBodyBreakdown } from "./preflight.js";
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, DEFAULT_DECIDE_MAX_TOKENS, DECIDE_TIMEOUT_MS, extractDecisionText, ladderMode, parseDecision, recordDecision, resolveDecisionRange, type DecideConfig, type DecisionOutcome } from "./nudge-decide.js";
import { gcConfigFromEnv, gcSessionFiles } from "./session-gc.js";
import { countImagesInParsedBody, countImagesInRawBody, imageTokensInRawBody, imageTokensInParsedBody, resolveImageBilling, upstreamHost, type ResolvedImageBilling } from "./image-tokens.js";
Expand Down Expand Up @@ -3777,20 +3777,29 @@ export function buildWindowShrinkNote(args: {
// tokens are charged separately by the caller (they sit in binary-carrying
// keys the walk excludes by design). Unparseable bodies return undefined —
// callers keep the byte-floor fallback for them.
// #7013: the meter prices only the wire-format's PROMPT mass (PROMPT_BODY_KEYS)
// — what the upstream tokenizer counts against the model window. Non-prompt
// fields (telemetry uploads like DSH's dsh_session_log, config scalars) ride the
// wire but consume no window; pricing them made a ~100K-token prompt with an
// 8 MiB upload field 413. The full-body `total` is still reported in the
// breakdown for the 413 message, so the operator sees where the mass is.
interface MeasuredOutbound {
/** #7013: the wire-format's prompt mass (PROMPT_BODY_KEYS) — the value the
* window-fit gates compare against the model window. */
tokens: number;
breakdown: RawBodyBreakdown;
}

// #5000 item 3: dimension clause for the measured-doom messages — which part
// of the body carries the mass, so "trim the oversized content" tells the
// operator WHERE (a 55.2 MiB #5000 body with 1.44 MiB of messages[] means the
// mass is tools[], not a giant paste).
// mass is tools[], not a giant paste). #7013: `other` is non-prompt mass (the
// 413 gate prices messages/tools/system only) — the clause says so.
export function measuredBreakdownClause(b: RawBodyBreakdown): string {
return `messages ~${b.messages.toLocaleString("en-US")} · tools[] ~${b.tools.toLocaleString("en-US")} · system ~${b.system.toLocaleString("en-US")} · other ~${b.other.toLocaleString("en-US")}`;
return `messages ~${b.messages.toLocaleString("en-US")} · tools[] ~${b.tools.toLocaleString("en-US")} · system ~${b.system.toLocaleString("en-US")} · other ~${b.other.toLocaleString("en-US")} (non-prompt, excluded from the window budget)`;
}

export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?: unknown }): MeasuredOutbound | undefined {
export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?: unknown; protocol: WireProtocol }): MeasuredOutbound | undefined {
let parsed: unknown = prepared.parsedBody;
if (parsed === undefined) {
const raw = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8");
Expand All @@ -3800,8 +3809,8 @@ export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?:
return undefined;
}
}
const breakdown = estimateRawBodyBreakdown(parsed);
return { tokens: breakdown.total, breakdown };
const breakdown = estimateRawBodyBreakdown(parsed, PROMPT_BODY_KEYS[prepared.protocol]);
return { tokens: breakdown.prompt, breakdown };
}

export async function preflightCompressIfNeeded(
Expand Down Expand Up @@ -4525,15 +4534,18 @@ export async function preflightCompressIfNeeded(
// the #554 side-request guard's own: it walks every string field of the
// serialized body (system + tools + arguments + messages, CJK-aware), so
// unlike the m.text byte floor it sees the mass that actually rides the
// wire. Parse failure leaves both readings undefined — the byte-floor
// fallbacks stay binding for those bodies.
// wire. #7013: it prices only the wire-format's PROMPT mass (PROMPT_BODY_KEYS
// for outbound.protocol) — non-prompt fields (telemetry uploads like DSH's
// dsh_session_log, config scalars) ride the wire but consume no window, so
// they must not doom a fitting prompt. Parse failure leaves both readings
// undefined — the byte-floor fallbacks stay binding for those bodies.
let outboundBudgetTokens: number | undefined;
let outboundRawTokens: number | undefined;
let outboundBreakdown: RawBodyBreakdown | undefined;
try {
const outBody: unknown = JSON.parse(typeof outbound.body === "string" ? outbound.body : outbound.body.toString("utf8"));
outboundBreakdown = estimateRawBodyBreakdown(outBody);
outboundRawTokens = outboundBreakdown.total;
outboundBreakdown = estimateRawBodyBreakdown(outBody, PROMPT_BODY_KEYS[outbound.protocol]);
outboundRawTokens = outboundBreakdown.prompt;
const outField = outputBudgetField(outBody);
const rawBudget = outField !== null ? readOutputBudget(outBody as Record<string, unknown>, outField) : undefined;
if (typeof rawBudget === "number" && rawBudget > 0) outboundBudgetTokens = rawBudget;
Expand Down Expand Up @@ -4563,13 +4575,15 @@ export async function preflightCompressIfNeeded(
// Images are NOT part of this comparison: an image-dominated overage is
// the #496/#1800 billing-arbitration domain (forward, let upstream bill
// or 400 for real evidence), not a doom — #857's screenshot turn pins
// that. The #554 meter excludes image payloads (image_url/data/url/...),
// so `outboundRawTokens` is text-only by construction.
// that. The #554 meter excludes image payloads (image_url/data/url/...)
// and, #7013, prices only the wire-format's prompt keys — so
// `outboundRawTokens` is prompt-text-only by construction (non-prompt
// telemetry uploads like DSH's dsh_session_log never doom here).
const armMeasuredBar = typeof resolvedNativeWindow === "number" && resolvedNativeWindow > 0 ? resolvedNativeWindow : limit;
if (outboundRawTokens !== undefined && outboundRawTokens > armMeasuredBar) {
const outMsgs = outbound.processedMessages.length > 0 ? outbound.processedMessages : outbound.originalMessages;
log("error", `[${session.id}] preflight 413 measured-body profile (post-fold): ${describeCoreTextDuplication(outMsgs)}`);
const measuredDoomMessage = `rebuilt outbound body measures ~${outboundRawTokens.toLocaleString("en-US")} TEXT tokens (${outboundBreakdown ? measuredBreakdownClause(outboundBreakdown) : "by field: n/a"}; CJK-aware count, the #554 guard meter; images billed separately, never doom here — #496/#1800/#857) vs the model window ${armMeasuredBar.toLocaleString("en-US")} (model=${model}); the body ITSELF exceeds the window, so it was NOT forwarded for upstream arbitration. Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`;
const measuredDoomMessage = `rebuilt outbound prompt measures ~${outboundRawTokens.toLocaleString("en-US")} tokens (${outboundBreakdown ? measuredBreakdownClause(outboundBreakdown) : "by field: n/a"}; CJK-aware count, the #554 guard meter; images billed separately, never doom here — #496/#1800/#857; non-prompt fields in the other bucket are excluded from the window budget — #7013) vs the model window ${armMeasuredBar.toLocaleString("en-US")} (model=${model}); the prompt ITSELF exceeds the window, so it was NOT forwarded for upstream arbitration. Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`;
log("error", `[${session.id}] preflight fail-fast 413 (retryable=false): ${measuredDoomMessage}`);
noteLocalRefusal(session, currentOrigin, model, limit);
return { failFast: true, status: 413, message: measuredDoomMessage, retryable: false, respond: !res.writableEnded };
Expand Down Expand Up @@ -4666,12 +4680,14 @@ export async function preflightCompressIfNeeded(
// false-refuse a natively-fitting body), the headroom limit as
// fallback. Measured tier first, byte floor below stays as the
// fallback for unparseable bodies.
// Same text-only rule as the #1839 arm above: image-dominated overage
// belongs to the #496/#1800 billing arbitration, never to a local doom.
// Same prompt-text-only rule as the #1839 arm above (#7013): image-
// dominated overage belongs to the #496/#1800 billing arbitration, and
// non-prompt telemetry uploads (DSH's dsh_session_log) never doom a
// fitting prompt, never to a local doom.
const gateMeasuredBar = typeof resolvedNativeWindow === "number" && resolvedNativeWindow > 0 ? resolvedNativeWindow : limit;
if (outboundRawTokens !== undefined && outboundRawTokens > gateMeasuredBar) {
log("error", `[${session.id}] preflight 413 measured-body profile: ${describeCoreTextDuplication(outMsgs)}`);
const measuredDoomMessage = `outbound body measures ~${outboundRawTokens.toLocaleString("en-US")} TEXT tokens even after preflight compression (${outboundBreakdown ? measuredBreakdownClause(outboundBreakdown) : "by field: n/a"}; CJK-aware count, the #554 guard meter; images billed separately, never doom here — #496/#1800/#857) vs the model window ${gateMeasuredBar.toLocaleString("en-US")} (model=${model}); the body ITSELF exceeds the window, so it was NOT forwarded. Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`;
const measuredDoomMessage = `outbound prompt measures ~${outboundRawTokens.toLocaleString("en-US")} tokens even after preflight compression (${outboundBreakdown ? measuredBreakdownClause(outboundBreakdown) : "by field: n/a"}; CJK-aware count, the #554 guard meter; images billed separately, never doom here — #496/#1800/#857; non-prompt fields in the other bucket are excluded from the window budget — #7013) vs the model window ${gateMeasuredBar.toLocaleString("en-US")} (model=${model}); the prompt ITSELF exceeds the window, so it was NOT forwarded. Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`;
log("error", `[${session.id}] preflight fail-fast 413 (retryable=false): ${measuredDoomMessage}`);
noteLocalRefusal(session, currentOrigin, model, limit);
return { failFast: true, status: 413, message: measuredDoomMessage, retryable: false, respond: !res.writableEnded };
Expand Down
6 changes: 5 additions & 1 deletion src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1036,11 +1036,15 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
}
return;
}
// #7013: the fuse prices the wire-format's PROMPT mass (PROMPT_BODY_KEYS
// for prepared.protocol) — non-prompt fields (telemetry uploads like
// DSH's dsh_session_log, config scalars) ride the wire but consume no
// window, so they must not 413 a fitting prompt.
const fuseMeasured = measuredOutbound(prepared);
const fuseRawTokens = fuseMeasured?.tokens;
if (fuseMeasured !== undefined && fuseRawTokens !== undefined && fuseRawTokens > fuseWindow) {
log("warn", `413: [${prepared.session.id}] egress window fuse measured-body profile: ${describeCoreTextDuplication(fuseMsgs)}`);
const fuseMessage = `outbound body measures ~${fuseRawTokens.toLocaleString("en-US")} tokens (${measuredBreakdownClause(fuseMeasured.breakdown)}; CJK-aware count, the #554 guard meter) vs the effective model window ${declaredWindow.toLocaleString("en-US")} (bars never assume below ${MIN_ASSUMED_WINDOW_TOKENS} tokens, #4079) (model=${prepared.model ?? "unknown"}); the body ITSELF exceeds the window, so the send was stopped at the egress fuse instead of being forwarded (#5000/#4543/#4068). Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`;
const fuseMessage = `outbound prompt measures ~${fuseRawTokens.toLocaleString("en-US")} tokens (${measuredBreakdownClause(fuseMeasured.breakdown)}; CJK-aware count, the #554 guard meter; non-prompt fields in the other bucket are excluded from the window budget — #7013) vs the effective model window ${declaredWindow.toLocaleString("en-US")} (bars never assume below ${MIN_ASSUMED_WINDOW_TOKENS} tokens, #4079) (model=${prepared.model ?? "unknown"}); the prompt ITSELF exceeds the window, so the send was stopped at the egress fuse instead of being forwarded (#5000/#4543/#4068). Remove or trim the oversized dimension named above (e.g. a giant paste or tool result in messages[], or tools[] schemas) and retry.`;
log("warn", `413: [${prepared.session.id}] egress window fuse: ${fuseMessage}`);
if (res.headersSent) emitStreamError(res, prepared.protocol, fuseMessage, undefined, opts.streamErrorShape);
else {
Expand Down
Loading
Loading