Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import { epochConfirmPending } from "./fold-reconcile.js";
import { deadSkipKeys, filterDeadRanges, parseStructuralEmptyVerdict, recordDeadRange } from "./dead-ranges.js";
import { peekRegistryOutputLimit } from "./registry.js";
import { safePrefix } from "./text-safe.js";
import { applyEstimateCalibration, currentCalibrationFactor } from "./util.js";
import { applyEstimateCalibration, currentCalibrationFactor, type WireProtocol } from "./util.js";
import { configuredSummaryPlan, type ConfiguredSummaryPlan } from "./external-summary-runtime.js";
import { messageTimestampSections } from "./external-summary-surface.js";
import { isCodexClient } from "./codex-compact.js";
Expand Down Expand Up @@ -474,11 +474,30 @@ function isNonModelTopLevelKey(key: string): boolean {
return NON_MODEL_TOP_LEVEL_KEYS.has(key) || key.startsWith("dsh_");
}

export function estimateRawBodyTokens(parsed: unknown): number {
// #6828: a dual-shape body carries the conversation under TWO top-level keys —
// the lane's native field (messages[] on openai/anthropic, input[] on
// responses, contents[] on google) plus another protocol's field (a
// Responses-native host such as dsh sends `input` alongside `messages`). The
// upstream bills only the native field, so counting the foreign copy double-
// counts the conversation and the egress fuse false-trips on the phantom total
// (1,815,699 measured vs ~380K billed, 4.77x). Skip the foreign conversation
// fields for the lane's protocol, TOP LEVEL ONLY — a nested key with the same
// name (a message's own `input` content) is real content and stays counted.
// Without a protocol the meter keeps its fail-closed count-everything behavior.
const FOREIGN_CONVERSATION_FIELDS: Record<WireProtocol, ReadonlySet<string>> = {
responses: new Set(["messages", "system"]),
openai: new Set(["input", "instructions"]),
anthropic: new Set(["input", "instructions"]),
google: new Set(["input", "instructions", "messages", "system"]),
};

export function estimateRawBodyTokens(parsed: unknown, protocol?: WireProtocol): number {
let tokens = 0;
const foreign = protocol === undefined ? undefined : FOREIGN_CONVERSATION_FIELDS[protocol];
const pending: [unknown, string | undefined][] = [];
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (foreign?.has(k)) continue;
if (isNonModelTopLevelKey(k)) continue;
pending.push([v, k]);
}
Expand Down Expand Up @@ -508,7 +527,7 @@ export function estimateRawBodyTokens(parsed: unknown): number {
* oversized content" is actively misleading for a 55.2 MiB body whose
* messages[] is only 1.44 MiB. Same walk + same exclusions as
* estimateRawBodyTokens, split on the TOP-LEVEL field, so `total` always
* equals estimateRawBodyTokens(parsed) exactly. */
* equals estimateRawBodyTokens(parsed, protocol) exactly for the same protocol. */
export interface RawBodyBreakdown {
total: number;
messages: number;
Expand All @@ -517,11 +536,13 @@ export interface RawBodyBreakdown {
other: number;
}

export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown {
export function estimateRawBodyBreakdown(parsed: unknown, protocol?: WireProtocol): RawBodyBreakdown {
const buckets: [number, number, number, number] = [0, 0, 0, 0]; // messages, tools, system, other
const foreign = protocol === undefined ? undefined : FOREIGN_CONVERSATION_FIELDS[protocol];
const pending: [unknown, string | undefined, number][] = [];
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (foreign?.has(k)) continue; // #6828: the other protocol's conversation field
if (isNonModelTopLevelKey(k)) continue;
const bucket = k === "messages" ? 0 : k === "tools" ? 1 : k === "system" ? 2 : 3;
pending.push([v, k, bucket]);
Expand Down
11 changes: 8 additions & 3 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3790,7 +3790,7 @@ export function measuredBreakdownClause(b: RawBodyBreakdown): string {
return `messages ~${b.messages.toLocaleString("en-US")} · tools[] ~${b.tools.toLocaleString("en-US")} · system ~${b.system.toLocaleString("en-US")} · other ~${b.other.toLocaleString("en-US")}`;
}

export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?: unknown }): MeasuredOutbound | undefined {
export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?: unknown; protocol?: WireProtocol }): MeasuredOutbound | undefined {
let parsed: unknown = prepared.parsedBody;
if (parsed === undefined) {
const raw = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8");
Expand All @@ -3800,7 +3800,10 @@ export function measuredOutbound(prepared: { body: string | Buffer; parsedBody?:
return undefined;
}
}
const breakdown = estimateRawBodyBreakdown(parsed);
// #6828: pass the lane's protocol so a dual-shape body's foreign conversation
// field (e.g. Responses `input` riding alongside OpenAI `messages`) is not
// metered — the upstream never bills it, and counting it false-trips the fuse.
const breakdown = estimateRawBodyBreakdown(parsed, prepared.protocol);
return { tokens: breakdown.total, breakdown };
}

Expand Down Expand Up @@ -4532,7 +4535,9 @@ export async function preflightCompressIfNeeded(
let outboundBreakdown: RawBodyBreakdown | undefined;
try {
const outBody: unknown = JSON.parse(typeof outbound.body === "string" ? outbound.body : outbound.body.toString("utf8"));
outboundBreakdown = estimateRawBodyBreakdown(outBody);
// #6828: same protocol-scoped meter as the relay fuse — a dual-shape
// body's foreign conversation field must not doom-gate the rebuild.
outboundBreakdown = estimateRawBodyBreakdown(outBody, prepared.protocol);
outboundRawTokens = outboundBreakdown.total;
const outField = outputBudgetField(outBody);
const rawBudget = outField !== null ? readOutputBudget(outBody as Record<string, unknown>, outField) : undefined;
Expand Down
6 changes: 5 additions & 1 deletion src/server/side-request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1044,6 +1044,10 @@ export function sideRequestGuard(
const maxOut = (field ? readOutputBudget(parsed as Record<string, unknown>, field) : undefined) ?? 0;
if (reserveHeadroom && limit > 0) limit = reserveOutputHeadroom(limit, maxOut, headroomCap);
const imageMass = imageReserve ?? imageTokensInParsedBody(protocol, parsed, imageBilling, configuredCap);
const estimate = estimateRawBodyTokens(parsed) + imageMass;
// #6828: the #554 meter must skip the lane's foreign conversation field —
// a dual-shape body (e.g. Responses `input` riding alongside OpenAI
// `messages`) double-counts the conversation and false-blocks the side
// request the upstream would bill only on the native field.
const estimate = estimateRawBodyTokens(parsed, protocol) + imageMass;
return { blocked: limit > 0 && estimate >= limit * SIDE_REQUEST_GUARD_TOLERANCE, estimate, limit };
}
104 changes: 104 additions & 0 deletions tests/issue6828-guard-meter-dual-shape.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import assert from "node:assert/strict";
import test from "node:test";
import { defaultCountTokens } from "acp-kernel";
import { estimateRawBodyBreakdown, estimateRawBodyTokens } from "../src/preflight.ts";

// #6828: a dual-shape body carries the conversation under TWO top-level keys —
// the lane's native field (messages[] on openai/anthropic, input[] on
// responses) plus the other protocol's field (a Responses-native host such as
// dsh sends `input` alongside `messages`). The upstream bills only the native
// field; the #554 meter walks every string field, so it counted the
// conversation twice and the egress fuse false-tripped on the phantom total
// (1,815,699 measured vs ~380K billed, 4.77x). The meter must skip the foreign
// conversation fields for the lane's protocol, top level only.
const t = (s: string): number => defaultCountTokens(s);
const WINDOW = 1_048_576;

test("openai lane: a Responses-shaped `input` copy is not metered (#6828)", () => {
const body = {
model: "deepseek-flash",
messages: [{ role: "user", content: "hello" }],
tools: [{ type: "function", function: { name: "compress", description: "fold" } }],
system: "be helpful",
input: [{ type: "message", role: "user", content: "the whole raw history, again" }],
instructions: "do the thing",
};
const bd = estimateRawBodyBreakdown(body, "openai");
// the walk counts every string leaf, roles/types included
assert.equal(bd.messages, t("user") + t("hello"));
assert.equal(bd.system, t("be helpful"));
assert.equal(bd.tools, t("function") + t("compress") + t("fold"), "tools still counted");
assert.equal(bd.other, t("deepseek-flash"), "other holds only the envelope strings, not input/instructions");
assert.equal(bd.total, bd.messages + bd.tools + bd.system + bd.other);
assert.equal(bd.total, estimateRawBodyTokens(body, "openai"), "total invariant holds with a protocol");
});

test("responses lane: stray `messages`/`system` copies are not metered (#6828)", () => {
const body = {
model: "deepseek-flash",
input: [{ type: "message", role: "user", content: "the conversation" }],
instructions: "be helpful",
tools: [{ type: "function", function: { name: "compress", description: "fold" } }],
messages: [{ role: "user", content: "the same conversation, openai-shaped" }],
system: "be helpful, again",
};
const bd = estimateRawBodyBreakdown(body, "responses");
assert.equal(bd.messages, 0, "foreign messages[] excluded on the responses lane");
assert.equal(bd.system, 0, "foreign system excluded on the responses lane");
assert.equal(bd.other, t("deepseek-flash") + t("message") + t("user") + t("the conversation") + t("be helpful"));
assert.equal(bd.total, estimateRawBodyTokens(body, "responses"), "total invariant holds with a protocol");
});

test("no protocol: the meter keeps its fail-closed count-everything behavior (#6828)", () => {
const body = {
messages: [{ role: "user", content: "hello" }],
input: [{ type: "message", role: "user", content: "the whole raw history, again" }],
};
const bd = estimateRawBodyBreakdown(body);
assert.equal(bd.messages, t("user") + t("hello"));
assert.equal(bd.other, t("message") + t("user") + t("the whole raw history, again"), "without a protocol nothing is excluded");
assert.equal(bd.total, estimateRawBodyTokens(body));
});

test("nested same-named keys stay counted: only the top level is skipped (#6828)", () => {
const body = {
messages: [{ role: "user", content: "hi", input: "nested input text" }],
input: "foreign top-level copy",
};
const bd = estimateRawBodyBreakdown(body, "openai");
assert.equal(bd.messages, t("user") + t("hi") + t("nested input text"), "a message's own `input` field is real content");
assert.equal(bd.other, 0, "only the top-level `input` is skipped");
});

test("reporter scale: the #6828 fuse reading measures ~213K, not ~1.8M (#6828)", () => {
// models the issue's fuse reading: messages ~126,254 · tools[] ~76,502 ·
// system ~10,641 · other ~1,602,302 · total ~1,815,699 vs window 1,048,576,
// while the upstream billed ~380,665 (inputTokens 249 + cacheRead 380,416).
const cjk = (n: number): string => "测".repeat(n);
const ascii = (n: number): string => "a".repeat(n);
const perMsg = Math.floor(126_254 / 25);
const messages = Array.from({ length: 25 }, (_, i) => ({
role: i % 2 === 0 ? "user" : "assistant",
content: cjk(perMsg),
}));
const tools = [
{ type: "function", function: { name: "compress", description: ascii(100_000), parameters: ascii(100_000) } },
{ type: "function", function: { name: "retrieve", description: ascii(106_008), parameters: "" } },
];
const perItem = Math.floor(1_602_302 / 40);
const input = Array.from({ length: 40 }, (_, i) => ({
type: "message",
role: i % 2 === 0 ? "user" : "assistant",
content: cjk(perItem),
}));
const body = { model: "deepseek-flash", messages, tools, system: cjk(10_641), input };
const unscoped = estimateRawBodyBreakdown(body);
assert.ok(unscoped.total > WINDOW, `unscoped meter still sees the phantom total (${unscoped.total})`);
assert.ok(Math.abs(unscoped.total / 380_665 - 4.77) < 0.05, `unscoped ratio reproduces the reported 4.77x (${(unscoped.total / 380_665).toFixed(2)}x)`);
const bd = estimateRawBodyBreakdown(body, "openai");
assert.ok(bd.total < WINDOW, `openai-lane meter fits the window (${bd.total} < ${WINDOW})`);
assert.ok(Math.abs(bd.messages - 126_254) < 200, `messages bucket matches the report (${bd.messages})`);
assert.ok(Math.abs(bd.tools - 76_502) < 200, `tools bucket matches the report (${bd.tools})`);
assert.equal(bd.system, 10_641, "system bucket matches the report");
assert.ok(bd.other < 100, `other is envelope-only, not the 1.6M copy (${bd.other})`);
});
16 changes: 16 additions & 0 deletions tests/side-request-isolation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -678,6 +678,22 @@ test("sideRequestGuard: raw-body fit against declared ∩ armed window minus out
assert.equal(sideRequestGuard(cjkBody, "anthropic", Math.floor(cjkEst / 1.20), undefined, undefined, 1).blocked, true, "genuinely oversized CJK still blocks");
});

test("sideRequestGuard: a dual-shape body's foreign field does not false-block (#6828)", () => {
// The #554 guard meters the raw body before forwarding a side request.
// On a dual-shape body the protocol-blind walk counted the foreign
// conversation field too, so the compress call — the very request that
// would shrink the session — false-blocked at the same phantom total.
const cjk = "测".repeat(400_000);
const body = {
messages: [{ role: "user", content: cjk }],
input: [{ type: "message", role: "user", content: cjk }],
};
const limit = 600_000;
const guard = sideRequestGuard(body, "openai", limit);
assert.ok(guard.estimate < limit * 1.15, `native field alone is under the guard bar (${guard.estimate} < ${Math.round(limit * 1.15)})`);
assert.equal(guard.blocked, false, "the dual-shape body's foreign field must not false-block the side request");
});

function okSse(inputTokens: number): string {
return (
`event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "m1", role: "assistant", usage: { input_tokens: inputTokens } } })}\n\n` +
Expand Down
Loading