diff --git a/README.md b/README.md index 6b4913dbd3..e731496f82 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The proxy runs in one of two modes, and **the mode decides who executes | Client | ACP-native agent with the bili extension (pi/omp) | Any OpenAI/Anthropic client, no extension | | Who executes `compress` | **The agent** (pi runs it locally) | **The proxy** (server-side compress loop) | | `compress` tool call in the re-sent history? | Yes — part of the agent's own conversation | No — ephemeral proxy-loop traffic | -| Preflight blocks (no tool call)? | No — overflow forces the model to call `compress` | Yes — `src/preflight.ts` compresses behind the client's back | +| Preflight blocks (no tool call)? | Last-resort backstop — the agent normally compresses on its own `compress` calls, but `src/preflight.ts` still fires (in both modes) when the input alone exceeds the window (#470) | Yes — `src/preflight.ts` compresses behind the client's back | | **Summary carrier on the wire** | **the `compress` tool call** | **an `acp_summary` user message** | | System messages on the wire | always exactly 1 (client + prompt) | always exactly 1 (client + prompt) — summaries ride on user messages | | SGLang "single system" 400 (#377) | cannot happen | cannot happen (summaries are user messages, not system) | diff --git a/src/preflight.ts b/src/preflight.ts index 302892578d..a31d5784ca 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -44,6 +44,8 @@ export interface PreflightDeps { log: (level: string, msg: string) => void; /** Constant floor on the forwarded-payload size for this request (image bytes, #488). Folding only ever removes images, so adding this to every text estimate keeps the fit decision sound for multimodal payloads. */ imageFloor?: number; + /** Constant wire overhead for this request (system prompt + tool definitions, #470). Folding never removes it, so every fit decision must add it — otherwise the loop stops with "text fits" while the billed input still overflows. */ + wireOverhead?: number; } export type PreflightFailureKind = "upstream" | "exhausted" | "aborted"; @@ -247,7 +249,7 @@ function noEmergencyTruncate(config: Config): Config { export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise { const limit = deps.config.modelContextLimit; - const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) }; + const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0) }; if (limit <= 0) return result; const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION)); // applyCompression rejects ranges below config.compress.minCompressRange @@ -284,10 +286,10 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // Floor on the session's measured input baseline: the upstream's // input_tokens also covers the system prompt + tool definitions, which // are not in turn.messages, so the direct estimate can undershoot. - currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0)); + currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0)); // The caller's forward/fail-fast gate uses the payload's own estimate // (the floor can be stale — see PreflightResult.payloadEstimate). - result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0); + result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0); if (startTokens < 0) startTokens = currentTokens; if (currentTokens < limit) break; const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []); diff --git a/src/server.ts b/src/server.ts index 1d1e19c518..d5f4094002 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1750,6 +1750,41 @@ export function estimateInputTokens(processedMessages: CoreMessage[], systemText return Math.max(lastInputTokens > 0 ? lastInputTokens : 0, est); } +/** #470: tokens the wire payload carries OUTSIDE the message array — + * system/instructions text and tool definitions (including the proxy-injected + * ACP tools). estimateCoreMessages only counts messages, so without this term + * the preflight trigger fires ~10-20K late on agent clients with big tool + * manifests: text alone "fits" while the real billed input already overflows + * the window. Same term estimateInputTokens applies to the output clamp (#467). */ +export function estimateWireOverhead(protocol: "anthropic" | "openai" | "responses", body: string | Buffer): number { + let parsed: Record; + try { + parsed = JSON.parse(typeof body === "string" ? body : body.toString("utf8")) as Record; + } catch { + return 0; + } + const sysRaw = protocol === "responses" ? parsed.instructions : parsed.system; + let sysText = ""; + if (typeof sysRaw === "string") { + sysText = sysRaw; + } else if (Array.isArray(sysRaw)) { + sysText = sysRaw + .map((part) => (typeof (part as { text?: unknown })?.text === "string" ? (part as { text: string }).text : "")) + .join("\n"); + } + // openai chat: the kernel hoists leading system/developer messages out of + // the array into the rebuilt body's system field — but raw clients that + // never went through a rebuild keep them in messages; count both shapes. + if (protocol === "openai" && Array.isArray(parsed.messages)) { + const hoisted = (parsed.messages as Array>) + .filter((m) => m.role === "system" || m.role === "developer") + .map((m) => (typeof m.content === "string" ? m.content : "")) + .join("\n"); + sysText = sysText ? `${sysText}\n${hoisted}` : hoisted; + } + return defaultCountTokens(sysText) + defaultCountTokens(JSON.stringify(parsed.tools ?? [])); +} + /** Output-budget cap so input+output <= window. Returns the clamped budget, or * undefined when no reduction is needed (requested already fits, or the cap * drops below OUTPUT_CLAMP_FLOOR — i.e. input alone nearly fills the window, @@ -2525,7 +2560,11 @@ async function preflightCompressIfNeeded( // add their cost to every size decision here (trigger, fit gates, self-heal). const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body); const textEstimate = estimateCoreMessages(prepared.processedMessages); - const payloadEstimate = textEstimate + imageTokens; + // #470: system + tool definitions ride the wire too but are invisible to + // estimateCoreMessages — without them the trigger fires late (text alone + // under the window while the billed input already overflows it). + const overheadEstimate = estimateWireOverhead(prepared.protocol, prepared.body); + const payloadEstimate = textEstimate + overheadEstimate + imageTokens; const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate); if (limit <= 0 || !model || tokenCount < limit) return prepared; // #496 forward-once-then-learn: the default image cost (base64/4) matches byte @@ -2600,6 +2639,7 @@ async function preflightCompressIfNeeded( signal: clientAbort.signal, log, imageFloor: imageTokens, + wireOverhead: overheadEstimate, }, prepared.originalMessages, ); @@ -2615,8 +2655,8 @@ async function preflightCompressIfNeeded( // runPrepare re-incremented stats.requests; the rebuild is internal // to this single client request. session.stats.requests -= 1; - if (estimateCoreMessages(rebuilt.processedMessages) + imageTokens < limit) return rebuilt; - } else if (estimateCoreMessages(prepared.processedMessages) + imageTokens < limit) { + if (estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit) return rebuilt; + } else if (estimateCoreMessages(prepared.processedMessages) + overheadEstimate + imageTokens < limit) { log("warn", `[${session.id}] preflight made no progress but the payload fits; forwarding as-is`); return prepared; } diff --git a/tests/issue453-backstop.test.ts b/tests/issue453-backstop.test.ts index 4f03fc0ef3..69adeb8a8a 100644 --- a/tests/issue453-backstop.test.ts +++ b/tests/issue453-backstop.test.ts @@ -162,27 +162,31 @@ function turn2Compressible(): Record[] { } test("#453 clamp: oversized max_tokens is reduced when input+system nearly fills the window", async () => { - // window 100k, requested max_tokens 40k -> reserved window 60k. The core - // conversation estimates ~48k (under 60k, so preflight skips — it ignores the - // system), but the 60k-char system pushes the TRUE input past 60k. Only the - // clamp sees the system, so it lowers max_tokens to keep input+output under 100k. + // window 400k, requested max_tokens 150k -> reserved window 250k. The + // conversation text ~40k + an 800k-char system (~200k tokens) puts the + // true input at ~240k: UNDER the 250k reserved window (preflight skips — + // it counts system+tools since #470, plus the injected compress prompt + // ~2.4k, and still fits), but input+output no longer fits the full 400k + // window. Only the clamp sees the collision, so it lowers max_tokens. + // The band between the clamp threshold (~238k: input+5% margin+150k > 400k) + // and the preflight threshold (~250k) is where the clamp acts alone. const fired = await drive({ sessionId: "clamp-fire", - window: 100_000, + window: 400_000, turn1PromptTokens: 5_000, turn2Body: { - model: "m", max_tokens: 40_000, + model: "m", max_tokens: 150_000, messages: [ - { role: "system", content: "z".repeat(60_000) }, + { role: "system", content: "z".repeat(800_000) }, { role: "user", content: "hi" }, - { role: "assistant", content: "y".repeat(192_000) }, + { role: "assistant", content: "y".repeat(160_000) }, { role: "user", content: "now" }, ], }, }); const out = typeof fired.turn2Body.max_tokens === "number" ? fired.turn2Body.max_tokens : undefined; assert.ok(out !== undefined, "forwarded body carries a numeric max_tokens"); - assert.ok(out! < 40_000, `max_tokens clamped below the 40k request (got ${out})`); + assert.ok(out! < 150_000, `max_tokens clamped below the 150k request (got ${out})`); assert.ok(out! > 0, "clamped budget stays positive"); // Control: no oversized system -> input fits -> max_tokens passes through. diff --git a/tests/preflight-fail-fast.test.ts b/tests/preflight-fail-fast.test.ts index 45b524859e..81e4dc50da 100644 --- a/tests/preflight-fail-fast.test.ts +++ b/tests/preflight-fail-fast.test.ts @@ -273,13 +273,17 @@ test("e2e #330: over-window payload whose only foldable content is in the protec try { // Three messages against a 10k window: a small opener plus two large - // recent messages (~8k tokens each, ~16k total). All three sit inside + // recent messages (~6k tokens each, ~12k total). All three sit inside // the soft-protected recent zone (preserveRecentMessages=5 covers all // three), so the normal pass finds nothing foldable. Before #330 this // 502'd forever; now preflight relaxes the soft zone, folds the oldest - // large message, and forwards the now-fitting payload. - const big1 = "A".repeat(32000); - const big2 = "B".repeat(32000); + // large message, and forwards the now-fitting payload. #470: the wire + // overhead (injected compress system prompt + ACP tools, ~2.4k tokens) + // now counts toward every size decision, so post-fold fit needs the + // second big message at 24k chars, not 32k (the pre-#470 test was + // forwarding a payload whose real billed input exceeded the window). + const big1 = "A".repeat(24000); + const big2 = "B".repeat(24000); const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", "x-acp-session": "preflight-relax-sess" }, @@ -304,3 +308,50 @@ test("e2e #330: over-window payload whose only foldable content is in the protec await once(upstream, "close"); } }); + +test("e2e #470: system + tools overhead counts in the preflight trigger — text fits, wire overflows → preflight folds", async () => { + const calls: Call[] = []; + const upstream = makeUpstreamOk(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } }); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + // The message text alone (~8.5k) fits the 10k window, but the wire + // payload carries more: user system (~2k) + user tool (~1.2k) + the + // injected compress system prompt + ACP tools (~2.4k) — the billed + // input overflows. Before #470 the trigger counted messages only and + // forwarded the payload verbatim (guaranteed upstream 400 later). + const big1 = "A".repeat(34_000); + const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "preflight-overhead-sess" }, + body: JSON.stringify({ + model: "claude-small", + max_tokens: 1024, + stream: true, + system: "S".repeat(8_000), + tools: [{ name: "lookup", description: "T".repeat(4_600), input_schema: { type: "object", properties: {} } }], + messages: [ + { role: "user", content: "start the task" }, + { role: "assistant", content: big1 }, + { role: "user", content: "wrap up" }, + ], + }), + }); + assert.equal(r.status, 200, "the over-window (incl. system+tools) payload is folded and forwarded"); + assert.ok(calls.filter((c) => !c.stream).length >= 1, "preflight made the summarization call — the trigger counted the wire overhead"); + assert.equal(calls.filter((c) => c.stream).length, 1, "the folded payload was forwarded upstream"); + const forwarded = calls.find((c) => c.stream)?.body ?? ""; + assert.ok(!forwarded.includes("A".repeat(100)), "the folded big message did NOT ride the forwarded payload"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); diff --git a/tests/wire-overhead.test.ts b/tests/wire-overhead.test.ts new file mode 100644 index 0000000000..59b704bdd1 --- /dev/null +++ b/tests/wire-overhead.test.ts @@ -0,0 +1,60 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { estimateWireOverhead } from "../src/server.ts"; + +// #470: the preflight trigger under-counted the wire payload — system text and +// tool definitions (incl. the proxy-injected compress prompt + ACP tools) ride +// the request but are invisible to estimateCoreMessages. estimateWireOverhead +// extracts them from the raw wire body per protocol. + +test("estimateWireOverhead: anthropic string system + tools", () => { + const body = JSON.stringify({ + model: "m", + system: "S".repeat(4000), + tools: [{ name: "t", description: "D".repeat(2000), input_schema: {} }], + messages: [], + }); + const oh = estimateWireOverhead("anthropic", body); + assert.ok(oh >= 1000 && oh <= 2200, `system ~1000 + tools ~500 + JSON punctuation (got ${oh})`); +}); + +test("estimateWireOverhead: anthropic array-form system blocks", () => { + const body = JSON.stringify({ + system: [{ type: "text", text: "S1 ".repeat(500) }, { type: "text", text: "S2 ".repeat(500) }], + }); + const oh = estimateWireOverhead("anthropic", body); + assert.ok(oh >= 500 && oh <= 1200, `array blocks joined (got ${oh})`); +}); + +test("estimateWireOverhead: openai hoisted system/developer messages counted, non-system not", () => { + const body = JSON.stringify({ + messages: [ + { role: "system", content: "SYS ".repeat(300) }, + { role: "developer", content: "DEV ".repeat(300) }, + { role: "user", content: "U".repeat(40000) }, + { role: "assistant", content: "A".repeat(40000) }, + ], + }); + const oh = estimateWireOverhead("openai", body); + // ~300 tokens of system text — NOT the 20k of user/assistant content + assert.ok(oh >= 250 && oh <= 800, `only system/developer messages count (got ${oh})`); +}); + +test("estimateWireOverhead: responses instructions", () => { + const body = JSON.stringify({ instructions: "I".repeat(8000) }); + const oh = estimateWireOverhead("responses", body); + assert.ok(oh >= 1800 && oh <= 2400, `instructions ~2000 (got ${oh})`); +}); + +test("estimateWireOverhead: unparseable body → 0, not a crash", () => { + assert.equal(estimateWireOverhead("anthropic", "not json {"), 0); + assert.equal(estimateWireOverhead("openai", Buffer.from("}{"), ), 0); +}); + +test("estimateWireOverhead: empty payload → ~0 (no system, no tools)", () => { + assert.ok(estimateWireOverhead("anthropic", JSON.stringify({ messages: [] })) <= 1); + assert.ok(estimateWireOverhead("responses", JSON.stringify({})) <= 1); +});