diff --git a/README.md b/README.md index 6b4913dbd3..693691ccdd 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The proxy runs in one of two modes, and **the mode decides who executes | Client | ACP-native agent with the bili extension (pi/omp) | Any OpenAI/Anthropic client, no extension | | Who executes `compress` | **The agent** (pi runs it locally) | **The proxy** (server-side compress loop) | | `compress` tool call in the re-sent history? | Yes — part of the agent's own conversation | No — ephemeral proxy-loop traffic | -| Preflight blocks (no tool call)? | No — overflow forces the model to call `compress` | Yes — `src/preflight.ts` compresses behind the client's back | +| Preflight blocks (no tool call)? | Yes — active here too as a last-resort backstop (e.g. #453); normally the agent's own `compress` fires first via the nudge | Yes — `src/preflight.ts` compresses behind the client's back | | **Summary carrier on the wire** | **the `compress` tool call** | **an `acp_summary` user message** | | System messages on the wire | always exactly 1 (client + prompt) | always exactly 1 (client + prompt) — summaries ride on user messages | | SGLang "single system" 400 (#377) | cannot happen | cannot happen (summaries are user messages, not system) | diff --git a/src/preflight.ts b/src/preflight.ts index 5390cacbc3..a39a9e0b3f 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -44,6 +44,8 @@ export interface PreflightDeps { log: (level: string, msg: string) => void; /** Constant floor on the forwarded-payload size for this request (image bytes, #488). Folding only ever removes images, so adding this to every text estimate keeps the fit decision sound for multimodal payloads. */ imageFloor?: number; + /** Constant billed-but-not-in-messages envelope cost (system/instructions + tools, #470). Folding never removes it, so it floors every size decision — without it the loop stops at "text fits" while the billed input overflows and the caller's overhead-aware gate fail-fasts a foldable payload. */ + wireOverhead?: number; } export type PreflightFailureKind = "upstream" | "exhausted" | "aborted"; @@ -226,7 +228,7 @@ const ABORTED_FAILURE: PreflightFailure = { kind: "aborted", detail: "the client export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise { const limit = deps.config.modelContextLimit; - const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) }; + const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0) }; if (limit <= 0) return result; const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION)); // applyCompression rejects ranges below config.compress.minCompressRange @@ -257,10 +259,10 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // Floor on the session's measured input baseline: the upstream's // input_tokens also covers the system prompt + tool definitions, which // are not in turn.messages, so the direct estimate can undershoot. - currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0)); + currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0)); // The caller's forward/fail-fast gate uses the payload's own estimate // (the floor can be stale — see PreflightResult.payloadEstimate). - result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0); + result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0); if (startTokens < 0) startTokens = currentTokens; if (currentTokens < limit) break; const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []); diff --git a/src/server.ts b/src/server.ts index a63362af16..f3d8cc076a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1761,6 +1761,55 @@ export function clampOutputBudget(requested: number, inputEstimate: number, nati return cap; } +function isObj(v: unknown): v is Record { + return typeof v === "object" && v !== null; +} + +/** #470: system/instructions + tools billed on top of the messages (which + * estimateCoreMessages omits). Read from the rebuilt body so it's mode-correct; + * text-only, so it never double-counts imageTokensInRawBody. Unparseable -> 0. */ +export function estimateWireOverhead(protocol: "anthropic" | "openai" | "responses", body: string | Buffer): number { + const s = typeof body === "string" ? body : body.toString("utf8"); + let parsed: unknown; + try { + parsed = JSON.parse(s); + } catch { + return 0; + } + if (!isObj(parsed)) return 0; + const textOf = (content: unknown): string => { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + let out = ""; + for (const part of content) if (isObj(part) && typeof part.text === "string") out += part.text; + return out; + }; + let systemText = ""; + if (protocol === "anthropic") { + const sys = parsed.system; + if (typeof sys === "string") systemText = sys; + else if (Array.isArray(sys)) for (const blk of sys) if (isObj(blk) && typeof blk.text === "string") systemText += blk.text; + } else if (protocol === "openai") { + const msgs = parsed.messages; + if (Array.isArray(msgs)) { + for (const m of msgs) { + if (!isObj(m) || (m.role !== "system" && m.role !== "developer")) continue; + systemText += textOf(m.content); + } + } + } else { + if (typeof parsed.instructions === "string") systemText += parsed.instructions; + const input = parsed.input; + if (Array.isArray(input)) { + for (const item of input) { + if (!isObj(item) || item.role !== "developer") continue; + systemText += textOf(item.content); + } + } + } + return defaultCountTokens(systemText) + defaultCountTokens(JSON.stringify(parsed.tools ?? [])); +} + // Only override genuine cadence silences: skip the kernel's deliberate // "nothing compressible to offer" suppression (empty ranges). export function emergencyNudge(nudge: NudgeDecision | null | undefined, escalationPct: number = EMERGENCY_NUDGE_ESCALATION_PCT): boolean { @@ -2524,8 +2573,10 @@ async function preflightCompressIfNeeded( // #488: images are forwarded verbatim but invisible to the kernel's text model — // add their cost to every size decision here (trigger, fit gates, self-heal). const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body); + // #470: system/instructions + tools are billed but invisible to estimateCoreMessages; count them or the trigger fires late and the post-fold gates pass over-window payloads. + const wireOverhead = estimateWireOverhead(prepared.protocol, prepared.body); const textEstimate = estimateCoreMessages(prepared.processedMessages); - const payloadEstimate = textEstimate + imageTokens; + const payloadEstimate = textEstimate + imageTokens + wireOverhead; const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate); if (limit <= 0 || !model || tokenCount < limit) return prepared; // #496 forward-once-then-learn: the default image cost (base64/4) matches byte @@ -2543,8 +2594,8 @@ async function preflightCompressIfNeeded( (model ? learnedMap?.[model] : undefined) ?? (session.metadata.learnedContextLimit as number | undefined); const noOverflowEvidence = session.stats.lastInputTokens < limit && learnedLimit === undefined; - if (imageTokens > 0 && textEstimate < limit && noOverflowEvidence) { - log("warn", `[${session.id}] image-dominated payload (~${textEstimate} text + ~${imageTokens} image tokens) exceeds window ${limit} by estimate only, no upstream overflow evidence — forwarding once so the upstream arbitrates billing (#496)`); + if (imageTokens > 0 && textEstimate + wireOverhead < limit && noOverflowEvidence) { + log("warn", `[${session.id}] image-dominated payload (~${textEstimate + wireOverhead} non-image + ~${imageTokens} image tokens) exceeds window ${limit} by estimate only, no upstream overflow evidence — forwarding once so the upstream arbitrates billing (#496)`); return prepared; } // #301: forwarding as-is is safe ONLY when the payload's own estimate @@ -2594,6 +2645,7 @@ async function preflightCompressIfNeeded( signal: clientAbort.signal, log, imageFloor: imageTokens, + wireOverhead, }, prepared.originalMessages, ); diff --git a/tests/codex-compact-e2e.test.ts b/tests/codex-compact-e2e.test.ts index b12effe37c..4076a08ca7 100644 --- a/tests/codex-compact-e2e.test.ts +++ b/tests/codex-compact-e2e.test.ts @@ -37,9 +37,9 @@ function fcEvents(callId: string, args: string): string { sse("response.output_item.done", { item: { type: "function_call", id: `fc_${callId}`, call_id: callId, name: "compress", arguments: args }, output_index: 0 }), ].join(""); } -// 7 messages × ~4400 chars (~1100 tokens each, ~7700 total). Below the 10k -// window so preflight never fires; above the recent-tail protection so -// m00001/m00002 are compressible by the model. +// 7 messages (~7700 tokens): with the billed wire envelope counted (#470) the +// turn is ~10.1k, under the 12k window so preflight never fires on turn 1; +// m00001/m00002 stay above the recent-tail protection so the model can fold them. function conversation() { const input: { type: string; role: string; content: string }[] = []; for (let i = 0; i < 7; i++) { @@ -47,7 +47,7 @@ function conversation() { } return input; } -// 14 messages ≈ 15.4k tokens — OVER the 10k window, so preflight WOULD fire +// 14 messages ≈ 15.4k tokens — OVER the 12k window, so preflight WOULD fire // if this payload went through the normal pipeline (#332 regression input). function bigConversation() { const input: { type: string; role: string; content: string }[] = []; @@ -104,9 +104,9 @@ async function withHarness(opts: { mode?: string; firstTurnTokens: number }, fn: port: 0, host: "127.0.0.1", upstream: "http://127.0.0.1", - routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-resp": { context: 10_000 } } } }, - modelContextLimit: 10_000, - kernelConfig: defaultConfig(10_000), + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-resp": { context: 12_000 } } } }, + modelContextLimit: 12_000, + kernelConfig: defaultConfig(12_000), compress: { injectTool: true, injectNudge: true }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", @@ -301,7 +301,7 @@ test("e2e E2 (endpoint form): intercept + healthy ACP → forged JSON {output}, // Issue #332: a compaction_trigger request that is NOT intercepted must reach // upstream byte-identical — no preflight overflow-compress, no payload // rebuild, no window clamp, no session-state folding as a side effect. The -// oversized payload (14 msgs ≈ 15.4k > 10k window) is what made the old +// oversized payload (14 msgs ≈ 15.4k > 12k window) is what made the old // ordering fire preflight before the compact detection ran. test("e2e #332 (trigger form): pass mode + oversized payload → forwarded byte-identical, no state mutation", async () => { diff --git a/tests/compaction-trigger-finality.test.ts b/tests/compaction-trigger-finality.test.ts index b81441eafa..ac5910922f 100644 --- a/tests/compaction-trigger-finality.test.ts +++ b/tests/compaction-trigger-finality.test.ts @@ -30,14 +30,14 @@ function completed(inputTokens: number): string { }); } -// 7 messages × ~4400 chars (~1100 tokens each, ~7700 total). Above the -// kernel's 5000-token recent-tail protection so m00001/m00002 stay -// compressible (a tier-1 pending range for the nudge), but below the 10k -// window so preflight never fires. +// 7 messages sized so the turn's billed input (text + system/instructions + +// tools, #470) stays under the 10k window (preflight never fires on turn 1) +// while m00001/m00002 remain above the recent-tail protection — the pending +// range the over-limit nudge points at. function conversation() { const input: { type: string; role: string; content: string }[] = []; for (let i = 0; i < 7; i++) { - input.push({ type: "message", role: i % 2 === 0 ? "user" : "assistant", content: `Message ${i} of the working session. ` + `WORK_${i}_content_`.repeat(290) }); + input.push({ type: "message", role: i % 2 === 0 ? "user" : "assistant", content: `Message ${i} of the working session. ` + `WORK_${i}_content_`.repeat(250) }); } return input; } diff --git a/tests/issue453-backstop.test.ts b/tests/issue453-backstop.test.ts index 4f03fc0ef3..396473e860 100644 --- a/tests/issue453-backstop.test.ts +++ b/tests/issue453-backstop.test.ts @@ -162,37 +162,38 @@ function turn2Compressible(): Record[] { } test("#453 clamp: oversized max_tokens is reduced when input+system nearly fills the window", async () => { - // window 100k, requested max_tokens 40k -> reserved window 60k. The core - // conversation estimates ~48k (under 60k, so preflight skips — it ignores the - // system), but the 60k-char system pushes the TRUE input past 60k. Only the - // clamp sees the system, so it lowers max_tokens to keep input+output under 100k. + // window 400k / max_tokens 150k -> reserved input budget 250k. Billed input + // (core+system+tools) is sized just under 250k so preflight skips, yet adding + // the 150k output pushes input+output over the window -> only the clamp acts. + // Re-sized from 100k/40k (#470): preflight now counts the envelope, so the old + // shape tripped preflight instead of isolating the clamp. const fired = await drive({ sessionId: "clamp-fire", - window: 100_000, + window: 400_000, turn1PromptTokens: 5_000, turn2Body: { - model: "m", max_tokens: 40_000, + model: "m", max_tokens: 150_000, messages: [ - { role: "system", content: "z".repeat(60_000) }, + { role: "system", content: "z".repeat(380_000) }, { role: "user", content: "hi" }, - { role: "assistant", content: "y".repeat(192_000) }, + { role: "assistant", content: "y".repeat(576_000) }, { role: "user", content: "now" }, ], }, }); const out = typeof fired.turn2Body.max_tokens === "number" ? fired.turn2Body.max_tokens : undefined; assert.ok(out !== undefined, "forwarded body carries a numeric max_tokens"); - assert.ok(out! < 40_000, `max_tokens clamped below the 40k request (got ${out})`); + assert.ok(out! < 150_000, `max_tokens clamped below the 150k request (got ${out})`); assert.ok(out! > 0, "clamped budget stays positive"); // Control: no oversized system -> input fits -> max_tokens passes through. const ctrl = await drive({ sessionId: "clamp-ctrl", - window: 100_000, + window: 400_000, turn1PromptTokens: 5_000, - turn2Body: { model: "m", max_tokens: 40_000, messages: turn2Compressible() }, + turn2Body: { model: "m", max_tokens: 150_000, messages: turn2Compressible() }, }); - assert.equal(ctrl.turn2Body.max_tokens, 40_000, "unclamped when input fits the window"); + assert.equal(ctrl.turn2Body.max_tokens, 150_000, "unclamped when input fits the window"); }); test("#453 escalation: nudge injected at the escalation line though the kernel cadence-stays silent", async () => { diff --git a/tests/preflight-fail-fast.test.ts b/tests/preflight-fail-fast.test.ts index bc8b3867e2..bfd3f588ff 100644 --- a/tests/preflight-fail-fast.test.ts +++ b/tests/preflight-fail-fast.test.ts @@ -85,6 +85,34 @@ function startProxy(upstreamPort: number, models: Record { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed: { stream?: boolean } = {}; + try { parsed = JSON.parse(raw); } catch { /* keep {} */ } + calls?.push({ stream: !!parsed.stream, body: raw }); + if (parsed.stream) { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(okSse(1000)); + } else { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + id: "msg_summary", type: "message", role: "assistant", model: "claude-small", + content: [{ type: "text", text: FOLD_SUMMARY }], stop_reason: "end_turn", + usage: { input_tokens: 500, output_tokens: 50 }, + })); + } + }); + }); +} + test("e2e #301: overflow + summary upstream 429 → structured 503, over-window payload NOT forwarded", async () => { const calls: Call[] = []; // The preflight summarization call hits the same rate-limited upstream as @@ -225,3 +253,42 @@ test("e2e #301: over-window payload with nothing compressible → structured 502 await once(upstream, "close"); } }); + +test("e2e #470: text fits the window alone but system+tools push the billed input over → preflight folds and forwards", async () => { + const calls: Call[] = []; + const upstream = makeUpstreamSummarize(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = upstream.address().port; + + // bigConversation() is 13.1k tokens: under a 15k window on TEXT alone + // (master's estimateCoreMessages would skip preflight), yet the injected + // compress prompt + ACP tools (~2.4k) push the BILLED input to 15.5k, past + // the window. Only counting the envelope makes preflight fire, fold, and + // forward — otherwise the over-window body goes up as-is (upstream 400). + const proxy = await startProxy(upstreamPort, { "claude-small": { context: 15_000 } }); + await once(proxy, "listening"); + const proxyPort = proxy.address().port; + + try { + const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "preflight-envelope-sess" }, + body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: bigConversation() }), + }); + assert.equal(r.status, 200, "envelope-driven overflow is folded and forwarded (not fail-fast, not blind forward)"); + await r.text(); + + assert.ok(calls.filter((c) => !c.stream).length >= 1, "preflight made a summarization call (master skipped preflight entirely)"); + const forwards = calls.filter((c) => c.stream); + const lastForward = forwards[forwards.length - 1]; + assert.ok(lastForward, "the folded payload was forwarded"); + assert.ok(!lastForward.body.includes("MARKER_1_"), "an early message was folded out of the forwarded body"); + assert.ok(lastForward.body.includes(FOLD_SUMMARY), "the folded summary is in the rebuilt payload"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); diff --git a/tests/wire-overhead.test.ts b/tests/wire-overhead.test.ts new file mode 100644 index 0000000000..a3a52fc789 --- /dev/null +++ b/tests/wire-overhead.test.ts @@ -0,0 +1,82 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { defaultCountTokens } from "acp-kernel"; +import { estimateWireOverhead } from "../src/server.ts"; + +// #470: recover the billed envelope (system/instructions + tools) the preflight +// trigger/gates omitted. Assertions use EXACT values from the same +// defaultCountTokens, so any over-count (leaked message / image / role) fails. + +const t = (s: string): number => defaultCountTokens(s); +const J = (o: unknown): string => JSON.stringify(o); + +test("anthropic: counts string system + tools, never the conversation messages", () => { + const body = { model: "m", max_tokens: 8, system: "S".repeat(4000), tools: [{ name: "read" }], messages: [{ role: "user", content: "M".repeat(40000) }] }; + const got = estimateWireOverhead("anthropic", J(body)); + assert.equal(got, t("S".repeat(4000)) + t(J([{ name: "read" }]))); + assert.ok(got < t("M".repeat(40000)), "the 40k-char conversation message must not leak into the envelope cost"); +}); + +test("anthropic: counts array-of-blocks system (text blocks only, images skipped)", () => { + const body = { + model: "m", + system: [ + { type: "text", text: "A".repeat(2000) }, + { type: "text", text: "B".repeat(2000) }, + { type: "image", source: { type: "url", url: "https://example.com/x.png" } }, + ], + messages: [], + }; + const got = estimateWireOverhead("anthropic", J(body)); + assert.equal(got, t("A".repeat(2000) + "B".repeat(2000)) + t("[]")); +}); + +test("openai: counts hoisted system/developer messages + tools, not other roles", () => { + const body = { + model: "m", + messages: [ + { role: "system", content: "S".repeat(3000) }, + { role: "developer", content: [{ type: "text", text: "D".repeat(1000) }] }, + { role: "user", content: "U".repeat(30000) }, + ], + tools: [{ type: "function", function: { name: "f" } }], + }; + const got = estimateWireOverhead("openai", J(body)); + assert.equal(got, t("S".repeat(3000) + "D".repeat(1000)) + t(J([{ type: "function", function: { name: "f" } }]))); +}); + +test("responses: counts top-level instructions + developer input item + tools", () => { + const body = { + model: "m", + instructions: "I".repeat(2000), + input: [ + { type: "message", role: "developer", content: [{ type: "input_text", text: "DEV".repeat(1500) }] }, + { type: "message", role: "user", content: [{ type: "input_text", text: "U".repeat(30000) }] }, + ], + tools: [{ type: "function", name: "g", parameters: {} }], + }; + const got = estimateWireOverhead("responses", J(body)); + assert.equal(got, t("I".repeat(2000) + "DEV".repeat(1500)) + t(J([{ type: "function", name: "g", parameters: {} }]))); +}); + +test("tools: larger definitions raise the overhead across all three protocols", () => { + const bigTools = Array.from({ length: 20 }, (_, i) => ({ type: "function", function: { name: `tool_${i}`, description: "d".repeat(200) } })); + const smallTools = [{ type: "function", function: { name: "tool_0" } }]; + for (const p of ["anthropic", "openai"] as const) { + const mk = (tools: unknown) => J({ model: "m", system: "", messages: [], tools }); + assert.ok(estimateWireOverhead(p, mk(bigTools)) > estimateWireOverhead(p, mk(smallTools)), `${p}: more tools -> more overhead`); + } + assert.ok( + estimateWireOverhead("responses", J({ model: "m", input: [], tools: bigTools })) > estimateWireOverhead("responses", J({ model: "m", input: [], tools: smallTools })), + "responses: tools counted even with no system/instructions", + ); +}); + +test("unparseable / non-object body -> 0; Buffer input behaves like a string", () => { + assert.equal(estimateWireOverhead("anthropic", "{not json"), 0); + assert.equal(estimateWireOverhead("openai", ""), 0); + assert.equal(estimateWireOverhead("responses", "null"), 0); + assert.equal(estimateWireOverhead("anthropic", "123"), 0); + const valid = J({ model: "m", system: "SS", tools: [{ name: "r" }] }); + assert.equal(estimateWireOverhead("anthropic", Buffer.from(valid, "utf8")), estimateWireOverhead("anthropic", valid)); +});