Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ The proxy runs in one of two modes, and **the mode decides who executes
| Client | ACP-native agent with the bili extension (pi/omp) | Any OpenAI/Anthropic client, no extension |
| Who executes `compress` | **The agent** (pi runs it locally) | **The proxy** (server-side compress loop) |
| `compress` tool call in the re-sent history? | Yes — part of the agent's own conversation | No — ephemeral proxy-loop traffic |
| Preflight blocks (no tool call)? | No — overflow forces the model to call `compress` | Yes — `src/preflight.ts` compresses behind the client's back |
| Preflight blocks (no tool call)? | Last-resort backstop — the agent normally compresses on its own `compress` calls, but `src/preflight.ts` still fires (in both modes) when the input alone exceeds the window (#470) | Yes — `src/preflight.ts` compresses behind the client's back |
| **Summary carrier on the wire** | **the `compress` tool call** | **an `acp_summary` user message** |
| System messages on the wire | always exactly 1 (client + prompt) | always exactly 1 (client + prompt) — summaries ride on user messages |
| SGLang "single system" 400 (#377) | cannot happen | cannot happen (summaries are user messages, not system) |
Expand Down
8 changes: 5 additions & 3 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ export interface PreflightDeps {
log: (level: string, msg: string) => void;
/** Constant floor on the forwarded-payload size for this request (image bytes, #488). Folding only ever removes images, so adding this to every text estimate keeps the fit decision sound for multimodal payloads. */
imageFloor?: number;
/** Constant wire overhead for this request (system prompt + tool definitions, #470). Folding never removes it, so every fit decision must add it — otherwise the loop stops with "text fits" while the billed input still overflows. */
wireOverhead?: number;
}

export type PreflightFailureKind = "upstream" | "exhausted" | "aborted";
Expand Down Expand Up @@ -247,7 +249,7 @@ function noEmergencyTruncate(config: Config): Config {

export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise<PreflightResult> {
const limit = deps.config.modelContextLimit;
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) };
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0) };
if (limit <= 0) return result;
const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
// applyCompression rejects ranges below config.compress.minCompressRange
Expand Down Expand Up @@ -284,10 +286,10 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// Floor on the session's measured input baseline: the upstream's
// input_tokens also covers the system prompt + tool definitions, which
// are not in turn.messages, so the direct estimate can undershoot.
currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0));
currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0));
// The caller's forward/fail-fast gate uses the payload's own estimate
// (the floor can be stale — see PreflightResult.payloadEstimate).
result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0);
result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0);
if (startTokens < 0) startTokens = currentTokens;
if (currentTokens < limit) break;
const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []);
Expand Down
46 changes: 43 additions & 3 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1750,6 +1750,41 @@ export function estimateInputTokens(processedMessages: CoreMessage[], systemText
return Math.max(lastInputTokens > 0 ? lastInputTokens : 0, est);
}

/** #470: tokens the wire payload carries OUTSIDE the message array —
* system/instructions text and tool definitions (including the proxy-injected
* ACP tools). estimateCoreMessages only counts messages, so without this term
* the preflight trigger fires ~10-20K late on agent clients with big tool
* manifests: text alone "fits" while the real billed input already overflows
* the window. Same term estimateInputTokens applies to the output clamp (#467). */
export function estimateWireOverhead(protocol: "anthropic" | "openai" | "responses", body: string | Buffer): number {
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(typeof body === "string" ? body : body.toString("utf8")) as Record<string, unknown>;
} catch {
return 0;
}
const sysRaw = protocol === "responses" ? parsed.instructions : parsed.system;
let sysText = "";
if (typeof sysRaw === "string") {
sysText = sysRaw;
} else if (Array.isArray(sysRaw)) {
sysText = sysRaw
.map((part) => (typeof (part as { text?: unknown })?.text === "string" ? (part as { text: string }).text : ""))
.join("\n");
}
// openai chat: the kernel hoists leading system/developer messages out of
// the array into the rebuilt body's system field — but raw clients that
// never went through a rebuild keep them in messages; count both shapes.
if (protocol === "openai" && Array.isArray(parsed.messages)) {
const hoisted = (parsed.messages as Array<Record<string, unknown>>)
.filter((m) => m.role === "system" || m.role === "developer")
.map((m) => (typeof m.content === "string" ? m.content : ""))
.join("\n");
sysText = sysText ? `${sysText}\n${hoisted}` : hoisted;
}
return defaultCountTokens(sysText) + defaultCountTokens(JSON.stringify(parsed.tools ?? []));
}

/** Output-budget cap so input+output <= window. Returns the clamped budget, or
* undefined when no reduction is needed (requested already fits, or the cap
* drops below OUTPUT_CLAMP_FLOOR — i.e. input alone nearly fills the window,
Expand Down Expand Up @@ -2525,7 +2560,11 @@ async function preflightCompressIfNeeded(
// add their cost to every size decision here (trigger, fit gates, self-heal).
const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body);
const textEstimate = estimateCoreMessages(prepared.processedMessages);
const payloadEstimate = textEstimate + imageTokens;
// #470: system + tool definitions ride the wire too but are invisible to
// estimateCoreMessages — without them the trigger fires late (text alone
// under the window while the billed input already overflows it).
const overheadEstimate = estimateWireOverhead(prepared.protocol, prepared.body);
const payloadEstimate = textEstimate + overheadEstimate + imageTokens;
const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate);
if (limit <= 0 || !model || tokenCount < limit) return prepared;
// #496 forward-once-then-learn: the default image cost (base64/4) matches byte
Expand Down Expand Up @@ -2600,6 +2639,7 @@ async function preflightCompressIfNeeded(
signal: clientAbort.signal,
log,
imageFloor: imageTokens,
wireOverhead: overheadEstimate,
},
prepared.originalMessages,
);
Expand All @@ -2615,8 +2655,8 @@ async function preflightCompressIfNeeded(
// runPrepare re-incremented stats.requests; the rebuild is internal
// to this single client request.
session.stats.requests -= 1;
if (estimateCoreMessages(rebuilt.processedMessages) + imageTokens < limit) return rebuilt;
} else if (estimateCoreMessages(prepared.processedMessages) + imageTokens < limit) {
if (estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit) return rebuilt;
} else if (estimateCoreMessages(prepared.processedMessages) + overheadEstimate + imageTokens < limit) {
log("warn", `[${session.id}] preflight made no progress but the payload fits; forwarding as-is`);
return prepared;
}
Expand Down
22 changes: 13 additions & 9 deletions tests/issue453-backstop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,27 +162,31 @@ function turn2Compressible(): Record<string, unknown>[] {
}

test("#453 clamp: oversized max_tokens is reduced when input+system nearly fills the window", async () => {
// window 100k, requested max_tokens 40k -> reserved window 60k. The core
// conversation estimates ~48k (under 60k, so preflight skips — it ignores the
// system), but the 60k-char system pushes the TRUE input past 60k. Only the
// clamp sees the system, so it lowers max_tokens to keep input+output under 100k.
// window 400k, requested max_tokens 150k -> reserved window 250k. The
// conversation text ~40k + an 800k-char system (~200k tokens) puts the
// true input at ~240k: UNDER the 250k reserved window (preflight skips —
// it counts system+tools since #470, plus the injected compress prompt
// ~2.4k, and still fits), but input+output no longer fits the full 400k
// window. Only the clamp sees the collision, so it lowers max_tokens.
// The band between the clamp threshold (~238k: input+5% margin+150k > 400k)
// and the preflight threshold (~250k) is where the clamp acts alone.
const fired = await drive({
sessionId: "clamp-fire",
window: 100_000,
window: 400_000,
turn1PromptTokens: 5_000,
turn2Body: {
model: "m", max_tokens: 40_000,
model: "m", max_tokens: 150_000,
messages: [
{ role: "system", content: "z".repeat(60_000) },
{ role: "system", content: "z".repeat(800_000) },
{ role: "user", content: "hi" },
{ role: "assistant", content: "y".repeat(192_000) },
{ role: "assistant", content: "y".repeat(160_000) },
{ role: "user", content: "now" },
],
},
});
const out = typeof fired.turn2Body.max_tokens === "number" ? fired.turn2Body.max_tokens : undefined;
assert.ok(out !== undefined, "forwarded body carries a numeric max_tokens");
assert.ok(out! < 40_000, `max_tokens clamped below the 40k request (got ${out})`);
assert.ok(out! < 150_000, `max_tokens clamped below the 150k request (got ${out})`);
assert.ok(out! > 0, "clamped budget stays positive");

// Control: no oversized system -> input fits -> max_tokens passes through.
Expand Down
59 changes: 55 additions & 4 deletions tests/preflight-fail-fast.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -273,13 +273,17 @@ test("e2e #330: over-window payload whose only foldable content is in the protec

try {
// Three messages against a 10k window: a small opener plus two large
// recent messages (~8k tokens each, ~16k total). All three sit inside
// recent messages (~6k tokens each, ~12k total). All three sit inside
// the soft-protected recent zone (preserveRecentMessages=5 covers all
// three), so the normal pass finds nothing foldable. Before #330 this
// 502'd forever; now preflight relaxes the soft zone, folds the oldest
// large message, and forwards the now-fitting payload.
const big1 = "A".repeat(32000);
const big2 = "B".repeat(32000);
// large message, and forwards the now-fitting payload. #470: the wire
// overhead (injected compress system prompt + ACP tools, ~2.4k tokens)
// now counts toward every size decision, so post-fold fit needs the
// second big message at 24k chars, not 32k (the pre-#470 test was
// forwarding a payload whose real billed input exceeded the window).
const big1 = "A".repeat(24000);
const big2 = "B".repeat(24000);
const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "preflight-relax-sess" },
Expand All @@ -304,3 +308,50 @@ test("e2e #330: over-window payload whose only foldable content is in the protec
await once(upstream, "close");
}
});

test("e2e #470: system + tools overhead counts in the preflight trigger — text fits, wire overflows → preflight folds", async () => {
const calls: Call[] = [];
const upstream = makeUpstreamOk(calls);
upstream.listen(0, "127.0.0.1");
await once(upstream, "listening");
const upstreamPort = upstream.address().port;

const proxy = await startProxy(upstreamPort, { "claude-small": { context: 10_000 } });
await once(proxy, "listening");
const proxyPort = proxy.address().port;

try {
// The message text alone (~8.5k) fits the 10k window, but the wire
// payload carries more: user system (~2k) + user tool (~1.2k) + the
// injected compress system prompt + ACP tools (~2.4k) — the billed
// input overflows. Before #470 the trigger counted messages only and
// forwarded the payload verbatim (guaranteed upstream 400 later).
const big1 = "A".repeat(34_000);
const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "preflight-overhead-sess" },
body: JSON.stringify({
model: "claude-small",
max_tokens: 1024,
stream: true,
system: "S".repeat(8_000),
tools: [{ name: "lookup", description: "T".repeat(4_600), input_schema: { type: "object", properties: {} } }],
messages: [
{ role: "user", content: "start the task" },
{ role: "assistant", content: big1 },
{ role: "user", content: "wrap up" },
],
}),
});
assert.equal(r.status, 200, "the over-window (incl. system+tools) payload is folded and forwarded");
assert.ok(calls.filter((c) => !c.stream).length >= 1, "preflight made the summarization call — the trigger counted the wire overhead");
assert.equal(calls.filter((c) => c.stream).length, 1, "the folded payload was forwarded upstream");
const forwarded = calls.find((c) => c.stream)?.body ?? "";
assert.ok(!forwarded.includes("A".repeat(100)), "the folded big message did NOT ride the forwarded payload");
} finally {
proxy.close();
await once(proxy, "close");
upstream.close();
await once(upstream, "close");
}
});
60 changes: 60 additions & 0 deletions tests/wire-overhead.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import assert from "node:assert/strict";
import test from "node:test";

process.env.NODE_ENV = "test";

import { estimateWireOverhead } from "../src/server.ts";

// #470: the preflight trigger under-counted the wire payload — system text and
// tool definitions (incl. the proxy-injected compress prompt + ACP tools) ride
// the request but are invisible to estimateCoreMessages. estimateWireOverhead
// extracts them from the raw wire body per protocol.

test("estimateWireOverhead: anthropic string system + tools", () => {
const body = JSON.stringify({
model: "m",
system: "S".repeat(4000),
tools: [{ name: "t", description: "D".repeat(2000), input_schema: {} }],
messages: [],
});
const oh = estimateWireOverhead("anthropic", body);
assert.ok(oh >= 1000 && oh <= 2200, `system ~1000 + tools ~500 + JSON punctuation (got ${oh})`);
});

test("estimateWireOverhead: anthropic array-form system blocks", () => {
const body = JSON.stringify({
system: [{ type: "text", text: "S1 ".repeat(500) }, { type: "text", text: "S2 ".repeat(500) }],
});
const oh = estimateWireOverhead("anthropic", body);
assert.ok(oh >= 500 && oh <= 1200, `array blocks joined (got ${oh})`);
});

test("estimateWireOverhead: openai hoisted system/developer messages counted, non-system not", () => {
const body = JSON.stringify({
messages: [
{ role: "system", content: "SYS ".repeat(300) },
{ role: "developer", content: "DEV ".repeat(300) },
{ role: "user", content: "U".repeat(40000) },
{ role: "assistant", content: "A".repeat(40000) },
],
});
const oh = estimateWireOverhead("openai", body);
// ~300 tokens of system text — NOT the 20k of user/assistant content
assert.ok(oh >= 250 && oh <= 800, `only system/developer messages count (got ${oh})`);
});

test("estimateWireOverhead: responses instructions", () => {
const body = JSON.stringify({ instructions: "I".repeat(8000) });
const oh = estimateWireOverhead("responses", body);
assert.ok(oh >= 1800 && oh <= 2400, `instructions ~2000 (got ${oh})`);
});

test("estimateWireOverhead: unparseable body → 0, not a crash", () => {
assert.equal(estimateWireOverhead("anthropic", "not json {"), 0);
assert.equal(estimateWireOverhead("openai", Buffer.from("}{"), ), 0);
});

test("estimateWireOverhead: empty payload → ~0 (no system, no tools)", () => {
assert.ok(estimateWireOverhead("anthropic", JSON.stringify({ messages: [] })) <= 1);
assert.ok(estimateWireOverhead("responses", JSON.stringify({})) <= 1);
});
Loading