Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ The proxy runs in one of two modes, and **the mode decides who executes
| Client | ACP-native agent with the bili extension (pi/omp) | Any OpenAI/Anthropic client, no extension |
| Who executes `compress` | **The agent** (pi runs it locally) | **The proxy** (server-side compress loop) |
| `compress` tool call in the re-sent history? | Yes — part of the agent's own conversation | No — ephemeral proxy-loop traffic |
| Preflight blocks (no tool call)? | No — overflow forces the model to call `compress` | Yes — `src/preflight.ts` compresses behind the client's back |
| Preflight blocks (no tool call)? | Yes — active here too as a last-resort backstop (e.g. #453); normally the agent's own `compress` fires first via the nudge | Yes — `src/preflight.ts` compresses behind the client's back |
| **Summary carrier on the wire** | **the `compress` tool call** | **an `acp_summary` user message** |
| System messages on the wire | always exactly 1 (client + prompt) | always exactly 1 (client + prompt) — summaries ride on user messages |
| SGLang "single system" 400 (#377) | cannot happen | cannot happen (summaries are user messages, not system) |
Expand Down
8 changes: 5 additions & 3 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ export interface PreflightDeps {
log: (level: string, msg: string) => void;
/** Constant floor on the forwarded-payload size for this request (image bytes, #488). Folding only ever removes images, so adding this to every text estimate keeps the fit decision sound for multimodal payloads. */
imageFloor?: number;
/** Constant billed-but-not-in-messages envelope cost (system/instructions + tools, #470). Folding never removes it, so it floors every size decision — without it the loop stops at "text fits" while the billed input overflows and the caller's overhead-aware gate fail-fasts a foldable payload. */
wireOverhead?: number;
}

export type PreflightFailureKind = "upstream" | "exhausted" | "aborted";
Expand Down Expand Up @@ -226,7 +228,7 @@ const ABORTED_FAILURE: PreflightFailure = { kind: "aborted", detail: "the client

export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise<PreflightResult> {
const limit = deps.config.modelContextLimit;
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) };
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0) };
if (limit <= 0) return result;
const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
// applyCompression rejects ranges below config.compress.minCompressRange
Expand Down Expand Up @@ -257,10 +259,10 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// Floor on the session's measured input baseline: the upstream's
// input_tokens also covers the system prompt + tool definitions, which
// are not in turn.messages, so the direct estimate can undershoot.
currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0));
currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0));
// The caller's forward/fail-fast gate uses the payload's own estimate
// (the floor can be stale — see PreflightResult.payloadEstimate).
result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0);
result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0);
if (startTokens < 0) startTokens = currentTokens;
if (currentTokens < limit) break;
const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []);
Expand Down
58 changes: 55 additions & 3 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1761,6 +1761,55 @@ export function clampOutputBudget(requested: number, inputEstimate: number, nati
return cap;
}

function isObj(v: unknown): v is Record<string, unknown> {
return typeof v === "object" && v !== null;
}

/** #470: system/instructions + tools billed on top of the messages (which
* estimateCoreMessages omits). Read from the rebuilt body so it's mode-correct;
* text-only, so it never double-counts imageTokensInRawBody. Unparseable -> 0. */
export function estimateWireOverhead(protocol: "anthropic" | "openai" | "responses", body: string | Buffer): number {
const s = typeof body === "string" ? body : body.toString("utf8");
let parsed: unknown;
try {
parsed = JSON.parse(s);
} catch {
return 0;
}
if (!isObj(parsed)) return 0;
const textOf = (content: unknown): string => {
if (typeof content === "string") return content;
if (!Array.isArray(content)) return "";
let out = "";
for (const part of content) if (isObj(part) && typeof part.text === "string") out += part.text;
return out;
};
let systemText = "";
if (protocol === "anthropic") {
const sys = parsed.system;
if (typeof sys === "string") systemText = sys;
else if (Array.isArray(sys)) for (const blk of sys) if (isObj(blk) && typeof blk.text === "string") systemText += blk.text;
} else if (protocol === "openai") {
const msgs = parsed.messages;
if (Array.isArray(msgs)) {
for (const m of msgs) {
if (!isObj(m) || (m.role !== "system" && m.role !== "developer")) continue;
systemText += textOf(m.content);
}
}
} else {
if (typeof parsed.instructions === "string") systemText += parsed.instructions;
const input = parsed.input;
if (Array.isArray(input)) {
for (const item of input) {
if (!isObj(item) || item.role !== "developer") continue;
systemText += textOf(item.content);
}
}
}
return defaultCountTokens(systemText) + defaultCountTokens(JSON.stringify(parsed.tools ?? []));
}

// Only override genuine cadence silences: skip the kernel's deliberate
// "nothing compressible to offer" suppression (empty ranges).
export function emergencyNudge(nudge: NudgeDecision | null | undefined, escalationPct: number = EMERGENCY_NUDGE_ESCALATION_PCT): boolean {
Expand Down Expand Up @@ -2524,8 +2573,10 @@ async function preflightCompressIfNeeded(
// #488: images are forwarded verbatim but invisible to the kernel's text model —
// add their cost to every size decision here (trigger, fit gates, self-heal).
const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body);
// #470: system/instructions + tools are billed but invisible to estimateCoreMessages; count them or the trigger fires late and the post-fold gates pass over-window payloads.
const wireOverhead = estimateWireOverhead(prepared.protocol, prepared.body);
const textEstimate = estimateCoreMessages(prepared.processedMessages);
const payloadEstimate = textEstimate + imageTokens;
const payloadEstimate = textEstimate + imageTokens + wireOverhead;
const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate);
if (limit <= 0 || !model || tokenCount < limit) return prepared;
// #496 forward-once-then-learn: the default image cost (base64/4) matches byte
Expand All @@ -2543,8 +2594,8 @@ async function preflightCompressIfNeeded(
(model ? learnedMap?.[model] : undefined) ??
(session.metadata.learnedContextLimit as number | undefined);
const noOverflowEvidence = session.stats.lastInputTokens < limit && learnedLimit === undefined;
if (imageTokens > 0 && textEstimate < limit && noOverflowEvidence) {
log("warn", `[${session.id}] image-dominated payload (~${textEstimate} text + ~${imageTokens} image tokens) exceeds window ${limit} by estimate only, no upstream overflow evidence — forwarding once so the upstream arbitrates billing (#496)`);
if (imageTokens > 0 && textEstimate + wireOverhead < limit && noOverflowEvidence) {
log("warn", `[${session.id}] image-dominated payload (~${textEstimate + wireOverhead} non-image + ~${imageTokens} image tokens) exceeds window ${limit} by estimate only, no upstream overflow evidence — forwarding once so the upstream arbitrates billing (#496)`);
return prepared;
}
// #301: forwarding as-is is safe ONLY when the payload's own estimate
Expand Down Expand Up @@ -2594,6 +2645,7 @@ async function preflightCompressIfNeeded(
signal: clientAbort.signal,
log,
imageFloor: imageTokens,
wireOverhead,
},
prepared.originalMessages,
);
Expand Down
16 changes: 8 additions & 8 deletions tests/codex-compact-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,17 @@ function fcEvents(callId: string, args: string): string {
sse("response.output_item.done", { item: { type: "function_call", id: `fc_${callId}`, call_id: callId, name: "compress", arguments: args }, output_index: 0 }),
].join("");
}
// 7 messages × ~4400 chars (~1100 tokens each, ~7700 total). Below the 10k
// window so preflight never fires; above the recent-tail protection so
// m00001/m00002 are compressible by the model.
// 7 messages (~7700 tokens): with the billed wire envelope counted (#470) the
// turn is ~10.1k, under the 12k window so preflight never fires on turn 1;
// m00001/m00002 stay above the recent-tail protection so the model can fold them.
function conversation() {
const input: { type: string; role: string; content: string }[] = [];
for (let i = 0; i < 7; i++) {
input.push({ type: "message", role: i % 2 === 0 ? "user" : "assistant", content: `Message ${i} of the working session. ` + `WORK_${i}_content_`.repeat(290) });
}
return input;
}
// 14 messages ≈ 15.4k tokens — OVER the 10k window, so preflight WOULD fire
// 14 messages ≈ 15.4k tokens — OVER the 12k window, so preflight WOULD fire
// if this payload went through the normal pipeline (#332 regression input).
function bigConversation() {
const input: { type: string; role: string; content: string }[] = [];
Expand Down Expand Up @@ -104,9 +104,9 @@ async function withHarness(opts: { mode?: string; firstTurnTokens: number }, fn:
port: 0,
host: "127.0.0.1",
upstream: "http://127.0.0.1",
routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-resp": { context: 10_000 } } } },
modelContextLimit: 10_000,
kernelConfig: defaultConfig(10_000),
routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-resp": { context: 12_000 } } } },
modelContextLimit: 12_000,
kernelConfig: defaultConfig(12_000),
compress: { injectTool: true, injectNudge: true },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
Expand Down Expand Up @@ -301,7 +301,7 @@ test("e2e E2 (endpoint form): intercept + healthy ACP → forged JSON {output},
// Issue #332: a compaction_trigger request that is NOT intercepted must reach
// upstream byte-identical — no preflight overflow-compress, no payload
// rebuild, no window clamp, no session-state folding as a side effect. The
// oversized payload (14 msgs ≈ 15.4k > 10k window) is what made the old
// oversized payload (14 msgs ≈ 15.4k > 12k window) is what made the old
// ordering fire preflight before the compact detection ran.

test("e2e #332 (trigger form): pass mode + oversized payload → forwarded byte-identical, no state mutation", async () => {
Expand Down
10 changes: 5 additions & 5 deletions tests/compaction-trigger-finality.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,14 @@ function completed(inputTokens: number): string {
});
}

// 7 messages × ~4400 chars (~1100 tokens each, ~7700 total). Above the
// kernel's 5000-token recent-tail protection so m00001/m00002 stay
// compressible (a tier-1 pending range for the nudge), but below the 10k
// window so preflight never fires.
// 7 messages sized so the turn's billed input (text + system/instructions +
// tools, #470) stays under the 10k window (preflight never fires on turn 1)
// while m00001/m00002 remain above the recent-tail protection — the pending
// range the over-limit nudge points at.
function conversation() {
const input: { type: string; role: string; content: string }[] = [];
for (let i = 0; i < 7; i++) {
input.push({ type: "message", role: i % 2 === 0 ? "user" : "assistant", content: `Message ${i} of the working session. ` + `WORK_${i}_content_`.repeat(290) });
input.push({ type: "message", role: i % 2 === 0 ? "user" : "assistant", content: `Message ${i} of the working session. ` + `WORK_${i}_content_`.repeat(250) });
}
return input;
}
Expand Down
25 changes: 13 additions & 12 deletions tests/issue453-backstop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,37 +162,38 @@ function turn2Compressible(): Record<string, unknown>[] {
}

test("#453 clamp: oversized max_tokens is reduced when input+system nearly fills the window", async () => {
// window 100k, requested max_tokens 40k -> reserved window 60k. The core
// conversation estimates ~48k (under 60k, so preflight skips — it ignores the
// system), but the 60k-char system pushes the TRUE input past 60k. Only the
// clamp sees the system, so it lowers max_tokens to keep input+output under 100k.
// window 400k / max_tokens 150k -> reserved input budget 250k. Billed input
// (core+system+tools) is sized just under 250k so preflight skips, yet adding
// the 150k output pushes input+output over the window -> only the clamp acts.
// Re-sized from 100k/40k (#470): preflight now counts the envelope, so the old
// shape tripped preflight instead of isolating the clamp.
const fired = await drive({
sessionId: "clamp-fire",
window: 100_000,
window: 400_000,
turn1PromptTokens: 5_000,
turn2Body: {
model: "m", max_tokens: 40_000,
model: "m", max_tokens: 150_000,
messages: [
{ role: "system", content: "z".repeat(60_000) },
{ role: "system", content: "z".repeat(380_000) },
{ role: "user", content: "hi" },
{ role: "assistant", content: "y".repeat(192_000) },
{ role: "assistant", content: "y".repeat(576_000) },
{ role: "user", content: "now" },
],
},
});
const out = typeof fired.turn2Body.max_tokens === "number" ? fired.turn2Body.max_tokens : undefined;
assert.ok(out !== undefined, "forwarded body carries a numeric max_tokens");
assert.ok(out! < 40_000, `max_tokens clamped below the 40k request (got ${out})`);
assert.ok(out! < 150_000, `max_tokens clamped below the 150k request (got ${out})`);
assert.ok(out! > 0, "clamped budget stays positive");

// Control: no oversized system -> input fits -> max_tokens passes through.
const ctrl = await drive({
sessionId: "clamp-ctrl",
window: 100_000,
window: 400_000,
turn1PromptTokens: 5_000,
turn2Body: { model: "m", max_tokens: 40_000, messages: turn2Compressible() },
turn2Body: { model: "m", max_tokens: 150_000, messages: turn2Compressible() },
});
assert.equal(ctrl.turn2Body.max_tokens, 40_000, "unclamped when input fits the window");
assert.equal(ctrl.turn2Body.max_tokens, 150_000, "unclamped when input fits the window");
});

test("#453 escalation: nudge injected at the escalation line though the kernel cadence-stays silent", async () => {
Expand Down
67 changes: 67 additions & 0 deletions tests/preflight-fail-fast.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,34 @@ function startProxy(upstreamPort: number, models: Record<string, { context: numb
} as ProxyOptions);
}

const FOLD_SUMMARY =
"PREFLIGHT SUMMARY: the segment covered a multi-step debugging session. " +
"Key decisions, files touched, and outcome are condensed here.";

function makeUpstreamSummarize(calls?: Call[]): http.Server {
return http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
const raw = Buffer.concat(chunks).toString("utf8");
let parsed: { stream?: boolean } = {};
try { parsed = JSON.parse(raw); } catch { /* keep {} */ }
calls?.push({ stream: !!parsed.stream, body: raw });
if (parsed.stream) {
res.writeHead(200, { "content-type": "text/event-stream" });
res.end(okSse(1000));
} else {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({
id: "msg_summary", type: "message", role: "assistant", model: "claude-small",
content: [{ type: "text", text: FOLD_SUMMARY }], stop_reason: "end_turn",
usage: { input_tokens: 500, output_tokens: 50 },
}));
}
});
});
}

test("e2e #301: overflow + summary upstream 429 → structured 503, over-window payload NOT forwarded", async () => {
const calls: Call[] = [];
// The preflight summarization call hits the same rate-limited upstream as
Expand Down Expand Up @@ -225,3 +253,42 @@ test("e2e #301: over-window payload with nothing compressible → structured 502
await once(upstream, "close");
}
});

test("e2e #470: text fits the window alone but system+tools push the billed input over → preflight folds and forwards", async () => {
const calls: Call[] = [];
const upstream = makeUpstreamSummarize(calls);
upstream.listen(0, "127.0.0.1");
await once(upstream, "listening");
const upstreamPort = upstream.address().port;

// bigConversation() is 13.1k tokens: under a 15k window on TEXT alone
// (master's estimateCoreMessages would skip preflight), yet the injected
// compress prompt + ACP tools (~2.4k) push the BILLED input to 15.5k, past
// the window. Only counting the envelope makes preflight fire, fold, and
// forward — otherwise the over-window body goes up as-is (upstream 400).
const proxy = await startProxy(upstreamPort, { "claude-small": { context: 15_000 } });
await once(proxy, "listening");
const proxyPort = proxy.address().port;

try {
const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "preflight-envelope-sess" },
body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: bigConversation() }),
});
assert.equal(r.status, 200, "envelope-driven overflow is folded and forwarded (not fail-fast, not blind forward)");
await r.text();

assert.ok(calls.filter((c) => !c.stream).length >= 1, "preflight made a summarization call (master skipped preflight entirely)");
const forwards = calls.filter((c) => c.stream);
const lastForward = forwards[forwards.length - 1];
assert.ok(lastForward, "the folded payload was forwarded");
assert.ok(!lastForward.body.includes("MARKER_1_"), "an early message was folded out of the forwarded body");
assert.ok(lastForward.body.includes(FOLD_SUMMARY), "the folded summary is in the rebuilt payload");
} finally {
proxy.close();
await once(proxy, "close");
upstream.close();
await once(upstream, "close");
}
});
Loading