Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 29 additions & 4 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5055,8 +5055,19 @@ async function preflightCompressIfNeeded(
// hatch on pixel-billing upstreams). With evidence present we trust the
// estimate and fall through to fold / fail-fast below.
const noOverflowEvidence = session.stats.lastInputTokens < limit || session.stats.lastInputTokensSource !== "usage";
if (imageTokens > 0 && payloadEstimate >= limit && textEstimate < limit && noOverflowEvidence) {
log("warn", `[${session.id}] image-dominated payload (~${textEstimate} text + ~${imageTokens} image tokens) exceeds window ${limit} by estimate only, no upstream overflow evidence — forwarding once so the upstream arbitrates billing (#496)`);
// #1800: images are the sole over-window component and we hold no overflow
// evidence → the base64/4 (or pixels-fallback) image cost clears the window on
// ESTIMATE alone while the real bill is far smaller, so we let the upstream
// arbitrate billing instead of fail-fast'ing. But do NOT unconditionally
// short-circuit here: that permanently disabled auto-compression — preflight
// never ran while the inflated estimate sat over-window, so a growing text
// payload was folded 0× for the whole session (#1800). Only take the immediate
// forward when there is literally NOTHING compressible; otherwise remember the
// arbitration and let preflightCompress fold the text portion first, re-applying
// this same forward-instead-of-fail-fast decision after compression (below).
const imageArbitration = imageTokens > 0 && payloadEstimate >= limit && textEstimate < limit && noOverflowEvidence;
if (imageArbitration && (prepared.nudge?.compressibleRanges ?? []).length === 0) {
log("warn", `[${session.id}] image-dominated payload (~${textEstimate} text + ~${imageTokens} image tokens) exceeds window ${limit} by estimate only, nothing compressible, no upstream overflow evidence — forwarding for the upstream to arbitrate billing (#496/#1800)`);
return prepared;
}
// #301: forwarding as-is is safe ONLY when the payload's own estimate
Expand Down Expand Up @@ -5202,12 +5213,14 @@ async function preflightCompressIfNeeded(
// would look "fitting" on its text estimate alone. Unknown-baseline
// sessions keep the loop's own upper-bound judgment (result.fitsWindow,
// #553) — the optimistic re-estimate is exactly what that regime distrusts.
let outbound: Prepared = prepared;
if (result.compressedRanges > 0) {
log("info", `[${session.id}] preflight compressed ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved (${tokenCount} → ${session.stats.lastInputTokens}) in ${Date.now() - started}ms; rebuilding payload`);
const rebuilt = await runPrepare();
// runPrepare re-incremented stats.requests; the rebuild is internal
// to this single client request.
session.stats.requests -= 1;
outbound = rebuilt;
const fits = unknownBaseline
? result.fitsWindow
: estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit;
Expand All @@ -5218,13 +5231,25 @@ async function preflightCompressIfNeeded(
log("warn", `[${session.id}] preflight made no progress but the payload fits; forwarding as-is`);
return prepared;
}
// The payload still overflows the window: fail fast with a diagnostic
// error instead of forwarding a guaranteed-400 payload (#301).
const f = result.failure;
if (f?.kind === "aborted") {
log("warn", `[${session.id}] preflight aborted (${f.detail}); not forwarding`);
return { failFast: true, status: 0, message: f.detail, retryable: false, respond: false };
}
// #1800: still over-window after compression, but the residual excess is carried
// ENTIRELY by the image estimate (text+overhead fits on its own) and we hold no
// upstream overflow evidence. Forward for the upstream to arbitrate billing
// instead of fail-fasting a payload whose real bill likely fits (#496). The text
// portion was already folded above when foldable; we do NOT re-loop.
if (imageArbitration) {
const outText = estimateCoreMessages(outbound.processedMessages);
if (outText + overheadEstimate < limit && outText + overheadEstimate + imageTokens >= limit) {
log("info", `[${session.id}] preflight folded ${result.compressedRanges} range(s) but images alone (~${imageTokens} tokens) keep the estimate over window ${limit} with no upstream overflow evidence — forwarding for the upstream to arbitrate billing (#496/#1800)`);
return outbound;
}
}
// The payload still overflows the window: fail fast with a diagnostic
// error instead of forwarding a guaranteed-400 payload (#301).
const status = f?.kind === "upstream" && f.status === 429 ? 503 : 502;
const retryable = f?.retryable === true || (f?.kind === "upstream" && f.status !== undefined && (f.status === 429 || f.status >= 500));
const ff = failFast(status, f?.detail ?? "the payload still exceeds the window after preflight compression", retryable, result.compressedRanges > 0 ? session.stats.lastInputTokens : undefined, result.rangesRemaining);
Expand Down
144 changes: 144 additions & 0 deletions tests/issue1800-image-arbitration-compress.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
import assert from "node:assert/strict";
import http from "node:http";
import { once } from "node:events";
import test from "node:test";

process.env.NODE_ENV = "test";

import { defaultConfig } from "acp-kernel";
import { startServer, type ProxyOptions } from "../src/server.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { _setForTest as setRegistryForTest } from "../src/registry.ts";

// Issue #1800: on bytes-billing upstreams (unknown hosts), the conservative
// base64/4 image estimate overstates real image cost. When images ALONE push
// the payload estimate over the window while the TEXT portion fits, the old
// #496 arbitration branch did `return prepared` UNCONDITIONALLY — skipping
// preflightCompress entirely and recording no compression state. Result: the
// text portion was never folded even though it was compressible, so
// auto-compression stayed permanently disabled for image sessions (the issue's
// session logged preflightCompress ran 0 times across the whole session).
//
// Fix under test: the arbitration branch must still fold the compressible
// TEXT portion when ranges exist, and only short-circuit (forward-for-the-
// upstream-to-arbitrate) when there is genuinely nothing to fold. The image
// rides along byte-exact either way.
//
// Conventions copied from tests/issue857-image-baseline.test.ts (mock Responses
// upstream + startServer e2e). Window 200K; one screenshot whose BYTES-mode
// cost (b64/4) alone exceeds the window so no amount of text folding can make
// the payload fit — forcing the post-compress image-arbitration forward.

const WINDOW = 200_000;
const MODEL = "llama-flash-1800";
const IMAGE_TOKENS_TARGET = 210_000;
const IMG_PAD = IMAGE_TOKENS_TARGET * 4 - 32;

function pngB64(w: number, h: number, padChars: number): string {
const b = Buffer.alloc(24);
b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0);
b.writeUInt32BE(13, 8);
b.write("IHDR", 12, "ascii");
b.writeUInt32BE(w, 16);
b.writeUInt32BE(h, 20);
return b.toString("base64") + "A".repeat(padChars);
}

interface MockStats { streamingForwards: number; imagesSeen: number; summaryCalls: number }

function startMockUpstream(): Promise<{ server: http.Server; port: number; stats: MockStats }> {
const stats: MockStats = { streamingForwards: 0, imagesSeen: 0, summaryCalls: 0 };
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
const raw = Buffer.concat(chunks).toString("utf8");
let parsed: { stream?: boolean } = {};
try { parsed = JSON.parse(raw); } catch { parsed = {}; }
if (parsed.stream === false) {
stats.summaryCalls += 1;
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ output_text: "PREFLIGHT SUMMARY: folded segment." }));
return;
}
stats.streamingForwards += 1;
stats.imagesSeen += (raw.match(/"input_image"/g) ?? []).length;
res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
res.write(`event: response.completed\ndata: ${JSON.stringify({ response: { id: "resp_done", status: "completed", output: [], usage: { input_tokens: 1000, output_tokens: 5, total_tokens: 1005 } } })}\n\n`);
res.end();
});
});
server.listen(0, "127.0.0.1");
return new Promise((resolve) => {
server.on("listening", () => resolve({ server, port: (server.address() as { port: number }).port, stats }));
});
}

async function startProxy(upstreamPort: number): Promise<{ proxy: http.Server; port: number }> {
_setStoreForTest(new SessionStore({ enabled: false }));
setRegistryForTest({});
const proxy = await startServer({
port: 0,
host: "127.0.0.1",
upstream: "http://127.0.0.1",
routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { [MODEL]: { context: WINDOW } } } },
modelContextLimit: WINDOW,
kernelConfig: defaultConfig(WINDOW),
compress: { injectTool: true, injectNudge: true },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
log: false,
debug: false,
passthrough: false,
autoUpdate: false,
mitm: { enabled: false, domains: [] },
} as ProxyOptions);
await once(proxy, "listening");
return { proxy, port: (proxy.address() as { port: number }).port };
}

function body(sessionId: string, input: unknown[]): string {
return JSON.stringify({ model: MODEL, stream: true, store: false, session_id: sessionId, instructions: "You are the test coding agent.", input, max_output_tokens: 1024 });
}

function fillerHistory(n: number): unknown[] {
const out: unknown[] = [];
for (let i = 0; i < n; i++) out.push({ type: "message", role: i % 2 ? "assistant" : "user", content: `MARKER_${i}: ${"lorem ipsum dolor sit amet ".repeat(400)}` });
return out;
}

function imageInput(): unknown[] {
return [{ type: "message", role: "user", content: [{ type: "input_text", text: "screenshot attached" }, { type: "input_image", image_url: `data:image/png;base64,${pngB64(1653, 2339, IMG_PAD)}` }] }];
}

test("#1800: image-dominated estimate with compressible text still folds the text portion (and forwards)", async () => {
const { server: upstream, port: uport, stats } = await startMockUpstream();
const { proxy, port } = await startProxy(uport);
try {
const url = `http://127.0.0.1:${port}/bili/http://127.0.0.1:${uport}/v1/responses`;
const headers = { "content-type": "application/json" };

// Turn 1: small text-only turn establishes a sub-window usage baseline
// (lastInputTokens=1000, source="usage") → noOverflowEvidence=true.
const r1 = await fetch(url, { method: "POST", headers, body: body("s1800a", [{ type: "message", role: "user", content: "hello there" }]) });
assert.equal(r1.status, 200);
await r1.text();

// Turn 2: enough filler history to create compressible ranges, plus one
// screenshot whose bytes-mode cost (~210K tokens) ALONE exceeds the
// 200K window → payloadEstimate over, textEstimate under.
const r2 = await fetch(url, { method: "POST", headers, body: body("s1800a", [...fillerHistory(60), ...imageInput()]) });
assert.equal(r2.status, 200, "the image-dominated payload forwards (for the upstream to arbitrate billing), not 502");
await r2.text();

// THE TOOTH: preflightCompress actually RAN and spent summarization
// calls folding the compressible text portion. Before the fix the old
// branch returned `prepared` before reaching preflightCompress, so
// summaryCalls was 0 and auto-compression stayed permanently disabled.
assert.ok(stats.summaryCalls >= 1, `preflight folded the compressible text portion (got ${stats.summaryCalls} summarization calls)`);
assert.ok(stats.imagesSeen >= 1, "the screenshot rode along byte-exact through the folded forward");
} finally {
upstream.close();
proxy.close();
}
});
Loading