diff --git a/src/server.ts b/src/server.ts index 73153a2511..2596373d4f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5055,8 +5055,19 @@ async function preflightCompressIfNeeded( // hatch on pixel-billing upstreams). With evidence present we trust the // estimate and fall through to fold / fail-fast below. const noOverflowEvidence = session.stats.lastInputTokens < limit || session.stats.lastInputTokensSource !== "usage"; - if (imageTokens > 0 && payloadEstimate >= limit && textEstimate < limit && noOverflowEvidence) { - log("warn", `[${session.id}] image-dominated payload (~${textEstimate} text + ~${imageTokens} image tokens) exceeds window ${limit} by estimate only, no upstream overflow evidence — forwarding once so the upstream arbitrates billing (#496)`); + // #1800: images are the sole over-window component and we hold no overflow + // evidence → the base64/4 (or pixels-fallback) image cost clears the window on + // ESTIMATE alone while the real bill is far smaller, so we let the upstream + // arbitrate billing instead of fail-fast'ing. But do NOT unconditionally + // short-circuit here: that permanently disabled auto-compression — preflight + // never ran while the inflated estimate sat over-window, so a growing text + // payload was folded 0× for the whole session (#1800). Only take the immediate + // forward when there is literally NOTHING compressible; otherwise remember the + // arbitration and let preflightCompress fold the text portion first, re-applying + // this same forward-instead-of-fail-fast decision after compression (below). + const imageArbitration = imageTokens > 0 && payloadEstimate >= limit && textEstimate < limit && noOverflowEvidence; + if (imageArbitration && (prepared.nudge?.compressibleRanges ?? []).length === 0) { + log("warn", `[${session.id}] image-dominated payload (~${textEstimate} text + ~${imageTokens} image tokens) exceeds window ${limit} by estimate only, nothing compressible, no upstream overflow evidence — forwarding for the upstream to arbitrate billing (#496/#1800)`); return prepared; } // #301: forwarding as-is is safe ONLY when the payload's own estimate @@ -5202,12 +5213,14 @@ async function preflightCompressIfNeeded( // would look "fitting" on its text estimate alone. Unknown-baseline // sessions keep the loop's own upper-bound judgment (result.fitsWindow, // #553) — the optimistic re-estimate is exactly what that regime distrusts. + let outbound: Prepared = prepared; if (result.compressedRanges > 0) { log("info", `[${session.id}] preflight compressed ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved (${tokenCount} → ${session.stats.lastInputTokens}) in ${Date.now() - started}ms; rebuilding payload`); const rebuilt = await runPrepare(); // runPrepare re-incremented stats.requests; the rebuild is internal // to this single client request. session.stats.requests -= 1; + outbound = rebuilt; const fits = unknownBaseline ? result.fitsWindow : estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit; @@ -5218,13 +5231,25 @@ async function preflightCompressIfNeeded( log("warn", `[${session.id}] preflight made no progress but the payload fits; forwarding as-is`); return prepared; } - // The payload still overflows the window: fail fast with a diagnostic - // error instead of forwarding a guaranteed-400 payload (#301). const f = result.failure; if (f?.kind === "aborted") { log("warn", `[${session.id}] preflight aborted (${f.detail}); not forwarding`); return { failFast: true, status: 0, message: f.detail, retryable: false, respond: false }; } + // #1800: still over-window after compression, but the residual excess is carried + // ENTIRELY by the image estimate (text+overhead fits on its own) and we hold no + // upstream overflow evidence. Forward for the upstream to arbitrate billing + // instead of fail-fasting a payload whose real bill likely fits (#496). The text + // portion was already folded above when foldable; we do NOT re-loop. + if (imageArbitration) { + const outText = estimateCoreMessages(outbound.processedMessages); + if (outText + overheadEstimate < limit && outText + overheadEstimate + imageTokens >= limit) { + log("info", `[${session.id}] preflight folded ${result.compressedRanges} range(s) but images alone (~${imageTokens} tokens) keep the estimate over window ${limit} with no upstream overflow evidence — forwarding for the upstream to arbitrate billing (#496/#1800)`); + return outbound; + } + } + // The payload still overflows the window: fail fast with a diagnostic + // error instead of forwarding a guaranteed-400 payload (#301). const status = f?.kind === "upstream" && f.status === 429 ? 503 : 502; const retryable = f?.retryable === true || (f?.kind === "upstream" && f.status !== undefined && (f.status === 429 || f.status >= 500)); const ff = failFast(status, f?.detail ?? "the payload still exceeds the window after preflight compression", retryable, result.compressedRanges > 0 ? session.stats.lastInputTokens : undefined, result.rangesRemaining); diff --git a/tests/issue1800-image-arbitration-compress.test.ts b/tests/issue1800-image-arbitration-compress.test.ts new file mode 100644 index 0000000000..9c0e4a55c0 --- /dev/null +++ b/tests/issue1800-image-arbitration-compress.test.ts @@ -0,0 +1,144 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { defaultConfig } from "acp-kernel"; +import { startServer, type ProxyOptions } from "../src/server.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; + +// Issue #1800: on bytes-billing upstreams (unknown hosts), the conservative +// base64/4 image estimate overstates real image cost. When images ALONE push +// the payload estimate over the window while the TEXT portion fits, the old +// #496 arbitration branch did `return prepared` UNCONDITIONALLY — skipping +// preflightCompress entirely and recording no compression state. Result: the +// text portion was never folded even though it was compressible, so +// auto-compression stayed permanently disabled for image sessions (the issue's +// session logged preflightCompress ran 0 times across the whole session). +// +// Fix under test: the arbitration branch must still fold the compressible +// TEXT portion when ranges exist, and only short-circuit (forward-for-the- +// upstream-to-arbitrate) when there is genuinely nothing to fold. The image +// rides along byte-exact either way. +// +// Conventions copied from tests/issue857-image-baseline.test.ts (mock Responses +// upstream + startServer e2e). Window 200K; one screenshot whose BYTES-mode +// cost (b64/4) alone exceeds the window so no amount of text folding can make +// the payload fit — forcing the post-compress image-arbitration forward. + +const WINDOW = 200_000; +const MODEL = "llama-flash-1800"; +const IMAGE_TOKENS_TARGET = 210_000; +const IMG_PAD = IMAGE_TOKENS_TARGET * 4 - 32; + +function pngB64(w: number, h: number, padChars: number): string { + const b = Buffer.alloc(24); + b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0); + b.writeUInt32BE(13, 8); + b.write("IHDR", 12, "ascii"); + b.writeUInt32BE(w, 16); + b.writeUInt32BE(h, 20); + return b.toString("base64") + "A".repeat(padChars); +} + +interface MockStats { streamingForwards: number; imagesSeen: number; summaryCalls: number } + +function startMockUpstream(): Promise<{ server: http.Server; port: number; stats: MockStats }> { + const stats: MockStats = { streamingForwards: 0, imagesSeen: 0, summaryCalls: 0 }; + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed: { stream?: boolean } = {}; + try { parsed = JSON.parse(raw); } catch { parsed = {}; } + if (parsed.stream === false) { + stats.summaryCalls += 1; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ output_text: "PREFLIGHT SUMMARY: folded segment." })); + return; + } + stats.streamingForwards += 1; + stats.imagesSeen += (raw.match(/"input_image"/g) ?? []).length; + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(`event: response.completed\ndata: ${JSON.stringify({ response: { id: "resp_done", status: "completed", output: [], usage: { input_tokens: 1000, output_tokens: 5, total_tokens: 1005 } } })}\n\n`); + res.end(); + }); + }); + server.listen(0, "127.0.0.1"); + return new Promise((resolve) => { + server.on("listening", () => resolve({ server, port: (server.address() as { port: number }).port, stats })); + }); +} + +async function startProxy(upstreamPort: number): Promise<{ proxy: http.Server; port: number }> { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const proxy = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { [MODEL]: { context: WINDOW } } } }, + modelContextLimit: WINDOW, + kernelConfig: defaultConfig(WINDOW), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + await once(proxy, "listening"); + return { proxy, port: (proxy.address() as { port: number }).port }; +} + +function body(sessionId: string, input: unknown[]): string { + return JSON.stringify({ model: MODEL, stream: true, store: false, session_id: sessionId, instructions: "You are the test coding agent.", input, max_output_tokens: 1024 }); +} + +function fillerHistory(n: number): unknown[] { + const out: unknown[] = []; + for (let i = 0; i < n; i++) out.push({ type: "message", role: i % 2 ? "assistant" : "user", content: `MARKER_${i}: ${"lorem ipsum dolor sit amet ".repeat(400)}` }); + return out; +} + +function imageInput(): unknown[] { + return [{ type: "message", role: "user", content: [{ type: "input_text", text: "screenshot attached" }, { type: "input_image", image_url: `data:image/png;base64,${pngB64(1653, 2339, IMG_PAD)}` }] }]; +} + +test("#1800: image-dominated estimate with compressible text still folds the text portion (and forwards)", async () => { + const { server: upstream, port: uport, stats } = await startMockUpstream(); + const { proxy, port } = await startProxy(uport); + try { + const url = `http://127.0.0.1:${port}/bili/http://127.0.0.1:${uport}/v1/responses`; + const headers = { "content-type": "application/json" }; + + // Turn 1: small text-only turn establishes a sub-window usage baseline + // (lastInputTokens=1000, source="usage") → noOverflowEvidence=true. + const r1 = await fetch(url, { method: "POST", headers, body: body("s1800a", [{ type: "message", role: "user", content: "hello there" }]) }); + assert.equal(r1.status, 200); + await r1.text(); + + // Turn 2: enough filler history to create compressible ranges, plus one + // screenshot whose bytes-mode cost (~210K tokens) ALONE exceeds the + // 200K window → payloadEstimate over, textEstimate under. + const r2 = await fetch(url, { method: "POST", headers, body: body("s1800a", [...fillerHistory(60), ...imageInput()]) }); + assert.equal(r2.status, 200, "the image-dominated payload forwards (for the upstream to arbitrate billing), not 502"); + await r2.text(); + + // THE TOOTH: preflightCompress actually RAN and spent summarization + // calls folding the compressible text portion. Before the fix the old + // branch returned `prepared` before reaching preflightCompress, so + // summaryCalls was 0 and auto-compression stayed permanently disabled. + assert.ok(stats.summaryCalls >= 1, `preflight folded the compressible text portion (got ${stats.summaryCalls} summarization calls)`); + assert.ok(stats.imagesSeen >= 1, "the screenshot rode along byte-exact through the folded forward"); + } finally { + upstream.close(); + proxy.close(); + } +});