From f7deccc677d2158eec52486a9e8719c764e823cc Mon Sep 17 00:00:00 2001 From: ework-agent Date: Thu, 3 Sep 2026 16:17:09 +0800 Subject: [PATCH] fix: count image bytes in payload-size decisions; keep store:false in Responses preflight summary (#488) --- src/image-tokens.ts | 94 +++++++++++ src/preflight.ts | 11 +- src/server.ts | 33 ++-- tests/fix-488-multimodal.test.ts | 268 +++++++++++++++++++++++++++++++ 4 files changed, 393 insertions(+), 13 deletions(-) create mode 100644 src/image-tokens.ts create mode 100644 tests/fix-488-multimodal.test.ts diff --git a/src/image-tokens.ts b/src/image-tokens.ts new file mode 100644 index 0000000000..a8f0361a36 --- /dev/null +++ b/src/image-tokens.ts @@ -0,0 +1,94 @@ +// #488: images are invisible to the kernel's token model (wire codecs move them +// out of CoreMessage.text into sidecars), yet they ARE forwarded verbatim — so +// every fit decision built on the text-only estimate undercounts image-bearing +// payloads up to 15×. This module estimates the image share of a RAW request +// body so callers can add it to the text estimate. Known base64 payloads are +// charged ceil(base64Length/4) — the same chars/4 rule defaultCountTokens uses, +// matching how byte-counting relays actually bill (#488: 7 screenshots ≈ 1.4M +// tokens); remote URLs we cannot size get a flat conservative cost. Pixel-tile +// upstreams (official Anthropic/OpenAI) will see an overestimate — set +// BILI_IMAGE_TOKEN_CAP to clamp the per-image cost. + +export const REMOTE_IMAGE_TOKENS = 4096; + +function imageTokenCap(): number { + const v = Number(process.env.BILI_IMAGE_TOKEN_CAP ?? ""); + return Number.isInteger(v) && v > 0 ? v : 0; +} + +function applyCap(cost: number): number { + const cap = imageTokenCap(); + return cap > 0 ? Math.min(cost, cap) : cost; +} + +function costForUrl(url: string): number { + if (url.startsWith("data:")) { + const i = url.indexOf("base64,"); + if (i >= 0) return applyCap(Math.ceil((url.length - i - "base64,".length) / 4)); + } + return applyCap(REMOTE_IMAGE_TOKENS); +} + +function isObj(v: unknown): v is Record { + return typeof v === "object" && v !== null; +} + +function urlOf(v: unknown): string | undefined { + if (typeof v === "string") return v; + if (isObj(v) && typeof v.url === "string") return v.url; + return undefined; +} + +export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "responses", body: unknown): number { + if (!isObj(body)) return 0; + let total = 0; + if (protocol === "responses") { + const input = body.input; + if (!Array.isArray(input)) return 0; + for (const item of input) { + if (!isObj(item) || !Array.isArray(item.content)) continue; + for (const part of item.content) { + if (!isObj(part) || part.type !== "input_image") continue; + const url = urlOf(part.image_url); + if (url) total += costForUrl(url); + } + } + return total; + } + const messages = body.messages; + if (!Array.isArray(messages)) return 0; + for (const m of messages) { + if (!isObj(m) || !Array.isArray(m.content)) continue; + for (const part of m.content) { + if (!isObj(part)) continue; + if (protocol === "openai") { + if (part.type !== "image_url") continue; + const url = urlOf(part.image_url); + if (url) total += costForUrl(url); + } else { + if (part.type !== "image") continue; + const src = part.source; + if (isObj(src) && src.type === "base64" && typeof src.data === "string") total += applyCap(Math.ceil(src.data.length / 4)); + else if (isObj(src) && src.type === "url" && typeof src.url === "string") total += costForUrl(src.url); + } + } + } + return total; +} + +// Cheap gate: most bodies carry no images — skip the JSON parse entirely then. +// prepared.body is bili's own compact JSON.stringify, but client raw buffers +// may carry spaces, so probe both forms. +export function imageTokensInRawBody(protocol: "anthropic" | "openai" | "responses", raw: string | Buffer): number { + const s = typeof raw === "string" ? raw : raw.toString("utf8"); + const probe = + protocol === "responses" ? s.includes("input_image") + : protocol === "openai" ? s.includes("image_url") + : s.includes('"type":"image"') || s.includes('"type": "image"'); + if (!probe) return 0; + try { + return imageTokensInParsedBody(protocol, JSON.parse(s)); + } catch { + return 0; + } +} diff --git a/src/preflight.ts b/src/preflight.ts index b1e22ddb84..5390cacbc3 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -42,6 +42,8 @@ export interface PreflightDeps { proxyUrl?: string; signal?: AbortSignal; log: (level: string, msg: string) => void; + /** Constant floor on the forwarded-payload size for this request (image bytes, #488). Folding only ever removes images, so adding this to every text estimate keeps the fit decision sound for multimodal payloads. */ + imageFloor?: number; } export type PreflightFailureKind = "upstream" | "exhausted" | "aborted"; @@ -145,7 +147,8 @@ function summaryPayload(protocol: PreflightProtocol, model: string, system: stri if (protocol === "openai") { return { model, max_tokens: MAX_SUMMARY_OUTPUT_TOKENS, messages: [{ role: "system", content: system }, { role: "user", content }], stream: false }; } - return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false }; + // #488: codex relays reject Responses calls without store:false ("Store must be set to false"). + return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false, store: false }; } function extractSummaryText(protocol: PreflightProtocol, json: Record): string { @@ -223,7 +226,7 @@ const ABORTED_FAILURE: PreflightFailure = { kind: "aborted", detail: "the client export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise { const limit = deps.config.modelContextLimit; - const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) }; + const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) }; if (limit <= 0) return result; const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION)); // applyCompression rejects ranges below config.compress.minCompressRange @@ -254,10 +257,10 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // Floor on the session's measured input baseline: the upstream's // input_tokens also covers the system prompt + tool definitions, which // are not in turn.messages, so the direct estimate can undershoot. - currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages)); + currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0)); // The caller's forward/fail-fast gate uses the payload's own estimate // (the floor can be stale — see PreflightResult.payloadEstimate). - result.payloadEstimate = estimateCoreMessages(turn.messages); + result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0); if (startTokens < 0) startTokens = currentTokens; if (currentTokens < limit) break; const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []); diff --git a/src/server.ts b/src/server.ts index 7061889fbc..941bbd5b9d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -46,6 +46,7 @@ import { COMPRESS_TOOL, ACP_TOOLS_ANTHROPIC, ACP_TOOLS_OPENAI, ACP_TOOLS_RESPONS import { rewriteJsonResponse, type RewriteCtx } from "./stream.js"; import { applyRanges } from "./stream.js"; import { preflightCompress, estimateCoreMessages } from "./preflight.js"; +import { imageTokensInRawBody, imageTokensInParsedBody } from "./image-tokens.js"; import { renderUI, handleConfigGet, handleConfigPut } from "./web/index.js"; import { reapOrphanBlocks } from "./orphan-gc.js"; import { getStore } from "./persist.js"; @@ -1559,13 +1560,15 @@ export function emergencyNudge(nudge: NudgeDecision | null | undefined, escalati function clampOutgoingOutput( rebuilt: Record, field: "max_tokens" | "max_completion_tokens" | "max_output_tokens", - ctx: { systemText: string; tools: unknown; processedMessages: CoreMessage[]; lastInputTokens: number; nativeWindow: number }, + ctx: { systemText: string; tools: unknown; processedMessages: CoreMessage[]; lastInputTokens: number; nativeWindow: number; imageTokens: number }, sessionId: string, log: (level: string, msg: string) => void, ): void { const raw = rebuilt[field]; if (typeof raw !== "number") return; - const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens); + // #488: images ride along in the rebuilt body but are invisible to the text model — + // without them the cap is too generous and input+output can still overflow. + const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens) + ctx.imageTokens; const capped = clampOutputBudget(raw, inputEstimate, ctx.nativeWindow); if (capped !== undefined) { rebuilt[field] = capped; @@ -1675,7 +1678,7 @@ function prepareOpenai( } const rebuilt: OpenAIRequestBody = { ...parsed, messages: rebuiltMessages, tools: toolsOut as OpenAITool[] | undefined }; - clampOutgoingOutput(rebuilt as Record, typeof (parsed as Record).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow }, sessionId, log); + clampOutgoingOutput(rebuilt as Record, typeof (parsed as Record).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow, imageTokens: imageTokensInParsedBody("openai", rebuilt) }, sessionId, log); // prompt_cache_retention is an OpenAI-host-only cache directive; the dsh // launcher forces PI_CACHE_RETENTION=long (for the session-id // prompt_cache_key) which makes the client also emit it. Third-party @@ -1884,7 +1887,7 @@ function prepareResponses( const rebuilt: ResponsesRequestBody = { ...parsed, input: rebuiltInput, tools: toolsOut }; if (!isCompactionTrigger) { - clampOutgoingOutput(rebuilt as Record, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow }, sessionId, log); + clampOutgoingOutput(rebuilt as Record, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow, imageTokens: imageTokensInParsedBody("responses", rebuilt) }, sessionId, log); } // Route with the upstream THIS request goes to — session.meta.upstreamOrigin // is first-wins and would keep injecting pck toward a relay we switched @@ -2292,7 +2295,10 @@ async function preflightCompressIfNeeded( // A fresh session (id rotated, e.g. after a model switch) has // lastInputTokens = 0 while still carrying a full raw history; size the // trigger on the real post-fold payload too. - const payloadEstimate = estimateCoreMessages(prepared.processedMessages); + // #488: images are forwarded verbatim but invisible to the kernel's text model — + // add their cost to every size decision here (trigger, fit gates, self-heal). + const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body); + const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + imageTokens; const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate); if (limit <= 0 || !model || tokenCount < limit) return prepared; // #301: forwarding as-is is safe ONLY when the payload's own estimate @@ -2301,10 +2307,14 @@ async function preflightCompressIfNeeded( // double-counted usage report (#300) — and must not turn a fitting // payload into a fail-fast false positive. const failFast = (status: number, detail: string, retryable: boolean): PreflightFailFast => { + const imageNote = imageTokens >= limit + ? ` Images alone account for ~${imageTokens} tokens (≥ window ${limit}); compression cannot remove them — shrink or remove the images, or raise the window.` + : ""; const message = `context ~${tokenCount} tokens exceeds the model window ${limit} (model=${model}) ` + - `and preflight compression could not bring it under: ${detail}. ` + - `The over-window payload was NOT forwarded.`; + `and preflight compression could not bring it under: ${detail}.` + + imageNote + + ` The over-window payload was NOT forwarded.`; log("error", `[${session.id}] preflight fail-fast ${status} (retryable=${retryable}): ${message}`); return { failFast: true, status, message, retryable, respond: !res.writableEnded }; }; @@ -2337,6 +2347,7 @@ async function preflightCompressIfNeeded( proxyUrl, signal: clientAbort.signal, log, + imageFloor: imageTokens, }, prepared.originalMessages, ); @@ -2560,9 +2571,13 @@ async function forward( // model that produced the overflow (per-model scoping: a stale // limit from another model must not cap this one). let reqModel: string | undefined; + let rejectedImageTokens = 0; try { const rawBody = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8"); - reqModel = (JSON.parse(rawBody) as { model?: string }).model; + const parsedBody = JSON.parse(rawBody) as Record; + reqModel = typeof parsedBody.model === "string" ? parsedBody.model : undefined; + // #488: the rejected payload's size must include its images (they were forwarded verbatim). + rejectedImageTokens = imageTokensInParsedBody(prepared.protocol, parsedBody); } catch { reqModel = undefined; // non-JSON body — fall back to the legacy scalar } @@ -2583,7 +2598,7 @@ async function forward( // so the next turn re-centers the limit and the pre-flight // compresses below it. Only shrink, never grow: a previously // learned (smaller) value is the tighter bound. - const payloadEstimate = estimateCoreMessages(prepared.processedMessages); + const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + rejectedImageTokens; const prev = (reqModel ? learnedMap[reqModel] : undefined) ?? (s.metadata.learnedContextLimit as number | undefined); if (payloadEstimate >= 1000 && (prev === undefined || payloadEstimate < prev)) { if (reqModel) learnedMap[reqModel] = payloadEstimate; diff --git a/tests/fix-488-multimodal.test.ts b/tests/fix-488-multimodal.test.ts new file mode 100644 index 0000000000..e45626d23c --- /dev/null +++ b/tests/fix-488-multimodal.test.ts @@ -0,0 +1,268 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { defaultConfig } from "acp-kernel"; +import { startServer, type ProxyOptions } from "../src/server.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions } from "../src/session.ts"; +import { REMOTE_IMAGE_TOKENS, imageTokensInParsedBody, imageTokensInRawBody } from "../src/image-tokens.ts"; + +// Issue #488: (A) image bytes were invisible to every payload-size decision +// while being forwarded verbatim, so multimodal payloads blew past the window +// unintercepted; (B) the preflight summary request for the Responses protocol +// omitted store:false, which codex relays reject with 400, killing the rescue. + +const B64_8K = "A".repeat(8000); +const DATA_URL = `data:image/png;base64,${B64_8K}`; + +test("image-tokens: base64 data URLs count as ceil(b64len/4) across protocols", () => { + assert.equal( + imageTokensInParsedBody("responses", { input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }, { type: "input_image", image_url: DATA_URL }] }] }), + 2000); + assert.equal( + imageTokensInParsedBody("openai", { messages: [{ role: "user", content: [{ type: "text", text: "hi" }, { type: "image_url", image_url: { url: DATA_URL } }] }] }), + 2000); + assert.equal( + imageTokensInParsedBody("openai", { messages: [{ role: "user", content: [{ type: "image_url", image_url: DATA_URL }] }] }), + 2000); + assert.equal( + imageTokensInParsedBody("anthropic", { messages: [{ role: "user", content: [{ type: "text", text: "hi" }, { type: "image", source: { type: "base64", media_type: "image/png", data: B64_8K } }] }] }), + 2000); +}); + +test("image-tokens: remote URLs get a flat cost; multiple images sum; string content ignored", () => { + assert.equal(imageTokensInParsedBody("responses", { input: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: "https://example.com/a.png" }] }] }), REMOTE_IMAGE_TOKENS); + assert.equal(imageTokensInParsedBody("anthropic", { messages: [{ role: "user", content: [{ type: "image", source: { type: "url", url: "https://example.com/a.png" } }] }] }), REMOTE_IMAGE_TOKENS); + const du = `data:image/png;base64,${"B".repeat(4000)}`; + assert.equal( + imageTokensInParsedBody("responses", { input: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: du }, { type: "input_image", image_url: du }] }] }), + 2000); + assert.equal(imageTokensInParsedBody("responses", { input: [{ type: "message", role: "user", content: "plain string body" }] }), 0); + assert.equal(imageTokensInParsedBody("openai", { messages: [{ role: "user", content: "plain string body" }] }), 0); + assert.equal(imageTokensInParsedBody("anthropic", { messages: [{ role: "user", content: [{ type: "text", text: "no images here" }] }] }), 0); +}); + +test("image-tokens: raw-body gate skips parsing when no image marker is present", () => { + const noImage = JSON.stringify({ model: "m", input: [{ type: "message", role: "user", content: "hello world, no pictures" }] }); + assert.equal(imageTokensInRawBody("responses", noImage), 0); + const du = `data:image/png;base64,${"C".repeat(12000)}`; + const withImage = JSON.stringify({ model: "m", input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "look" }, { type: "input_image", image_url: du }] }] }); + assert.equal(imageTokensInRawBody("responses", withImage), 3000); + assert.equal(imageTokensInRawBody("responses", Buffer.from(withImage)), 3000); + assert.equal(imageTokensInRawBody("responses", '{"input_image": broken'), 0); +}); + +test("image-tokens: BILI_IMAGE_TOKEN_CAP clamps each image's cost", () => { + process.env.BILI_IMAGE_TOKEN_CAP = "500"; + try { + assert.equal( + imageTokensInParsedBody("responses", { input: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: DATA_URL }, { type: "input_image", image_url: DATA_URL }] }] }), + 1000); + assert.equal( + imageTokensInParsedBody("responses", { input: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: "https://example.com/a.png" }] }] }), + 500); + } finally { + delete process.env.BILI_IMAGE_TOKEN_CAP; + } +}); + +const SUMMARY_TEXT = + "PREFLIGHT SUMMARY: the segment covered a multi-step debugging session. Key decisions: chose the preflight approach over lossy truncation because the payload must stay coherent. Files touched: src/a.ts:10, src/b.ts:20. Outcome: fixed and verified by tests."; + +function sse(type: string, data: unknown): string { + return `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; +} + +function completed(inputTokens: number): string { + return sse("response.completed", { + response: { id: "resp_done", status: "completed", output: [], usage: { input_tokens: inputTokens, output_tokens: 5, total_tokens: inputTokens + 5 } }, + }); +} + +function longInput() { + const input: { type: string; role: string; content: string }[] = []; + for (let i = 0; i < 12; i++) { + input.push({ type: "message", role: i % 2 === 0 ? "user" : "assistant", content: `Message ${i} of the long conversation. ` + `MARKER_${i}_content_`.repeat(250) }); + } + return input; +} + +async function startProxy(upstreamPort: number): Promise<{ proxy: http.Server; url: string }> { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const proxy = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-resp": { context: 10_000 } } } }, + modelContextLimit: 10_000, + kernelConfig: defaultConfig(10_000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + return { proxy, url: `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/responses` }; +} + +test("e2e #488-B (Responses): preflight summary requests carry store:false so codex relays accept them", async () => { + let summaryCalls = 0; + const summaryBodies: unknown[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { stream?: boolean }; + if (parsed.stream === false) { + summaryCalls += 1; + summaryBodies.push(parsed); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ output_text: SUMMARY_TEXT })); + return; + } + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(completed(1000)); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const { proxy, url } = await startProxy(upstreamPort); + + try { + // Fresh session replaying an oversized raw history (~13k > 10k window, + // lastInputTokens=0) — preflight must run its summarization rescue. + const r = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-resp", stream: true, session_id: "resp-sess-b", instructions: "You are the test coding agent.", input: longInput() }), + }); + assert.equal(r.status, 200, "oversized restored session's request succeeds"); + await r.text(); + + assert.ok(summaryCalls >= 1, `preflight summarization ran (got ${summaryCalls})`); + for (const b of summaryBodies) { + assert.equal((b as { store?: unknown }).store, false, "preflight summary request keeps store:false"); + } + + const s = listSessions().find((x) => x.meta.label === "resp-sess-b"); + assert.ok(s, "session exists"); + assert.ok((s!.state.blocks ?? []).some((b) => b.active), "compression block recorded from preflight"); + } finally { + await closeAll(proxy, upstream); + } +}); + +test("e2e #488-A (Responses): images alone over the window are withheld with an actionable error, never forwarded", async () => { + const streamForwards: boolean[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { stream?: boolean }; + streamForwards.push(parsed.stream === true); + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(completed(100)); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const { proxy, url } = await startProxy(upstreamPort); + + try { + // 7 screenshots × 60k base64 chars = 7 × 15_000 = 105_000 image tokens, + // against a 10_000 window — no amount of text folding can fix this. + const bigB64 = "A".repeat(60_000); + const input = [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "here are seven screenshots of the failing screen" }, + ...Array.from({ length: 7 }, () => ({ type: "input_image", image_url: `data:image/png;base64,${bigB64}` })), + ], + }, + ]; + const r = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-resp", stream: true, session_id: "img-sess-a", instructions: "You are the test coding agent.", input }), + }); + assert.equal(r.status, 502, "over-window multimodal payload is withheld"); + const err = JSON.parse(await r.text()) as { error?: { code?: string; message?: string; retryable?: boolean } }; + assert.equal(err.error?.code, "preflight_compress_failed"); + assert.match(err.error?.message ?? "", /Images alone account for ~105000 tokens/); + assert.ok(!streamForwards.includes(true), "the over-window payload was never forwarded upstream"); + } finally { + await closeAll(proxy, upstream); + } +}); + +test("e2e #488-A2 (Responses): image tokens count toward the output-budget clamp", async () => { + const forwardedMaxOutput: number[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { stream?: boolean; max_output_tokens?: number }; + if (parsed.stream === true && typeof parsed.max_output_tokens === "number") forwardedMaxOutput.push(parsed.max_output_tokens); + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(completed(100)); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const { proxy, url } = await startProxy(upstreamPort); + + try { + // Control: text-only input vs the 10k window — a 5k output request fits, no clamp. + const r1 = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-resp", stream: true, session_id: "img-sess-c1", instructions: "You are the test coding agent.", input: [{ type: "message", role: "user", content: "hello there" }], max_output_tokens: 5_000 }), + }); + assert.equal(r1.status, 200); + await r1.text(); + + // Same shape plus one 16k-base64-char screenshot (4_000 image tokens): input+output + // would overflow the 10k window, so the outgoing max_output_tokens must be clamped. + const bigB64 = "A".repeat(16_000); + const r2 = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-resp", stream: true, session_id: "img-sess-c2", instructions: "You are the test coding agent.", input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "screenshot attached" }, { type: "input_image", image_url: `data:image/png;base64,${bigB64}` }] }], max_output_tokens: 5_000 }), + }); + assert.equal(r2.status, 200); + await r2.text(); + + assert.equal(forwardedMaxOutput[0], 5_000, "text-only request keeps its requested output budget"); + assert.ok(forwardedMaxOutput.length >= 2, "image-bearing request was forwarded"); + assert.ok(forwardedMaxOutput[1] < 5_000, `image tokens shrink the output budget (got ${forwardedMaxOutput[1]})`); + assert.ok(forwardedMaxOutput[1] >= 1024, "clamp stays above the floor"); + } finally { + await closeAll(proxy, upstream); + } +}); + +async function closeAll(...servers: http.Server[]): Promise { + for (const s of servers) { + s.close(); + await once(s, "close"); + } +}