From 9bfdd9224fabf81fe4bcc563c8709bef112f2716 Mon Sep 17 00:00:00 2001 From: Afloat Date: Tue, 10 Nov 2026 22:50:34 +0800 Subject: [PATCH 1/3] fix(google): account for images nested in tool responses --- src/image-note.ts | 20 ++++++++- src/image-tokens.ts | 18 ++++++-- tests/google-tool-image-accounting.test.ts | 51 ++++++++++++++++++++++ tests/preflight-google-summary.test.ts | 46 +++++++++++++++++++ 4 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 tests/google-tool-image-accounting.test.ts diff --git a/src/image-note.ts b/src/image-note.ts index 301c8b2335..0488c67a55 100644 --- a/src/image-note.ts +++ b/src/image-note.ts @@ -12,7 +12,7 @@ import type { CoreMessage } from "acp-kernel"; import { parseDataUrl, type BiliMessage } from "acp-kernel/wire"; -import { decodeImageDims } from "./image-tokens.js"; +import { decodeImageDims, googleMediaParts } from "./image-tokens.js"; import { responsesToolImageParts } from "./responses-tool-output.js"; /** The codec's own placeholder for an anthropic image block (wire/index.js). @@ -93,6 +93,24 @@ export function messageImages(m: CoreMessage): ImageRef[] { } if (refs.length > 0) return refs; } + // Gemini tool responses keep their images in rawGoogleParts rather than + // the singular image sidecar; inspect all parts before that fallback. + if (Array.isArray(mm.rawGoogleParts)) { + const refs: ImageRef[] = []; + for (const part of googleMediaParts(mm.rawGoogleParts)) { + const inline = part.inlineData; + if (isObj(inline) && typeof inline.mimeType === "string" && inline.mimeType.startsWith("image/") + && typeof inline.data === "string") { + refs.push({ mediaType: inline.mimeType, b64: inline.data }); + continue; + } + const file = part.fileData; + if (isObj(file) && typeof file.mimeType === "string" && file.mimeType.startsWith("image/")) { + refs.push(refFromDataUrl(file.fileUri) ?? { mediaType: file.mimeType }); + } + } + if (refs.length > 0) return refs; + } // OpenAI chat: a single data-URL image lands in imageBase64 (+media type), // multi-image messages in rawOpenaiContentParts. rawOpenaiContent can also // be a developer-role message object — gate on the part shape. Checked diff --git a/src/image-tokens.ts b/src/image-tokens.ts index 91d85e16b9..94a147224c 100644 --- a/src/image-tokens.ts +++ b/src/image-tokens.ts @@ -224,6 +224,18 @@ function opaquePartCost(part: Record, configuredCap?: number): return applyCap(Math.ceil(JSON.stringify(part).length / 4), configuredCap); } +/** Gemini tool responses carry media beside their JSON result, in parts. */ +export function* googleMediaParts(parts: unknown[]): Generator> { + for (const part of parts) { + if (!isObj(part)) continue; + yield part; + const response = part.functionResponse; + if (isObj(response) && Array.isArray(response.parts)) { + for (const nested of response.parts) if (isObj(nested)) yield nested; + } + } +} + export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "responses" | "google", body: unknown, billing: ResolvedImageBilling = "bytes", configuredCap?: number): number { if (!isObj(body)) return 0; let total = 0; @@ -235,8 +247,7 @@ export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "resp if (!Array.isArray(contents)) return 0; for (const c of contents) { if (!isObj(c) || !Array.isArray(c.parts)) continue; - for (const part of c.parts) { - if (!isObj(part)) continue; + for (const part of googleMediaParts(c.parts)) { const inline = part.inlineData; if (isObj(inline) && typeof inline.data === "string") { total += base64ImageCost(inline.data, billing, configuredCap); continue; } const file = part.fileData; @@ -395,8 +406,7 @@ export function countImagesInParsedBody(protocol: "anthropic" | "openai" | "resp if (!Array.isArray(contents)) return 0; for (const c of contents) { if (!isObj(c) || !Array.isArray(c.parts)) continue; - for (const part of c.parts) { - if (!isObj(part)) continue; + for (const part of googleMediaParts(c.parts)) { const inline = part.inlineData; if (isObj(inline) && typeof inline.data === "string") { n += 1; continue; } const file = part.fileData; diff --git a/tests/google-tool-image-accounting.test.ts b/tests/google-tool-image-accounting.test.ts new file mode 100644 index 0000000000..6ae4eb536e --- /dev/null +++ b/tests/google-tool-image-accounting.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { googleToCore, type GoogleRequestBody } from "acp-kernel/wire"; +import { imagePlaceholders, imagePlaceholdersForSummary } from "../src/image-note.js"; +import { countImagesInParsedBody, countImagesInRawBody, imageTokensInParsedBody, imageTokensInRawBody, pixelTileEstimate, REMOTE_IMAGE_TOKENS } from "../src/image-tokens.js"; + +const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; +const image = { inlineData: { mimeType: "image/png", data: png } }; +const remote = { fileData: { mimeType: "image/jpeg", fileUri: "https://example.com/screenshot.jpg" } }; +const toolBody: GoogleRequestBody = { + contents: [{ role: "user", parts: [{ functionResponse: { + name: "screenshot", response: { output: "Captured two views" }, parts: [image, image], + } }] }], +}; + +test("Google tool-result image notes retain every image and its recovery ref", () => { + const message = googleToCore(toolBody).msgs[0]; + assert.equal(message.text, "Captured two views"); + assert.deepEqual(imagePlaceholders(message), ["[image: png 1x1]", "[image: png 1x1]"]); + assert.deepEqual(imagePlaceholdersForSummary(message, "m00002"), ["[image: png 1x1 · m00002]", "[image: png 1x1 · m00002]"]); + assert.ok(!imagePlaceholdersForSummary(message, "m00002").join(" ").includes(png)); +}); + +test("Google remote image notes are retained without fetching or exposing the URL", () => { + const message = googleToCore({ contents: [{ role: "user", parts: [remote] }] }).msgs[0]; + assert.deepEqual(imagePlaceholders(message), ["[image: jpeg]"]); +}); + +test("Google nested image bytes and pixels are counted in both parsed and raw entry points", () => { + const bytes = 2 * Math.ceil(png.length / 4); + const pixels = 2 * pixelTileEstimate(1, 1); + assert.equal(imageTokensInParsedBody("google", toolBody, "bytes"), bytes); + assert.equal(imageTokensInRawBody("google", JSON.stringify(toolBody), "bytes"), bytes); + assert.equal(imageTokensInParsedBody("google", toolBody, "pixels"), pixels); + assert.equal(imageTokensInRawBody("google", JSON.stringify(toolBody), "pixels"), pixels); + assert.equal(countImagesInParsedBody("google", toolBody), 2); + assert.equal(countImagesInRawBody("google", JSON.stringify(toolBody)), 2); + assert.equal(imageTokensInParsedBody("google", toolBody, "pixels", 100), 200); +}); + +test("Google tool images share top-level media billing without charging the result text twice", () => { + const body: GoogleRequestBody = { contents: [{ role: "user", parts: [image, { functionResponse: { + name: "screenshot", response: { output: "result text ".repeat(1000) }, parts: [remote, image], + } }] }] }; + assert.equal(countImagesInParsedBody("google", body), 3); + assert.equal(imageTokensInParsedBody("google", body, "pixels"), 2 * pixelTileEstimate(1, 1) + REMOTE_IMAGE_TOKENS); + const textOnly: GoogleRequestBody = { contents: [{ role: "user", parts: [{ functionResponse: { name: "text", response: { output: "plain" } } }] }] }; + assert.equal(imageTokensInParsedBody("google", textOnly), 0); + assert.equal(countImagesInParsedBody("google", textOnly), 0); + assert.deepEqual(imagePlaceholders(googleToCore(textOnly).msgs[0]), []); +}); diff --git a/tests/preflight-google-summary.test.ts b/tests/preflight-google-summary.test.ts index 56ee02cf27..7e0b307aca 100644 --- a/tests/preflight-google-summary.test.ts +++ b/tests/preflight-google-summary.test.ts @@ -6,6 +6,7 @@ import test from "node:test"; process.env.NODE_ENV = "test"; import { defaultConfig } from "acp-kernel"; +import type { GoogleRequestBody } from "acp-kernel/wire"; import { startServer } from "../src/server.ts"; import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; @@ -193,3 +194,48 @@ test("e2e preflight (#829): the JSON summary of a non-streaming Gemini call is r await once(upstream, "close"); } }); + + +test("Gemini tool screenshots reach the real preflight summary as recovery notes", async () => { + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + const calls: Call[] = []; + const upstream = makeGeminiUpstream(calls); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startProxy(upstreamPort); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + try { + const contents: NonNullable = longGeminiContents(); + contents[1].parts!.push({ functionCall: { name: "screenshot", args: {} } }); + contents[2].parts!.unshift({ functionResponse: { + name: "screenshot", response: { output: "Captured two views" }, + parts: [ + { inlineData: { mimeType: "image/png", data: png } }, + { inlineData: { mimeType: "image/png", data: png } }, + ], + } }); + const response = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1beta/models/gemini-test:streamGenerateContent?alt=sse`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "google-tool-image-notes" }, + body: JSON.stringify({ contents, systemInstruction: { parts: [{ text: "You are a test assistant." }] }, generationConfig: { maxOutputTokens: 4096 } }), + }); + const body = await response.text(); + assert.equal(response.status, 200, body.slice(0, 300)); + const summaries = calls.filter((call) => call.summary).map((call) => call.raw).join("\n"); + assert.ok(summaries.includes("Captured two views"), "the tool result belongs to the folded range"); + assert.equal((summaries.match(/\[image: png 1x1 · m\d{5}\]/g) ?? []).length, 2); + assert.ok(!summaries.includes(png), "summary input carries notes instead of image bytes"); + const forwards = calls.filter((call) => !call.summary).map((call) => call.raw); + assert.ok(forwards.some((raw) => raw.includes(SUMMARY_TEXT.slice(0, 40))), "the real proxy applied the fold"); + assert.ok(forwards.every((raw) => !raw.includes(png)), "folded pixels leave the model request"); + } finally { + proxy.close(); + proxy.closeIdleConnections?.(); + await once(proxy, "close"); + upstream.close(); + upstream.closeIdleConnections?.(); + await once(upstream, "close"); + } +}); From 5f3d479dbfa222a68e16bfa5ddf4358d027afd42 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 11 Nov 2026 07:04:55 +0800 Subject: [PATCH 2/3] test(google): pin the image/* note gate negative direction for nested tool media (#6146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review supplement to PR #6147: #3617 gate discipline requires both directions of a gate. The image/* mime filter on the Gemini note path had only its positive case pinned. Adds a nested application/pdf case asserting it is billed and counted but earns no recovery note — the note carries a decompress({imageRef}) recovery ref, and the restore channel indexes nested google media for image/* mimes only (src/image-restore.ts), so a note there would promise a restore it cannot deliver. Billing stays mime-agnostic. Also drops a stray double blank line in tests/preflight-google-summary.test.ts. --- tests/google-tool-image-accounting.test.ts | 15 +++++++++++++++ tests/preflight-google-summary.test.ts | 1 - 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/google-tool-image-accounting.test.ts b/tests/google-tool-image-accounting.test.ts index 6ae4eb536e..ca02e6b9db 100644 --- a/tests/google-tool-image-accounting.test.ts +++ b/tests/google-tool-image-accounting.test.ts @@ -49,3 +49,18 @@ test("Google tool images share top-level media billing without charging the resu assert.equal(countImagesInParsedBody("google", textOnly), 0); assert.deepEqual(imagePlaceholders(googleToCore(textOnly).msgs[0]), []); }); + +test("Google nested non-image media is billed and counted but earns no recovery note", () => { + const data = "JVBERi0xLjQKJcOkw7zDtsOfCg=="; + const body: GoogleRequestBody = { contents: [{ role: "user", parts: [{ functionResponse: { + name: "export_doc", response: { output: "written" }, parts: [{ inlineData: { mimeType: "application/pdf", data } }], + } }] }] }; + assert.equal(countImagesInParsedBody("google", body), 1); + assert.equal(imageTokensInParsedBody("google", body, "bytes"), Math.ceil(data.length / 4)); + // Negative direction of the image/* note gate (#3617): the note carries a + // decompress({ imageRef }) recovery ref, and the restore channel indexes + // NESTED google media only for image/* mimes (src/image-restore.ts) — an + // note here would promise a restore it cannot deliver. Billing stays + // mime-agnostic: every forwarded byte reaches upstream. + assert.deepEqual(imagePlaceholders(googleToCore(body).msgs[0]), []); +}); diff --git a/tests/preflight-google-summary.test.ts b/tests/preflight-google-summary.test.ts index 7e0b307aca..058a9e0476 100644 --- a/tests/preflight-google-summary.test.ts +++ b/tests/preflight-google-summary.test.ts @@ -195,7 +195,6 @@ test("e2e preflight (#829): the JSON summary of a non-streaming Gemini call is r } }); - test("Gemini tool screenshots reach the real preflight summary as recovery notes", async () => { const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; const calls: Call[] = []; From 378ba55151ab257e8b638e1b688ae1f04260317d Mon Sep 17 00:00:00 2001 From: Afloat Date: Wed, 11 Nov 2026 12:39:45 +0800 Subject: [PATCH 3/3] fix: index Gemini inline images referenced by function responses --- src/image-restore.ts | 21 +++++++-------------- tests/image-restore.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/image-restore.ts b/src/image-restore.ts index fdac387535..31971ee455 100644 --- a/src/image-restore.ts +++ b/src/image-restore.ts @@ -155,20 +155,13 @@ export function messageImageBytes(m: CoreMessage): RestorableImage[] { } const fr = part.functionResponse; if (isObj(fr) && Array.isArray(fr.parts)) { - // Review ⑤: mirror the strip side's Gemini-3 $ref guard (from the - // removed kernel strip-images.ts; kept for parity) — when - // `response` points at a part via - // {"$ref": displayName} the bytes are not inline, - // so there is nothing to restore: skip indexing (harmless waste, - // not corruption). - const refGuarded = JSON.stringify(fr.response ?? null).includes('"$ref"'); - if (!refGuarded) { - for (const np of fr.parts) { - if (!isObj(np)) continue; - const ni = np.inlineData; - if (isObj(ni) && typeof ni.data === "string" && typeof ni.mimeType === "string" && ni.mimeType.startsWith("image/")) { - out.push({ mediaType: ni.mimeType, b64: ni.data, bytes: b64Bytes(ni.data) }); - } + // Gemini's $ref names an inlineData part in this same response: + // https://ai.google.dev/api/generate-content#FunctionResponse + for (const np of fr.parts) { + if (!isObj(np)) continue; + const ni = np.inlineData; + if (isObj(ni) && typeof ni.data === "string" && typeof ni.mimeType === "string" && ni.mimeType.startsWith("image/")) { + out.push({ mediaType: ni.mimeType, b64: ni.data, bytes: b64Bytes(ni.data) }); } } } diff --git a/tests/image-restore.test.ts b/tests/image-restore.test.ts index d0611aea23..2bffee226a 100644 --- a/tests/image-restore.test.ts +++ b/tests/image-restore.test.ts @@ -14,6 +14,7 @@ import { type RestorableImage, } from "../src/image-restore.ts"; import { resolveDecompress, type ProxyToolCtx } from "../src/decompress-shared.ts"; +import { imagePlaceholdersForSummary } from "../src/image-note.ts"; import type { WireProtocol } from "../src/util.js"; // Isolate stateDir() (hence restoreExportDir()) under a throwaway dir before any @@ -300,3 +301,29 @@ test("buildIncomingImageIndex: end-to-end nested tool_result recovery (anthropic assert.ok(existsSync(imgs[0].path), "nested image spilled at index time"); assert.deepEqual(readFileSync(imgs[0].path), Buffer.from(PNG, "base64"), "nested pixels round-trip"); }); + +test("buildIncomingImageIndex: Gemini $ref image notes resolve to their inline pixels", () => { + const body = { contents: [{ role: "user", parts: [{ functionResponse: { + name: "screenshot", response: { image_ref: { $ref: "capture.png" } }, + parts: [{ inlineData: { mimeType: "image/png", displayName: "capture.png", data: PNG } }], + } }] }] }; + const original = JSON.stringify(body); + const message = googleToCore(body).msgs[0]; + const index = buildIndex("google", body); + assert.equal(index.size, 1); + const [ref, images] = [...index.entries()][0]; + assert.deepEqual(imagePlaceholdersForSummary(message, ref), [`[image: png 1x1 · ${ref}]`]); + assert.match(resolveDecompress({ imageRef: ref }, ctxWith(index)).text, /Restored 1 image\(s\)/); + assert.deepEqual(readFileSync(images[0].path), Buffer.from(PNG, "base64")); + assert.equal(JSON.stringify(body), original, "indexing preserves the complete wire response"); +}); + +test("buildIncomingImageIndex: a Gemini $ref without inline bytes adds no image", () => { + const body = { contents: [{ role: "user", parts: [{ functionResponse: { + name: "screenshot", response: { image_ref: { $ref: "capture.png" } }, + } }] }] }; + const message = googleToCore(body).msgs[0]; + assert.deepEqual(messageImageBytes(message), []); + assert.deepEqual(imagePlaceholdersForSummary(message, "m00001"), []); + assert.equal(buildIndex("google", body).size, 0); +});