Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion src/image-note.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

import type { CoreMessage } from "acp-kernel";
import { parseDataUrl, type BiliMessage } from "acp-kernel/wire";
import { decodeImageDims } from "./image-tokens.js";
import { decodeImageDims, googleMediaParts } from "./image-tokens.js";
import { responsesToolImageParts } from "./responses-tool-output.js";

/** The codec's own placeholder for an anthropic image block (wire/index.js).
Expand Down Expand Up @@ -93,6 +93,24 @@ export function messageImages(m: CoreMessage): ImageRef[] {
}
if (refs.length > 0) return refs;
}
// Gemini tool responses keep their images in rawGoogleParts rather than
// the singular image sidecar; inspect all parts before that fallback.
if (Array.isArray(mm.rawGoogleParts)) {
const refs: ImageRef[] = [];
for (const part of googleMediaParts(mm.rawGoogleParts)) {
const inline = part.inlineData;
if (isObj(inline) && typeof inline.mimeType === "string" && inline.mimeType.startsWith("image/")
&& typeof inline.data === "string") {
refs.push({ mediaType: inline.mimeType, b64: inline.data });
continue;
}
const file = part.fileData;
if (isObj(file) && typeof file.mimeType === "string" && file.mimeType.startsWith("image/")) {
refs.push(refFromDataUrl(file.fileUri) ?? { mediaType: file.mimeType });
}
}
if (refs.length > 0) return refs;
}
// OpenAI chat: a single data-URL image lands in imageBase64 (+media type),
// multi-image messages in rawOpenaiContentParts. rawOpenaiContent can also
// be a developer-role message object — gate on the part shape. Checked
Expand Down
21 changes: 7 additions & 14 deletions src/image-restore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,20 +155,13 @@ export function messageImageBytes(m: CoreMessage): RestorableImage[] {
}
const fr = part.functionResponse;
if (isObj(fr) && Array.isArray(fr.parts)) {
// Review ⑤: mirror the strip side's Gemini-3 $ref guard (from the
// removed kernel strip-images.ts; kept for parity) — when
// `response` points at a part via
// {"$ref": displayName} the bytes are not inline,
// so there is nothing to restore: skip indexing (harmless waste,
// not corruption).
const refGuarded = JSON.stringify(fr.response ?? null).includes('"$ref"');
if (!refGuarded) {
for (const np of fr.parts) {
if (!isObj(np)) continue;
const ni = np.inlineData;
if (isObj(ni) && typeof ni.data === "string" && typeof ni.mimeType === "string" && ni.mimeType.startsWith("image/")) {
out.push({ mediaType: ni.mimeType, b64: ni.data, bytes: b64Bytes(ni.data) });
}
// Gemini's $ref names an inlineData part in this same response:
// https://ai.google.dev/api/generate-content#FunctionResponse
for (const np of fr.parts) {
if (!isObj(np)) continue;
const ni = np.inlineData;
if (isObj(ni) && typeof ni.data === "string" && typeof ni.mimeType === "string" && ni.mimeType.startsWith("image/")) {
out.push({ mediaType: ni.mimeType, b64: ni.data, bytes: b64Bytes(ni.data) });
}
}
}
Expand Down
18 changes: 14 additions & 4 deletions src/image-tokens.ts
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,18 @@ function opaquePartCost(part: Record<string, unknown>, configuredCap?: number):
return applyCap(Math.ceil(JSON.stringify(part).length / 4), configuredCap);
}

/** Gemini tool responses carry media beside their JSON result, in parts. */
export function* googleMediaParts(parts: unknown[]): Generator<Record<string, unknown>> {
for (const part of parts) {
if (!isObj(part)) continue;
yield part;
const response = part.functionResponse;
if (isObj(response) && Array.isArray(response.parts)) {
for (const nested of response.parts) if (isObj(nested)) yield nested;
}
}
}

export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "responses" | "google", body: unknown, billing: ResolvedImageBilling = "bytes", configuredCap?: number): number {
if (!isObj(body)) return 0;
let total = 0;
Expand All @@ -235,8 +247,7 @@ export function imageTokensInParsedBody(protocol: "anthropic" | "openai" | "resp
if (!Array.isArray(contents)) return 0;
for (const c of contents) {
if (!isObj(c) || !Array.isArray(c.parts)) continue;
for (const part of c.parts) {
if (!isObj(part)) continue;
for (const part of googleMediaParts(c.parts)) {
const inline = part.inlineData;
if (isObj(inline) && typeof inline.data === "string") { total += base64ImageCost(inline.data, billing, configuredCap); continue; }
const file = part.fileData;
Expand Down Expand Up @@ -395,8 +406,7 @@ export function countImagesInParsedBody(protocol: "anthropic" | "openai" | "resp
if (!Array.isArray(contents)) return 0;
for (const c of contents) {
if (!isObj(c) || !Array.isArray(c.parts)) continue;
for (const part of c.parts) {
if (!isObj(part)) continue;
for (const part of googleMediaParts(c.parts)) {
const inline = part.inlineData;
if (isObj(inline) && typeof inline.data === "string") { n += 1; continue; }
const file = part.fileData;
Expand Down
66 changes: 66 additions & 0 deletions tests/google-tool-image-accounting.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import assert from "node:assert/strict";
import test from "node:test";
import { googleToCore, type GoogleRequestBody } from "acp-kernel/wire";
import { imagePlaceholders, imagePlaceholdersForSummary } from "../src/image-note.js";
import { countImagesInParsedBody, countImagesInRawBody, imageTokensInParsedBody, imageTokensInRawBody, pixelTileEstimate, REMOTE_IMAGE_TOKENS } from "../src/image-tokens.js";

const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==";
const image = { inlineData: { mimeType: "image/png", data: png } };
const remote = { fileData: { mimeType: "image/jpeg", fileUri: "https://example.com/screenshot.jpg" } };
const toolBody: GoogleRequestBody = {
contents: [{ role: "user", parts: [{ functionResponse: {
name: "screenshot", response: { output: "Captured two views" }, parts: [image, image],
} }] }],
};

test("Google tool-result image notes retain every image and its recovery ref", () => {
const message = googleToCore(toolBody).msgs[0];
assert.equal(message.text, "Captured two views");
assert.deepEqual(imagePlaceholders(message), ["[image: png 1x1]", "[image: png 1x1]"]);
assert.deepEqual(imagePlaceholdersForSummary(message, "m00002"), ["[image: png 1x1 · m00002]", "[image: png 1x1 · m00002]"]);
assert.ok(!imagePlaceholdersForSummary(message, "m00002").join(" ").includes(png));
});

test("Google remote image notes are retained without fetching or exposing the URL", () => {
const message = googleToCore({ contents: [{ role: "user", parts: [remote] }] }).msgs[0];
assert.deepEqual(imagePlaceholders(message), ["[image: jpeg]"]);
});

test("Google nested image bytes and pixels are counted in both parsed and raw entry points", () => {
const bytes = 2 * Math.ceil(png.length / 4);
const pixels = 2 * pixelTileEstimate(1, 1);
assert.equal(imageTokensInParsedBody("google", toolBody, "bytes"), bytes);
assert.equal(imageTokensInRawBody("google", JSON.stringify(toolBody), "bytes"), bytes);
assert.equal(imageTokensInParsedBody("google", toolBody, "pixels"), pixels);
assert.equal(imageTokensInRawBody("google", JSON.stringify(toolBody), "pixels"), pixels);
assert.equal(countImagesInParsedBody("google", toolBody), 2);
assert.equal(countImagesInRawBody("google", JSON.stringify(toolBody)), 2);
assert.equal(imageTokensInParsedBody("google", toolBody, "pixels", 100), 200);
});

test("Google tool images share top-level media billing without charging the result text twice", () => {
const body: GoogleRequestBody = { contents: [{ role: "user", parts: [image, { functionResponse: {
name: "screenshot", response: { output: "result text ".repeat(1000) }, parts: [remote, image],
} }] }] };
assert.equal(countImagesInParsedBody("google", body), 3);
assert.equal(imageTokensInParsedBody("google", body, "pixels"), 2 * pixelTileEstimate(1, 1) + REMOTE_IMAGE_TOKENS);
const textOnly: GoogleRequestBody = { contents: [{ role: "user", parts: [{ functionResponse: { name: "text", response: { output: "plain" } } }] }] };
assert.equal(imageTokensInParsedBody("google", textOnly), 0);
assert.equal(countImagesInParsedBody("google", textOnly), 0);
assert.deepEqual(imagePlaceholders(googleToCore(textOnly).msgs[0]), []);
});

test("Google nested non-image media is billed and counted but earns no recovery note", () => {
const data = "JVBERi0xLjQKJcOkw7zDtsOfCg==";
const body: GoogleRequestBody = { contents: [{ role: "user", parts: [{ functionResponse: {
name: "export_doc", response: { output: "written" }, parts: [{ inlineData: { mimeType: "application/pdf", data } }],
} }] }] };
assert.equal(countImagesInParsedBody("google", body), 1);
assert.equal(imageTokensInParsedBody("google", body, "bytes"), Math.ceil(data.length / 4));
// Negative direction of the image/* note gate (#3617): the note carries a
// decompress({ imageRef }) recovery ref, and the restore channel indexes
// NESTED google media only for image/* mimes (src/image-restore.ts) — an
// note here would promise a restore it cannot deliver. Billing stays
// mime-agnostic: every forwarded byte reaches upstream.
assert.deepEqual(imagePlaceholders(googleToCore(body).msgs[0]), []);
});
27 changes: 27 additions & 0 deletions tests/image-restore.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
type RestorableImage,
} from "../src/image-restore.ts";
import { resolveDecompress, type ProxyToolCtx } from "../src/decompress-shared.ts";
import { imagePlaceholdersForSummary } from "../src/image-note.ts";
import type { WireProtocol } from "../src/util.js";

// Isolate stateDir() (hence restoreExportDir()) under a throwaway dir before any
Expand Down Expand Up @@ -300,3 +301,29 @@ test("buildIncomingImageIndex: end-to-end nested tool_result recovery (anthropic
assert.ok(existsSync(imgs[0].path), "nested image spilled at index time");
assert.deepEqual(readFileSync(imgs[0].path), Buffer.from(PNG, "base64"), "nested pixels round-trip");
});

test("buildIncomingImageIndex: Gemini $ref image notes resolve to their inline pixels", () => {
const body = { contents: [{ role: "user", parts: [{ functionResponse: {
name: "screenshot", response: { image_ref: { $ref: "capture.png" } },
parts: [{ inlineData: { mimeType: "image/png", displayName: "capture.png", data: PNG } }],
} }] }] };
const original = JSON.stringify(body);
const message = googleToCore(body).msgs[0];
const index = buildIndex("google", body);
assert.equal(index.size, 1);
const [ref, images] = [...index.entries()][0];
assert.deepEqual(imagePlaceholdersForSummary(message, ref), [`[image: png 1x1 · ${ref}]`]);
assert.match(resolveDecompress({ imageRef: ref }, ctxWith(index)).text, /Restored 1 image\(s\)/);
assert.deepEqual(readFileSync(images[0].path), Buffer.from(PNG, "base64"));
assert.equal(JSON.stringify(body), original, "indexing preserves the complete wire response");
});

test("buildIncomingImageIndex: a Gemini $ref without inline bytes adds no image", () => {
const body = { contents: [{ role: "user", parts: [{ functionResponse: {
name: "screenshot", response: { image_ref: { $ref: "capture.png" } },
} }] }] };
const message = googleToCore(body).msgs[0];
assert.deepEqual(messageImageBytes(message), []);
assert.deepEqual(imagePlaceholdersForSummary(message, "m00001"), []);
assert.equal(buildIndex("google", body).size, 0);
});
45 changes: 45 additions & 0 deletions tests/preflight-google-summary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import test from "node:test";
process.env.NODE_ENV = "test";

import { defaultConfig } from "acp-kernel";
import type { GoogleRequestBody } from "acp-kernel/wire";
import { startServer } from "../src/server.ts";
import type { ProxyOptions } from "../src/config.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
Expand Down Expand Up @@ -193,3 +194,47 @@ test("e2e preflight (#829): the JSON summary of a non-streaming Gemini call is r
await once(upstream, "close");
}
});

test("Gemini tool screenshots reach the real preflight summary as recovery notes", async () => {
const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==";
const calls: Call[] = [];
const upstream = makeGeminiUpstream(calls);
upstream.listen(0, "127.0.0.1");
await once(upstream, "listening");
const upstreamPort = (upstream.address() as { port: number }).port;
const proxy = await startProxy(upstreamPort);
await once(proxy, "listening");
const proxyPort = (proxy.address() as { port: number }).port;
try {
const contents: NonNullable<GoogleRequestBody["contents"]> = longGeminiContents();
contents[1].parts!.push({ functionCall: { name: "screenshot", args: {} } });
contents[2].parts!.unshift({ functionResponse: {
name: "screenshot", response: { output: "Captured two views" },
parts: [
{ inlineData: { mimeType: "image/png", data: png } },
{ inlineData: { mimeType: "image/png", data: png } },
],
} });
const response = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1beta/models/gemini-test:streamGenerateContent?alt=sse`, {
method: "POST",
headers: { "content-type": "application/json", "x-acp-session": "google-tool-image-notes" },
body: JSON.stringify({ contents, systemInstruction: { parts: [{ text: "You are a test assistant." }] }, generationConfig: { maxOutputTokens: 4096 } }),
});
const body = await response.text();
assert.equal(response.status, 200, body.slice(0, 300));
const summaries = calls.filter((call) => call.summary).map((call) => call.raw).join("\n");
assert.ok(summaries.includes("Captured two views"), "the tool result belongs to the folded range");
assert.equal((summaries.match(/\[image: png 1x1 · m\d{5}\]/g) ?? []).length, 2);
assert.ok(!summaries.includes(png), "summary input carries notes instead of image bytes");
const forwards = calls.filter((call) => !call.summary).map((call) => call.raw);
assert.ok(forwards.some((raw) => raw.includes(SUMMARY_TEXT.slice(0, 40))), "the real proxy applied the fold");
assert.ok(forwards.every((raw) => !raw.includes(png)), "folded pixels leave the model request");
} finally {
proxy.close();
proxy.closeIdleConnections?.();
await once(proxy, "close");
upstream.close();
upstream.closeIdleConnections?.();
await once(upstream, "close");
}
});
Loading