From 54a9cda24863690aa350cb2f91c67299e3e2cefc Mon Sep 17 00:00:00 2001 From: qiangu2020 <57162279+qiangu2020@users.noreply.github.com> Date: Wed, 21 Oct 2026 12:53:47 +0800 Subject: [PATCH 1/2] fix: preserve original images in PTC tool results (#4070) --- src/egress-shed.ts | 39 ++++++---- src/image-rescale.ts | 39 +++++++--- tests/issue3400-image-rescale.test.ts | 108 ++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 25 deletions(-) diff --git a/src/egress-shed.ts b/src/egress-shed.ts index 6aac645c30..2551dae464 100644 --- a/src/egress-shed.ts +++ b/src/egress-shed.ts @@ -18,7 +18,7 @@ import { currentFetchTransport, isWsLaneTransport } from "./fetch-transport.js"; import { limitsWsOutboundBytes } from "./knobs.js"; import { decodeImageDims } from "./image-tokens.js"; import { READ_ORIGINAL_IMAGE_TOOL_NAME } from "./compress-tool.js"; -import { refForItemBytes, shedDownscaleImage } from "./image-rescale.js"; +import { matchesOriginalImageReceipt, refForItemBytes, shedDownscaleImage } from "./image-rescale.js"; import { ResponsesWsUpstreamError } from "./responses-ws.js"; type Part = Record; @@ -132,6 +132,24 @@ function isExemptOutput(protocol: WireProtocol, item: Part, names: Map { + parts.forEach((part, pos) => { + if (!isObj(part) || part.type !== "image") return; + const src = part.source; + if (!isObj(src) || src.type !== "base64" || typeof src.data !== "string" || src.data.length === 0) return; + if (toolOutput && matchesOriginalImageReceipt(src.data, parts)) return; + const mediaType = typeof src.media_type === "string" ? src.media_type : "image/png"; + occ.push({ + entryIndex, + b64: src.data, + mediaType, + protectedTail: false, + archived: false, + applyArchive: (note) => { parts[pos] = { type: "text", text: note }; }, + applyShrink: (b64) => { src.data = b64; }, + }); + }); + }; if (protocol === "google") { const contents = body.contents; if (!Array.isArray(contents)) return occ; @@ -169,7 +187,7 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] { parts.forEach((part, pos) => { if (!isObj(part) || part.type !== "input_image" || typeof part.image_url !== "string") return; const d = parseDataUrl(part.image_url); - if (!d) return; + if (!d || (partsKey === "output" && matchesOriginalImageReceipt(d.base64, parts))) return; occ.push({ entryIndex, b64: d.base64, @@ -193,6 +211,7 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] { if (protocol === "anthropic" && Array.isArray(m.content) && m.content.some((b) => isObj(b) && b.type === "tool_result" && isExemptOutput(protocol, b, names))) return; if (!Array.isArray(m.content)) return; const content = m.content as unknown[]; + if (protocol === "anthropic") addAnthropicImages(content, entryIndex, false); content.forEach((part, pos) => { const p = isObj(part) ? part : undefined; if (!p) return; @@ -212,20 +231,8 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] { applyArchive: (note) => { content[pos] = { type: "text", text: note }; }, applyShrink: (b64) => { p.image_url = typeof iu === "string" ? dataUrl(d.mediaType, b64) : { ...(iu ?? {}), url: dataUrl(d.mediaType, b64) }; }, }); - } else { - if (p.type !== "image") return; - const src = p.source; - if (!isObj(src) || src.type !== "base64" || typeof src.data !== "string" || src.data.length === 0) return; - const mediaType = typeof src.media_type === "string" ? src.media_type : "image/png"; - occ.push({ - entryIndex, - b64: src.data, - mediaType, - protectedTail: false, - archived: false, - applyArchive: (note) => { content[pos] = { type: "text", text: note }; }, - applyShrink: (b64) => { src.data = b64; }, - }); + } else if (p.type === "tool_result" && Array.isArray(p.content)) { + addAnthropicImages(p.content, entryIndex, true); } }); }); diff --git a/src/image-rescale.ts b/src/image-rescale.ts index 15ee5b3893..3b64eaa81a 100644 --- a/src/image-rescale.ts +++ b/src/image-rescale.ts @@ -274,6 +274,22 @@ function asPart(v: unknown): Part | undefined { // never mutated (session state and persistence keep the originals). // --------------------------------------------------------------------------- +/** Non-security, caller-fabricable convention: only exact decoded image bytes in + * this tool output match. Never search other messages or cache an exemption. */ +export function matchesOriginalImageReceipt(b64: string, parts: unknown[]): boolean { + const hashes = new Set(); + for (const part of parts) { + const p = asPart(part); + if ((p?.type !== "text" && p?.type !== "input_text") || typeof p.text !== "string") continue; + for (const line of p.text.split(/\r?\n/)) { + const match = /^Billion original image receipt v1 sha256=([a-f0-9]{64})$/.exec(line); + if (match) hashes.add(match[1]); + } + } + if (hashes.size === 0) return false; + return hashes.has(createHash("sha256").update(Buffer.from(b64, "base64")).digest("hex")); +} + async function rescaleAnthropic(m: BiliMessage, ctx: RescaleCtx): Promise { const block = asPart(m.rawAnthropicBlock); if (!block) return undefined; @@ -300,7 +316,7 @@ async function rescaleAnthropic(m: BiliMessage, ctx: RescaleCtx): Promise { const item = asPart(m.rawResponsesItem); - if (!item || item.type !== "message" || !Array.isArray(item.content)) return undefined; - const content = item.content as unknown[]; + if (!item) return undefined; + const toolOutput = item.type === "function_call_output" || item.type === "custom_tool_call_output"; + const partsKey = toolOutput ? "output" : item.type === "message" ? "content" : undefined; + if (partsKey === undefined || !Array.isArray(item[partsKey])) return undefined; + const content = item[partsKey] as unknown[]; const changed: unknown[] = []; let first: RescaledImage | undefined; const ref = refFor(ctx.session, m.id); @@ -418,7 +437,7 @@ async function rescaleResponsesItem(m: BiliMessage, ctx: RescaleCtx): Promise, session: Session, @@ -624,7 +645,7 @@ export async function executeReadOriginalImage( // carries TEXT only — the host agent opens this file with its own tools. const dims = `${entry.width ?? "?"}x${entry.height ?? "?"}`; return { - ...toolOk(`Original image for ${ref} restored (${dims}, ${formatKb(buf.length)}KB${entry.mediaType ? `, ${entry.mediaType}` : ""}). Local copy: ${entry.path}. It is larger than the downscaled copy you saw — use it for fine detail (small text, colors, alignment).`), + ...toolOk(`Original image for ${ref} restored (${dims}, ${formatKb(buf.length)}KB${entry.mediaType ? `, ${entry.mediaType}` : ""}). Local copy: ${entry.path}. It is larger than the downscaled copy you saw — use it for fine detail (small text, colors, alignment).\nBillion original image receipt v1 sha256=${createHash("sha256").update(buf).digest("hex")}`), image: { base64: buf.toString("base64"), mediaType: entry.mediaType }, }; } diff --git a/tests/issue3400-image-rescale.test.ts b/tests/issue3400-image-rescale.test.ts index a85f67d4b7..6f5f797c23 100644 --- a/tests/issue3400-image-rescale.test.ts +++ b/tests/issue3400-image-rescale.test.ts @@ -1,4 +1,7 @@ import { test } from "node:test"; +import { createHash } from "node:crypto"; +import { responsesToCoreWithToolImages, coreToResponsesWithToolImages } from "../src/responses-tool-output.js"; +import { shedWireImages } from "../src/egress-shed.js"; import assert from "node:assert/strict"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -375,3 +378,108 @@ test("persistence invariant: remembered session view keeps ORIGINAL image bytes await fake.close(); } }); + +test("PTC original receipt: real restore survives an Anthropic ipython tool result", async () => { + const { session, ctx } = ctxFor("s-ptc-original"); + const imgPath = path.join(root, "ptc-original.png"); + const buf = Buffer.from(BIG.b64, "base64"); + writeFileSync(imgPath, buf); + session.incomingImageIndex = new Map([["m12345", [{ mediaType: "image/png", bytes: buf.length, width: 2400, height: 1800, path: imgPath }]]]); + const restored = await executeReadOriginalImage({ ref: "m12345" }, session, log); + assert.ok(restored.image); + const input: BiliMessage[] = [{ + id: "ptc-result", role: "tool", contentType: "tool-result", toolName: "ipython", text: restored.text, + rawAnthropicBlock: { type: "tool_result", tool_use_id: "ptc", content: [ + { type: "text", text: restored.text }, + { type: "image", source: { type: "base64", media_type: restored.image.mediaType, data: restored.image.base64 } }, + ] }, + }]; + const before = JSON.stringify(input); + const out = await applyImageRescalePass(input, ctx); + assert.equal(out[0], input[0], "receipt-bound original must survive PTC unchanged"); + assert.equal(JSON.stringify(input), before, "history remains byte-exact"); + assert.match(restored.text, /^Original image for m12345 restored .*\nBillion original image receipt v1 sha256=[a-f0-9]{64}$/); +}); + +const RECEIPT = `Billion original image receipt v1 sha256=${createHash("sha256").update(Buffer.from(BIG.b64, "base64")).digest("hex")}`; +const OTHER = (await makePng(2200, 1700)).b64; +type ImageTestPart = { type: string; text?: string; image_url?: string }; +const responseImage = (b64: string): ImageTestPart => ({ type: "input_image", image_url: `data:image/png;base64,${b64}` }); + +test("PTC original receipt: Responses steady and rebuilt outputs protect only matching images", async () => { + const { ctx } = ctxFor("s-ptc-responses"); + for (const type of ["function_call_output", "custom_tool_call_output"]) { + const input = [{ type, call_id: "ptc", output: [ + { type: "input_text", text: RECEIPT }, responseImage(BIG.b64), responseImage(OTHER), + ] }]; + const before = JSON.stringify(input); + const steady = await applyImageRescaleItems(input, ctx) as typeof input; + assert.ok(steady[0].output[1].image_url === responseImage(BIG.b64).image_url, "matching original survives steady egress"); + assert.ok(steady[0].output[2].image_url !== responseImage(OTHER).image_url, "unrelated PTC image still shrinks"); + const projection = responsesToCoreWithToolImages({ input }); + const generic = await applyImageRescalePass(projection.msgs, ctx); + const rebuilt = coreToResponsesWithToolImages(generic, projection.customToolCallIds) as typeof input; + assert.ok(rebuilt[0].output[1].image_url === responseImage(BIG.b64).image_url, "matching original survives generic/rebuilt egress"); + assert.ok(rebuilt[0].output[2].image_url !== responseImage(OTHER).image_url, "unrelated rebuilt image still shrinks"); + assert.equal(JSON.stringify(input), before, "history is untouched"); + } +}); + +test("PTC original receipt: byte-shed keeps matching originals but sheds unrelated images", async () => { + for (const tail of [0, 9]) { + for (const protocol of ["responses", "anthropic"] as const) { + const output = protocol === "responses" + ? [{ type: "input_text", text: RECEIPT }, responseImage(BIG.b64), responseImage(OTHER)] + : [{ type: "text", text: RECEIPT }, + { type: "image", source: { type: "base64", media_type: "image/png", data: BIG.b64 } }, + { type: "image", source: { type: "base64", media_type: "image/png", data: OTHER } }]; + const body = protocol === "responses" + ? { input: [{ type: "function_call_output", call_id: "ptc", output }, ...Array.from({ length: tail }, () => ({ type: "message", role: "user", content: "padding" }))] } + : { messages: [{ role: "user", content: [{ type: "tool_result", tool_use_id: "ptc", content: output }] }, ...Array.from({ length: tail }, () => ({ role: "user", content: "padding" }))] }; + const result = await shedWireImages(JSON.stringify(body), protocol, 1000); + assert.ok(result.body.includes(BIG.b64), "original survives both archiving and forced downscale"); + assert.ok(!result.body.includes(OTHER), "unrelated image remains eligible for byte-shed"); + assert.ok(tail ? result.stats.archived > 0 : result.stats.downscaled > 0); + } + } +}); + +test("PTC original receipt: missing, malformed, mismatched and unrelated receipts never exempt", async () => { + const { ctx } = ctxFor("s-ptc-negative"); + const cases = ["", RECEIPT.replace("v1", "v2"), RECEIPT.slice(0, -1), `${RECEIPT} extra`, + `Billion original image receipt v1 sha256=${"0".repeat(64)}`]; + for (const receipt of cases) { + const input = [{ type: "function_call_output", call_id: "ptc", output: [ + { type: "input_text", text: receipt }, responseImage(BIG.b64), + ] }]; + const steady = await applyImageRescaleItems(input, ctx) as typeof input; + assert.ok(steady[0].output[1].image_url !== responseImage(BIG.b64).image_url, "invalid receipt follows ordinary shrink"); + const generic = await applyImageRescalePass(responsesToCoreWithToolImages({ input }).msgs, ctx); + const rebuilt = coreToResponsesWithToolImages(generic) as typeof input; + assert.ok(rebuilt[0].output[1].image_url !== responseImage(BIG.b64).image_url); + const anth: BiliMessage = { id: "negative", role: "tool", contentType: "tool-result", toolName: "run_code", text: receipt, + rawAnthropicBlock: { type: "tool_result", tool_use_id: "ptc", content: [ + { type: "text", text: receipt }, { type: "image", source: { type: "base64", media_type: "image/png", data: BIG.b64 } }, + ] } }; + const out = await applyImageRescalePass([anth], ctx); + assert.ok(out[0] !== anth, "Anthropic invalid receipt follows ordinary shrink"); + } + const otherOutput = [ + { type: "function_call_output", call_id: "receipt-only", output: [{ type: "input_text", text: RECEIPT }] }, + { type: "function_call_output", call_id: "image-only", output: [responseImage(BIG.b64)] }, + { type: "message", role: "user", content: [{ type: "input_text", text: RECEIPT }, responseImage(BIG.b64)] }, + ]; + const before = JSON.stringify(otherOutput); + const out = await applyImageRescaleItems(otherOutput, ctx); + assert.ok(!JSON.stringify(out).includes(BIG.b64), "receipt elsewhere or in a user message grants no exemption"); + assert.equal(JSON.stringify(otherOutput), before); + clearRescaleEncodeCache(); + const freshCtx = ctxFor("s-fresh-process").ctx; + const valid = [{ type: "function_call_output", call_id: "ptc", output: [{ type: "input_text", text: RECEIPT }, responseImage(BIG.b64)] }]; + assert.deepEqual(await applyImageRescaleItems(valid, freshCtx), valid, "receipt is stateless after cache/session reset"); + const ordinary = [{ type: "function_call_output", call_id: "ordinary", output: [responseImage(BIG.b64)] }]; + const normal = await applyImageRescaleItems(ordinary, ctxFor("s-other-session").ctx) as typeof ordinary; + assert.ok(normal[0].output[0].image_url !== responseImage(BIG.b64).image_url, "exemption never leaks into encode cache"); + const shed = await shedWireImages(JSON.stringify({ input: otherOutput }), "responses", 1000); + assert.ok(!shed.body.includes(BIG.b64), "byte-shed does not search unrelated outputs or user text for receipts"); +}); From 9534fb5cefe475c4e08722ae0c5b6fa57f91691c Mon Sep 17 00:00:00 2001 From: awork Date: Wed, 21 Oct 2026 14:33:09 +0800 Subject: [PATCH 2/2] fix(google): apply original-image receipt exemption to the Google wire (#4070) Reviewer-added completion of PR #4102: the receipt predicate landed for Anthropic and Responses but not Google, so a restored original attached by a PTC host beside its entry-level inlineData part would still be rescaled/shed on that wire. Same mechanism, same fix shape: - rescaleGoogle skips inlineData whose decoded bytes match a receipt line in the same content-entry parts; the egress-shed google collector applies the same skip before archiving/downscaling. - matchesOriginalImageReceipt now accepts any string .text part (the Google text part is untyped {text}); typed-wire behavior is unchanged. - Images nested inside functionResponse.parts were already invisible to both passes and remain so (unchanged). Tests: positive+negative rescale rows plus both shed rungs added to tests/issue3400-image-rescale.test.ts. Local: typecheck clean, targeted files green. --- src/egress-shed.ts | 1 + src/image-rescale.ts | 10 +++++-- tests/issue3400-image-rescale.test.ts | 40 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/egress-shed.ts b/src/egress-shed.ts index 2551dae464..a7b44b8721 100644 --- a/src/egress-shed.ts +++ b/src/egress-shed.ts @@ -160,6 +160,7 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] { if (!isObj(part)) return; const inline = part.inlineData; if (!isObj(inline) || typeof inline.data !== "string" || inline.data.length === 0) return; + if (matchesOriginalImageReceipt(inline.data, parts)) return; const mediaType = typeof inline.mimeType === "string" ? inline.mimeType : "image/png"; occ.push({ entryIndex, diff --git a/src/image-rescale.ts b/src/image-rescale.ts index 3b64eaa81a..f7c6a55c1d 100644 --- a/src/image-rescale.ts +++ b/src/image-rescale.ts @@ -275,12 +275,16 @@ function asPart(v: unknown): Part | undefined { // --------------------------------------------------------------------------- /** Non-security, caller-fabricable convention: only exact decoded image bytes in - * this tool output match. Never search other messages or cache an exemption. */ + * this tool output match. The receipt line may ride any text part of the same + * output — typed on the Anthropic/Responses wires ({type:"text"} / + * {type:"input_text"}), untyped on the Google wire ({text}) — so the scan is + * shape-agnostic over string .text fields. Never search other messages or + * cache an exemption. */ export function matchesOriginalImageReceipt(b64: string, parts: unknown[]): boolean { const hashes = new Set(); for (const part of parts) { const p = asPart(part); - if ((p?.type !== "text" && p?.type !== "input_text") || typeof p.text !== "string") continue; + if (typeof p?.text !== "string") continue; for (const line of p.text.split(/\r?\n/)) { const match = /^Billion original image receipt v1 sha256=([a-f0-9]{64})$/.exec(line); if (match) hashes.add(match[1]); @@ -402,7 +406,7 @@ async function rescaleGoogle(m: BiliMessage, ctx: RescaleCtx): Promise { + const { ctx } = ctxFor("s-ptc-google"); + const pos: BiliMessage = { + id: "g-pos", role: "tool", contentType: "tool-result", toolName: "ipython", text: "", + rawGoogleParts: [ + { text: RECEIPT }, + { inlineData: { mimeType: "image/png", data: BIG.b64 } }, + { inlineData: { mimeType: "image/png", data: OTHER } }, + ], + }; + const out = await applyImageRescalePass([pos], ctx); + assert.equal(out.length, 1); + assert.notEqual(out[0], pos, "unrelated image must still produce a rescaled copy"); + const gp = out[0].rawGoogleParts as Array<{ inlineData?: { data?: string } }>; + assert.equal(gp[1].inlineData?.data, BIG.b64, "receipt-bound original survives Google egress byte-identical"); + assert.notEqual(gp[2].inlineData?.data, OTHER, "unrelated PTC image still shrinks"); + const neg: BiliMessage = { + id: "g-neg", role: "tool", contentType: "tool-result", toolName: "ipython", text: "", + rawGoogleParts: [ + { text: RECEIPT.replace("v1", "v2") }, + { inlineData: { mimeType: "image/png", data: BIG.b64 } }, + ], + }; + const nout = await applyImageRescalePass([neg], ctx); + assert.notEqual(nout[0], neg, "invalid receipt follows ordinary shrink on the Google wire"); + for (const tail of [0, 9]) { + const entry = { role: "user", parts: [ + { functionResponse: { name: "ipython", response: {} } }, + { text: RECEIPT }, + { inlineData: { mimeType: "image/png", data: BIG.b64 } }, + { inlineData: { mimeType: "image/png", data: OTHER } }, + ] }; + const body = { contents: [entry, ...Array.from({ length: tail }, () => ({ role: "user", parts: [{ text: "padding" }] }))] }; + const result = await shedWireImages(JSON.stringify(body), "google", 1000); + assert.ok(result.body.includes(BIG.b64), "original survives both archiving and forced downscale"); + assert.ok(!result.body.includes(OTHER), "unrelated image remains eligible for byte-shed"); + assert.ok(tail ? result.stats.archived > 0 : result.stats.downscaled > 0); + } +});