Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 24 additions & 16 deletions src/egress-shed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { currentFetchTransport, isWsLaneTransport } from "./fetch-transport.js";
import { limitsWsOutboundBytes } from "./knobs.js";
import { decodeImageDims } from "./image-tokens.js";
import { READ_ORIGINAL_IMAGE_TOOL_NAME } from "./compress-tool.js";
import { refForItemBytes, shedDownscaleImage } from "./image-rescale.js";
import { matchesOriginalImageReceipt, refForItemBytes, shedDownscaleImage } from "./image-rescale.js";
import { ResponsesWsUpstreamError } from "./responses-ws.js";

type Part = Record<string, unknown>;
Expand Down Expand Up @@ -132,6 +132,24 @@ function isExemptOutput(protocol: WireProtocol, item: Part, names: Map<string, s

function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] {
const occ: Occurrence[] = [];
const addAnthropicImages = (parts: unknown[], entryIndex: number, toolOutput: boolean): void => {
parts.forEach((part, pos) => {
if (!isObj(part) || part.type !== "image") return;
const src = part.source;
if (!isObj(src) || src.type !== "base64" || typeof src.data !== "string" || src.data.length === 0) return;
if (toolOutput && matchesOriginalImageReceipt(src.data, parts)) return;
const mediaType = typeof src.media_type === "string" ? src.media_type : "image/png";
occ.push({
entryIndex,
b64: src.data,
mediaType,
protectedTail: false,
archived: false,
applyArchive: (note) => { parts[pos] = { type: "text", text: note }; },
applyShrink: (b64) => { src.data = b64; },
});
});
};
if (protocol === "google") {
const contents = body.contents;
if (!Array.isArray(contents)) return occ;
Expand All @@ -142,6 +160,7 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] {
if (!isObj(part)) return;
const inline = part.inlineData;
if (!isObj(inline) || typeof inline.data !== "string" || inline.data.length === 0) return;
if (matchesOriginalImageReceipt(inline.data, parts)) return;
const mediaType = typeof inline.mimeType === "string" ? inline.mimeType : "image/png";
occ.push({
entryIndex,
Expand Down Expand Up @@ -169,7 +188,7 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] {
parts.forEach((part, pos) => {
if (!isObj(part) || part.type !== "input_image" || typeof part.image_url !== "string") return;
const d = parseDataUrl(part.image_url);
if (!d) return;
if (!d || (partsKey === "output" && matchesOriginalImageReceipt(d.base64, parts))) return;
occ.push({
entryIndex,
b64: d.base64,
Expand All @@ -193,6 +212,7 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] {
if (protocol === "anthropic" && Array.isArray(m.content) && m.content.some((b) => isObj(b) && b.type === "tool_result" && isExemptOutput(protocol, b, names))) return;
if (!Array.isArray(m.content)) return;
const content = m.content as unknown[];
if (protocol === "anthropic") addAnthropicImages(content, entryIndex, false);
content.forEach((part, pos) => {
const p = isObj(part) ? part : undefined;
if (!p) return;
Expand All @@ -212,20 +232,8 @@ function collectOccurrences(protocol: WireProtocol, body: Part): Occurrence[] {
applyArchive: (note) => { content[pos] = { type: "text", text: note }; },
applyShrink: (b64) => { p.image_url = typeof iu === "string" ? dataUrl(d.mediaType, b64) : { ...(iu ?? {}), url: dataUrl(d.mediaType, b64) }; },
});
} else {
if (p.type !== "image") return;
const src = p.source;
if (!isObj(src) || src.type !== "base64" || typeof src.data !== "string" || src.data.length === 0) return;
const mediaType = typeof src.media_type === "string" ? src.media_type : "image/png";
occ.push({
entryIndex,
b64: src.data,
mediaType,
protectedTail: false,
archived: false,
applyArchive: (note) => { content[pos] = { type: "text", text: note }; },
applyShrink: (b64) => { src.data = b64; },
});
} else if (p.type === "tool_result" && Array.isArray(p.content)) {
addAnthropicImages(p.content, entryIndex, true);
}
});
});
Expand Down
45 changes: 35 additions & 10 deletions src/image-rescale.ts
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,26 @@ function asPart(v: unknown): Part | undefined {
// never mutated (session state and persistence keep the originals).
// ---------------------------------------------------------------------------

/** Non-security, caller-fabricable convention: only exact decoded image bytes in
* this tool output match. The receipt line may ride any text part of the same
* output — typed on the Anthropic/Responses wires ({type:"text"} /
* {type:"input_text"}), untyped on the Google wire ({text}) — so the scan is
* shape-agnostic over string .text fields. Never search other messages or
* cache an exemption. */
export function matchesOriginalImageReceipt(b64: string, parts: unknown[]): boolean {
const hashes = new Set<string>();
for (const part of parts) {
const p = asPart(part);
if (typeof p?.text !== "string") continue;
for (const line of p.text.split(/\r?\n/)) {
const match = /^Billion original image receipt v1 sha256=([a-f0-9]{64})$/.exec(line);
if (match) hashes.add(match[1]);
}
}
if (hashes.size === 0) return false;
return hashes.has(createHash("sha256").update(Buffer.from(b64, "base64")).digest("hex"));
}

async function rescaleAnthropic(m: BiliMessage, ctx: RescaleCtx): Promise<BiliMessage[] | undefined> {
const block = asPart(m.rawAnthropicBlock);
if (!block) return undefined;
Expand All @@ -300,7 +320,7 @@ async function rescaleAnthropic(m: BiliMessage, ctx: RescaleCtx): Promise<BiliMe
const p = asPart(part);
if (p?.type === "image") {
const src = asPart(p.source);
if (src?.type === "base64" && typeof src.data === "string") {
if (src?.type === "base64" && typeof src.data === "string" && !matchesOriginalImageReceipt(src.data, content)) {
const r = await rescaleImage(typeof src.media_type === "string" ? src.media_type : "image/png", src.data, ctx.maxDim);
if (r) {
any = true;
Expand Down Expand Up @@ -386,7 +406,7 @@ async function rescaleGoogle(m: BiliMessage, ctx: RescaleCtx): Promise<BiliMessa
for (const part of parts) {
const p = asPart(part);
const inline = p ? asPart(p.inlineData) : undefined;
if (inline && typeof inline.data === "string") {
if (inline && typeof inline.data === "string" && !matchesOriginalImageReceipt(inline.data, parts)) {
const r = await rescaleImage(typeof inline.mimeType === "string" ? inline.mimeType : "image/png", inline.data, ctx.maxDim);
if (r) {
any = true;
Expand All @@ -409,16 +429,19 @@ async function rescaleGoogle(m: BiliMessage, ctx: RescaleCtx): Promise<BiliMessa

async function rescaleResponsesItem(m: BiliMessage, ctx: RescaleCtx): Promise<BiliMessage[] | undefined> {
const item = asPart(m.rawResponsesItem);
if (!item || item.type !== "message" || !Array.isArray(item.content)) return undefined;
const content = item.content as unknown[];
if (!item) return undefined;
const toolOutput = item.type === "function_call_output" || item.type === "custom_tool_call_output";
const partsKey = toolOutput ? "output" : item.type === "message" ? "content" : undefined;
if (partsKey === undefined || !Array.isArray(item[partsKey])) return undefined;
const content = item[partsKey] as unknown[];
const changed: unknown[] = [];
let first: RescaledImage | undefined;
const ref = refFor(ctx.session, m.id);
for (const part of content) {
const p = asPart(part);
if (p?.type === "input_image" && typeof p.image_url === "string" && p.image_url.startsWith("data:")) {
const d = parseDataUrl(p.image_url);
if (d) {
if (d && !(toolOutput && matchesOriginalImageReceipt(d.base64, content))) {
const r = await rescaleImage(d.mediaType, d.base64, ctx.maxDim);
if (r) {
if (!first) first = r;
Expand All @@ -436,7 +459,8 @@ async function rescaleResponsesItem(m: BiliMessage, ctx: RescaleCtx): Promise<Bi
changed.push(part);
}
if (!first) return undefined;
const copy: BiliMessage = { ...m, rawResponsesItem: { ...item, content: changed } };
const copy: BiliMessage = { ...m, rawResponsesItem: { ...item, [partsKey]: changed } };
if (toolOutput) return [{ ...copy, text: `${m.text ?? ""}\n\n${noteFor(ctx, ref, first)}` }];
const noteMsg: BiliMessage = { id: `acp_imgnote_${m.id}`, role: "user", contentType: "text", text: noteFor(ctx, ref, first) };
return [copy, noteMsg];
}
Expand Down Expand Up @@ -555,7 +579,7 @@ export async function applyImageRescaleItems(items: unknown[], ctx: RescaleCtx):
const p = asPart(part);
if (p?.type === "input_image" && typeof p.image_url === "string" && p.image_url.startsWith("data:")) {
const d = parseDataUrl(p.image_url);
if (d) {
if (d && !(partsKey === "output" && matchesOriginalImageReceipt(d.base64, parts))) {
const r = await rescaleImage(d.mediaType, d.base64, ctx.maxDim);
if (r) {
any = true;
Expand Down Expand Up @@ -588,8 +612,9 @@ const REF_PATTERN = /^m\d{5,}$/;
/** Execute acp_read_original_image: fetch the ORIGINAL bytes for a message ref
* from the local session storage (incoming-image index + spill dir) and
* return them as an image attachment — the tool result itself stays TEXT.
* The caller attaches the bytes to an ephemeral carrier message whose
* toolName marks it structurally exempt from further rescaling. */
* Direct callers use the structural toolName exemption. PTC hosts must forward
* the receipt text beside the actual image in the SAME tool output. The digest
* is a caller-fabricable convention, not authenticated restore provenance. */
export async function executeReadOriginalImage(
args: Record<string, unknown>,
session: Session,
Expand Down Expand Up @@ -624,7 +649,7 @@ export async function executeReadOriginalImage(
// carries TEXT only — the host agent opens this file with its own tools.
const dims = `${entry.width ?? "?"}x${entry.height ?? "?"}`;
return {
...toolOk(`Original image for ${ref} restored (${dims}, ${formatKb(buf.length)}KB${entry.mediaType ? `, ${entry.mediaType}` : ""}). Local copy: ${entry.path}. It is larger than the downscaled copy you saw — use it for fine detail (small text, colors, alignment).`),
...toolOk(`Original image for ${ref} restored (${dims}, ${formatKb(buf.length)}KB${entry.mediaType ? `, ${entry.mediaType}` : ""}). Local copy: ${entry.path}. It is larger than the downscaled copy you saw — use it for fine detail (small text, colors, alignment).\nBillion original image receipt v1 sha256=${createHash("sha256").update(buf).digest("hex")}`),
image: { base64: buf.toString("base64"), mediaType: entry.mediaType },
};
}
Loading
Loading