From 18fe6365056672eaf87b7e0930529d9afac3450c Mon Sep 17 00:00:00 2001 From: Afloat Date: Sat, 14 Nov 2026 18:36:04 +0800 Subject: [PATCH] fix: recognize escaped JSON image markers --- src/image-restore.ts | 23 ++++-- tests/image-marker-escapes.test.ts | 111 +++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 tests/image-marker-escapes.test.ts diff --git a/src/image-restore.ts b/src/image-restore.ts index 4282aeb106..5c59fe4d40 100644 --- a/src/image-restore.ts +++ b/src/image-restore.ts @@ -203,6 +203,20 @@ function coreMsgsOf(result: unknown): CoreMessage[] | undefined { return undefined; } +function jsonAsciiMarker(text: string, foldCase = false): string { + return [...text].map((character) => { + const variants = foldCase ? [...new Set([character.toLowerCase(), character.toUpperCase()])] : [character]; + return `(?:${variants.flatMap((value) => { + const hex = value.charCodeAt(0).toString(16).padStart(4, "0").replace(/[a-f]/g, (digit) => `[${digit}${digit.toUpperCase()}]`); + return [value, `\\\\u${hex}`]; + }).join("|")})`; + }).join(""); +} + +const IMAGE_DATA_URL_MARKER = new RegExp(`"${jsonAsciiMarker("data:", true)}[^;,"]+(?:${jsonAsciiMarker(";base64", true)})?${jsonAsciiMarker(",")}`); +const ANTHROPIC_BASE64_MARKER = new RegExp(`"${jsonAsciiMarker("type")}"\\s*:\\s*"${jsonAsciiMarker("base64")}"`); +const GOOGLE_INLINE_DATA_MARKER = new RegExp(`"${jsonAsciiMarker("inlineData")}"\\s*:`); + // #2824: raw-text pre-check — can this body carry indexable image bytes at // ALL? buildIncomingImageIndex pays a full wire→core conversion of the whole // history (twice per request with the post-prepare refresh) only to discover @@ -215,15 +229,16 @@ function coreMsgsOf(result: unknown): CoreMessage[] | undefined { // - anthropic source.type "base64": raw base64 payload, NO data: prefix; // - google inlineData parts (top-level AND nested in functionResponse): // raw base64 .data field, NO data: prefix. -// Keyed markers tolerate JSON whitespace ("type": "base64" vs "type":"base64"). +// JSON string characters may use Unicode escapes (RFC 8259 section 7); match the ASCII +// markers directly without decoding or copying the complete request. export function bodyMightCarryImages(bodyText: string, protocol: WireProtocol): boolean { switch (protocol) { case "anthropic": - return /"data:[^;,"]+(?:;base64)?,/i.test(bodyText) || /"type"\s*:\s*"base64"/.test(bodyText); + return IMAGE_DATA_URL_MARKER.test(bodyText) || ANTHROPIC_BASE64_MARKER.test(bodyText); case "google": - return /"inlineData"\s*:/.test(bodyText); + return GOOGLE_INLINE_DATA_MARKER.test(bodyText); default: - return /"data:[^;,"]+(?:;base64)?,/i.test(bodyText); + return IMAGE_DATA_URL_MARKER.test(bodyText); } } diff --git a/tests/image-marker-escapes.test.ts b/tests/image-marker-escapes.test.ts new file mode 100644 index 0000000000..a3c53d3252 --- /dev/null +++ b/tests/image-marker-escapes.test.ts @@ -0,0 +1,111 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInitialState, defaultConfig } from "acp-kernel"; +import { anthropicToCore, googleToCore, openaiToCore, responsesToCore } from "acp-kernel/wire"; +import { bodyMightCarryImages, buildIncomingImageIndex } from "../src/image-restore.ts"; +import { startServer } from "../src/server.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions } from "../src/session.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import type { WireProtocol } from "../src/util.ts"; + +const root = mkdtempSync(join(tmpdir(), "bili-image-markers-")); +process.env.XDG_STATE_HOME = join(root, "state"); +process.env.XDG_DATA_HOME = join(root, "data"); +process.env.XDG_CACHE_HOME = join(root, "cache"); +process.env.XDG_CONFIG_HOME = join(root, "config"); +test.after(() => rmSync(root, { recursive: true, force: true })); + +const PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; +const dataUrl = `data:image/png;base64,${PNG}`; +const openai = { model: "gpt-fixture", stream: false, messages: [{ role: "user", content: [{ type: "text", text: "Describe this image." }, { type: "image_url", image_url: { url: dataUrl } }] }] }; +const fixtures: Array<{ protocol: WireProtocol; body: unknown; marker: string; escaped: string }> = [ + { protocol: "openai", body: openai, marker: "data:", escaped: "\\u0064ata\\u003a" }, + { protocol: "responses", body: { model: "gpt-fixture", input: [{ type: "message", role: "user", content: [{ type: "input_image", image_url: dataUrl }] }] }, marker: "data:", escaped: "\\u0064ata\\u003A" }, + { protocol: "anthropic", body: { model: "claude-fixture", max_tokens: 10, messages: [{ role: "user", content: [{ type: "image", source: { type: "base64", media_type: "image/png", data: PNG } }] }] }, marker: '"base64"', escaped: '"base\\u00364"' }, + { protocol: "google", body: { contents: [{ role: "user", parts: [{ inlineData: { mimeType: "image/png", data: PNG } }] }] }, marker: "inlineData", escaped: "\\u0069nlineD\\u0061ta" }, +]; +const converters = { anthropic: anthropicToCore, openai: openaiToCore, responses: responsesToCore, google: googleToCore }; + +function escapeStrings(body: unknown): string { + return JSON.stringify(body).replace(/"(?:\\.|[^"\\])*"/g, (token) => { + const value = JSON.parse(token) as string; + return `"${[...value].map((character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`).join("")}"`; + }); +} + +for (const fixture of fixtures) { + for (const escaped of [false, true]) { + test(`${fixture.protocol} image indexing survives ${escaped ? "escaped" : "literal"} JSON markers`, () => { + const plain = JSON.stringify(fixture.body); + const wire = escaped ? plain.replace(fixture.marker, fixture.escaped) : plain; + const parsed = JSON.parse(wire); + assert.deepEqual(parsed, fixture.body); + const converted = (converters[fixture.protocol] as (body: unknown) => unknown)(parsed); + const messages = (Array.isArray(converted) ? converted : (converted as { msgs: unknown[] }).msgs) as Array<{ id: string }>; + const state = createInitialState(); + messages.forEach((message, index) => { state.messageRefs.byRaw[message.id] = `m${String(index + 1).padStart(5, "0")}`; }); + const expected = buildIncomingImageIndex(parsed, fixture.protocol, state, `fixture-${fixture.protocol}`); + assert.equal(expected.size, 1, "the protocol converter exposes actual recoverable image bytes"); + for (const images of expected.values()) assert.deepEqual(readFileSync(images[0]!.path), Buffer.from(PNG, "base64")); + const guarded = bodyMightCarryImages(wire, fixture.protocol) + ? buildIncomingImageIndex(parsed, fixture.protocol, state, `fixture-${fixture.protocol}`) : new Map(); + assert.deepEqual(guarded, expected); + }); + } + test(`${fixture.protocol} escaped text remains an image-free fast path`, () => { + const text = '{"text":"\\u0068ello \\u4e16\\u754c and \\\\u0061 literal"}'; + assert.equal(bodyMightCarryImages(text, fixture.protocol), false); + }); +} + +test("fully escaped JSON keys, media markers, and separators remain detectable", () => { + for (const fixture of fixtures) { + const wire = escapeStrings(fixture.body); + assert.deepEqual(JSON.parse(wire), fixture.body); + assert.equal(bodyMightCarryImages(wire, fixture.protocol), true, fixture.protocol); + } +}); + +test("encoded data URL tokens keep the existing case-insensitive recognition", () => { + const body = { image_url: `DATA:IMAGE/PNG;BASE64,${PNG}` }; + assert.equal(bodyMightCarryImages(JSON.stringify(body), "openai"), true); + assert.equal(bodyMightCarryImages(escapeStrings(body), "openai"), true); +}); + +test("real HTTP preparation archives the same pixels with escaped and literal data URL markers", async () => { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const upstream = http.createServer(async (request, response) => { + for await (const _chunk of request) { /* consume the actual forwarded request */ } + response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ id: "fixture", object: "chat.completion", model: "gpt-fixture", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 } })); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const origin = `http://127.0.0.1:${(upstream.address() as { port: number }).port}`; + const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: origin, routes: { [origin]: { models: { "gpt-fixture": { context: 100_000 } } } }, modelContextLimit: 100_000, kernelConfig: defaultConfig(100_000), compress: { injectTool: true, injectNudge: true }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", log: false, debug: false, passthrough: false, passthroughSource: null, autoUpdate: false, autoRestartOnUpdate: false, updateTag: "latest", advisoryCheck: false, releaseNotesCheck: false, compat: { roles: {} }, streamErrorShape: "protocol", mitm: { enabled: false, domains: [] } } as ProxyOptions); + if (!proxy.listening) await once(proxy, "listening"); + try { + for (const escaped of [false, true]) { + const label = `image-markers-${escaped}`; + const plain = JSON.stringify(openai); + const wire = escaped ? plain.replace("data:", "\\u0064ata\\u003a") : plain; + const response = await fetch(`http://127.0.0.1:${(proxy.address() as { port: number }).port}/bili/${origin}/v1/chat/completions`, { method: "POST", headers: { "content-type": "application/json", "x-acp-session": label }, body: wire, signal: AbortSignal.timeout(10_000) }); + assert.equal(response.status, 200, await response.text()); + const session = listSessions().find((candidate) => candidate.meta.label === label); + assert.ok(session, "the real preparation path tracked the conversation"); + assert.equal(session.incomingImageIndex?.size, 1, label); + const image = [...session.incomingImageIndex!.values()][0]![0]!; + assert.deepEqual(readFileSync(image.path), Buffer.from(PNG, "base64")); + } + } finally { + proxy.closeAllConnections(); upstream.closeAllConnections(); + await Promise.all([new Promise(resolve => proxy.close(() => resolve())), new Promise(resolve => upstream.close(() => resolve()))]); + } +});