diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 2f4e0d8499..52a0b57e61 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -28,6 +28,7 @@ import type { CoreMessage, CompressionBlock } from "../../kernel/src/types.js"; import type { BlockView } from "../session.js"; import { refToIndex } from "../../kernel/src/refs.js"; +import { splitMediaContent } from "./media.js"; import { sha256Hex, type BlockRow, type CcrEntryRow, type IngestSessionInput, type MessageRow, type RefRow, type SessionRow } from "./store.js"; /** Field order of mergeState() output (src/persist.ts) — the loader must @@ -94,31 +95,6 @@ export const ingestSerializations = { count: 0 }; /** One messages-log entry with its ABSOLUTE seq already assigned (intern * assigns first-seen order; the delta planner continues after the trusted * prefix). */ -/** - * #3762 media CAS split: pull the top-level imageBase64 payload out of the - * serialized message. The stored content keeps the key POSITION but carries - * a {"$media": sha256(bytes)} reference; the bytes themselves live in the - * content CAS, refcounted via message_media rows. Key order is preserved by - * JSON.parse, so hydrate-and-restringify is byte-identical — every existing - * reader (getMessages hydrates before returning) sees the verbatim original. - * No media (or empty) → passthrough with the plain content hash. - */ -function splitMediaContent(content: string): { content: string; hash: string; media: { hash: string; bytes: string }[] } { - if (!content.includes("\"imageBase64\"")) return { content, hash: sha256Hex(content), media: [] }; - let parsed: { imageBase64?: unknown }; - try { - parsed = JSON.parse(content) as { imageBase64?: unknown }; - } catch { - return { content, hash: sha256Hex(content), media: [] }; - } - const bytes = parsed.imageBase64; - if (typeof bytes !== "string" || bytes.length === 0) return { content, hash: sha256Hex(content), media: [] }; - const mediaHash = sha256Hex(bytes); - parsed.imageBase64 = { $media: mediaHash }; - const split = JSON.stringify(parsed); - return { content: split, hash: sha256Hex(split), media: [{ hash: mediaHash, bytes }] }; -} - interface LogEntry { seq: number; msgId: string; diff --git a/src/storage/media.ts b/src/storage/media.ts new file mode 100644 index 0000000000..3e38df4f76 --- /dev/null +++ b/src/storage/media.ts @@ -0,0 +1,321 @@ +/** + * #3762/#4069 media CAS split — pure helpers shared by the ingest planners + * (split side) and the store hydrators (read side). + * + * Every wire codec stores each image TWICE on one core message: the + * `imageBase64`/`imageMediaType` sidecar PLUS a verbatim raw carrier — openai + * `rawOpenaiContent`/`rawOpenaiContentParts` parts, anthropic + * `rawAnthropicBlock.source.data`, google `rawGoogleParts[].inlineData.data`, + * responses `rawResponsesItem.content[].input_image` urls (kernel/src/wire/*.ts + * — the rebuild prefers the raw carrier over the sidecar, so both must + * exist). Storing both copies inline defeats #3762's dedup: on a real 11 GB + * corpus the split yielded 1.89× instead of 33.8× (#4069), and master's + * codecs still produce both fields, so fresh installs inherit the + * duplication. Both copies therefore ride the content CAS: + * + * - the sidecar slot becomes {"$media": sha256(b64)} (key position kept); + * - every raw-wire data URL becomes `data:;base64,$media:` — + * the reference is embedded in the url STRING itself (`$` is outside the + * base64 alphabet, so a placeholder can never collide with real payload + * bytes). No sibling key, no new top-level slot: the stored JSON keeps + * the exact key set of the original, so hydrate-and-restringify stays + * byte-identical to the pre-split serialization and every reader that + * parses urls as strings keeps working unhydrated-shape-free; + * - bare base64 values (anthropic source.data, google inlineData.data) + * become `$media:` in place — the stored value IS the reference. + * + * Splitting is gated twice: + * - SIZE: below MIN_SPLIT_MEDIA_CHARS the 77-char reference plus the CAS + * row and its mapping row cost more than the payload; + * - STRICT CANONICAL FORM: reconstruction `data:;base64,` + * must be byte-exact. The kernel's loose parseDataUrl also accepts a + * missing ;base64 marker, extra params and case-folded schemes — those + * round-trip to DIFFERENT strings through the sidecar fallback, so such + * urls stay inline untouched (the raw copy may be the only carrier). + */ + +import { createHash } from "node:crypto"; + +/** #4069: base64 payloads shorter than this (in characters) stay inline. + * Splitting adds a constant overhead (~77-char ref + CAS row + mapping + * row ≈ 200 B) independent of payload size; measured corpora put >99% of + * image bytes in blobs far above this floor. */ +export const MIN_SPLIT_MEDIA_CHARS = 65_536; + +// One-line local copy rather than importing from ./store.js — store.ts +// imports the hydrate helpers from HERE, and a crypto one-liner does not +// justify breaking the cycle the other way. +const sha256Hex = (s: string): string => createHash("sha256").update(s).digest("hex"); + +/** Strict canonical data URL. No /i (the kernel's parseDataUrl is loose on + * purpose; the split gate must not be): reconstruction is byte-exact by + * construction for anything this matches. */ +const CANONICAL_DATA_URL = /^data:([^;,]+);base64,(.+)$/; + +/** A split raw-wire url: `data:;base64,$media:`. */ +const SPLIT_MEDIA_URL = /^data:([^;,]+);base64,\$media:([0-9a-f]{64})$/; + +/** A bare base64-slot reference (anthropic source.data / google + * inlineData.data): the stored value IS `$media:`. */ +const BARE_MEDIA_REF = /^\$media:([0-9a-f]{64})$/; + +/** Fast-path markers: content lacking ALL of these cannot carry splittable + * media (split side) or hydrated refs (read side) — skip the JSON.parse. + * The quoted "rawOpenaiContent" does NOT match the Parts variant, hence + * both openai entries are listed separately. */ +const SPLIT_MARKERS = [ + "\"imageBase64\"", "\"rawOpenaiContent\"", "\"rawOpenaiContentParts\"", + "\"rawAnthropicBlock\"", "\"rawGoogleParts\"", "\"rawResponsesItem\"", +]; +const HYDRATE_MARKERS = ["\"$media\"", "$media:"]; + +interface MediaRef { hash: string; bytes: string } + +interface RawOpenaiImagePart { image_url: { url: unknown } } + +/** True when v is a split-form slot value: an object whose only job is to + * carry a CAS hash under $media. */ +function isMediaRef(v: unknown): v is { $media: string } { + return v !== null && typeof v === "object" && !Array.isArray(v) && typeof (v as { $media?: unknown }).$media === "string"; +} + +/** Collects the raw OpenAI wire image parts of a parsed message — the + * singular carrier plus the plural array (legacy multi-image shapes). + * Returned objects are references INTO the parsed message: mutating their + * image_url.url lands on the message itself. Developer-role messages reuse + * rawOpenaiContent for the whole original message object — filtered out by + * the type check below. */ +function collectRawOpenaiImageParts(msg: Record): RawOpenaiImagePart[] { + const out: RawOpenaiImagePart[] = []; + const push = (c: unknown): void => { + if (c === null || typeof c !== "object") return; + const p = c as { type?: unknown; image_url?: unknown }; + if (p.type !== "image_url" || p.image_url === null || typeof p.image_url !== "object") return; + out.push(p as unknown as RawOpenaiImagePart); + }; + if (msg.rawOpenaiContent !== undefined) push(msg.rawOpenaiContent); + if (Array.isArray(msg.rawOpenaiContentParts)) for (const c of msg.rawOpenaiContentParts) push(c); + return out; +} + +/** Collects the base64 source slots of a raw Anthropic block: the top-level + * source (image AND document/unknown-type blocks share the shape) plus image + * blocks NESTED in tool_result content arrays. The predicate mirrors the + * codec's own mediaSidecar condition (kernel/src/wire/anthropic.ts), so the + * split covers exactly what the codec would lift into the sidecar. Returned + * objects alias the parsed message — mutating .data lands on it. */ +function collectRawAnthropicDataSlots(msg: Record): { data: string }[] { + const out: { data: string }[] = []; + const consider = (s: unknown): void => { + if (s === null || typeof s !== "object" || Array.isArray(s)) return; + const src = s as { type?: unknown; media_type?: unknown; data?: unknown }; + if (src.type !== "base64" || typeof src.media_type !== "string" || typeof src.data !== "string") return; + out.push(src as { data: string }); + }; + const b = msg.rawAnthropicBlock; + if (b === null || typeof b !== "object") return out; + consider((b as { source?: unknown }).source); + const content = (b as { content?: unknown }).content; + if (Array.isArray(content)) for (const c of content) consider((c as { source?: unknown })?.source); + return out; +} + +/** Collects raw Google inlineData slots (fileData is a URI, never base64 — + * not media here). Returned objects alias the parsed message. */ +function collectRawGoogleInlineDataSlots(msg: Record): { data: string }[] { + const out: { data: string }[] = []; + const parts = msg.rawGoogleParts; + if (!Array.isArray(parts)) return out; + for (const p of parts) { + if (p === null || typeof p !== "object") continue; + const inline = (p as { inlineData?: unknown }).inlineData; + if (inline === null || typeof inline !== "object" || Array.isArray(inline)) continue; + const d = inline as { mimeType?: unknown; data?: unknown }; + if (typeof d.mimeType !== "string" || typeof d.data !== "string") continue; + out.push(d as { data: string }); + } + return out; +} + +/** Collects input_image url holders inside a raw Responses item (the item + * carries ALL its images; the sidecar only the first). Returned objects + * alias the parsed message. */ +function collectRawResponseImageUrls(msg: Record): { image_url: string }[] { + const out: { image_url: string }[] = []; + const item = msg.rawResponsesItem; + if (item === null || typeof item !== "object") return out; + const content = (item as { content?: unknown }).content; + if (!Array.isArray(content)) return out; + for (const p of content) { + if (p === null || typeof p !== "object") continue; + const part = p as { type?: unknown; image_url?: unknown }; + if (part.type !== "input_image" || typeof part.image_url !== "string") continue; + out.push(part as { image_url: string }); + } + return out; +} + +/** #3762/#4069: pull every qualifying media payload out of the serialized + * message. See the module header for the two stored reference shapes and + * the size/canonical gates. Same-hash payloads are interned into ONE CAS + * entry (the dual-field case: sidecar and raw url share a row). No + * qualifying media → passthrough with the plain content hash, original + * bytes untouched. */ +export function splitMediaContent(content: string): { content: string; hash: string; media: MediaRef[] } { + if (!SPLIT_MARKERS.some((k) => content.includes(k))) return { content, hash: sha256Hex(content), media: [] }; + let parsed: Record; + try { + parsed = JSON.parse(content) as Record; + } catch { + return { content, hash: sha256Hex(content), media: [] }; + } + const byHash = new Map(); + const intern = (b64: string): string => { + const h = sha256Hex(b64); + if (!byHash.has(h)) byHash.set(h, b64); + return h; + }; + // Slot 1: the top-level imageBase64 sidecar (#3762). + const b64 = parsed.imageBase64; + if (typeof b64 === "string" && b64.length >= MIN_SPLIT_MEDIA_CHARS) { + parsed.imageBase64 = { $media: intern(b64) }; + } + // Slot 2: the raw OpenAI wire data URLs carrying the same payloads (#4069). + for (const part of collectRawOpenaiImageParts(parsed)) { + const url = part.image_url.url; + if (typeof url !== "string") continue; + const m = CANONICAL_DATA_URL.exec(url); + if (!m || m[2].length < MIN_SPLIT_MEDIA_CHARS) continue; + part.image_url.url = `data:${m[1]};base64,$media:${intern(m[2])}`; + } + // Slots 3+4: anthropic/google raw base64 payloads — bare `$media:` marker, + // no url wrapper to preserve (the stored value IS the base64). + for (const slot of [...collectRawAnthropicDataSlots(parsed), ...collectRawGoogleInlineDataSlots(parsed)]) { + if (slot.data.length < MIN_SPLIT_MEDIA_CHARS) continue; + slot.data = `$media:${intern(slot.data)}`; + } + // Slot 5: responses item input_image data URLs (embedded-url form like openai; + // every image of the item, not just the sidecarred first one). + for (const part of collectRawResponseImageUrls(parsed)) { + const m = CANONICAL_DATA_URL.exec(part.image_url); + if (!m || m[2].length < MIN_SPLIT_MEDIA_CHARS) continue; + part.image_url = `data:${m[1]};base64,$media:${intern(m[2])}`; + } + if (byHash.size === 0) return { content, hash: sha256Hex(content), media: [] }; + const split = JSON.stringify(parsed); + return { content: split, hash: sha256Hex(split), media: [...byHash.entries()].map(([hash, bytes]) => ({ hash, bytes })) }; +} + +/** Read-side inverse of splitMediaContent: swap every media reference back + * to its CAS bytes — the {"$media"} sidecar slot AND the embedded + * `$media:` refs inside raw-wire urls. Returns the ORIGINAL string + * unchanged when nothing hydrated (missing CAS rows fall back to the split + * form rather than throwing — visible, not silent, same discipline as + * #3762). Key order is preserved by parse/stringify and no key is ever + * added or removed, so the result is byte-identical to the pre-split + * serialization. */ +export function hydrateMediaRefs(content: string, getBytes: (hash: string) => string | null | undefined): string { + if (!HYDRATE_MARKERS.some((k) => content.includes(k))) return content; + let parsed: Record; + try { + parsed = JSON.parse(content) as Record; + } catch { + return content; + } + let changed = false; + const slot = parsed.imageBase64; + if (isMediaRef(slot)) { + const bytes = getBytes(slot.$media); + if (typeof bytes === "string" && bytes.length > 0) { + parsed.imageBase64 = bytes; + changed = true; + } + } + for (const part of collectRawOpenaiImageParts(parsed)) { + const m = typeof part.image_url.url === "string" ? SPLIT_MEDIA_URL.exec(part.image_url.url) : null; + if (!m) continue; + const bytes = getBytes(m[2]); + if (typeof bytes === "string" && bytes.length > 0) { + part.image_url.url = `data:${m[1]};base64,${bytes}`; + changed = true; + } + } + for (const slot of [...collectRawAnthropicDataSlots(parsed), ...collectRawGoogleInlineDataSlots(parsed)]) { + const m = BARE_MEDIA_REF.exec(slot.data); + if (!m) continue; + const bytes = getBytes(m[1]); + if (typeof bytes === "string" && bytes.length > 0) { + slot.data = bytes; + changed = true; + } + } + for (const part of collectRawResponseImageUrls(parsed)) { + const m = SPLIT_MEDIA_URL.exec(part.image_url); + if (!m) continue; + const bytes = getBytes(m[2]); + if (typeof bytes === "string" && bytes.length > 0) { + part.image_url = `data:${m[1]};base64,${bytes}`; + changed = true; + } + } + if (!changed) return content; + try { + return JSON.stringify(parsed); + } catch { + return content; + } +} + +/** Async twin of hydrateMediaRefs for the mysql backend (the sqlite store is + * synchronous). Same contract: original string back when nothing hydrated. */ +export async function hydrateMediaRefsAsync(content: string, getBytes: (hash: string) => Promise): Promise { + if (!HYDRATE_MARKERS.some((k) => content.includes(k))) return content; + let parsed: Record; + try { + parsed = JSON.parse(content) as Record; + } catch { + return content; + } + let changed = false; + const slot = parsed.imageBase64; + if (isMediaRef(slot)) { + const bytes = await getBytes(slot.$media); + if (typeof bytes === "string" && bytes.length > 0) { + parsed.imageBase64 = bytes; + changed = true; + } + } + for (const part of collectRawOpenaiImageParts(parsed)) { + const m = typeof part.image_url.url === "string" ? SPLIT_MEDIA_URL.exec(part.image_url.url) : null; + if (!m) continue; + const bytes = await getBytes(m[2]); + if (typeof bytes === "string" && bytes.length > 0) { + part.image_url.url = `data:${m[1]};base64,${bytes}`; + changed = true; + } + } + for (const slot of [...collectRawAnthropicDataSlots(parsed), ...collectRawGoogleInlineDataSlots(parsed)]) { + const m = BARE_MEDIA_REF.exec(slot.data); + if (!m) continue; + const bytes = await getBytes(m[1]); + if (typeof bytes === "string" && bytes.length > 0) { + slot.data = bytes; + changed = true; + } + } + for (const part of collectRawResponseImageUrls(parsed)) { + const m = SPLIT_MEDIA_URL.exec(part.image_url); + if (!m) continue; + const bytes = await getBytes(m[2]); + if (typeof bytes === "string" && bytes.length > 0) { + part.image_url = `data:${m[1]};base64,${bytes}`; + changed = true; + } + } + if (!changed) return content; + try { + return JSON.stringify(parsed); + } catch { + return content; + } +} diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 79de335a6c..ee20bbecd5 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -32,6 +32,7 @@ import { mkdirSync, writeFileSync, appendFileSync, readFileSync, existsSync } fr import { join } from "node:path"; import { log as loggerLog } from "../logger.js"; import { sha256Hex } from "./store.js"; +import { hydrateMediaRefsAsync } from "./media.js"; import type { MessageContentStore } from "acp-kernel"; import type { StorageBackend } from "./backend.js"; import type { @@ -825,22 +826,9 @@ export class MysqlBackend implements StorageBackend { return out; } + /** #3762/#4069: async twin of the sqlite hydrate (contract in media.ts). */ private async hydrateMediaAsync(content: string): Promise { - if (!content.includes('"$media"')) return content; - try { - const parsed = JSON.parse(content) as { imageBase64?: unknown }; - const slot = parsed.imageBase64; - if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { - const bytes = await this.getContent(this.pool, (slot as { $media: string }).$media); - if (typeof bytes === "string" && bytes.length > 0) { - parsed.imageBase64 = bytes; - return JSON.stringify(parsed); - } - } - return content; - } catch { - return content; - } + return hydrateMediaRefsAsync(content, (h) => this.getContent(this.pool, h)); } async getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { diff --git a/src/storage/schema.ts b/src/storage/schema.ts index a354971ca2..7fe53cfb06 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -249,10 +249,15 @@ CREATE TABLE IF NOT EXISTS kv_store ( PRIMARY KEY (namespace, key) ) WITHOUT ROWID; --- #3762 media CAS: one row per (message, media blob) reference. The message --- content stores {"$media": hash} in the imageBase64 slot; the bytes live in --- the content CAS, refcounted by these mapping rows. Swept with the session --- (and on idempotent re-receipt undo, same discipline as messages.content). +-- #3762/#4069 media CAS: one row per (message, media blob) reference. Message +-- content stores media refs in three shapes: {"$media": hash} in the +-- imageBase64 slot; data:;base64,$media: embedded in raw wire +-- data URLs (openai parts, responses input_image); bare $media: as a +-- base64 value (anthropic source.data, google inlineData). The codecs carry +-- most payloads in BOTH the sidecar and their raw carrier, so both ride the +-- CAS; the bytes live in the content CAS, refcounted by these mapping rows. +-- Swept with the session (and on idempotent re-receipt undo, same discipline +-- as messages.content). CREATE TABLE IF NOT EXISTS message_media ( session_id TEXT NOT NULL, seq INTEGER NOT NULL, diff --git a/src/storage/store.ts b/src/storage/store.ts index 140e042a46..de1fbf2070 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -22,6 +22,7 @@ import { dirname, join } from "node:path"; import { log as loggerLog } from "../logger.js"; import { openSqliteDatabase, type SqliteDatabase, type SqliteEngineName, type SqliteEngineSource } from "./driver.js"; import { migrateSchema, storagePragmas, WAL_JOURNAL_SIZE_LIMIT } from "./schema.js"; +import { hydrateMediaRefs } from "./media.js"; import type { MessageContentStore } from "acp-kernel"; /** #3250: run a PASSIVE wal_checkpoint every N successful ingests. The @@ -1793,27 +1794,9 @@ export class UnifiedStore { })); } - /** #3762: swap {"$media": hash} references back to the CAS bytes. Only - * touches content that carries the marker; key order is preserved by - * parse/stringify so the result is byte-identical to the pre-split - * serialization. A missing CAS row (corruption / manual surgery) falls - * back to the split form rather than throwing — visible, not silent. */ + /** #3762/#4069: swap media references back to the CAS bytes (see media.ts). */ private hydrateMedia(content: string): string { - if (!content.includes("\"$media\"")) return content; - try { - const parsed = JSON.parse(content) as { imageBase64?: unknown }; - const slot = parsed.imageBase64; - if (slot !== null && typeof slot === "object" && typeof (slot as { $media?: unknown }).$media === "string") { - const bytes = this.getContent((slot as { $media: string }).$media); - if (typeof bytes === "string" && bytes.length > 0) { - parsed.imageBase64 = bytes; - return JSON.stringify(parsed); - } - } - return content; - } catch { - return content; - } + return hydrateMediaRefs(content, (h) => this.getContent(h)); } /** #3762 test/ops helper: refcount of the content row holding exactly diff --git a/tests/media-cas.test.ts b/tests/media-cas.test.ts index dea8486d45..45be79d98c 100644 --- a/tests/media-cas.test.ts +++ b/tests/media-cas.test.ts @@ -3,18 +3,22 @@ import assert from "node:assert/strict"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import { planIngest, loadLegacyView, type LegacySessionLike } from "../src/storage/ingest.ts"; -import { UnifiedStore } from "../src/storage/store.ts"; +import { planIngest, planIngestDelta, loadLegacyView, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore, sha256Hex } from "../src/storage/store.ts"; +import { splitMediaContent } from "../src/storage/media.ts"; /** - * #3901 / #3762 — media-level content addressing. + * #3901 / #3762 / #4069 — media-level content addressing. * * The same image posted into two conversations (or twice into one) is stored - * once: ingest splits media sidecars (imageBase64) out of the message JSON - * into content-CAS media rows; the stored message content keeps the key - * POSITION and carries a media reference value; getMessages() hydrates the - * bytes back so every reader sees the ORIGINAL verbatim JSON (byte-identical - * round-trip). Deleting the last referencing session releases the media row. + * once: ingest splits EVERY copy of the payload out of the message JSON into + * content-CAS media rows — both the top-level imageBase64 sidecar AND the raw + * OpenAI wire data URL that carries the SAME bytes (#4069: the codec writes + * the payload twice, so splitting only the sidecar halves at best); the + * stored message keeps key positions and carries media references; + * getMessages() hydrates the bytes back so every reader sees the ORIGINAL + * verbatim JSON (byte-identical round-trip). Deleting the last referencing + * session releases the media row. */ process.env.BILI_PERSIST_ZSTD = "0"; @@ -24,7 +28,9 @@ process.env.BILI_CONFIG_FILE = path.join( "no-config.json", ); -const IMG_B64 = "a".repeat(4096); // stand-in for base64 image bytes +// Above MIN_SPLIT_MEDIA_CHARS so these payloads actually split (#4069 floor). +const BIG_B64 = "a".repeat(70_000); // stand-in for base64 image bytes +const SMALL_B64 = "a".repeat(4096); // below the floor: stays inline interface CoreMsg { id: string; role: string; contentType: string; text: string; imageBase64?: string; imageMediaType?: string } @@ -55,14 +61,14 @@ test("media CAS: same image in two sessions stored once; round-trip verbatim; re const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-")); try { const store = UnifiedStore.open(path.join(dir, "index.db")); - const mA = mkMsg("m-a", "look at this in session A", IMG_B64); - const mB = mkMsg("m-b", "same screenshot in session B", IMG_B64); + const mA = mkMsg("m-a", "look at this in session A", BIG_B64); + const mB = mkMsg("m-b", "same screenshot in session B", BIG_B64); const mC = mkMsg("m-c", "plain text message"); store.ingestSession(planIngest(mkSession("sess-A", [mA, mC]))); store.ingestSession(planIngest(mkSession("sess-B", [mB]))); // (1) Media bytes exist exactly once, referenced by both messages. - assert.equal(store.mediaRefCount(IMG_B64), 2, "one media row, two references"); + assert.equal(store.mediaRefCount(BIG_B64), 2, "one media row, two references"); // (2) STORED (pre-hydration) message content does NOT inline the // base64 — query the table directly; getMessages always hydrates. @@ -74,19 +80,19 @@ test("media CAS: same image in two sessions stored once; round-trip verbatim; re const body = (store as unknown as { db: { prepare(q: string): { get(...a: unknown[]): { bytes: Uint8Array } | undefined } } }) .db.prepare("SELECT bytes FROM content WHERE hash = ?").get(row.content_hash); assert.ok(body, `content row for ${sid}`); - assert.ok(!Buffer.from(body.bytes).toString("utf8").includes(IMG_B64), `stored content must not inline media bytes (${sid})`); + assert.ok(!Buffer.from(body.bytes).toString("utf8").includes(BIG_B64), `stored content must not inline media bytes (${sid})`); } } // (3) …and getMessages hydrates back to the ORIGINAL verbatim JSON. const rowsA = store.getMessages("sess-A"); - assert.equal(JSON.parse(rowsA[0].content).imageBase64, IMG_B64); + assert.equal(JSON.parse(rowsA[0].content).imageBase64, BIG_B64); assert.equal(rowsA[0].content, JSON.stringify(mA), "byte-identical round-trip"); // (4) Deleting one session keeps the shared media row; deleting both releases it. store.deleteSession("sess-A"); - assert.equal(store.mediaRefCount(IMG_B64), 1, "shared media survives one deletion"); + assert.equal(store.mediaRefCount(BIG_B64), 1, "shared media survives one deletion"); store.deleteSession("sess-B"); - assert.equal(store.mediaRefCount(IMG_B64), 0, "last reference gone → media row released"); + assert.equal(store.mediaRefCount(BIG_B64), 0, "last reference gone → media row released"); // (5) Legacy rows (inline bytes, pre-split) still read back verbatim. store.close(); @@ -99,7 +105,7 @@ test("media CAS: viewFor round-trip through loadLegacyView carries hydrated mess const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-2-")); try { const store = UnifiedStore.open(path.join(dir, "index.db")); - const m = mkMsg("m-img", "image turn", IMG_B64); + const m = mkMsg("m-img", "image turn", BIG_B64); store.ingestSession(planIngest(mkSession("sess-img", [m]))); const view = loadLegacyView({ session: store.getSessionRow("sess-img")!, @@ -110,7 +116,7 @@ test("media CAS: viewFor round-trip through loadLegacyView carries hydrated mess }) as LegacySessionLike & { lastMessages?: CoreMsg[] }; const tail = (view.lastMessages ?? []) as CoreMsg[]; assert.equal(tail.length, 1); - assert.equal(tail[0].imageBase64, IMG_B64, "view path hydrates media bytes"); + assert.equal(tail[0].imageBase64, BIG_B64, "view path hydrates media bytes"); store.close(); } finally { try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } @@ -121,17 +127,17 @@ test("media CAS: stale-row sweep releases media refs and mappings (full re-inges const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-3-")); try { const store = UnifiedStore.open(path.join(dir, "index.db")); - const mImg = mkMsg("m-img", "image turn", IMG_B64); + const mImg = mkMsg("m-img", "image turn", BIG_B64); const mTxt = mkMsg("m-txt", "plain turn"); store.ingestSession(planIngest(mkSession("sess-sweep", [mImg, mTxt])), undefined, { completeLog: true }); - assert.equal(store.mediaRefCount(IMG_B64), 1, "one media reference before re-ingest"); + assert.equal(store.mediaRefCount(BIG_B64), 1, "one media reference before re-ingest"); // The session's raw tail no longer contains the image message: the // full path sweeps the stale row — its media mapping and CAS refcount // must go with it, or the bytes leak forever (#3762 review, #3901). store.ingestSession(planIngest(mkSession("sess-sweep", [mTxt])), undefined, { completeLog: true }); - assert.equal(store.mediaRefCount(IMG_B64), 0, "swept message released its media ref"); + assert.equal(store.mediaRefCount(BIG_B64), 0, "swept message released its media ref"); const mappings = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number }[] } } }) .db.prepare("SELECT COUNT(*) AS n FROM message_media WHERE session_id = ?").all("sess-sweep"); assert.equal(mappings[0].n, 0, "swept media mappings deleted"); @@ -143,3 +149,356 @@ test("media CAS: stale-row sweep releases media refs and mappings (full re-inges try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } } }); + +// ---- #4069: the OpenAI codec stores the SAME payload twice per image — the +// top-level sidecar AND the raw wire part (rawOpenaiContent / plural array). +// Splitting only the sidecar caps the win at ~2× on real data; both copies +// must ride the CAS for the full dedup. + +const RAW_IMG = (b64: string): { type: string; image_url: { url: string } } => ({ + type: "image_url", + image_url: { url: `data:image/png;base64,${b64}` }, +}); + +interface DualMsg extends CoreMsg { rawOpenaiContent?: unknown; rawOpenaiContentParts?: unknown[] } + +const mkDualMsg = (id: string, text: string, b64: string): DualMsg => ({ + id, + role: "user", + contentType: "image", + text, + rawOpenaiContent: RAW_IMG(b64), + imageBase64: b64, + imageMediaType: "image/png", +}); + +type DbShape = { + prepare(q: string): { + all(...a: unknown[]): { content_hash: string }[]; + get(...a: unknown[]): { bytes: Uint8Array } | undefined; + }; +}; + +const storedBodies = (store: UnifiedStore, sid: string): string[] => { + const db = (store as unknown as { db: DbShape }).db; + return db.prepare("SELECT content_hash FROM messages WHERE session_id = ?").all(sid) + .map((r) => Buffer.from(db.prepare("SELECT bytes FROM content WHERE hash = ?").get(r.content_hash)!.bytes).toString("utf8")); +}; + +test("#4069: dual-field message splits BOTH copies into ONE CAS row; round-trip byte-identical", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-4-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const m = mkDualMsg("m-dual", "screenshot turn", BIG_B64); + store.ingestSession(planIngest(mkSession("sess-dual", [m]))); + + assert.equal(store.mediaRefCount(BIG_B64), 1, "sidecar and raw url share one CAS row"); + for (const body of storedBodies(store, "sess-dual")) { + assert.ok(!body.includes(BIG_B64), "stored content must not inline the payload in either shape"); + assert.ok(body.includes("$media:"), "raw url carries the embedded ref"); + } + const rows = store.getMessages("sess-dual"); + assert.equal(rows[0].content, JSON.stringify(m), "byte-identical round-trip"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: same dual-field image across two sessions → one shared CAS row, refcount 2", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-5-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + store.ingestSession(planIngest(mkSession("sess-a", [mkDualMsg("m-a", "A", BIG_B64)]))); + store.ingestSession(planIngest(mkSession("sess-b", [mkDualMsg("m-b", "B", BIG_B64)]))); + assert.equal(store.mediaRefCount(BIG_B64), 2, "one shared CAS row, two references"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: plural carrier (multi-image legacy shape) splits every part", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-6-")); + const PA = "b".repeat(70_000); + const PB = "c".repeat(70_000); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const m: DualMsg = { + id: "m-multi", + role: "user", + contentType: "image", + text: "two pics", + rawOpenaiContentParts: [RAW_IMG(PA), { type: "text", text: "between" }, RAW_IMG(PB)], + }; + store.ingestSession(planIngest(mkSession("sess-multi", [m]))); + assert.equal(store.mediaRefCount(PA), 1); + assert.equal(store.mediaRefCount(PB), 1); + for (const body of storedBodies(store, "sess-multi")) { + assert.ok(!body.includes(PA) && !body.includes(PB), "no part stays inline"); + } + const rows = store.getMessages("sess-multi"); + assert.equal(rows[0].content, JSON.stringify(m), "byte-identical round-trip"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: non-canonical data URLs stay inline (reconstruction not provably exact)", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-7-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + // Extra params: the kernel's loose parseDataUrl rejects these entirely + // (no sidecar produced) — nothing to dedup against, url stays verbatim. + const mParams: DualMsg = { + id: "m-params", role: "user", contentType: "image", text: "svg", + rawOpenaiContent: { type: "image_url", image_url: { url: `data:image/svg+xml;charset=utf8;base64,${BIG_B64}` } }, + }; + // Missing ;base64 / uppercase scheme: the loose parser ACCEPTS them + // (the sidecar exists) but the strict gate refuses — the sidecar + // splits, the raw url stays inline untouched. + const mLoose: DualMsg = { + id: "m-loose", role: "user", contentType: "image", text: "loose", + rawOpenaiContent: { type: "image_url", image_url: { url: `data:image/png,${BIG_B64}` } }, + imageBase64: BIG_B64, imageMediaType: "image/png", + }; + const mUpper: DualMsg = { + id: "m-upper", role: "user", contentType: "image", text: "upper", + rawOpenaiContent: { type: "image_url", image_url: { url: `DATA:image/png;base64,${BIG_B64}` } }, + imageBase64: BIG_B64, imageMediaType: "image/png", + }; + store.ingestSession(planIngest(mkSession("sess-noncanon", [mParams, mLoose, mUpper]))); + + const rows = store.getMessages("sess-noncanon"); + assert.equal(rows[0].content, JSON.stringify(mParams), "extra-param url stays verbatim"); + assert.equal(rows[1].content, JSON.stringify(mLoose), "missing-;base64 url stays verbatim"); + assert.equal(rows[2].content, JSON.stringify(mUpper), "uppercase-scheme url stays verbatim"); + assert.equal(store.mediaRefCount(BIG_B64), 2, "sidecars of the loose messages still ride the CAS"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: remote URL image source passes through untouched", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-8-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const m: DualMsg = { + id: "m-remote", role: "user", contentType: "image", text: "linked", + rawOpenaiContent: { type: "image_url", image_url: { url: "https://example.com/pic.png" } }, + }; + store.ingestSession(planIngest(mkSession("sess-remote", [m]))); + assert.equal(storedBodies(store, "sess-remote")[0], JSON.stringify(m), "no refs → plain passthrough serialization"); + assert.equal(store.getMessages("sess-remote")[0].content, JSON.stringify(m)); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: sub-floor payloads stay fully inline (ref overhead would exceed the payload)", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-9-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const m = mkDualMsg("m-small", "tiny icon", SMALL_B64); + store.ingestSession(planIngest(mkSession("sess-small", [m]))); + assert.equal(store.mediaRefCount(SMALL_B64), 0, "nothing interned below the floor"); + assert.equal(storedBodies(store, "sess-small")[0], JSON.stringify(m), "stored verbatim, unsplit"); + assert.equal(store.getMessages("sess-small")[0].content, JSON.stringify(m)); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: stale-row sweep releases dual-slot media refs", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-10-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const mImg = mkDualMsg("m-img", "image turn", BIG_B64); + const mTxt = mkMsg("m-txt", "plain turn"); + store.ingestSession(planIngest(mkSession("sess-sweep2", [mImg, mTxt])), undefined, { completeLog: true }); + assert.equal(store.mediaRefCount(BIG_B64), 1, "first ingest keeps the ref"); + store.ingestSession(planIngest(mkSession("sess-sweep2", [mTxt])), undefined, { completeLog: true }); + assert.equal(store.mediaRefCount(BIG_B64), 0, "swept dual-slot message released its ref"); + const rows = (store as unknown as { db: { prepare(q: string): { all(...a: unknown[]): { n: number }[] } } }) + .db.prepare("SELECT COUNT(*) AS n FROM message_media WHERE session_id = ?").all("sess-sweep2"); + assert.equal(rows[0].n, 0, "no orphan mapping rows"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: delta planner verifies an image boundary through the split hash", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-11-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const mImg = mkDualMsg("m-img", "boundary image", BIG_B64); + const mTxt = mkMsg("m-txt", "after the image"); + store.ingestSession(planIngest(mkSession("sess-delta", [mImg]))); + + const storedHash = (store as unknown as { db: DbShape }).db + .prepare("SELECT content_hash FROM messages WHERE msg_id = ?").all("m-img")[0]!.content_hash; + const session2 = mkSession("sess-delta", [mImg, mTxt]); + // Positive: the stored boundary hash covers the SPLIT serialization; + // the probe re-splits identically and accepts the delta. + const ok = planIngestDelta(session2, 1, [1], { msgId: "m-img", contentHash: storedHash }, true); + assert.ok(ok, "image boundary verifies through the split hash"); + // Negative: the plain (unsplit) hash must NOT verify an image row. + const bad = planIngestDelta(session2, 1, [1], { msgId: "m-img", contentHash: sha256Hex(JSON.stringify(mImg)) }, true); + assert.equal(bad, null, "unsplit hash fails the boundary probe"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +// ---- #4069 follow-up: the OTHER codecs' raw carriers carry the same payloads +// a second time too (anthropic source.data, google inlineData, responses +// item input_image urls) — they must ride the CAS as well, or a +// multi-protocol install gets a half-split store. + +const ANTH_B64 = "e".repeat(70_000); +const GOOG_A = "f".repeat(70_000); +const GOOG_B = "g".repeat(70_000); +const RESP_A = "h".repeat(70_000); +const RESP_B = "i".repeat(70_000); + +test("#4069: anthropic rawAnthropicBlock source.data splits; dual-field dedups to one CAS row; round-trip byte-exact", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + // Shape as produced by kernel/src/wire/anthropic.ts case "image": + // rawAnthropicBlock carries the WHOLE block incl. source.data, and the + // sidecar lifts the same bytes. + const m = { + id: "m-anth", role: "user", contentType: "image", text: "[image]", imagePart: true, + rawAnthropicBlock: { type: "image", source: { type: "base64", media_type: "image/png", data: ANTH_B64 } }, + imageBase64: ANTH_B64, imageMediaType: "image/png", + }; + store.ingestSession(planIngest(mkSession("sess-anth", [m]))); + assert.equal(store.mediaRefCount(ANTH_B64), 1, "sidecar + raw slot share ONE CAS row"); + const bodies = storedBodies(store, "sess-anth"); + assert.ok(bodies.some((b) => !b.includes(ANTH_B64) && b.includes("$media:"), "stored content must not inline the base64")); + assert.equal(store.getMessages("sess-anth")[0].content, JSON.stringify(m), "byte-identical round-trip"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: anthropic tool_result nested image blocks split; redacted_thinking .data NEVER touched", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + // tool_result whose content array nests an image block (kernel + // anthropic.ts :175 stores such blocks verbatim under rawAnthropicBlock). + const nested = { + id: "m-nested", role: "tool", contentType: "tool_result", text: "", + rawAnthropicBlock: { + type: "tool_result", tool_use_id: "tu_1", + content: [{ type: "image", source: { type: "base64", media_type: "image/png", data: ANTH_B64 } }], + }, + }; + // KDD#10 negative control: thinking-family bytes are signature-verified + // upstream and must stay verbatim ON THE WIRE — the storage split does + // not touch them (hydrate restores anyway, but the gate must not claim + // them either). Above floor on purpose: size must not unlock it. + const REDACTED_B64 = "j".repeat(70_000); + const redacted = { + id: "m-redacted", role: "assistant", contentType: "message", text: "", + rawAnthropicBlock: { type: "redacted_thinking", data: REDACTED_B64 }, + }; + store.ingestSession(planIngest(mkSession("sess-anth2", [nested, redacted]))); + assert.equal(store.mediaRefCount(ANTH_B64), 1, "nested tool_result image split"); + assert.equal(store.mediaRefCount(REDACTED_B64), 0, "redacted_thinking never split"); + const rows = store.getMessages("sess-anth2"); + assert.equal(rows[0].content, JSON.stringify(nested), "nested round-trip byte-exact"); + assert.equal(rows[1].content, JSON.stringify(redacted), "redacted round-trip verbatim"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: google rawGoogleParts inlineData splits per part; fileData URI untouched; round-trip byte-exact", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + // Two inlineData parts (plural carrier) + a fileData part (URI — not + // base64 media, must pass through untouched). + const m = { + id: "m-goog", role: "user", contentType: "image", text: "[images]", imagePart: true, + rawGoogleParts: [ + { inlineData: { mimeType: "image/png", data: GOOG_A } }, + { text: "between" }, + { inlineData: { mimeType: "image/jpeg", data: GOOG_B } }, + { fileData: { mimeType: "application/pdf", fileUri: "https://files.example/x.pdf" } }, + ], + imageBase64: GOOG_A, imageMediaType: "image/png", + }; + store.ingestSession(planIngest(mkSession("sess-goog", [m]))); + assert.equal(store.mediaRefCount(GOOG_A), 1, "first image: sidecar + inlineData share one row"); + assert.equal(store.mediaRefCount(GOOG_B), 1, "second image split from its own part"); + const bodies = storedBodies(store, "sess-goog"); + assert.ok(bodies.some((b) => !b.includes(GOOG_A) && !b.includes(GOOG_B) && b.includes("$media:")), "stored content must not inline either payload"); + assert.equal(store.getMessages("sess-goog")[0].content, JSON.stringify(m), "byte-identical round-trip"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: responses rawResponsesItem input_image urls split (n×m amplification); remote url untouched; round-trip byte-exact", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + // The item carries BOTH images; the sidecar lifts only the FIRST + // (kernel responses.ts) — the second rides the raw item alone, which is + // exactly the n×m amplification the issue calls out. + const m = { + id: "m-resp", role: "user", contentType: "image", text: "[images]", imagePart: true, + rawResponsesItem: { + type: "message", role: "user", + content: [ + { type: "input_image", image_url: `data:image/png;base64,${RESP_A}` }, + { type: "input_text", text: "and" }, + { type: "input_image", image_url: `data:image/jpeg;base64,${RESP_B}` }, + { type: "input_image", image_url: "https://img.example/r.png" }, + ], + }, + imageBase64: RESP_A, imageMediaType: "image/png", + }; + store.ingestSession(planIngest(mkSession("sess-resp", [m]))); + assert.equal(store.mediaRefCount(RESP_A), 1, "first image: sidecar + item url share one row"); + assert.equal(store.mediaRefCount(RESP_B), 1, "second image split from the item alone"); + const bodies = storedBodies(store, "sess-resp"); + assert.ok(bodies.some((b) => !b.includes(RESP_A) && !b.includes(RESP_B) && b.includes("$media:") && b.includes("https://img.example/r.png")), + "both data URLs out, remote URL kept verbatim"); + assert.equal(store.getMessages("sess-resp")[0].content, JSON.stringify(m), "byte-identical round-trip"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4069: sub-floor raw-carrier payloads stay fully inline (no refs at all)", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "storage-media-cas-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const m = { + id: "m-small", role: "user", contentType: "image", text: "[image]", imagePart: true, + rawAnthropicBlock: { type: "image", source: { type: "base64", media_type: "image/png", data: SMALL_B64 } }, + imageBase64: SMALL_B64, imageMediaType: "image/png", + }; + store.ingestSession(planIngest(mkSession("sess-small2", [m]))); + assert.equal(store.mediaRefCount(SMALL_B64), 0, "below floor: nothing split"); + assert.equal(store.getMessages("sess-small2")[0].content, JSON.stringify(m), "verbatim passthrough"); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +});