diff --git a/scripts/bench-message-content-read.ts b/scripts/bench-message-content-read.ts new file mode 100644 index 0000000000..28a0942025 --- /dev/null +++ b/scripts/bench-message-content-read.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { renderSessionJsonlFromTables } from "../src/export.ts"; +import { SqliteBackend } from "../src/storage/backend.ts"; +import { UnifiedStore, type IngestSessionInput } from "../src/storage/store.ts"; + +const count = Number(process.argv[2] ?? 10000); +assert.ok(Number.isSafeInteger(count) && count > 0); +const dir = mkdtempSync(join(tmpdir(), "message-content-read-")); +const store = UnifiedStore.open(join(dir, "index.db"), "node:sqlite"); +try { + const input: IngestSessionInput = { + session: { + sessionId: "benchmark", createdAt: 1, updatedAt: 2, nextBlockId: 1, nextRunId: 1, + metaJson: "{}", statsJson: "{}", metadataJson: "{}", kstateJson: "{}", ccrJson: null, + tailJson: JSON.stringify({ tailSeqs: Array.from({ length: count }, (_, seq) => seq) }), + }, + messages: Array.from({ length: count }, (_, seq) => ({ + seq, msgId: `message-${seq}`, ref: `m${seq + 1}`, refNum: seq + 1, role: "user", kind: "text", ts: 1000 + seq, + content: JSON.stringify({ id: `message-${seq}`, role: "user", contentType: "text", text: `${seq}: ${"stored original 日本語 😀 ".repeat(20)}` }), + })), + blocks: [], deadRefs: [], refs: [], ccrEntries: [], + }; + store.ingestSession(input); + const selected = input.messages.filter((row) => row.seq % 2 === 1).map((row) => row.seq).reverse(); + const covered = Array.from({ length: Math.ceil(count / 100) }, (_, i) => [i * 100, Math.min(i * 100 + 49, count - 1)] as [number, number]); + const uncovered = input.messages.filter((row) => row.seq % 100 >= 50).length; + const backend = new SqliteBackend(store); + const workloads: { name: string; rows: number; run(): unknown | Promise }[] = [ + { name: "getMessages", rows: count, run: () => store.getMessages("benchmark") }, + { name: "getMessagesAt", rows: selected.length, run: () => store.getMessagesAt("benchmark", selected) }, + { name: "uncoveredRows", rows: uncovered, run: () => store.uncoveredRows("benchmark", covered) }, + { name: "jsonlExport", rows: count, run: () => renderSessionJsonlFromTables(backend, "benchmark", false) }, + ]; + const measurements = []; + for (const workload of workloads) { + await workload.run(); + const cpuStart = process.cpuUsage(); + const started = performance.now(); + const result = await workload.run(); + const wallMs = performance.now() - started; + const cpu = process.cpuUsage(cpuStart); + const serialized = typeof result === "string" ? result : JSON.stringify(result); + if (typeof result === "string") { + const records = result.trimEnd().split("\n").map((line) => JSON.parse(line) as { type: string; message?: unknown }); + assert.equal(records.filter((row) => row.type === "message").length, workload.rows); + assert.deepEqual(records.at(-1)!.message, JSON.parse(input.messages.at(-1)!.content)); + } else { + assert.ok(Array.isArray(result)); + assert.equal(result.length, workload.rows); + for (const row of result as { seq: number; content: string }[]) assert.equal(row.content, input.messages[row.seq]!.content); + } + measurements.push({ name: workload.name, rows: workload.rows, wallMs, cpuMs: (cpu.user + cpu.system) / 1000, + outputSha256: createHash("sha256").update(serialized).digest("hex") }); + } + process.stdout.write(`${JSON.stringify({ count, warmups: 1, measurements })}\n`); +} finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); +} diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 5ac91211f9..36ab35b657 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1563,7 +1563,8 @@ export class MysqlBackend implements StorageBackend { const chunk = [...new Set(pointers.slice(i, i + CHUNK))]; if (chunk.length === 0) continue; const [rows] = (await this.pool.query( - `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts FROM messages + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts, c.bytes AS content_bytes FROM messages + LEFT JOIN content c ON c.hash = messages.content_hash WHERE session_id = ? AND retired_at IS NULL AND seq IN (${chunk.map(() => "?").join(", ")})`, [sessionId, ...chunk], )) as unknown as Row[][]; @@ -1578,7 +1579,7 @@ export class MysqlBackend implements StorageBackend { kind: str(r.kind), contentHash, ts: nullableNum(r.ts), - content: await this.hydrateMediaAsync((await this.getContent(this.pool, contentHash)) ?? "", this.pool), + content: await this.hydrateMediaAsync(str(r.content_bytes), this.pool), }); } } @@ -1606,20 +1607,31 @@ export class MysqlBackend implements StorageBackend { [sessionId, ...(opts?.seqs ?? [])], )) as unknown as Row[][]; const out: (MessageRow & { contentHash: string; retiredAt: number | null })[] = []; - for (const r of rows) { - const contentHash = str(r.content_hash); - out.push({ - seq: num(r.seq), - msgId: str(r.msg_id), - ref: nullableStr(r.ref), - refNum: nullableNum(r.ref_num), - role: str(r.role), - kind: str(r.kind), - contentHash, - retiredAt: nullableNum(r.retired_at), - ts: nullableNum(r.ts), - content: await this.hydrateMediaAsync((await this.getContent(conn, contentHash)) ?? "", conn), - }); + // Keep body batches bounded even when the selected metadata spans a + // whole historian log. Every query stays on the caller's connection. + const CHUNK = 500; + for (let i = 0; i < rows.length; i += CHUNK) { + const chunk = rows.slice(i, i + CHUNK); + const hashes = [...new Set(chunk.map((row) => str(row.content_hash)))]; + const [contents] = (await conn.query( + `SELECT hash, bytes FROM content WHERE hash IN (${hashes.map(() => "?").join(",")})`, hashes, + )) as unknown as Row[][]; + const byHash = new Map(contents.map((row) => [str(row.hash), str(row.bytes)])); + for (const r of chunk) { + const contentHash = str(r.content_hash); + out.push({ + seq: num(r.seq), + msgId: str(r.msg_id), + ref: nullableStr(r.ref), + refNum: nullableNum(r.ref_num), + role: str(r.role), + kind: str(r.kind), + contentHash, + retiredAt: nullableNum(r.retired_at), + ts: nullableNum(r.ts), + content: await this.hydrateMediaAsync(byHash.get(contentHash) ?? "", conn), + }); + } } return out; } diff --git a/src/storage/store.ts b/src/storage/store.ts index ca4d95e0a4..1e71740687 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -79,7 +79,7 @@ export function rewriteArchiveLog(file: string, text: string): void { } } /** #4739 test instrumentation: message rows materialized by getMessages / - * getMessagesAt (each row = one CAS read + media hydration). Production + * getMessagesAt (including content decoding + media hydration). Production * code never reads this; tests assert ordinary loads stop decoding the * whole historical log. */ export const messageRowReads = { count: 0 }; @@ -3340,7 +3340,7 @@ export class UnifiedStore { + ` AND (${clauses.join(" OR ")}) ORDER BY seq`; for (const row of this.db.prepare(sql).all(...params) as Record[]) rows.push(row); } - return rows.map((r) => ({ + return Array.from(this.messageContents(rows), ([r, content]) => ({ seq: r.seq as number, msgId: r.msg_id as string, ref: (r.ref as string | null) ?? null, @@ -3349,10 +3349,28 @@ export class UnifiedStore { kind: r.kind as string, contentHash: r.content_hash as string, ts: (r.ts as number | null) ?? null, - content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), + content: this.hydrateMedia(content), })); } + /** Fetch bodies for captured message rows in bounded batches. Keeping + * the metadata query separate avoids holding a whole history's BLOBs + * alongside all of its decoded strings. Hashes preserve row identity. */ + private *messageContents(rows: readonly Record[]): Generator, string]> { + const CHUNK = 500; + for (let i = 0; i < rows.length; i += CHUNK) { + const chunk = rows.slice(i, i + CHUNK); + const hashes = [...new Set(chunk.map((row) => row.content_hash as string))]; + const contents = this.db.prepare(`SELECT hash, bytes FROM content WHERE hash IN (${hashes.map(() => "?").join(",")})`) + .all(...hashes) as { hash: string; bytes: Uint8Array }[]; + const byHash = new Map(contents.map((row) => [row.hash, row.bytes])); + for (const row of chunk) { + const bytes = byHash.get(row.content_hash as string); + yield [row, bytes === undefined ? "" : Buffer.from(bytes).toString("utf8")]; + } + } + } + getMessages(sessionId: string, opts?: { includeRetired?: boolean; seqs?: readonly number[]; fromSeqExclusive?: number }): (MessageRow & { contentHash: string; retiredAt: number | null })[] { // #4390: retired rows (rewound/undo history) stay out of the unified // replay by default — they are history, not view. Callers that need @@ -3371,7 +3389,7 @@ export class UnifiedStore { `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, retired_at, ts FROM messages WHERE session_id = ?${retiredFilter}${seqFilter}${afterClause} ORDER BY seq`, ) .all(...params) as Record[]; - return rows.map((r) => { + return Array.from(this.messageContents(rows), ([r, content]) => { messageRowReads.count++; return { seq: r.seq as number, @@ -3383,7 +3401,7 @@ export class UnifiedStore { contentHash: r.content_hash as string, retiredAt: (r.retired_at as number | null) ?? null, ts: (r.ts as number | null) ?? null, - content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), + content: this.hydrateMedia(content), }; }); } @@ -3403,7 +3421,8 @@ export class UnifiedStore { const chunk = unique.slice(i, i + CHUNK); const rows = this.db .prepare( - `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts FROM messages + `SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, ts, c.bytes AS content_bytes FROM messages + LEFT JOIN content c ON c.hash = messages.content_hash WHERE session_id = ? AND retired_at IS NULL AND seq IN (${chunk.map(() => "?").join(", ")})`, ) .all(sessionId, ...chunk) as Record[]; @@ -3418,7 +3437,7 @@ export class UnifiedStore { kind: r.kind as string, contentHash: r.content_hash as string, ts: (r.ts as number | null) ?? null, - content: this.hydrateMedia(this.getContent(r.content_hash as string) ?? ""), + content: this.hydrateMedia(r.content_bytes === null ? "" : Buffer.from(r.content_bytes as Uint8Array).toString("utf8")), }); } } diff --git a/tests/issue4739-storage-read-snapshot.test.ts b/tests/issue4739-storage-read-snapshot.test.ts index ad259e4717..25edb0149a 100644 --- a/tests/issue4739-storage-read-snapshot.test.ts +++ b/tests/issue4739-storage-read-snapshot.test.ts @@ -155,7 +155,7 @@ test("mysql snapshot pins SQL, CAS and media hydration to its read-only connecti if (sql.includes("FROM content")) { const bytes = values?.[0] === "msg" ? JSON.stringify({ imageBase64: { $media: mediaHash } }) : values?.[0] === mediaHash ? "image-bytes" : '{"text":"summary","count":1}'; - return [[{ bytes }]]; + return [[{ hash: values?.[0], bytes }]]; } return [[]]; }, diff --git a/tests/storage-message-content-join.test.ts b/tests/storage-message-content-join.test.ts new file mode 100644 index 0000000000..e060e8059d --- /dev/null +++ b/tests/storage-message-content-join.test.ts @@ -0,0 +1,210 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { openSqliteDatabase, type SqliteDatabase } from "../src/storage/driver.ts"; +import { splitMediaContent } from "../src/storage/identity.ts"; +import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; +import { UnifiedStore, sha256Hex, type IngestSessionInput } from "../src/storage/store.ts"; + +function input(count = 8, sessionId = "target"): IngestSessionInput { + return { + session: { + sessionId, createdAt: 1, updatedAt: 2, nextBlockId: 1, nextRunId: 1, + metaJson: "{}", statsJson: "{}", metadataJson: "{}", kstateJson: "{}", + tailJson: JSON.stringify({ tailSeqs: Array.from({ length: count }, (_, i) => i) }), ccrJson: null, + }, + messages: Array.from({ length: count }, (_, seq) => ({ + seq, msgId: `m${seq}`, ref: `m${String(seq + 1).padStart(5, "0")}`, refNum: seq + 1, + role: seq % 2 ? "assistant" : "user", kind: "text", ts: 1000 + seq, + content: JSON.stringify({ id: `m${seq}`, role: seq % 2 ? "assistant" : "user", contentType: "text", text: `${sessionId} ${seq} 日本語 😀\u0000` }), + })), + blocks: [], refs: [], deadRefs: [], ccrEntries: [], + }; +} + +function expected(value = input()) { + return value.messages.map((row) => ({ ...row, contentHash: sha256Hex(row.content), retiredAt: null })); +} + +function withoutRetired(rows: T[]) { + return rows.map(({ retiredAt: _retiredAt, ...row }) => row); +} + +function setup(t: TestContext, engine: "node:sqlite" | "better-sqlite3" = "node:sqlite", count = 8) { + const dir = mkdtempSync(join(tmpdir(), "bili-message-join-")); + const file = join(dir, "index.db"); + let store: UnifiedStore; + try { store = UnifiedStore.open(file, engine); } + catch (error) { + rmSync(dir, { recursive: true, force: true }); + if (engine === "better-sqlite3" && String(error).includes("[storage:better-sqlite3]")) { + t.skip(String(error)); + return; + } + throw error; + } + const writer = openSqliteDatabase(file, engine).db; + t.after(() => { writer.close(); store.close(); rmSync(dir, { recursive: true, force: true }); }); + store.ingestSession(input(count)); + return { store, writer }; +} + +function countSelects(t: TestContext, store: UnifiedStore) { + const db = (store as unknown as { db: SqliteDatabase }).db; + const prepare = db.prepare.bind(db); + let count = 0; + let maxPayloadRows = 0; + t.mock.method(db, "prepare", (sql: string) => { + if (/^SELECT\b/.test(sql.trim())) count++; + const statement = prepare(sql); + return { + get: statement.get.bind(statement), run: statement.run.bind(statement), + all(...args: unknown[]) { + const rows = statement.all(...args); + const payloadRows = rows.filter((row) => row !== null && typeof row === "object" && ("bytes" in row || "content_bytes" in row)).length; + maxPayloadRows = Math.max(maxPayloadRows, payloadRows); + return rows; + }, + }; + }); + return () => ({ count, maxPayloadRows }); +} + +for (const engine of ["node:sqlite", "better-sqlite3"] as const) { + for (const read of ["all", "pointers", "uncovered", "uncovered-full"] as const) { + test(`${engine}: ${read} message reads fetch base content in bounded statements`, (t) => { + const fixture = setup(t, engine, 1003); + if (!fixture) return; + const { store } = fixture; + const count = countSelects(t, store); + const rows = read === "all" ? store.getMessages("target") + : read === "pointers" ? store.getMessagesAt("target", Array.from({ length: 1003 }, (_, i) => 1002 - i)) + : store.uncoveredRows("target", read === "uncovered-full" ? [] : Array.from({ length: 502 }, (_, i) => [i * 2, i * 2] as [number, number])); + assert.equal(rows.length, read === "uncovered" ? 501 : 1003); + assert.equal(count().count, read === "pointers" ? 3 : read === "uncovered" ? 5 : 4); + assert.ok(count().maxPayloadRows <= 500, "body batches must not grow with the whole history"); + }); + } + + test(`${engine}: filters, ordering, timestamps and full message fields stay intact`, (t) => { + const fixture = setup(t, engine); + if (!fixture) return; + const { store, writer } = fixture; + const rows = expected(); + assert.deepEqual(store.getMessages("target"), rows); + store.ingestSession(input(8, "other")); + writer.prepare("UPDATE messages SET retired_at = 5000 WHERE session_id = ? AND seq = 2").run("target"); + const live = rows.filter((row) => row.seq !== 2); + assert.deepEqual(store.getMessages("target"), live); + assert.deepEqual(store.getMessages("target", { includeRetired: true }), rows.map((row) => row.seq === 2 ? { ...row, retiredAt: 5000 } : row)); + assert.deepEqual(store.getMessages("target", { seqs: [7, 2, 1, 7, 99], fromSeqExclusive: 6 }), [rows[1], rows[7]]); + assert.deepEqual(store.getMessages("target", { fromSeqExclusive: 4 }), rows.slice(5)); + assert.deepEqual(store.getMessagesAt("target", [7, 2, 1, 7, 99]), withoutRetired([rows[1]!, rows[7]!])); + assert.deepEqual(store.uncoveredRows("target", [[5, 7], [2, 0], [0, 1], [1, 2]]), withoutRetired(rows.slice(3, 5))); + assert.deepEqual(store.sessionViewRows("target")!.messages, withoutRetired(live)); + assert.deepEqual(store.getMessages("missing"), []); + assert.deepEqual(store.getMessages("target", { seqs: [] }), []); + assert.deepEqual(store.getMessagesAt("target", []), []); + assert.deepEqual(store.getMessages("other"), expected(input(8, "other"))); + }); + + test(`${engine}: missing, empty and invalid UTF-8 content retain the old decoding`, (t) => { + const fixture = setup(t, engine); + if (!fixture) return; + const { store, writer } = fixture; + const hashes = input().messages.map((row) => sha256Hex(row.content)); + writer.exec("PRAGMA foreign_keys=OFF"); + writer.prepare("DELETE FROM content WHERE hash = ?").run(hashes[0]); + writer.prepare("UPDATE content SET bytes = ?, byte_len = 0 WHERE hash = ?").run(Buffer.alloc(0), hashes[1]); + const invalid = Buffer.from([0x66, 0x80, 0x00, 0x6f]); + writer.prepare("UPDATE content SET bytes = ?, byte_len = ? WHERE hash = ?").run(invalid, invalid.length, hashes[2]); + const wanted = ["", "", invalid.toString("utf8")]; + assert.deepEqual(store.getMessages("target").slice(0, 3).map((row) => row.content), wanted); + assert.deepEqual(store.getMessagesAt("target", [0, 1, 2]).map((row) => row.content), wanted); + assert.deepEqual(store.uncoveredRows("target", [[3, 7]]).map((row) => row.content), wanted); + }); + + test(`${engine}: joined message bytes still hydrate both media representations`, (t) => { + const fixture = setup(t, engine, 0); + if (!fixture) return; + const { store, writer } = fixture; + const value = input(2); + const payload = "YWJj".repeat(600); + const originals = value.messages.map((row) => JSON.stringify({ id: row.msgId, imageBase64: payload, + rawOpenaiContent: { image_url: { url: `data:image/png;base64,${payload}` } } })); + value.messages = value.messages.map((row, i) => { + const split = splitMediaContent(originals[i]!); + return { ...row, content: split.content, contentHash: split.hash, media: split.media }; + }); + store.ingestSession(value); + assert.deepEqual(store.getMessages("target").map((row) => row.content), originals); + assert.deepEqual(store.getMessagesAt("target", [0, 1]).map((row) => row.content), originals); + assert.deepEqual(store.uncoveredRows("target", []).map((row) => row.content), originals); + writer.exec("PRAGMA foreign_keys=OFF"); + writer.prepare("DELETE FROM content WHERE hash = ?").run(sha256Hex(payload)); + assert.deepEqual(store.getMessages("target").map((row) => row.content), value.messages.map((row) => row.content)); + }); +} + +function mysqlWithSqlite(t: TestContext, count = 8) { + const fixture = setup(t, "node:sqlite", count)!; + const calls: { source: string; sql: string; payloadRows?: number }[] = []; + const query = (source: string) => async (sql: string, params: unknown[] = []) => { + calls.push({ source, sql }); + if (sql.startsWith("SET TRANSACTION") || sql.startsWith("START TRANSACTION")) return [[]]; + const rows = fixture.writer.prepare(sql).all(...params) as Record[]; + calls.at(-1)!.payloadRows = rows.filter((row) => "bytes" in row || "content_bytes" in row).length; + return [rows.map((row) => Object.fromEntries(Object.entries(row).map(([key, value]) => [key, value instanceof Uint8Array ? Buffer.from(value) : value])))]; + }; + const conn = { query: query("connection"), rollback: async () => {}, release: () => {} }; + const driver = { createPool: () => ({ query: query("pool"), getConnection: async () => conn, end: async () => {} }) } as unknown as Mysql2Module; + const backend = new MysqlBackend("mysql://query-contract", driver); + t.after(() => backend.close()); + return { ...fixture, backend, calls }; +} + +for (const read of ["all", "pointers"] as const) { + test(`mysql ${read} readers execute SQL against SQLite without per-message CAS queries`, async (t) => { + const { backend, calls } = mysqlWithSqlite(t, 1003); + const rows = read === "all" ? await backend.getMessages("target") : (await backend.sessionViewRows("target"))!.messages; + assert.equal(rows.length, 1003); + assert.equal(calls.filter((call) => /FROM messages\b|FROM content\b/.test(call.sql)).length, read === "all" ? 4 : 3); + assert.ok(calls.every((call) => (call.payloadRows ?? 0) <= 500), "body batches must stay bounded on the SQL backend"); + }); +} + +test("mysql joined reads preserve row shapes, filters, media and the snapshot connection", async (t) => { + const { store, writer, backend, calls } = mysqlWithSqlite(t); + writer.prepare("UPDATE messages SET retired_at = 5000 WHERE session_id = ? AND seq = 2").run("target"); + for (const opts of [undefined, { includeRetired: true }, { seqs: [7, 2, 1, 99, 7] }, { seqs: [] }]) { + assert.deepEqual(await backend.getMessages("target", opts), store.getMessages("target", opts)); + } + assert.deepEqual((await backend.sessionViewRows("target"))!.messages, store.sessionViewRows("target")!.messages); + const value = input(1, "media"); + const payload = "YWJj".repeat(600); + const original = JSON.stringify({ imageBase64: payload, text: "kept", rawOpenaiContent: { url: `data:image/png;base64,${payload}` } }); + const split = splitMediaContent(original); + value.messages[0] = { ...value.messages[0]!, content: split.content, contentHash: split.hash, media: split.media }; + store.ingestSession(value); + calls.length = 0; + await backend.readSnapshot(async (reader) => { + assert.equal((await reader.getMessages("media"))[0]!.content, original); + }); + assert.ok(calls.length > 2); + assert.ok(calls.every((call) => call.source === "connection")); + assert.deepEqual(await backend.getMessages("missing"), []); +}); + +test("mysql message reads keep missing/empty CAS fallbacks and Buffer decoding", async (t) => { + const { store, writer, backend } = mysqlWithSqlite(t); + const hashes = input().messages.map((row) => sha256Hex(row.content)); + writer.exec("PRAGMA foreign_keys=OFF"); + writer.prepare("DELETE FROM content WHERE hash = ?").run(hashes[0]); + writer.prepare("UPDATE content SET bytes = ?, byte_len = 0 WHERE hash = ?").run(Buffer.alloc(0), hashes[1]); + const invalid = Buffer.from([0x66, 0x80, 0x00, 0x6f]); + writer.prepare("UPDATE content SET bytes = ?, byte_len = ? WHERE hash = ?").run(invalid, invalid.length, hashes[2]); + assert.deepEqual(await backend.getMessages("target"), store.getMessages("target")); + assert.deepEqual((await backend.sessionViewRows("target"))!.messages, store.sessionViewRows("target")!.messages); +});