diff --git a/src/storage/schema.ts b/src/storage/schema.ts index 6f253e7c36..b152f4ccce 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -281,6 +281,7 @@ export function migrateSchema(db: { const row = db.prepare("PRAGMA user_version").get() as { user_version: number } | undefined; const from = row?.user_version ?? 0; if (from >= STORAGE_SCHEMA_VERSION) return { from, to: from }; + let needV8Backfill = false; db.transaction(() => { if (from < 1) db.exec(STORAGE_DDL_V1); if (from < 2) { @@ -358,69 +359,105 @@ export function migrateSchema(db: { db.exec("ALTER TABLE messages ADD COLUMN retired_at INTEGER"); } db.exec("CREATE INDEX IF NOT EXISTS idx_messages_identity ON messages (session_id, msg_id, identity)"); - // Backfill from stored bytes, healing the wireIndex-split - // generations the polluted formula minted: same (msg_id, - // identity) after the strip → keep the FIRST arrival (lowest - // seq), drop the later duplicates and hand their CAS refs back. - // Message bytes live in the content CAS (messages carry only the - // hash) — join in, decompress nothing (bytes are stored raw). - const rows = db - .prepare("SELECT m.session_id AS session_id, m.seq AS seq, m.msg_id AS msg_id, m.content_hash AS content_hash, c.bytes AS content FROM messages m JOIN content c ON c.hash = m.content_hash") - .all() as { session_id: string; seq: number; msg_id: string; content: string; content_hash: string }[]; - const keep = new Map(); - const drop: { session_id: string; seq: number; content_hash: string }[] = []; - const rawByIdentityKey = new Map(); + // The identity backfill runs AFTER this transaction (see + // migrateV8Identities below), one session per transaction: + // v0.1.207 loaded EVERY message's content bytes into memory in a + // single .all() here inside one transaction, so multi-GB stores + // OOM'd on first open — and because the user_version bump sat in + // the same transaction, every subsequent open re-ran the whole + // migration and re-OMM'd: permanently unopenable stores (#4677). + needV8Backfill = true; + } + }); + if (needV8Backfill) migrateV8Identities(db); + db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`); + return { from, to: STORAGE_SCHEMA_VERSION }; +} + +/** Same v7 shape as the wireIndex-split generations the polluted formula + * minted: same (msg_id, identity) after the strip → keep the FIRST arrival + * (lowest seq), drop the later duplicates and hand their CAS refs back — + * but streamed one session per transaction (#4677): a crash between sessions + * leaves the done sessions committed and user_version still 7, so the next + * open skips them (no rows with identity = '') and resumes; memory holds one + * page window + one session's keeper keys, never the store's bytes. Rows + * whose content bytes are missing from the CAS get the ABSENT marker — a + * "absent:" prefix — a string no sha256 hex output can equal (it carries a + * ':' and is shorter than 64 hex chars), so their session terminates + * instead of rescanning forever; joins treat it exactly like the v0.1.207 + * leftover '' (neither can match a live-computed identity). Printable on + * purpose: node:sqlite truncates bound strings at an embedded NUL, so the + * \u0000-shaped marker silently bound as '' — the row looked unmigrated + * forever and got rescanned on every open. */ +const ABSENT_IDENTITY = "absent:"; +const V8_BACKFILL_PAGE = 500; + +function migrateV8Identities(db: { + exec(sql: string): void; + prepare(sql: string): { get(...a: unknown[]): unknown; all(...a: unknown[]): unknown[]; run(...a: unknown[]): unknown }; + transaction(fn: () => T): T; +}): void { + const sessions = db.prepare("SELECT DISTINCT session_id FROM messages WHERE identity = ''").all() as { session_id: string }[]; + if (sessions.length === 0) return; + const page = db.prepare( + "SELECT m.seq AS seq, m.msg_id AS msg_id, m.content_hash AS content_hash, c.bytes AS content " + + "FROM messages m LEFT JOIN content c ON c.hash = m.content_hash " + + "WHERE m.session_id = ? AND m.identity = '' AND m.seq > ? ORDER BY m.seq LIMIT ?", + ); + const upd = db.prepare("UPDATE messages SET identity = ? WHERE session_id = ? AND seq = ?"); + const decRef = db.prepare("UPDATE content SET refcount = refcount - 1 WHERE hash = ?"); + const delMedia = db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?"); + const mediaOf = db.prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?"); + const delMsg = db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?"); + for (const s of sessions) { + const keep = new Set(); + const updates: { seq: number; identity: string }[] = []; + const drops: { seq: number; content_hash: string }[] = []; + let cursor = -1; + for (;;) { + const rows = page.all(s.session_id, cursor, V8_BACKFILL_PAGE) as { + seq: number; + msg_id: string; + content_hash: string; + content: string | Uint8Array | null; + }[]; + if (rows.length === 0) break; for (const r of rows) { + cursor = r.seq; // node:sqlite returns BLOB as Uint8Array — normalize to utf8 // before hashing (JSON.parse would coerce it to a comma list // and silently hash the wrong input). - const raw = typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"); - const identity = identityOfStoredContent(raw); - rawByIdentityKey.set(`${r.session_id}\u0000${r.msg_id}\u0000${identity}`, raw); - const key = `${r.session_id}\u0000${r.msg_id}\u0000${identity}`; - const seen = keep.get(key); - if (seen === undefined) { - keep.set(key, { session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); - } else if (r.seq < seen.seq) { - // an earlier arrival discovered late — swap the keeper - drop.push({ session_id: seen.session_id, seq: seen.seq, content_hash: seen.content_hash }); - keep.set(key, { session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); - } else { - drop.push({ session_id: r.session_id, seq: r.seq, content_hash: r.content_hash }); + const identity = + r.content == null + ? ABSENT_IDENTITY + : identityOfStoredContent( + typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"), + ); + const key = `${r.msg_id}\u0000${identity}`; + if (keep.has(key)) drops.push({ seq: r.seq, content_hash: r.content_hash }); + else { + keep.add(key); + updates.push({ seq: r.seq, identity }); } } - const identityBySeq = new Map(); - for (const r of rows) { - const raw = rawByIdentityKey.get(`${r.session_id}\u0000${r.msg_id}\u0000${identityOfStoredContent(typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"))}`) ?? (typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8")); - const identity = identityOfStoredContent(raw); - const key = `${r.session_id}\u0000${r.msg_id}\u0000${identity}`; - if (keep.get(key)?.seq === r.seq) identityBySeq.set(`${r.session_id}\u0000${r.seq}`, identity); - } - const upd = db.prepare("UPDATE messages SET identity = ? WHERE session_id = ? AND seq = ?"); - for (const [k, identity] of identityBySeq) { - const [sessionId, seqStr] = k.split("\u0000"); - upd.run(identity, sessionId, Number(seqStr)); - } - // Duplicates: delete their media mappings + message rows and - // decrement CAS refcounts (delete referencing rows FIRST — the - // #4242 discipline; refcount may reach 0, the next gc run - // collects those content rows). - const decRef = db.prepare("UPDATE content SET refcount = refcount - 1 WHERE hash = ?"); - const delMedia = db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?"); - const mediaOf = db.prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?"); - const delMsg = db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?"); - for (const d of drop) { - for (const mm of mediaOf.all(d.session_id, d.seq) as { media_hash: string }[]) { + if (rows.length < V8_BACKFILL_PAGE) break; + } + // One transaction per session: keepers get their identity, the later + // duplicates hand their media mappings + message rows + CAS refs back + // (delete referencing rows FIRST — the #4242 discipline; refcount may + // reach 0, the next gc run collects those content rows). + db.transaction(() => { + for (const u of updates) upd.run(u.identity, s.session_id, u.seq); + for (const d of drops) { + for (const mm of mediaOf.all(s.session_id, d.seq) as { media_hash: string }[]) { decRef.run(mm.media_hash); } - delMedia.run(d.session_id, d.seq); - delMsg.run(d.session_id, d.seq); + delMedia.run(s.session_id, d.seq); + delMsg.run(s.session_id, d.seq); decRef.run(d.content_hash); } - } - db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`); - }); - return { from, to: STORAGE_SCHEMA_VERSION }; + }); + } } /** Test-only alias of the fresh-install DDL (unused-export gate: INTERNAL_ONLY diff --git a/tests/issue4677-v8-migration-streaming.test.ts b/tests/issue4677-v8-migration-streaming.test.ts new file mode 100644 index 0000000000..8bfbb0d467 --- /dev/null +++ b/tests/issue4677-v8-migration-streaming.test.ts @@ -0,0 +1,199 @@ +/** + * #4677: the v8 identity backfill must stream, one session per transaction — + * not load every message's content bytes into memory in a single .all() + * inside one transaction (the v0.1.207 shape, which OOM'd multi-GB stores on + * first open and, because the user_version bump sat in the same transaction, + * re-ran and re-OMM'd on every subsequent open: permanently unopenable). + * + * Discriminators asserted here: + * 1. backfill + wireIndex-split dedup still produces the v0.1.207 OUTCOME + * (keeper = lowest seq, later duplicates dropped, their CAS refs handed + * back) for rows whose bytes differ only in id/wireIndex; + * 2. crash resume: a session whose rows already carry identities is never + * re-processed (planted stale identity survives a re-run), user_version + * still ends at 8; + * 3. content-less rows terminate: they get the ABSENT marker (never equal + * to a sha256 hex identity), so the session is not rescanned forever; + * 4. dedup groups span page boundaries (keeper at the head of the session, + * duplicate past the 500-row page edge); + * 5. an empty store migrates to v8 without touching anything. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as crypto from "node:crypto"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; +import { migrateSchema } from "../src/storage/schema.ts"; +import { identityOfStoredContent } from "../src/storage/identity.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4677-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json"); + +type Db = ReturnType["db"]; + +function sha256(text: string): string { + return crypto.createHash("sha256").update(text, "utf8").digest("hex"); +} + +function tmpDb(): { db: Db; file: string } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4677-")); + const file = path.join(dir, "sessions.db"); + const db = openSqliteDatabase(file).db; + migrateSchema(db); + return { db, file }; +} + +/** Rewind a fully-migrated DB into the pre-v8 state: columns present + * (probe-first path), every identity blanked, version stamped 7. */ +function downgradeToV7(db: Db): void { + db.exec("UPDATE messages SET identity = ''"); + db.exec("PRAGMA user_version = 7"); +} + +/** Insert a content row and return its hash (refcount starts at 1). */ +function contentPut(db: Db, bytes: string): string { + const hash = sha256(bytes); + db.prepare("INSERT INTO content (hash, byte_len, refcount, algo, bytes) VALUES (?, ?, 1, 'sha256', ?)").run( + hash, + Buffer.byteLength(bytes, "utf8"), + Buffer.from(bytes, "utf8"), + ); + return hash; +} + +function msgPut(db: Db, sessionId: string, seq: number, msgId: string, role: string, contentHash: string): void { + db.prepare( + "INSERT INTO messages (session_id, seq, msg_id, role, kind, content_hash) VALUES (?, ?, ?, ?, 'message', ?)", + ).run(sessionId, seq, msgId, role, contentHash); +} + +function identityOf(db: Db, sessionId: string, seq: number): string { + return (db.prepare("SELECT identity FROM messages WHERE session_id = ? AND seq = ?").get(sessionId, seq) as { + identity: string; + }).identity; +} + +function refcountOf(db: Db, hash: string): number | undefined { + return (db.prepare("SELECT refcount FROM content WHERE hash = ?").get(hash) as { refcount: number } | undefined) + ?.refcount; +} + +/** A stored-message serialization whose bytes differ only in id/wireIndex — + * the wireIndex-split shape: different content_hash, same identity. */ +function storedMessage(id: string, text: string): string { + return JSON.stringify({ id, wireIndex: 0, role: "user", content: text }); +} + +test("#4677 backfill sets identities and drops wireIndex-split duplicates (lowest seq keeps)", () => { + const { db } = tmpDb(); + const a1 = contentPut(db, storedMessage("id-1", "hello")); + const a2 = contentPut(db, storedMessage("id-2", "hello")); + msgPut(db, "s1", 1, "m-keep", "user", a1); + msgPut(db, "s1", 2, "m-keep", "user", a2); + downgradeToV7(db); + + const { from, to } = migrateSchema(db); + + assert.equal(from, 7); + assert.equal(to, 8); + const identity = identityOfStoredContent(storedMessage("stripped", "hello")); + assert.equal(identityOf(db, "s1", 1), identity); + // the later duplicate row is gone and its content CAS ref handed back + assert.equal( + (db.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = 's1'").get() as { n: number }).n, + 1, + ); + assert.equal(refcountOf(db, a2), 0); + assert.equal(refcountOf(db, a1), 1); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 8); + db.close(); +}); + +test("#4677 crash resume: sessions that already carry identities are skipped, not recomputed", () => { + const { db } = tmpDb(); + const done = contentPut(db, storedMessage("done", "already migrated")); + msgPut(db, "s-done", 1, "m-done", "user", done); + const pending = contentPut(db, storedMessage("pending", "still fresh")); + msgPut(db, "s-pending", 1, "m-pending", "user", pending); + downgradeToV7(db); + // s-done was completed by an earlier (crashed) run: its rows already + // carry identities — plant a deliberately stale one; a re-processing + // implementation would overwrite it with the recomputed hash. + db.prepare("UPDATE messages SET identity = 'stale-planted' WHERE session_id = 's-done'").run(); + + migrateSchema(db); + + assert.equal(identityOf(db, "s-done", 1), "stale-planted"); + assert.equal(identityOf(db, "s-pending", 1), identityOfStoredContent(storedMessage("pending", "still fresh"))); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 8); + db.close(); +}); + +test("#4677 rows without content bytes terminate with the ABSENT marker instead of rescanning forever", () => { + const { db } = tmpDb(); + const ghost = contentPut(db, storedMessage("ghost", "will lose its bytes")); + msgPut(db, "s-ghost", 1, "m-ghost", "user", ghost); + downgradeToV7(db); + // fabricate the orphaned-row state (content GC'd while the message row + // stayed): the FK is enforced on write, so lift it just for this delete + db.exec("PRAGMA foreign_keys = OFF"); + db.prepare("DELETE FROM content WHERE hash = ?").run(ghost); + db.exec("PRAGMA foreign_keys = ON"); + + migrateSchema(db); + const identity = identityOf(db, "s-ghost", 1); + + assert.equal(identity, "absent:"); + // second run is a no-op — the marker is not identity = '', so the + // session is not selected again + assert.equal(migrateSchema(db).from, 8); + assert.equal(identityOf(db, "s-ghost", 1), "absent:"); + db.close(); +}); + +test("#4677 dedup groups span page boundaries (keeper head, duplicate past the 500-row page edge)", () => { + const { db } = tmpDb(); + const keeperHash = contentPut(db, storedMessage("page-keeper", "shared text")); + msgPut(db, "s-page", 1, "m-page", "user", keeperHash); + // fill past one page window with unique messages (one transaction — the + // per-insert autocommit fsync made this the suite's slowest test) + db.transaction(() => { + for (let i = 2; i <= 502; i++) { + const h = contentPut(db, storedMessage(`filler-${i}`, `filler ${i}`)); + msgPut(db, "s-page", i, `m-filler-${i}`, "user", h); + } + }); + // the duplicate lands past the page boundary: same identity (bytes + // differ only in id), different content_hash + const dupHash = contentPut(db, storedMessage("page-dup", "shared text")); + msgPut(db, "s-page", 503, "m-page", "user", dupHash); + downgradeToV7(db); + + migrateSchema(db); + + assert.equal(identityOf(db, "s-page", 1), identityOfStoredContent(storedMessage("x", "shared text"))); + const gone = db + .prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = 's-page' AND seq = 503") + .get() as { n: number }; + assert.equal(gone.n, 0); + assert.equal(refcountOf(db, dupHash), 0); + const total = db + .prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = 's-page'") + .get() as { n: number }; + assert.equal(total.n, 502); + db.close(); +}); + +test("#4677 empty store migrates to v8 untouched", () => { + const { db } = tmpDb(); + downgradeToV7(db); + const { from, to } = migrateSchema(db); + assert.equal(from, 7); + assert.equal(to, 8); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 8); + db.close(); +});