Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 90 additions & 53 deletions src/storage/schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,7 @@ export function migrateSchema(db: {
const row = db.prepare("PRAGMA user_version").get() as { user_version: number } | undefined;
const from = row?.user_version ?? 0;
if (from >= STORAGE_SCHEMA_VERSION) return { from, to: from };
let needV8Backfill = false;
db.transaction(() => {
if (from < 1) db.exec(STORAGE_DDL_V1);
if (from < 2) {
Expand Down Expand Up @@ -358,69 +359,105 @@ export function migrateSchema(db: {
db.exec("ALTER TABLE messages ADD COLUMN retired_at INTEGER");
}
db.exec("CREATE INDEX IF NOT EXISTS idx_messages_identity ON messages (session_id, msg_id, identity)");
// Backfill from stored bytes, healing the wireIndex-split
// generations the polluted formula minted: same (msg_id,
// identity) after the strip → keep the FIRST arrival (lowest
// seq), drop the later duplicates and hand their CAS refs back.
// Message bytes live in the content CAS (messages carry only the
// hash) — join in, decompress nothing (bytes are stored raw).
const rows = db
.prepare("SELECT m.session_id AS session_id, m.seq AS seq, m.msg_id AS msg_id, m.content_hash AS content_hash, c.bytes AS content FROM messages m JOIN content c ON c.hash = m.content_hash")
.all() as { session_id: string; seq: number; msg_id: string; content: string; content_hash: string }[];
const keep = new Map<string, { session_id: string; seq: number; content_hash: string }>();
const drop: { session_id: string; seq: number; content_hash: string }[] = [];
const rawByIdentityKey = new Map<string, string>();
// The identity backfill runs AFTER this transaction (see
// migrateV8Identities below), one session per transaction:
// v0.1.207 loaded EVERY message's content bytes into memory in a
// single .all() here inside one transaction, so multi-GB stores
// OOM'd on first open — and because the user_version bump sat in
// the same transaction, every subsequent open re-ran the whole
// migration and re-OMM'd: permanently unopenable stores (#4677).
needV8Backfill = true;
}
});
if (needV8Backfill) migrateV8Identities(db);
db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`);
return { from, to: STORAGE_SCHEMA_VERSION };
}

/** Same v7 shape as the wireIndex-split generations the polluted formula
* minted: same (msg_id, identity) after the strip → keep the FIRST arrival
* (lowest seq), drop the later duplicates and hand their CAS refs back —
* but streamed one session per transaction (#4677): a crash between sessions
* leaves the done sessions committed and user_version still 7, so the next
* open skips them (no rows with identity = '') and resumes; memory holds one
* page window + one session's keeper keys, never the store's bytes. Rows
* whose content bytes are missing from the CAS get the ABSENT marker — a
* "absent:" prefix — a string no sha256 hex output can equal (it carries a
* ':' and is shorter than 64 hex chars), so their session terminates
* instead of rescanning forever; joins treat it exactly like the v0.1.207
* leftover '' (neither can match a live-computed identity). Printable on
* purpose: node:sqlite truncates bound strings at an embedded NUL, so the
* \u0000-shaped marker silently bound as '' — the row looked unmigrated
* forever and got rescanned on every open. */
const ABSENT_IDENTITY = "absent:";
const V8_BACKFILL_PAGE = 500;

function migrateV8Identities(db: {
exec(sql: string): void;
prepare(sql: string): { get(...a: unknown[]): unknown; all(...a: unknown[]): unknown[]; run(...a: unknown[]): unknown };
transaction<T>(fn: () => T): T;
}): void {
const sessions = db.prepare("SELECT DISTINCT session_id FROM messages WHERE identity = ''").all() as { session_id: string }[];
if (sessions.length === 0) return;
const page = db.prepare(
"SELECT m.seq AS seq, m.msg_id AS msg_id, m.content_hash AS content_hash, c.bytes AS content " +
"FROM messages m LEFT JOIN content c ON c.hash = m.content_hash " +
"WHERE m.session_id = ? AND m.identity = '' AND m.seq > ? ORDER BY m.seq LIMIT ?",
);
const upd = db.prepare("UPDATE messages SET identity = ? WHERE session_id = ? AND seq = ?");
const decRef = db.prepare("UPDATE content SET refcount = refcount - 1 WHERE hash = ?");
const delMedia = db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?");
const mediaOf = db.prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?");
const delMsg = db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?");
for (const s of sessions) {
const keep = new Set<string>();
const updates: { seq: number; identity: string }[] = [];
const drops: { seq: number; content_hash: string }[] = [];
let cursor = -1;
for (;;) {
const rows = page.all(s.session_id, cursor, V8_BACKFILL_PAGE) as {
seq: number;
msg_id: string;
content_hash: string;
content: string | Uint8Array | null;
}[];
if (rows.length === 0) break;
for (const r of rows) {
cursor = r.seq;
// node:sqlite returns BLOB as Uint8Array — normalize to utf8
// before hashing (JSON.parse would coerce it to a comma list
// and silently hash the wrong input).
const raw = typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8");
const identity = identityOfStoredContent(raw);
rawByIdentityKey.set(`${r.session_id}\u0000${r.msg_id}\u0000${identity}`, raw);
const key = `${r.session_id}\u0000${r.msg_id}\u0000${identity}`;
const seen = keep.get(key);
if (seen === undefined) {
keep.set(key, { session_id: r.session_id, seq: r.seq, content_hash: r.content_hash });
} else if (r.seq < seen.seq) {
// an earlier arrival discovered late — swap the keeper
drop.push({ session_id: seen.session_id, seq: seen.seq, content_hash: seen.content_hash });
keep.set(key, { session_id: r.session_id, seq: r.seq, content_hash: r.content_hash });
} else {
drop.push({ session_id: r.session_id, seq: r.seq, content_hash: r.content_hash });
const identity =
r.content == null
? ABSENT_IDENTITY
: identityOfStoredContent(
typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"),
);
const key = `${r.msg_id}\u0000${identity}`;
if (keep.has(key)) drops.push({ seq: r.seq, content_hash: r.content_hash });
else {
keep.add(key);
updates.push({ seq: r.seq, identity });
}
}
const identityBySeq = new Map<string, string>();
for (const r of rows) {
const raw = rawByIdentityKey.get(`${r.session_id}\u0000${r.msg_id}\u0000${identityOfStoredContent(typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"))}`) ?? (typeof r.content === "string" ? r.content : Buffer.from(r.content).toString("utf8"));
const identity = identityOfStoredContent(raw);
const key = `${r.session_id}\u0000${r.msg_id}\u0000${identity}`;
if (keep.get(key)?.seq === r.seq) identityBySeq.set(`${r.session_id}\u0000${r.seq}`, identity);
}
const upd = db.prepare("UPDATE messages SET identity = ? WHERE session_id = ? AND seq = ?");
for (const [k, identity] of identityBySeq) {
const [sessionId, seqStr] = k.split("\u0000");
upd.run(identity, sessionId, Number(seqStr));
}
// Duplicates: delete their media mappings + message rows and
// decrement CAS refcounts (delete referencing rows FIRST — the
// #4242 discipline; refcount may reach 0, the next gc run
// collects those content rows).
const decRef = db.prepare("UPDATE content SET refcount = refcount - 1 WHERE hash = ?");
const delMedia = db.prepare("DELETE FROM message_media WHERE session_id = ? AND seq = ?");
const mediaOf = db.prepare("SELECT media_hash FROM message_media WHERE session_id = ? AND seq = ?");
const delMsg = db.prepare("DELETE FROM messages WHERE session_id = ? AND seq = ?");
for (const d of drop) {
for (const mm of mediaOf.all(d.session_id, d.seq) as { media_hash: string }[]) {
if (rows.length < V8_BACKFILL_PAGE) break;
}
// One transaction per session: keepers get their identity, the later
// duplicates hand their media mappings + message rows + CAS refs back
// (delete referencing rows FIRST — the #4242 discipline; refcount may
// reach 0, the next gc run collects those content rows).
db.transaction(() => {
for (const u of updates) upd.run(u.identity, s.session_id, u.seq);
for (const d of drops) {
for (const mm of mediaOf.all(s.session_id, d.seq) as { media_hash: string }[]) {
decRef.run(mm.media_hash);
}
delMedia.run(d.session_id, d.seq);
delMsg.run(d.session_id, d.seq);
delMedia.run(s.session_id, d.seq);
delMsg.run(s.session_id, d.seq);
decRef.run(d.content_hash);
}
}
db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`);
});
return { from, to: STORAGE_SCHEMA_VERSION };
});
}
}

/** Test-only alias of the fresh-install DDL (unused-export gate: INTERNAL_ONLY
Expand Down
199 changes: 199 additions & 0 deletions tests/issue4677-v8-migration-streaming.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
/**
* #4677: the v8 identity backfill must stream, one session per transaction —
* not load every message's content bytes into memory in a single .all()
* inside one transaction (the v0.1.207 shape, which OOM'd multi-GB stores on
* first open and, because the user_version bump sat in the same transaction,
* re-ran and re-OMM'd on every subsequent open: permanently unopenable).
*
* Discriminators asserted here:
* 1. backfill + wireIndex-split dedup still produces the v0.1.207 OUTCOME
* (keeper = lowest seq, later duplicates dropped, their CAS refs handed
* back) for rows whose bytes differ only in id/wireIndex;
* 2. crash resume: a session whose rows already carry identities is never
* re-processed (planted stale identity survives a re-run), user_version
* still ends at 8;
* 3. content-less rows terminate: they get the ABSENT marker (never equal
* to a sha256 hex identity), so the session is not rescanned forever;
* 4. dedup groups span page boundaries (keeper at the head of the session,
* duplicate past the 500-row page edge);
* 5. an empty store migrates to v8 without touching anything.
*/

import { test } from "node:test";
import assert from "node:assert/strict";
import * as crypto from "node:crypto";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { openSqliteDatabase } from "../src/storage/driver.ts";
import { migrateSchema } from "../src/storage/schema.ts";
import { identityOfStoredContent } from "../src/storage/identity.ts";

process.env.BILI_PERSIST_ZSTD = "0";
const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4677-hermetic-"));
process.env.BILI_CONFIG_FILE = path.join(hermetic, "no-config.json");

type Db = ReturnType<typeof openSqliteDatabase>["db"];

function sha256(text: string): string {
return crypto.createHash("sha256").update(text, "utf8").digest("hex");
}

function tmpDb(): { db: Db; file: string } {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4677-"));
const file = path.join(dir, "sessions.db");
const db = openSqliteDatabase(file).db;
migrateSchema(db);
return { db, file };
}

/** Rewind a fully-migrated DB into the pre-v8 state: columns present
* (probe-first path), every identity blanked, version stamped 7. */
function downgradeToV7(db: Db): void {
db.exec("UPDATE messages SET identity = ''");
db.exec("PRAGMA user_version = 7");
}

/** Insert a content row and return its hash (refcount starts at 1). */
function contentPut(db: Db, bytes: string): string {
const hash = sha256(bytes);
db.prepare("INSERT INTO content (hash, byte_len, refcount, algo, bytes) VALUES (?, ?, 1, 'sha256', ?)").run(
hash,
Buffer.byteLength(bytes, "utf8"),
Buffer.from(bytes, "utf8"),
);
return hash;
}

function msgPut(db: Db, sessionId: string, seq: number, msgId: string, role: string, contentHash: string): void {
db.prepare(
"INSERT INTO messages (session_id, seq, msg_id, role, kind, content_hash) VALUES (?, ?, ?, ?, 'message', ?)",
).run(sessionId, seq, msgId, role, contentHash);
}

function identityOf(db: Db, sessionId: string, seq: number): string {
return (db.prepare("SELECT identity FROM messages WHERE session_id = ? AND seq = ?").get(sessionId, seq) as {
identity: string;
}).identity;
}

function refcountOf(db: Db, hash: string): number | undefined {
return (db.prepare("SELECT refcount FROM content WHERE hash = ?").get(hash) as { refcount: number } | undefined)
?.refcount;
}

/** A stored-message serialization whose bytes differ only in id/wireIndex —
* the wireIndex-split shape: different content_hash, same identity. */
function storedMessage(id: string, text: string): string {
return JSON.stringify({ id, wireIndex: 0, role: "user", content: text });
}

test("#4677 backfill sets identities and drops wireIndex-split duplicates (lowest seq keeps)", () => {
const { db } = tmpDb();
const a1 = contentPut(db, storedMessage("id-1", "hello"));
const a2 = contentPut(db, storedMessage("id-2", "hello"));
msgPut(db, "s1", 1, "m-keep", "user", a1);
msgPut(db, "s1", 2, "m-keep", "user", a2);
downgradeToV7(db);

const { from, to } = migrateSchema(db);

assert.equal(from, 7);
assert.equal(to, 8);
const identity = identityOfStoredContent(storedMessage("stripped", "hello"));
assert.equal(identityOf(db, "s1", 1), identity);
// the later duplicate row is gone and its content CAS ref handed back
assert.equal(
(db.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = 's1'").get() as { n: number }).n,
1,
);
assert.equal(refcountOf(db, a2), 0);
assert.equal(refcountOf(db, a1), 1);
assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 8);
db.close();
});

test("#4677 crash resume: sessions that already carry identities are skipped, not recomputed", () => {
const { db } = tmpDb();
const done = contentPut(db, storedMessage("done", "already migrated"));
msgPut(db, "s-done", 1, "m-done", "user", done);
const pending = contentPut(db, storedMessage("pending", "still fresh"));
msgPut(db, "s-pending", 1, "m-pending", "user", pending);
downgradeToV7(db);
// s-done was completed by an earlier (crashed) run: its rows already
// carry identities — plant a deliberately stale one; a re-processing
// implementation would overwrite it with the recomputed hash.
db.prepare("UPDATE messages SET identity = 'stale-planted' WHERE session_id = 's-done'").run();

migrateSchema(db);

assert.equal(identityOf(db, "s-done", 1), "stale-planted");
assert.equal(identityOf(db, "s-pending", 1), identityOfStoredContent(storedMessage("pending", "still fresh")));
assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 8);
db.close();
});

test("#4677 rows without content bytes terminate with the ABSENT marker instead of rescanning forever", () => {
const { db } = tmpDb();
const ghost = contentPut(db, storedMessage("ghost", "will lose its bytes"));
msgPut(db, "s-ghost", 1, "m-ghost", "user", ghost);
downgradeToV7(db);
// fabricate the orphaned-row state (content GC'd while the message row
// stayed): the FK is enforced on write, so lift it just for this delete
db.exec("PRAGMA foreign_keys = OFF");
db.prepare("DELETE FROM content WHERE hash = ?").run(ghost);
db.exec("PRAGMA foreign_keys = ON");

migrateSchema(db);
const identity = identityOf(db, "s-ghost", 1);

assert.equal(identity, "absent:");
// second run is a no-op — the marker is not identity = '', so the
// session is not selected again
assert.equal(migrateSchema(db).from, 8);
assert.equal(identityOf(db, "s-ghost", 1), "absent:");
db.close();
});

test("#4677 dedup groups span page boundaries (keeper head, duplicate past the 500-row page edge)", () => {
const { db } = tmpDb();
const keeperHash = contentPut(db, storedMessage("page-keeper", "shared text"));
msgPut(db, "s-page", 1, "m-page", "user", keeperHash);
// fill past one page window with unique messages (one transaction — the
// per-insert autocommit fsync made this the suite's slowest test)
db.transaction(() => {
for (let i = 2; i <= 502; i++) {
const h = contentPut(db, storedMessage(`filler-${i}`, `filler ${i}`));
msgPut(db, "s-page", i, `m-filler-${i}`, "user", h);
}
});
// the duplicate lands past the page boundary: same identity (bytes
// differ only in id), different content_hash
const dupHash = contentPut(db, storedMessage("page-dup", "shared text"));
msgPut(db, "s-page", 503, "m-page", "user", dupHash);
downgradeToV7(db);

migrateSchema(db);

assert.equal(identityOf(db, "s-page", 1), identityOfStoredContent(storedMessage("x", "shared text")));
const gone = db
.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = 's-page' AND seq = 503")
.get() as { n: number };
assert.equal(gone.n, 0);
assert.equal(refcountOf(db, dupHash), 0);
const total = db
.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = 's-page'")
.get() as { n: number };
assert.equal(total.n, 502);
db.close();
});

test("#4677 empty store migrates to v8 untouched", () => {
const { db } = tmpDb();
downgradeToV7(db);
const { from, to } = migrateSchema(db);
assert.equal(from, 7);
assert.equal(to, 8);
assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 8);
db.close();
});
Loading