Skip to content
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 10 additions & 3 deletions src/storage/mysql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1219,7 +1219,9 @@ export class MysqlBackend implements StorageBackend {
const exportPath = join(opts.archiveDir, MysqlBackend.ARCHIVE_LOG_NAME);
if (row.status === "archived") return { ok: true, exportPath };
const q = async (sql: string): Promise<Row[]> => ((await this.pool.query(sql, [sessionId])) as unknown as Row[][])[0];
const messages = await q("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq");
// #4665: carry the v8 row state (identity JOIN key + historian
// tombstone) across archive/restore — sqlite twin, same reason.
const messages = await q("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq");
const mediaMaps = await q("SELECT seq, media_hash FROM message_media WHERE session_id = ?");
const blocks = await q("SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json FROM blocks WHERE session_id = ? ORDER BY ord");
const refsRows = await q("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord");
Expand Down Expand Up @@ -1321,9 +1323,14 @@ export class MysqlBackend implements StorageBackend {
await this.tx(async (c) => {
for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash);
for (const m of dump.messages) {
// #4665: pre-fix dumps lack both fields — recompute identity
// from the CAS body (same formula as the v8 migration
// backfill); retire state is unrecoverable there → live.
const stored = typeof m.content_hash === "string" ? dump.content[m.content_hash] : undefined;
const identity = typeof m.identity === "string" && m.identity !== "" ? m.identity : stored !== undefined ? identityOfStoredContent(stored) : "";
await c.query(
"INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
[sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, m.identity ?? ""],
"INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
[sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, identity, typeof m.retired_at === "number" ? m.retired_at : null],
);
}
for (const mm of (dump as { mediaMaps?: Record<string, unknown>[] }).mediaMaps ?? []) {
Expand Down
16 changes: 13 additions & 3 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1622,7 +1622,10 @@ export class UnifiedStore {
const exportPath = join(opts.archiveDir, UnifiedStore.ARCHIVE_LOG_NAME);
if (row.status === "archived") return { ok: true, exportPath };
// --- read everything the session owns (before any mutation) ---
const messages = (this.db.prepare("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq").all(sessionId) as Record<string, unknown>[]);
// #4665: carry the v8 row state (identity JOIN key + historian
// tombstone) across archive/restore — dropping them made restored
// rows re-mint on the next full ingest and resurrect retired rows.
const messages = (this.db.prepare("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq").all(sessionId) as Record<string, unknown>[]);
const mediaMaps = (this.db.prepare("SELECT seq, media_hash FROM message_media WHERE session_id = ?").all(sessionId) as Record<string, unknown>[]);
const blocks = (this.db.prepare("SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json FROM blocks WHERE session_id = ? ORDER BY ord").all(sessionId) as Record<string, unknown>[]);
const refsRows = (this.db.prepare("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord").all(sessionId) as Record<string, unknown>[]);
Expand Down Expand Up @@ -1722,8 +1725,15 @@ export class UnifiedStore {
const sess: NonNullable<typeof dump.session> = dump.session;
this.db.transaction(() => {
for (const [hash, body] of Object.entries(dump.content)) this.putContent(Buffer.from(body, "utf8"), hash);
const insMsg = this.db.prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
for (const m of dump.messages) insMsg.run(sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash);
const insMsg = this.db.prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
for (const m of dump.messages) {
// #4665: pre-fix dumps lack both fields — recompute identity
// from the CAS body (same formula as the v8 migration
// backfill); retire state is unrecoverable there → live.
const stored = typeof m.content_hash === "string" ? dump.content[m.content_hash] : undefined;
const identity = typeof m.identity === "string" && m.identity !== "" ? m.identity : stored !== undefined ? identityOfStoredContent(stored) : "";
insMsg.run(sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, identity, typeof m.retired_at === "number" ? m.retired_at : null);
}
const insMedia = this.db.prepare("INSERT OR IGNORE INTO message_media (session_id, seq, media_hash) VALUES (?, ?, ?)");
for (const mm of (dump as { mediaMaps?: Record<string, unknown>[] }).mediaMaps ?? []) insMedia.run(sessionId, mm.seq, mm.media_hash);
const insBlock = this.db.prepare(`INSERT INTO blocks (block_id, session_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json)
Expand Down
247 changes: 247 additions & 0 deletions tests/issue4665-archive-restore-identity.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,247 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts";
import { UnifiedStore } from "../src/storage/store.ts";
import { identityOfStoredContent } from "../src/storage/identity.ts";
import { openBackend, type StorageBackend } from "../src/storage/backend.ts";

/**
* #4665 — archive/restore must carry the v8 row state (#4390) across:
*
* The dump SELECT and restore INSERT predated the identity/retired_at
* columns, so every restored row landed with identity='' (the full-ingest
* JOIN key — which #4390 moved off content_hash — can never match again,
* so the next full save re-mints every message at a fresh seq beside the
* restored copy = duplicate messages until the sweep converges) and with
* retired_at=NULL (a rolled-back row resurrects as a live message).
*
* Pinned here: the JSONL dump carries both fields verbatim; restore writes
* them back; a re-ingest after restore is seq-stable (no re-mint); a
* pre-fix dump lacking both fields falls back to recomputing identity from
* the CAS body (the v8 migration backfill formula).
*/

process.env.BILI_PERSIST_ZSTD = "0";
process.env.BILI_STORAGE_UNIFIED = "full";
process.env.BILI_CONFIG_FILE = path.join(
fs.mkdtempSync(path.join(os.tmpdir(), "bili-issue4665-")),
"no-config.json",
);

interface CoreMsg { id: string; role: string; contentType: string; text: string }
const mkMsg = (id: string, text: string): CoreMsg => ({ id, role: "user", contentType: "text", text });

const mkSession = (id: string, msgs: CoreMsg[]): LegacySessionLike => ({
id,
createdAt: 1_000,
lastSeen: 2_000,
meta: { protocol: "openai" },
stats: {},
metadata: { label: `label-${id}` },
state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } },
blockContents: new Map(),
lastMessages: msgs as unknown as LegacySessionLike["lastMessages"],
lastMessagesFolded: false,
pluginSnapshot: undefined,
contentStore: undefined,
} as unknown as LegacySessionLike);

type Row = { seq: number; msg_id: string; identity: string; retired_at: number | null };

function rowsOf(store: UnifiedStore, sessionId: string): Row[] {
const db = (store as unknown as { db: { prepare: (q: string) => { all: (...a: unknown[]) => unknown[] } } }).db;
return db
.prepare("SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq")
.all(sessionId) as Row[];
}

type DumpLine = {
sessionId: string;
content: Record<string, string>;
messages: Record<string, unknown>[];
};

function readDumpLine(exportPath: string, sessionId: string): DumpLine {
const lines = fs.readFileSync(exportPath, "utf8").split("\n").filter((l) => l.trim());
for (const l of lines) {
const parsed = JSON.parse(l) as DumpLine & { sessionId?: string };
if (parsed.sessionId === sessionId) return parsed;
}
throw new Error(`no dump line for ${sessionId}`);
}

test("#4665: dump carries identity + retired_at; restore round-trips both; re-ingest stays seq-stable", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-rt-"));
try {
const store = UnifiedStore.open(path.join(dir, "index.db"));
const sid = "s4665-rt";
const a = mkMsg("m-a", "alpha ".repeat(30));
const b = mkMsg("m-b", "bravo ".repeat(30));
const c = mkMsg("m-c", "charlie ".repeat(30));
store.ingestSession(planIngest(mkSession(sid, [a, b, c])), undefined, { completeLog: true });
// Head truncation retires m-a (historian tombstone, #4390).
store.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true });

const before = rowsOf(store, sid);
assert.equal(before.length, 3, "3 rows: 2 live + 1 retired");
for (const r of before) assert.ok(r.identity.length > 0, `${r.msg_id} has a real identity pre-archive`);
const retA = before.find((r) => r.msg_id === "m-a")!;
assert.ok(retA.retired_at !== null, "m-a is retired pre-archive");
assert.equal(before.find((r) => r.msg_id === "m-b")!.retired_at, null, "m-b live pre-archive");

const archiveDir = path.join(dir, "archive");
assert.ok(store.archiveSession(sid, { archiveDir }).ok);
const line = readDumpLine(path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME), sid);
assert.equal(line.messages.length, 3, "dump carries retired rows too (historian bytes stay)");
for (const m of line.messages) {
const src = before.find((r) => r.msg_id === m.msg_id)!;
assert.equal(m.identity, src.identity, `${m.msg_id}: dump carries the stored identity`);
assert.equal(m.retired_at ?? null, src.retired_at, `${m.msg_id}: dump carries the retire state`);
}

assert.ok(store.restoreArchived(sid, { archiveDir }).ok);
const after = rowsOf(store, sid);
assert.equal(after.length, 3, "restore brings exactly the archived rows back");
for (const r of before) {
const back = after.find((x) => x.msg_id === r.msg_id)!;
assert.equal(back.seq, r.seq, `${r.msg_id}: original seq`);
assert.equal(back.identity, r.identity, `${r.msg_id}: identity survives archive/restore`);
assert.equal(back.retired_at, r.retired_at, `${r.msg_id}: retire state survives archive/restore`);
}
// The replay surface agrees: m-a stays out, no resurrection.
assert.deepEqual(
store.getMessages(sid).map((r) => r.msgId),
["m-b", "m-c"],
"restored retired row does NOT resurrect into the live view",
);

// The actual bug: with restored identities intact the next full
// ingest JOINs onto the same rows — no re-mint, no duplicate window.
store.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true });
const stable = rowsOf(store, sid);
assert.equal(stable.length, 3, "re-ingest after restore mints nothing new");
for (const r of before) {
const now = stable.find((x) => x.msg_id === r.msg_id)!;
assert.equal(now.seq, r.seq, `${r.msg_id}: seq unchanged across archive→restore→re-ingest`);
}
store.close();
} finally {
try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ }
}
});

test("#4665: pre-fix dump (no identity/retired_at fields) restores via recomputed identity", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-legacy-"));
try {
const store = UnifiedStore.open(path.join(dir, "index.db"));
const sid = "s4665-legacy";
const x = mkMsg("m-x", "x-ray ".repeat(30));
const y = mkMsg("m-y", "yankee ".repeat(30));
store.ingestSession(planIngest(mkSession(sid, [x, y])), undefined, { completeLog: true });

const archiveDir = path.join(dir, "archive");
assert.ok(store.archiveSession(sid, { archiveDir }).ok);
const logPath = path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME);
// Simulate an archive written before this fix: strip both fields.
const stripped = fs
.readFileSync(logPath, "utf8")
.split("\n")
.map((l) => {
if (!l.trim()) return l;
const parsed = JSON.parse(l) as DumpLine;
if (parsed.sessionId !== sid) return l;
for (const m of parsed.messages) { delete m.identity; delete m.retired_at; }
return JSON.stringify(parsed);
})
.join("\n");
fs.writeFileSync(logPath, stripped);
// Capture BEFORE restore: the restore rewrite drops the restored
// session's line from the log.
const line = readDumpLine(logPath, sid);
assert.ok(!("identity" in line.messages[0]), "fixture really lacks the fields");

assert.ok(store.restoreArchived(sid, { archiveDir }).ok);
const rows = rowsOf(store, sid);
assert.equal(rows.length, 2);
for (const r of rows) {
const dumped = line.messages.find((m) => m.msg_id === r.msg_id)!;
const expected = identityOfStoredContent(line.content[dumped.content_hash as string]);
assert.equal(r.identity, expected, `${r.msg_id}: legacy dump falls back to the CAS-body identity`);
assert.equal(r.retired_at, null, `${r.msg_id}: legacy dumps cannot recover retire state → live (documented boundary)`);
}
// The fallback re-connects the JOIN: a full re-ingest lands on the
// same rows instead of minting duplicates.
store.ingestSession(planIngest(mkSession(sid, [x, y])), undefined, { completeLog: true });
const stable = rowsOf(store, sid);
assert.equal(stable.length, 2, "legacy-restore re-ingest mints nothing new");
assert.deepEqual(
stable.map((r) => [r.msg_id, r.seq]),
rows.map((r) => [r.msg_id, r.seq]),
"seqs stable after the fallback restore",
);
store.close();
} finally {
try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ }
}
});

const MYSQL_DSN = process.env.BILI_MYSQL_TEST_DSN ?? "";

test("mysql lane: #4665 round-trip carries identity + retired_at", { skip: MYSQL_DSN.length === 0 ? "set BILI_MYSQL_TEST_DSN to run" : false }, async () => {
// The interface hides includeRetired/retiredAt; the concrete backend
// exposes them (same seam the contract suite asserts through).
const { MysqlBackend } = await import("../src/storage/mysql.ts");
const opened = await openBackend("mysql", MYSQL_DSN);
assert.ok(opened instanceof MysqlBackend, "mysql lane opens the real backend");
const backend = opened;
await backend.init?.();
try {
const sid = `s4665-${Date.now()}`;
const a = mkMsg("m-a", "alpha ".repeat(30));
const b = mkMsg("m-b", "bravo ".repeat(30));
const c = mkMsg("m-c", "charlie ".repeat(30));
assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [a, b, c])), undefined, { completeLog: true }));
assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true }));

const archiveDir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-mysql-"));
try {
const archived = await backend.archiveSession(sid, { archiveDir });
assert.ok(archived.ok && archived.exportPath, `archive ok (${archived.reason ?? ""})`);
const line = readDumpLine(archived.exportPath!, sid);
assert.equal(line.messages.length, 3, "mysql dump carries retired rows too");
for (const m of line.messages) {
assert.equal(typeof m.identity, "string", `${m.msg_id}: mysql dump carries identity`);
assert.ok((m.identity as string).length > 0, `${m.msg_id}: non-empty identity`);
assert.ok("retired_at" in m, `${m.msg_id}: mysql dump carries retired_at`);
}

assert.ok(await backend.restoreArchived(sid, { archiveDir }), "restore ok");
const all = await backend.getMessages(sid, { includeRetired: true });
assert.equal(all.length, 3, "all rows back incl. the retired one");
const retA = all.find((r) => r.msgId === "m-a")!;
assert.ok(retA.retiredAt !== null, "retired state survives the mysql round-trip");
assert.deepEqual(
(await backend.getMessages(sid)).map((r) => r.msgId),
["m-b", "m-c"],
"live view excludes the restored retired row",
);
// Re-ingest stability: the JOIN reconnects, nothing re-minted.
assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true }));
const stable = await backend.getMessages(sid, { includeRetired: true });
assert.equal(stable.length, 3, "mysql re-ingest after restore mints nothing new");
assert.deepEqual(
stable.map((r) => [r.seq, r.msgId]).sort(),
all.map((r) => [r.seq, r.msgId]).sort(),
"seqs unchanged across archive→restore→re-ingest",
);
} finally {
fs.rmSync(archiveDir, { recursive: true, force: true });
}
await backend.deleteSession(sid);
} finally {
await backend.close();
}
});
Loading