diff --git a/kernel/package-lock.json b/kernel/package-lock.json index da011d94e1..4c5e2d79e9 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.164", + "version": "0.0.165", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.164", + "version": "0.0.165", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/package-lock.json b/package-lock.json index 380212c96e..15a6e3f4e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.163", + "version": "0.0.165", "dev": true, "license": "MIT", "devDependencies": { diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index dd7fc1195f..44f9af4adf 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1219,7 +1219,9 @@ export class MysqlBackend implements StorageBackend { const exportPath = join(opts.archiveDir, MysqlBackend.ARCHIVE_LOG_NAME); if (row.status === "archived") return { ok: true, exportPath }; const q = async (sql: string): Promise => ((await this.pool.query(sql, [sessionId])) as unknown as Row[][])[0]; - const messages = await q("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq"); + // #4665: carry the v8 row state (identity JOIN key + historian + // tombstone) across archive/restore — sqlite twin, same reason. + const messages = await q("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq"); const mediaMaps = await q("SELECT seq, media_hash FROM message_media WHERE session_id = ?"); const blocks = await q("SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json FROM blocks WHERE session_id = ? ORDER BY ord"); const refsRows = await q("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord"); @@ -1321,9 +1323,14 @@ export class MysqlBackend implements StorageBackend { await this.tx(async (c) => { for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash); for (const m of dump.messages) { + // #4665: pre-fix dumps lack both fields — recompute identity + // from the CAS body (same formula as the v8 migration + // backfill); retire state is unrecoverable there → live. + const stored = typeof m.content_hash === "string" ? dump.content[m.content_hash] : undefined; + const identity = typeof m.identity === "string" && m.identity !== "" ? m.identity : stored !== undefined ? identityOfStoredContent(stored) : ""; await c.query( - "INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - [sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, m.identity ?? ""], + "INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, identity, typeof m.retired_at === "number" ? m.retired_at : null], ); } for (const mm of (dump as { mediaMaps?: Record[] }).mediaMaps ?? []) { diff --git a/src/storage/store.ts b/src/storage/store.ts index 234958e002..4a21d57223 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1622,7 +1622,10 @@ export class UnifiedStore { const exportPath = join(opts.archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); if (row.status === "archived") return { ok: true, exportPath }; // --- read everything the session owns (before any mutation) --- - const messages = (this.db.prepare("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq").all(sessionId) as Record[]); + // #4665: carry the v8 row state (identity JOIN key + historian + // tombstone) across archive/restore — dropping them made restored + // rows re-mint on the next full ingest and resurrect retired rows. + const messages = (this.db.prepare("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq").all(sessionId) as Record[]); const mediaMaps = (this.db.prepare("SELECT seq, media_hash FROM message_media WHERE session_id = ?").all(sessionId) as Record[]); const blocks = (this.db.prepare("SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json FROM blocks WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); const refsRows = (this.db.prepare("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); @@ -1722,8 +1725,15 @@ export class UnifiedStore { const sess: NonNullable = dump.session; this.db.transaction(() => { for (const [hash, body] of Object.entries(dump.content)) this.putContent(Buffer.from(body, "utf8"), hash); - const insMsg = this.db.prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); - for (const m of dump.messages) insMsg.run(sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash); + const insMsg = this.db.prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + for (const m of dump.messages) { + // #4665: pre-fix dumps lack both fields — recompute identity + // from the CAS body (same formula as the v8 migration + // backfill); retire state is unrecoverable there → live. + const stored = typeof m.content_hash === "string" ? dump.content[m.content_hash] : undefined; + const identity = typeof m.identity === "string" && m.identity !== "" ? m.identity : stored !== undefined ? identityOfStoredContent(stored) : ""; + insMsg.run(sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, identity, typeof m.retired_at === "number" ? m.retired_at : null); + } const insMedia = this.db.prepare("INSERT OR IGNORE INTO message_media (session_id, seq, media_hash) VALUES (?, ?, ?)"); for (const mm of (dump as { mediaMaps?: Record[] }).mediaMaps ?? []) insMedia.run(sessionId, mm.seq, mm.media_hash); const insBlock = this.db.prepare(`INSERT INTO blocks (block_id, session_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json) diff --git a/tests/issue4665-archive-restore-identity.test.ts b/tests/issue4665-archive-restore-identity.test.ts new file mode 100644 index 0000000000..d5e85f9441 --- /dev/null +++ b/tests/issue4665-archive-restore-identity.test.ts @@ -0,0 +1,247 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { identityOfStoredContent } from "../src/storage/identity.ts"; +import { openBackend, type StorageBackend } from "../src/storage/backend.ts"; + +/** + * #4665 — archive/restore must carry the v8 row state (#4390) across: + * + * The dump SELECT and restore INSERT predated the identity/retired_at + * columns, so every restored row landed with identity='' (the full-ingest + * JOIN key — which #4390 moved off content_hash — can never match again, + * so the next full save re-mints every message at a fresh seq beside the + * restored copy = duplicate messages until the sweep converges) and with + * retired_at=NULL (a rolled-back row resurrects as a live message). + * + * Pinned here: the JSONL dump carries both fields verbatim; restore writes + * them back; a re-ingest after restore is seq-stable (no re-mint); a + * pre-fix dump lacking both fields falls back to recomputing identity from + * the CAS body (the v8 migration backfill formula). + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "full"; +process.env.BILI_CONFIG_FILE = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "bili-issue4665-")), + "no-config.json", +); + +interface CoreMsg { id: string; role: string; contentType: string; text: string } +const mkMsg = (id: string, text: string): CoreMsg => ({ id, role: "user", contentType: "text", text }); + +const mkSession = (id: string, msgs: CoreMsg[]): LegacySessionLike => ({ + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "openai" }, + stats: {}, + metadata: { label: `label-${id}` }, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(), + lastMessages: msgs as unknown as LegacySessionLike["lastMessages"], + lastMessagesFolded: false, + pluginSnapshot: undefined, + contentStore: undefined, +} as unknown as LegacySessionLike); + +type Row = { seq: number; msg_id: string; identity: string; retired_at: number | null }; + +function rowsOf(store: UnifiedStore, sessionId: string): Row[] { + const db = (store as unknown as { db: { prepare: (q: string) => { all: (...a: unknown[]) => unknown[] } } }).db; + return db + .prepare("SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq") + .all(sessionId) as Row[]; +} + +type DumpLine = { + sessionId: string; + content: Record; + messages: Record[]; +}; + +function readDumpLine(exportPath: string, sessionId: string): DumpLine { + const lines = fs.readFileSync(exportPath, "utf8").split("\n").filter((l) => l.trim()); + for (const l of lines) { + const parsed = JSON.parse(l) as DumpLine & { sessionId?: string }; + if (parsed.sessionId === sessionId) return parsed; + } + throw new Error(`no dump line for ${sessionId}`); +} + +test("#4665: dump carries identity + retired_at; restore round-trips both; re-ingest stays seq-stable", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-rt-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const sid = "s4665-rt"; + const a = mkMsg("m-a", "alpha ".repeat(30)); + const b = mkMsg("m-b", "bravo ".repeat(30)); + const c = mkMsg("m-c", "charlie ".repeat(30)); + store.ingestSession(planIngest(mkSession(sid, [a, b, c])), undefined, { completeLog: true }); + // Head truncation retires m-a (historian tombstone, #4390). + store.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true }); + + const before = rowsOf(store, sid); + assert.equal(before.length, 3, "3 rows: 2 live + 1 retired"); + for (const r of before) assert.ok(r.identity.length > 0, `${r.msg_id} has a real identity pre-archive`); + const retA = before.find((r) => r.msg_id === "m-a")!; + assert.ok(retA.retired_at !== null, "m-a is retired pre-archive"); + assert.equal(before.find((r) => r.msg_id === "m-b")!.retired_at, null, "m-b live pre-archive"); + + const archiveDir = path.join(dir, "archive"); + assert.ok(store.archiveSession(sid, { archiveDir }).ok); + const line = readDumpLine(path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME), sid); + assert.equal(line.messages.length, 3, "dump carries retired rows too (historian bytes stay)"); + for (const m of line.messages) { + const src = before.find((r) => r.msg_id === m.msg_id)!; + assert.equal(m.identity, src.identity, `${m.msg_id}: dump carries the stored identity`); + assert.equal(m.retired_at ?? null, src.retired_at, `${m.msg_id}: dump carries the retire state`); + } + + assert.ok(store.restoreArchived(sid, { archiveDir }).ok); + const after = rowsOf(store, sid); + assert.equal(after.length, 3, "restore brings exactly the archived rows back"); + for (const r of before) { + const back = after.find((x) => x.msg_id === r.msg_id)!; + assert.equal(back.seq, r.seq, `${r.msg_id}: original seq`); + assert.equal(back.identity, r.identity, `${r.msg_id}: identity survives archive/restore`); + assert.equal(back.retired_at, r.retired_at, `${r.msg_id}: retire state survives archive/restore`); + } + // The replay surface agrees: m-a stays out, no resurrection. + assert.deepEqual( + store.getMessages(sid).map((r) => r.msgId), + ["m-b", "m-c"], + "restored retired row does NOT resurrect into the live view", + ); + + // The actual bug: with restored identities intact the next full + // ingest JOINs onto the same rows — no re-mint, no duplicate window. + store.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true }); + const stable = rowsOf(store, sid); + assert.equal(stable.length, 3, "re-ingest after restore mints nothing new"); + for (const r of before) { + const now = stable.find((x) => x.msg_id === r.msg_id)!; + assert.equal(now.seq, r.seq, `${r.msg_id}: seq unchanged across archive→restore→re-ingest`); + } + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4665: pre-fix dump (no identity/retired_at fields) restores via recomputed identity", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-legacy-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const sid = "s4665-legacy"; + const x = mkMsg("m-x", "x-ray ".repeat(30)); + const y = mkMsg("m-y", "yankee ".repeat(30)); + store.ingestSession(planIngest(mkSession(sid, [x, y])), undefined, { completeLog: true }); + + const archiveDir = path.join(dir, "archive"); + assert.ok(store.archiveSession(sid, { archiveDir }).ok); + const logPath = path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); + // Simulate an archive written before this fix: strip both fields. + const stripped = fs + .readFileSync(logPath, "utf8") + .split("\n") + .map((l) => { + if (!l.trim()) return l; + const parsed = JSON.parse(l) as DumpLine; + if (parsed.sessionId !== sid) return l; + for (const m of parsed.messages) { delete m.identity; delete m.retired_at; } + return JSON.stringify(parsed); + }) + .join("\n"); + fs.writeFileSync(logPath, stripped); + // Capture BEFORE restore: the restore rewrite drops the restored + // session's line from the log. + const line = readDumpLine(logPath, sid); + assert.ok(!("identity" in line.messages[0]), "fixture really lacks the fields"); + + assert.ok(store.restoreArchived(sid, { archiveDir }).ok); + const rows = rowsOf(store, sid); + assert.equal(rows.length, 2); + for (const r of rows) { + const dumped = line.messages.find((m) => m.msg_id === r.msg_id)!; + const expected = identityOfStoredContent(line.content[dumped.content_hash as string]); + assert.equal(r.identity, expected, `${r.msg_id}: legacy dump falls back to the CAS-body identity`); + assert.equal(r.retired_at, null, `${r.msg_id}: legacy dumps cannot recover retire state → live (documented boundary)`); + } + // The fallback re-connects the JOIN: a full re-ingest lands on the + // same rows instead of minting duplicates. + store.ingestSession(planIngest(mkSession(sid, [x, y])), undefined, { completeLog: true }); + const stable = rowsOf(store, sid); + assert.equal(stable.length, 2, "legacy-restore re-ingest mints nothing new"); + assert.deepEqual( + stable.map((r) => [r.msg_id, r.seq]), + rows.map((r) => [r.msg_id, r.seq]), + "seqs stable after the fallback restore", + ); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +const MYSQL_DSN = process.env.BILI_MYSQL_TEST_DSN ?? ""; + +test("mysql lane: #4665 round-trip carries identity + retired_at", { skip: MYSQL_DSN.length === 0 ? "set BILI_MYSQL_TEST_DSN to run" : false }, async () => { + // The interface hides includeRetired/retiredAt; the concrete backend + // exposes them (same seam the contract suite asserts through). + const { MysqlBackend } = await import("../src/storage/mysql.ts"); + const opened = await openBackend("mysql", MYSQL_DSN); + assert.ok(opened instanceof MysqlBackend, "mysql lane opens the real backend"); + const backend = opened; + await backend.init?.(); + try { + const sid = `s4665-${Date.now()}`; + const a = mkMsg("m-a", "alpha ".repeat(30)); + const b = mkMsg("m-b", "bravo ".repeat(30)); + const c = mkMsg("m-c", "charlie ".repeat(30)); + assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [a, b, c])), undefined, { completeLog: true })); + assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true })); + + const archiveDir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-mysql-")); + try { + const archived = await backend.archiveSession(sid, { archiveDir }); + assert.ok(archived.ok && archived.exportPath, `archive ok (${archived.reason ?? ""})`); + const line = readDumpLine(archived.exportPath!, sid); + assert.equal(line.messages.length, 3, "mysql dump carries retired rows too"); + for (const m of line.messages) { + assert.equal(typeof m.identity, "string", `${m.msg_id}: mysql dump carries identity`); + assert.ok((m.identity as string).length > 0, `${m.msg_id}: non-empty identity`); + assert.ok("retired_at" in m, `${m.msg_id}: mysql dump carries retired_at`); + } + + assert.ok(await backend.restoreArchived(sid, { archiveDir }), "restore ok"); + const all = await backend.getMessages(sid, { includeRetired: true }); + assert.equal(all.length, 3, "all rows back incl. the retired one"); + const retA = all.find((r) => r.msgId === "m-a")!; + assert.ok(retA.retiredAt !== null, "retired state survives the mysql round-trip"); + assert.deepEqual( + (await backend.getMessages(sid)).map((r) => r.msgId), + ["m-b", "m-c"], + "live view excludes the restored retired row", + ); + // Re-ingest stability: the JOIN reconnects, nothing re-minted. + assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true })); + const stable = await backend.getMessages(sid, { includeRetired: true }); + assert.equal(stable.length, 3, "mysql re-ingest after restore mints nothing new"); + assert.deepEqual( + stable.map((r) => [r.seq, r.msgId]).sort(), + all.map((r) => [r.seq, r.msgId]).sort(), + "seqs unchanged across archive→restore→re-ingest", + ); + } finally { + fs.rmSync(archiveDir, { recursive: true, force: true }); + } + await backend.deleteSession(sid); + } finally { + await backend.close(); + } +});