diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index d1ba085308..0a165573a3 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -611,14 +611,29 @@ interface LegacyView { type BlockContents = LegacyView["blockContents"]; -/** Shared by full-view and resident-field reads; each page is released after parsing. */ -export function appendBlockContents(contents: BlockContents, rows: Pick[]): void { +/** Shared by full-view and resident-field reads; each page is released after + * parsing. #7337: a corrupt block summary/one CAS row is skipped and returned + * (not thrown) so one bad row cannot abort the whole load — before this it + * threw out of finish()/readResidentFieldsFor and the session fell back to the + * legacy file (or was discarded when none exists). */ +export function appendBlockContents( + contents: BlockContents, + rows: Pick[], +): { blockId: string; error: string }[] { + const skipped: { blockId: string; error: string }[] = []; for (const b of rows) { - contents.set(b.blockId, { - one: b.one === null ? null : (JSON.parse(b.one) as BlockView), - full: JSON.parse(b.summary) as BlockView, - }); + let one: BlockView | null; + let full: BlockView; + try { + one = b.one === null ? null : (JSON.parse(b.one) as BlockView); + full = JSON.parse(b.summary) as BlockView; + } catch (err) { + skipped.push({ blockId: b.blockId, error: String(err) }); + continue; + } + contents.set(b.blockId, { one, full }); } + return skipped; } export function mergeOrphanBlockContents(contents: BlockContents, orphansJson: string | null | undefined): void { @@ -647,6 +662,11 @@ export class LegacyViewBuilder { private readonly blocks: unknown[] = []; private readonly blockContents: BlockContents = new Map(); private readonly messagesBySeq = new Map(); + /** #7335 poison-row isolation: rows whose stored bytes no longer parse as + * JSON are skipped and recorded here instead of aborting the whole load — + * before this, one bad row threw out of finish() and the session was + * rebuilt from scratch (all blocks and refs discarded). */ + readonly skippedCorruptRows: { table: "messages" | "blocks"; key: string; error: string }[] = []; constructor( private readonly session: LegacyViewRows["session"], @@ -697,12 +717,26 @@ export class LegacyViewBuilder { } addBlocks(rows: LegacyViewRows["blocks"]): void { - for (const b of rows) this.blocks.push(JSON.parse(b.payloadJson)); - appendBlockContents(this.blockContents, rows); + for (const b of rows) { + try { + this.blocks.push(JSON.parse(b.payloadJson)); + } catch (err) { + this.skippedCorruptRows.push({ table: "blocks", key: String(b.blockId), error: String(err) }); + } + } + for (const s of appendBlockContents(this.blockContents, rows)) { + this.skippedCorruptRows.push({ table: "blocks", key: String(s.blockId), error: s.error }); + } } addMessages(rows: LegacyViewRows["messages"]): void { - for (const m of rows) this.messagesBySeq.set(m.seq, JSON.parse(m.content) as CoreMessage); + for (const m of rows) { + try { + this.messagesBySeq.set(m.seq, JSON.parse(m.content) as CoreMessage); + } catch (err) { + this.skippedCorruptRows.push({ table: "messages", key: `seq=${m.seq}`, error: String(err) }); + } + } } finish(): LegacyView { diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index 0fcb26775a..e082a0a890 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -596,7 +596,15 @@ export class MysqlBackend implements StorageBackend { /** Atomic refcount-aware CAS insert-or-bump. */ private async putContent(c: PoolConnection | Pool, content: string, precomputedHash?: string): Promise { const bytes = Buffer.from(content, "utf8"); - const hash = precomputedHash ?? sha256Hex(bytes); + // #7335: verify the caller-supplied digest against the bytes — the CAS + // key is what every reader resolves by, so a mismatched pair poisons + // the store (sqlite twin has the same guard). + const hash = sha256Hex(bytes); + if (precomputedHash !== undefined && precomputedHash !== hash) { + throw new Error( + `content CAS digest mismatch: caller supplied ${precomputedHash} but ${bytes.byteLength} bytes hash to ${hash} — refusing to persist an unverified row`, + ); + } await c.query( "INSERT INTO content (hash, byte_len, refcount, algo, bytes) VALUES (?, ?, 1, 'sha256', ?) " + "ON DUPLICATE KEY UPDATE refcount = refcount + 1", diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index c70be67ba9..892c67cebb 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -727,6 +727,7 @@ export class UnifiedShadow { for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) { builder.addMessages(store.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) })); } + warnCorruptSkips(id, builder); // The builder preserves every MERGE_STATE_FIELDS key; its // generic state declaration only needs widening here. return builder.finish() as LegacySessionLike; @@ -756,12 +757,21 @@ export class UnifiedShadow { } if (fields.blockContents) { const contents: NonNullable = new Map(); + const skipped: { blockId: string; error: string }[] = []; for (let offset = 0; ; offset += VIEW_READ_BATCH_SIZE) { const page = store.getBlockContents(id, { limit: VIEW_READ_BATCH_SIZE, offset }); const blocks = Array.isArray(page) ? page : []; - appendBlockContents(contents, blocks); + for (const s of appendBlockContents(contents, blocks)) skipped.push(s); if (blocks.length < VIEW_READ_BATCH_SIZE) break; } + if (skipped.length > 0) { + loggerLog( + "warn", + `[storage] resident block contents degraded for ${id}: ${skipped.length} unparseable row(s) skipped (${skipped + .map((s) => `${s.blockId}: ${s.error}`) + .join("; ")}); loaded without them (#7337)`, + ); + } mergeOrphanBlockContents(contents, store.getOrphansJson(id)); result.blockContents = contents; } @@ -1312,6 +1322,7 @@ export class UnifiedShadow { for (let offset = 0; offset < builder.messageSeqs.length; offset += VIEW_READ_BATCH_SIZE) { builder.addMessages(await reader.getMessages(id, { seqs: builder.messageSeqs.slice(offset, offset + VIEW_READ_BATCH_SIZE) })); } + warnCorruptSkips(id, builder); return builder.finish() as LegacySessionLike; }); if (!loaded) return null; @@ -1537,6 +1548,25 @@ function warnTailGaps(id: string, view: { tailGaps?: number }): void { } } +/** #7335: poison-row isolation report — unparseable rows the builder skipped + * so a single corrupt row cannot abort the whole load (previously one bad + * row threw out of finish(), the session was rebuilt from scratch and every + * block and ref was discarded). */ +function warnCorruptSkips( + id: string, + builder: { skippedCorruptRows: { table: string; key: string; error: string }[] }, +): void { + const rows = builder.skippedCorruptRows; + if (rows.length > 0) { + loggerLog( + "warn", + `[storage] unified view degraded for ${id}: ${rows.length} unparseable row(s) skipped (${rows + .map((r) => `${r.table} ${r.key}: ${r.error}`) + .join("; ")}); the session loaded without them (#7335)`, + ); + } +} + function shadowDigest(view: unknown): string { return canonicalDigest(view); } @@ -1620,7 +1650,17 @@ class BatchedViewVerification { addMessages(rows: MessageRow[]): void { for (const row of rows) { - const raw = JSON.parse(row.content); + let raw: unknown; + try { + raw = JSON.parse(row.content); + } catch { + // #7335: a poisoned row must not abort the whole save — the + // verify-sample throw retried on every subsequent save until + // the store was cleaned. Skip it and mark drift so the + // comparison surfaces the disagreement loudly instead. + this.changed("messages"); + continue; + } this.messages.set(row.seq, { sig: messageSignature(raw), raw }); } } diff --git a/src/storage/store.ts b/src/storage/store.ts index e525a24202..51b5d169c8 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -959,11 +959,17 @@ export class UnifiedStore { /** Content CAS: insert-if-absent + refcount bump; returns hash. */ private putContent(content: string | Buffer, precomputedHash?: string): string { const bytes = typeof content === "string" ? Buffer.from(content, "utf8") : content; - // #3133: planner-produced rows carry the digest from their intern or - // delta pass — skip the per-message re-hash here (same value: sha256 - // over the same utf8 bytes). Only direct callers without a planner - // pay the hash at this layer. - const hash = precomputedHash ?? sha256Hex(bytes); + // #3133 kept the planner digest as the CAS key to skip the per-message + // re-hash; #7335 turns the same digest into a VERIFIED one — the CAS + // key is what every reader resolves by, so a caller-supplied hash that + // does not match its bytes silently poisons the store (observed: 26 + // mismatched rows from an older dev build killed an entire session). + const hash = sha256Hex(bytes); + if (precomputedHash !== undefined && precomputedHash !== hash) { + throw new Error( + `content CAS digest mismatch: caller supplied ${precomputedHash} but ${bytes.byteLength} bytes hash to ${hash} — refusing to persist an unverified row`, + ); + } const existing = this.db.prepare("SELECT refcount FROM content WHERE hash = ?").get(hash) as | { refcount: number } | undefined; diff --git a/tests/storage-poison-row-isolation.test.ts b/tests/storage-poison-row-isolation.test.ts new file mode 100644 index 0000000000..a2ce776033 --- /dev/null +++ b/tests/storage-poison-row-isolation.test.ts @@ -0,0 +1,170 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test, type TestContext } from "node:test"; +import { openSqliteDatabase } from "../src/storage/driver.ts"; +import { appendBlockContents, planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import type { BlockView } from "../src/session.js"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; + +// #7335: one corrupt row must not destroy a session. Field data: 26 content +// rows of a single pi session (poisoned by an older dev build — CAS hash did +// not match the stored bytes, JSON unparseable) made the unified cold load +// throw, the legacy fallback found no file, the state was discarded and a +// 600K-token context re-folded from scratch (32 serial summary rounds, +// 67–84s each — user-visible freeze). Two structural gaps fixed here: +// 1. writer: putContent trusted caller-supplied digests without verifying +// them against the bytes (#3133 skipped the re-hash); a mismatched pair +// now fails the save loudly instead of persisting poison. +// 2. reader: LegacyViewBuilder addBlocks/addMessages parsed every row +// without isolation; an unparseable row now degrades the view (recorded +// in skippedCorruptRows, counted into tailGaps, logged) instead of +// aborting the whole load. + +function seed(id: string): LegacySessionLike { + const messages = Array.from({ length: 4 }, (_, i) => ({ id: `m${i}`, role: "user" as const, contentType: "text" as const, text: `original ${i}`, ts: 1000 + i })); + const blocks: LegacySessionLike["state"]["blocks"] = messages.map((message, i) => ({ + blockId: `b${i}`, runId: "r1", tier: 1, summary: `summary ${i}`, directMessageIds: [message.id], + effectiveMessageIds: [message.id], directBlockIds: [], compressedTokens: 1, createdAt: 1, + survivedCount: 0, generation: "young", active: i % 2 === 0, + })); + return { + id, createdAt: 1, lastSeen: 2, meta: {}, stats: {}, metadata: {}, + state: { blocks, nextBlockId: 5, nextRunId: 2, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(blocks.map((block, i) => [block.blockId, { + full: { text: `full 海🌊 ${i}`, count: 1 }, one: { text: `one ${i}`, count: 1 }, + }])), lastMessages: messages, lastMessagesFolded: false, + }; +} + +function fixture(t: TestContext): { store: UnifiedStore; db: ReturnType["db"]; dir: string } { + const dir = mkdtempSync(join(tmpdir(), "bili-poison-row-")); + process.env.BILI_CONFIG_FILE = join(dir, "no-config.json"); + const store = UnifiedStore.open(join(dir, "index.db")); + const db = openSqliteDatabase(join(dir, "index.db"), "node:sqlite").db; + t.after(() => { store.close(); db.close(); rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); }); + return { store, db, dir }; +} + +test("writer: mismatched precomputed digest is refused, nothing persists", (t) => { + const { store, db } = fixture(t); + const clean = planIngest(seed("target")); + assert.equal(store.ingestSession(clean, undefined, { completeLog: true }), true); + + const poisoned = planIngest(seed("target2")); + assert.ok(poisoned.messages.length > 0); + poisoned.messages[0] = { ...poisoned.messages[0], contentHash: "0".repeat(64) }; + assert.throws( + () => store.ingestSession(poisoned, undefined, { completeLog: true }), + /content CAS digest mismatch/, + ); + // The failed save must leave no row behind (transaction rolled back). + const rows = db.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ?").get("target2") as { n: number }; + assert.equal(rows.n, 0); + const cas = db.prepare("SELECT COUNT(*) AS n FROM content WHERE hash = ?").get("0".repeat(64)) as { n: number }; + assert.equal(cas.n, 0); +}); + +test("reader: one unparseable message row degrades the view instead of aborting the load", (t) => { + const { store, db, dir } = fixture(t); + const plan = planIngest(seed("target")); + assert.equal(store.ingestSession(plan, undefined, { completeLog: true }), true); + + const row = db.prepare("SELECT seq, content_hash FROM messages WHERE session_id = ? ORDER BY seq DESC LIMIT 1").get("target") as { seq: number; content_hash: string }; + db.prepare("UPDATE content SET bytes = X'7b6f6f7073' WHERE hash = ?").run(row.content_hash); + + const shadow = UnifiedShadow.maybeOpen(dir, { ...process.env, BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + t.after(() => shadow.close()); + const view = shadow.viewFor("target") as LegacySessionLike & { tailGaps?: number } | null; + assert.ok(view, "the session must still load with one corrupt row"); + // The corrupt row is skipped; everything else survives. + assert.ok((view!.tailGaps ?? 0) >= 1, "corrupt tail row counted as a gap"); + assert.equal(view!.lastMessages!.length, 3); + assert.equal(view!.state.blocks!.length, 4, "blocks survive a corrupt message row"); +}); + +test("reader: one unparseable block payload degrades the view instead of aborting the load", (t) => { + const { store, db, dir } = fixture(t); + const plan = planIngest(seed("target")); + assert.equal(store.ingestSession(plan, undefined, { completeLog: true }), true); + + db.prepare("UPDATE blocks SET payload_json = X'7b6f6f7073' WHERE session_id = ? AND block_id = ?").run("target", "b0"); + + const shadow = UnifiedShadow.maybeOpen(dir, { ...process.env, BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + t.after(() => shadow.close()); + const view = shadow.viewFor("target") as LegacySessionLike & { tailGaps?: number } | null; + assert.ok(view, "the session must still load with one corrupt block row"); + assert.equal(view!.lastMessages!.length, 4, "messages survive a corrupt block row"); + assert.equal(view!.state.blocks!.length, 3); + // Block summaries live in the content CAS (summary_hash/one_hash), not in + // payload_json — they survive the payload corruption. + assert.equal(view!.blockContents.size, 4); +}); + +// #7337: the block summary/one CAS content (appendBlockContents) was the one +// poison carrier #7335 left unisolated. A corrupt summary/one row — bad CAS +// bytes, or a missing CAS row leaving summary="" — threw out of the shared +// helper and aborted the whole load on BOTH the full-view and resident-field +// paths. Now it degrades the same way as the message/payload rows: the row is +// skipped, recorded, and the rest of the session loads. + +test("appendBlockContents: corrupt summary/one CAS rows are skipped and returned, not thrown", (t) => { + const contents = new Map(); + const skipped = appendBlockContents(contents, [ + { blockId: "ok", summary: '{"text":"full","count":1}', one: '{"text":"one","count":1}' }, + { blockId: "bad-summary", summary: '{"oops', one: '{"text":"one","count":1}' }, + { blockId: "empty-summary", summary: "", one: '{"text":"one","count":1}' }, + { blockId: "bad-one", summary: '{"text":"full","count":1}', one: '{"oops' }, + { blockId: "missing-one", summary: '{"text":"full","count":1}', one: null }, + ]); + // The three corrupt rows are skipped and returned; the two good rows land in the map. + assert.equal(skipped.length, 3); + assert.deepEqual(skipped.map((s) => s.blockId).sort(), ["bad-one", "bad-summary", "empty-summary"]); + assert.ok(skipped.every((s) => s.error.length > 0)); + assert.equal(contents.size, 2); + assert.ok(contents.has("ok")); + assert.ok(contents.has("missing-one")); + assert.equal((contents.get("missing-one") as { one: unknown }).one, null); +}); + +test("reader: one unparseable block summary CAS row degrades the full view instead of aborting", (t) => { + const { store, db, dir } = fixture(t); + const plan = planIngest(seed("target")); + assert.equal(store.ingestSession(plan, undefined, { completeLog: true }), true); + + // Poison the block summary's CAS bytes (the full/one content — NOT payload_json). + const row = db.prepare("SELECT summary_hash FROM blocks WHERE session_id = ? AND block_id = ?").get("target", "b0") as { summary_hash: string }; + db.prepare("UPDATE content SET bytes = X'7b6f6f7073' WHERE hash = ?").run(row.summary_hash); + + const shadow = UnifiedShadow.maybeOpen(dir, { ...process.env, BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + t.after(() => shadow.close()); + const view = shadow.viewFor("target") as LegacySessionLike & { tailGaps?: number } | null; + assert.ok(view, "the session must still load with one corrupt block summary CAS row"); + assert.equal(view!.state.blocks!.length, 4, "block metadata survives a corrupt summary CAS row"); + assert.equal(view!.lastMessages!.length, 4, "messages survive a corrupt summary CAS row"); + assert.equal(view!.blockContents.size, 3, "the corrupt block's content row is skipped"); + assert.ok(!view!.blockContents.has("b0"), "the corrupt block's content is dropped"); +}); + +test("reader: one unparseable block summary CAS row degrades the resident-field read instead of aborting", (t) => { + const { store, db, dir } = fixture(t); + const plan = planIngest(seed("target")); + assert.equal(store.ingestSession(plan, undefined, { completeLog: true }), true); + + const row = db.prepare("SELECT summary_hash FROM blocks WHERE session_id = ? AND block_id = ?").get("target", "b0") as { summary_hash: string }; + db.prepare("UPDATE content SET bytes = X'7b6f6f7073' WHERE hash = ?").run(row.summary_hash); + + const shadow = UnifiedShadow.maybeOpen(dir, { ...process.env, BILI_STORAGE_UNIFIED: "full" }); + assert.ok(shadow); + t.after(() => shadow.close()); + const fields = shadow.readResidentFieldsFor("target", { blockContents: true }); + assert.ok(fields, "the resident fields must still load with one corrupt block summary CAS row"); + assert.equal(fields!.blockContents!.size, 3, "the corrupt block's content row is skipped"); + assert.ok(!fields!.blockContents!.has("b0"), "the corrupt block's content is dropped"); + assert.ok(fields!.blockContents!.has("b1"), "the other blocks' content survives"); +});