From 7270c8e012dc516d3ab71b5ecae03499af979764 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 18:04:55 +0800 Subject: [PATCH 01/35] fix: jsonl export includes retired (rewound) rows via historian read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #4664: since #4390 the unified store's getMessages filters retired_at IS NULL by default (tombstone-based rewind history), so the JSONL export silently dropped every rewound message. The store's own design comment names export a historian caller that must opt into includeRetired — it never did. - widen the StorageBackend contract: getMessages(sessionId, opts?) with includeRetired and retiredAt on every row (both backends already implemented this natively; SqliteBackend now forwards it) - renderSessionJsonlFromTables reads with { includeRetired: true } and emits retiredAt on tombstoned rows only — live rows omit the field, so output for sessions without retirements is byte-identical to before Regression test in tests/export-jsonl.test.ts drives the real ingestSession sweep (full ingest -> truncated re-ingest retires row one). --- src/export.ts | 9 ++++-- src/storage/backend.ts | 9 ++++-- tests/export-jsonl.test.ts | 63 +++++++++++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/src/export.ts b/src/export.ts index 1b52781ae1..05f723a8ae 100644 --- a/src/export.ts +++ b/src/export.ts @@ -201,7 +201,8 @@ function safeJson(raw: string | null | undefined): unknown { /** JSONL dump rendered from the unified tables (#2671 Phase 2c). One JSON * record per line: a session header, then messages ordered by seq (CCR - * placeholders dereferenced to their stored originals when `full`), then + * placeholders dereferenced to their stored originals when `full`; retired + * rows included carrying `retiredAt`, live rows omit it — #4390/#4664), then * blocks ordered by ord, then dead refs. Deterministic for a given store * state; a generated artifact, never a source of truth. */ // #3931: consumes the StorageBackend contract (not UnifiedStore) — the @@ -235,7 +236,9 @@ export async function renderSessionJsonlFromTables(backend: StorageBackend, sess // store by design (multi-MB rewrite cost, see session.ts contentStore). const ccr = new Map((await backend.getCcrEntries(sessionId)).map((e) => [e.key, e.content] as const)); if (extraCcr) for (const [k, v] of extraCcr) if (!ccr.has(k)) ccr.set(k, v); - for (const m of await backend.getMessages(sessionId)) { + // #4664: historian read — include retired (tombstoned) rows so a rewound + // session exports its full history; live rows omit `retiredAt`. + for (const m of await backend.getMessages(sessionId, { includeRetired: true })) { let message: Record; try { message = JSON.parse(m.content) as Record; @@ -247,7 +250,7 @@ export async function renderSessionJsonlFromTables(backend: StorageBackend, sess const payload = ph ? ccr.get(ph.ref) : undefined; if (payload !== undefined) message = { ...message, text: payload, dereferencedFrom: ph!.ref }; } - lines.push(JSON.stringify({ type: "message", seq: m.seq, rawId: m.msgId, ref: m.ref, refNum: m.refNum ?? undefined, role: m.role, kind: m.kind ?? undefined, message })); + lines.push(JSON.stringify({ type: "message", seq: m.seq, rawId: m.msgId, ref: m.ref, refNum: m.refNum ?? undefined, role: m.role, kind: m.kind ?? undefined, ...(m.retiredAt != null ? { retiredAt: m.retiredAt } : {}), message })); } for (const b of await backend.getBlocks(sessionId)) { lines.push( diff --git a/src/storage/backend.ts b/src/storage/backend.ts index dd8473893a..f6c80c890b 100644 --- a/src/storage/backend.ts +++ b/src/storage/backend.ts @@ -73,7 +73,10 @@ export interface StorageBackend { seqByRaw(sessionId: string): Promise | null>; /** #4316 PR3/3: fork adoption row clone (idempotent). */ cloneSessionRows(parentId: string, childId: string, throughSeq: number): Promise; - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]>; + /** #4390/#4664: retired (tombstoned) rows stay out of the unified replay + * by default; historian callers (export/diagnostics) opt in via + * `includeRetired`. Every row carries `retiredAt` (null = live). */ + getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]>; getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]>; getDeadRefs(sessionId: string): Promise<{ ord: number; ref: string; refNum: number; recordedAt: number }[]>; getRefs(sessionId: string): Promise; @@ -186,8 +189,8 @@ export class SqliteBackend implements StorageBackend { this.store.cloneSessionRows(parentId, childId, throughSeq); return Promise.resolve(true); } - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { - return Promise.resolve(this.store.getMessages(sessionId)); + getMessages(sessionId: string, opts?: { includeRetired?: boolean }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { + return Promise.resolve(this.store.getMessages(sessionId, opts)); } getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { return Promise.resolve(this.store.getBlocks(sessionId)); diff --git a/tests/export-jsonl.test.ts b/tests/export-jsonl.test.ts index ef01ca5312..bf7e1bcef2 100644 --- a/tests/export-jsonl.test.ts +++ b/tests/export-jsonl.test.ts @@ -7,7 +7,8 @@ import { SessionStore } from "../src/persist.ts"; import { UnifiedStore } from "../src/storage/store.ts"; import { exportSession, renderSessionJsonlFromTables, renderSessionJsonlFromSession } from "../src/export.ts"; import { SqliteBackend } from "../src/storage/backend.ts"; -import { buildStoredPlaceholder } from "acp-kernel"; +import { buildStoredPlaceholder, createInitialState } from "acp-kernel"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; import type { Session } from "../src/session.ts"; /** @@ -97,6 +98,66 @@ test("--jsonl renders from the unified tables: header, blocks, dead refs (folded }); }); +/** CoreMessage-shaped fixture; wireIndex simulates the positional metadata the + * wire codecs stamp on every parse (mirrors tests/storage-review-p2s.test.ts). */ +function wmsg(id: string, text: string, wireIndex: number) { + return { id, role: "user", contentType: "text", text, wireIndex }; +} + +function histLike(id: string, requests: number, msgs: { id: string; text: string; wireIndex: number }[]): LegacySessionLike { + const state = { ...createInitialState(), requests }; + state.nextBlockId = 1; + return { + id, + createdAt: Date.now(), + lastSeen: Date.now(), + meta: { label: id, protocol: "anthropic", upstreamOrigin: "https://api.anthropic.com" }, + stats: { requests }, + metadata: { pluginAgent: "proxy" }, + state: state as never, + blockContents: new Map() as never, + lastMessages: msgs.map((m) => wmsg(m.id, m.text, m.wireIndex)) as never, + lastMessagesFolded: false, + }; +} + +test("--jsonl includes retired (rewound) rows marked by retiredAt; live rows omit it (#4664)", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-jsonl-retired-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + const msgs = [ + { id: "id-a", text: "alpha", wireIndex: 0 }, + { id: "id-b", text: "bravo", wireIndex: 1 }, + { id: "id-c", text: "charlie", wireIndex: 2 }, + ]; + store.ingestSession(planIngest(histLike("retired-1", 10, msgs)), undefined, { completeLog: true }); + // The client rewound past id-a → the historian sweep tombstones it (#4390). + const rewound = msgs.slice(1).map((m, i) => ({ ...m, wireIndex: i })); + store.ingestSession(planIngest(histLike("retired-1", 11, rewound)), undefined, { completeLog: true }); + + // Precondition: default read hides the retired row, opt-in sees it. + assert.equal(store.getMessages("retired-1").length, 2, "default getMessages filters retired"); + const all = store.getMessages("retired-1", { includeRetired: true }); + assert.equal(all.length, 3); + assert.ok(all[0]!.retiredAt != null, "id-a is retired at the store level"); + + const lines = parseLines(await renderSessionJsonlFromTables(new SqliteBackend(store), "retired-1", false)); + assert.equal(lines[0]!.type, "session"); + const msgsOut = lines.filter((l) => l.type === "message") as Record[]; + assert.equal(msgsOut.length, 3, "the export must include the retired row (#4664)"); + for (let i = 1; i < msgsOut.length; i++) assert.ok((msgsOut[i]!.seq as number) > (msgsOut[i - 1]!.seq as number), "seq strictly increasing across live+retired"); + const a = msgsOut.find((l) => l.rawId === "id-a")!; + assert.ok(typeof a.retiredAt === "number" && (a.retiredAt as number) > 0, "retired row carries retiredAt"); + for (const l of msgsOut) if (l.rawId !== "id-a") assert.equal(l.retiredAt, undefined, "live rows omit retiredAt"); + } finally { + store.close(); + } + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows: an ephemeral stray handle must not fail CI */ } + } +}); + test("--jsonl falls back to the legacy tail when no index exists (deadrefs corpus)", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-jsonl-legacy-")); try { From b9f61124c0b7de7e66cd3cd299f057167a54bbf8 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 18:08:59 +0800 Subject: [PATCH 02/35] fix(#4665): carry identity + retired_at across archive/restore The archive dump SELECT and restore INSERT predated #4390's v8 row-state columns: every restored row landed with identity='' (the full-ingest JOIN key can never match again -> next full save re-mints every message beside the restored copy = duplicate messages) and retired_at=NULL (a rolled-back row resurrects as live). Both backends now carry both fields verbatim; pre-fix dumps lacking them fall back to recomputing identity from the CAS body (the v8 migration backfill formula) and restore retired rows as live (state unrecoverable there). --- src/storage/mysql.ts | 13 +- src/storage/store.ts | 16 +- ...issue4665-archive-restore-identity.test.ts | 247 ++++++++++++++++++ 3 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 tests/issue4665-archive-restore-identity.test.ts diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index f3c46ea373..69f799fa90 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -1168,7 +1168,9 @@ export class MysqlBackend implements StorageBackend { const exportPath = join(opts.archiveDir, MysqlBackend.ARCHIVE_LOG_NAME); if (row.status === "archived") return { ok: true, exportPath }; const q = async (sql: string): Promise => ((await this.pool.query(sql, [sessionId])) as unknown as Row[][])[0]; - const messages = await q("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq"); + // #4665: carry the v8 row state (identity JOIN key + historian + // tombstone) across archive/restore — sqlite twin, same reason. + const messages = await q("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq"); const mediaMaps = await q("SELECT seq, media_hash FROM message_media WHERE session_id = ?"); const blocks = await q("SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json FROM blocks WHERE session_id = ? ORDER BY ord"); const refsRows = await q("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord"); @@ -1270,9 +1272,14 @@ export class MysqlBackend implements StorageBackend { await this.tx(async (c) => { for (const [hash, body] of Object.entries(dump.content)) await this.putContent(c, body, hash); for (const m of dump.messages) { + // #4665: pre-fix dumps lack both fields — recompute identity + // from the CAS body (same formula as the v8 migration + // backfill); retire state is unrecoverable there → live. + const stored = typeof m.content_hash === "string" ? dump.content[m.content_hash] : undefined; + const identity = typeof m.identity === "string" && m.identity !== "" ? m.identity : stored !== undefined ? identityOfStoredContent(stored) : ""; await c.query( - "INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - [sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, m.identity ?? ""], + "INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, identity, typeof m.retired_at === "number" ? m.retired_at : null], ); } for (const mm of (dump as { mediaMaps?: Record[] }).mediaMaps ?? []) { diff --git a/src/storage/store.ts b/src/storage/store.ts index a13acf4edb..b6ba062916 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1542,7 +1542,10 @@ export class UnifiedStore { const exportPath = join(opts.archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); if (row.status === "archived") return { ok: true, exportPath }; // --- read everything the session owns (before any mutation) --- - const messages = (this.db.prepare("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash FROM messages WHERE session_id = ? ORDER BY seq").all(sessionId) as Record[]); + // #4665: carry the v8 row state (identity JOIN key + historian + // tombstone) across archive/restore — dropping them made restored + // rows re-mint on the next full ingest and resurrect retired rows. + const messages = (this.db.prepare("SELECT seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq").all(sessionId) as Record[]); const mediaMaps = (this.db.prepare("SELECT seq, media_hash FROM message_media WHERE session_id = ?").all(sessionId) as Record[]); const blocks = (this.db.prepare("SELECT block_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json FROM blocks WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); const refsRows = (this.db.prepare("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); @@ -1642,8 +1645,15 @@ export class UnifiedStore { const sess: NonNullable = dump.session; this.db.transaction(() => { for (const [hash, body] of Object.entries(dump.content)) this.putContent(Buffer.from(body, "utf8"), hash); - const insMsg = this.db.prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); - for (const m of dump.messages) insMsg.run(sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash); + const insMsg = this.db.prepare("INSERT INTO messages (session_id, seq, msg_id, ref, ref_num, role, kind, content_hash, identity, retired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + for (const m of dump.messages) { + // #4665: pre-fix dumps lack both fields — recompute identity + // from the CAS body (same formula as the v8 migration + // backfill); retire state is unrecoverable there → live. + const stored = typeof m.content_hash === "string" ? dump.content[m.content_hash] : undefined; + const identity = typeof m.identity === "string" && m.identity !== "" ? m.identity : stored !== undefined ? identityOfStoredContent(stored) : ""; + insMsg.run(sessionId, m.seq, m.msg_id, m.ref ?? null, m.ref_num ?? null, m.role, m.kind ?? "message", m.content_hash, identity, typeof m.retired_at === "number" ? m.retired_at : null); + } const insMedia = this.db.prepare("INSERT OR IGNORE INTO message_media (session_id, seq, media_hash) VALUES (?, ?, ?)"); for (const mm of (dump as { mediaMaps?: Record[] }).mediaMaps ?? []) insMedia.run(sessionId, mm.seq, mm.media_hash); const insBlock = this.db.prepare(`INSERT INTO blocks (block_id, session_id, ord, start_seq, end_seq, start_ref, end_ref, tier, summary_hash, one_hash, supersedes_block_id, active, created_at, covered_hash, payload_json) diff --git a/tests/issue4665-archive-restore-identity.test.ts b/tests/issue4665-archive-restore-identity.test.ts new file mode 100644 index 0000000000..d5e85f9441 --- /dev/null +++ b/tests/issue4665-archive-restore-identity.test.ts @@ -0,0 +1,247 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { identityOfStoredContent } from "../src/storage/identity.ts"; +import { openBackend, type StorageBackend } from "../src/storage/backend.ts"; + +/** + * #4665 — archive/restore must carry the v8 row state (#4390) across: + * + * The dump SELECT and restore INSERT predated the identity/retired_at + * columns, so every restored row landed with identity='' (the full-ingest + * JOIN key — which #4390 moved off content_hash — can never match again, + * so the next full save re-mints every message at a fresh seq beside the + * restored copy = duplicate messages until the sweep converges) and with + * retired_at=NULL (a rolled-back row resurrects as a live message). + * + * Pinned here: the JSONL dump carries both fields verbatim; restore writes + * them back; a re-ingest after restore is seq-stable (no re-mint); a + * pre-fix dump lacking both fields falls back to recomputing identity from + * the CAS body (the v8 migration backfill formula). + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "full"; +process.env.BILI_CONFIG_FILE = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "bili-issue4665-")), + "no-config.json", +); + +interface CoreMsg { id: string; role: string; contentType: string; text: string } +const mkMsg = (id: string, text: string): CoreMsg => ({ id, role: "user", contentType: "text", text }); + +const mkSession = (id: string, msgs: CoreMsg[]): LegacySessionLike => ({ + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "openai" }, + stats: {}, + metadata: { label: `label-${id}` }, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(), + lastMessages: msgs as unknown as LegacySessionLike["lastMessages"], + lastMessagesFolded: false, + pluginSnapshot: undefined, + contentStore: undefined, +} as unknown as LegacySessionLike); + +type Row = { seq: number; msg_id: string; identity: string; retired_at: number | null }; + +function rowsOf(store: UnifiedStore, sessionId: string): Row[] { + const db = (store as unknown as { db: { prepare: (q: string) => { all: (...a: unknown[]) => unknown[] } } }).db; + return db + .prepare("SELECT seq, msg_id, identity, retired_at FROM messages WHERE session_id = ? ORDER BY seq") + .all(sessionId) as Row[]; +} + +type DumpLine = { + sessionId: string; + content: Record; + messages: Record[]; +}; + +function readDumpLine(exportPath: string, sessionId: string): DumpLine { + const lines = fs.readFileSync(exportPath, "utf8").split("\n").filter((l) => l.trim()); + for (const l of lines) { + const parsed = JSON.parse(l) as DumpLine & { sessionId?: string }; + if (parsed.sessionId === sessionId) return parsed; + } + throw new Error(`no dump line for ${sessionId}`); +} + +test("#4665: dump carries identity + retired_at; restore round-trips both; re-ingest stays seq-stable", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-rt-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const sid = "s4665-rt"; + const a = mkMsg("m-a", "alpha ".repeat(30)); + const b = mkMsg("m-b", "bravo ".repeat(30)); + const c = mkMsg("m-c", "charlie ".repeat(30)); + store.ingestSession(planIngest(mkSession(sid, [a, b, c])), undefined, { completeLog: true }); + // Head truncation retires m-a (historian tombstone, #4390). + store.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true }); + + const before = rowsOf(store, sid); + assert.equal(before.length, 3, "3 rows: 2 live + 1 retired"); + for (const r of before) assert.ok(r.identity.length > 0, `${r.msg_id} has a real identity pre-archive`); + const retA = before.find((r) => r.msg_id === "m-a")!; + assert.ok(retA.retired_at !== null, "m-a is retired pre-archive"); + assert.equal(before.find((r) => r.msg_id === "m-b")!.retired_at, null, "m-b live pre-archive"); + + const archiveDir = path.join(dir, "archive"); + assert.ok(store.archiveSession(sid, { archiveDir }).ok); + const line = readDumpLine(path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME), sid); + assert.equal(line.messages.length, 3, "dump carries retired rows too (historian bytes stay)"); + for (const m of line.messages) { + const src = before.find((r) => r.msg_id === m.msg_id)!; + assert.equal(m.identity, src.identity, `${m.msg_id}: dump carries the stored identity`); + assert.equal(m.retired_at ?? null, src.retired_at, `${m.msg_id}: dump carries the retire state`); + } + + assert.ok(store.restoreArchived(sid, { archiveDir }).ok); + const after = rowsOf(store, sid); + assert.equal(after.length, 3, "restore brings exactly the archived rows back"); + for (const r of before) { + const back = after.find((x) => x.msg_id === r.msg_id)!; + assert.equal(back.seq, r.seq, `${r.msg_id}: original seq`); + assert.equal(back.identity, r.identity, `${r.msg_id}: identity survives archive/restore`); + assert.equal(back.retired_at, r.retired_at, `${r.msg_id}: retire state survives archive/restore`); + } + // The replay surface agrees: m-a stays out, no resurrection. + assert.deepEqual( + store.getMessages(sid).map((r) => r.msgId), + ["m-b", "m-c"], + "restored retired row does NOT resurrect into the live view", + ); + + // The actual bug: with restored identities intact the next full + // ingest JOINs onto the same rows — no re-mint, no duplicate window. + store.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true }); + const stable = rowsOf(store, sid); + assert.equal(stable.length, 3, "re-ingest after restore mints nothing new"); + for (const r of before) { + const now = stable.find((x) => x.msg_id === r.msg_id)!; + assert.equal(now.seq, r.seq, `${r.msg_id}: seq unchanged across archive→restore→re-ingest`); + } + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +test("#4665: pre-fix dump (no identity/retired_at fields) restores via recomputed identity", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-legacy-")); + try { + const store = UnifiedStore.open(path.join(dir, "index.db")); + const sid = "s4665-legacy"; + const x = mkMsg("m-x", "x-ray ".repeat(30)); + const y = mkMsg("m-y", "yankee ".repeat(30)); + store.ingestSession(planIngest(mkSession(sid, [x, y])), undefined, { completeLog: true }); + + const archiveDir = path.join(dir, "archive"); + assert.ok(store.archiveSession(sid, { archiveDir }).ok); + const logPath = path.join(archiveDir, UnifiedStore.ARCHIVE_LOG_NAME); + // Simulate an archive written before this fix: strip both fields. + const stripped = fs + .readFileSync(logPath, "utf8") + .split("\n") + .map((l) => { + if (!l.trim()) return l; + const parsed = JSON.parse(l) as DumpLine; + if (parsed.sessionId !== sid) return l; + for (const m of parsed.messages) { delete m.identity; delete m.retired_at; } + return JSON.stringify(parsed); + }) + .join("\n"); + fs.writeFileSync(logPath, stripped); + // Capture BEFORE restore: the restore rewrite drops the restored + // session's line from the log. + const line = readDumpLine(logPath, sid); + assert.ok(!("identity" in line.messages[0]), "fixture really lacks the fields"); + + assert.ok(store.restoreArchived(sid, { archiveDir }).ok); + const rows = rowsOf(store, sid); + assert.equal(rows.length, 2); + for (const r of rows) { + const dumped = line.messages.find((m) => m.msg_id === r.msg_id)!; + const expected = identityOfStoredContent(line.content[dumped.content_hash as string]); + assert.equal(r.identity, expected, `${r.msg_id}: legacy dump falls back to the CAS-body identity`); + assert.equal(r.retired_at, null, `${r.msg_id}: legacy dumps cannot recover retire state → live (documented boundary)`); + } + // The fallback re-connects the JOIN: a full re-ingest lands on the + // same rows instead of minting duplicates. + store.ingestSession(planIngest(mkSession(sid, [x, y])), undefined, { completeLog: true }); + const stable = rowsOf(store, sid); + assert.equal(stable.length, 2, "legacy-restore re-ingest mints nothing new"); + assert.deepEqual( + stable.map((r) => [r.msg_id, r.seq]), + rows.map((r) => [r.msg_id, r.seq]), + "seqs stable after the fallback restore", + ); + store.close(); + } finally { + try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handles */ } + } +}); + +const MYSQL_DSN = process.env.BILI_MYSQL_TEST_DSN ?? ""; + +test("mysql lane: #4665 round-trip carries identity + retired_at", { skip: MYSQL_DSN.length === 0 ? "set BILI_MYSQL_TEST_DSN to run" : false }, async () => { + // The interface hides includeRetired/retiredAt; the concrete backend + // exposes them (same seam the contract suite asserts through). + const { MysqlBackend } = await import("../src/storage/mysql.ts"); + const opened = await openBackend("mysql", MYSQL_DSN); + assert.ok(opened instanceof MysqlBackend, "mysql lane opens the real backend"); + const backend = opened; + await backend.init?.(); + try { + const sid = `s4665-${Date.now()}`; + const a = mkMsg("m-a", "alpha ".repeat(30)); + const b = mkMsg("m-b", "bravo ".repeat(30)); + const c = mkMsg("m-c", "charlie ".repeat(30)); + assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [a, b, c])), undefined, { completeLog: true })); + assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true })); + + const archiveDir = fs.mkdtempSync(path.join(os.tmpdir(), "issue4665-mysql-")); + try { + const archived = await backend.archiveSession(sid, { archiveDir }); + assert.ok(archived.ok && archived.exportPath, `archive ok (${archived.reason ?? ""})`); + const line = readDumpLine(archived.exportPath!, sid); + assert.equal(line.messages.length, 3, "mysql dump carries retired rows too"); + for (const m of line.messages) { + assert.equal(typeof m.identity, "string", `${m.msg_id}: mysql dump carries identity`); + assert.ok((m.identity as string).length > 0, `${m.msg_id}: non-empty identity`); + assert.ok("retired_at" in m, `${m.msg_id}: mysql dump carries retired_at`); + } + + assert.ok(await backend.restoreArchived(sid, { archiveDir }), "restore ok"); + const all = await backend.getMessages(sid, { includeRetired: true }); + assert.equal(all.length, 3, "all rows back incl. the retired one"); + const retA = all.find((r) => r.msgId === "m-a")!; + assert.ok(retA.retiredAt !== null, "retired state survives the mysql round-trip"); + assert.deepEqual( + (await backend.getMessages(sid)).map((r) => r.msgId), + ["m-b", "m-c"], + "live view excludes the restored retired row", + ); + // Re-ingest stability: the JOIN reconnects, nothing re-minted. + assert.ok(await backend.ingestSession(planIngest(mkSession(sid, [b, c])), undefined, { completeLog: true })); + const stable = await backend.getMessages(sid, { includeRetired: true }); + assert.equal(stable.length, 3, "mysql re-ingest after restore mints nothing new"); + assert.deepEqual( + stable.map((r) => [r.seq, r.msgId]).sort(), + all.map((r) => [r.seq, r.msgId]).sort(), + "seqs unchanged across archive→restore→re-ingest", + ); + } finally { + fs.rmSync(archiveDir, { recursive: true, force: true }); + } + await backend.deleteSession(sid); + } finally { + await backend.close(); + } +}); From ef2e7ed71eecda4501f5c38c3e8428ba5b31ebd3 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 08:04:05 +0800 Subject: [PATCH 03/35] fix: self-clear the stale split-brain verdict once the peer stops contesting (#4626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The surfaced contest was persisted into the storage runtime state at detection time and replayed verbatim on every /__bili/status read until a manual ack — but detection fires while the restarting peer's tombstone marker is still seconds fresh (its graceful-shutdown dispose() rewrites the marker), so a restart handover left an alarming card behind for hours after the peer pid was gone. The stored verdict now re-evaluates against CURRENT evidence at the status read, with the same never-freeze discipline as the sibling second-process warning: the server registers the very scan that produced the verdict (findContestedWriters over its own writers dir), and when that scan no longer reports the accused peer (pid gone + marker past its freshness grace, or the contest window lapsed) the entry clears itself with an info log line. Without a registered check the legacy replay behavior stands (standalone record/ack use). The fresh-tombstone case keeps persisting: a dead pid alone is not proof of absence on every platform, so the notice stays honest while marker freshness says the peer may be alive. --- src/server.ts | 10 ++- src/storage/runtime-status.ts | 20 +++++ tests/issue-split-brain-surface.test.ts | 114 +++++++++++++++++++++++- 3 files changed, 138 insertions(+), 6 deletions(-) diff --git a/src/server.ts b/src/server.ts index 3030b92888..bbe9df668b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -99,8 +99,8 @@ import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWri import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; -import { createServedMarkerWriter, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; -import { recordSplitBrainContest } from "./storage/runtime-status.js"; +import { createServedMarkerWriter, findContestedWriters, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; +import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; import { computeAnthropicMessageMarks, stampAnthropicSystemCacheControl, anthropicToolsCarryCacheControl } from "./loop/cache-control.js"; @@ -1231,6 +1231,12 @@ export async function startServer(opts: ProxyOptions): Promise { }, ); reapStaleWriterMarkers(markerDir, Date.now()); + // #4626: a surfaced verdict re-evaluates against CURRENT marker + // evidence at every status read — reuse of findContestedWriters + // keeps ONE definition of "still contested", so the restart- + // handover notice clears itself once the peer provably stops. + registerSplitBrainEvidenceCheck((contest) => + findContestedWriters(markerDir, instanceId, contest.sessionId, Date.now()).some((p) => p.instanceId === contest.peer.instanceId)); setGlobalSessionCommitListener((id) => servedWriter?.noteServed(id)); } const nOverrides = Object.keys(opts.routes).length; diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index f2c0aee365..1b683f3997 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -137,6 +137,18 @@ export function ackSplitBrainContest(sessionId: string): boolean { const MAX_SPLIT_BRAIN_ACKS = 64; +// #4626: the stored verdict must be a function of CURRENT evidence, not a +// replay of one moment's snapshot — at detection time the restart-handover +// tombstone is seconds fresh, but it must not pin the notice forever. The +// server registers the same writer-marker scan that produced the verdict +// (findContestedWriters); without a registered check the stored entry is +// replayed as-is (legacy behavior, standalone record/ack use). +let splitBrainEvidenceCheck: ((contest: SplitBrainContestInput) => boolean) | null = null; + +export function registerSplitBrainEvidenceCheck(check: ((contest: SplitBrainContestInput) => boolean) | null): void { + splitBrainEvidenceCheck = check; +} + /** #4175: the last successful VACUUM INTO backup. */ export function recordBackupOutcome(lastPath: string, lastBytes: number): void { state.backup = { lastPath, lastAt: Date.now(), lastBytes }; @@ -145,6 +157,7 @@ export function recordBackupOutcome(lastPath: string, lastBytes: number): void { export function _resetStorageRuntimeStatusForTest(): void { state = freshState(); cooldownCache = null; + splitBrainEvidenceCheck = null; } /** #4175: true when a pid belongs to a live process (EPERM = alive, foreign user). */ @@ -271,6 +284,13 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus { state.secondProcess = null; loggerLog("info", `[storage] second-process pid ${sp.pid} has exited — stale warning cleared`); } + // #4626: same never-freeze discipline as above — once the peer no longer + // contests (its marker evidence is gone), the stored verdict clears itself. + const sb = state.splitBrain; + if (sb !== null && splitBrainEvidenceCheck !== null && !splitBrainEvidenceCheck(sb)) { + state.splitBrain = null; + loggerLog("info", `[split-brain] session ${sb.sessionId} notice cleared — peer ${sb.peer.instanceId} (pid ${sb.peer.pid}) no longer contests it`); + } return { requested, effective, diff --git a/tests/issue-split-brain-surface.test.ts b/tests/issue-split-brain-surface.test.ts index e6b368cdeb..07bd6051ed 100644 --- a/tests/issue-split-brain-surface.test.ts +++ b/tests/issue-split-brain-surface.test.ts @@ -12,6 +12,9 @@ // G2 the surfaced entry is dismissable (POST /__bili/storage/split-brain/ // ack): ack clears it, the SAME contest stays quiet, a NEW contest // re-surfaces, an unknown session answers 400. +// G3 the stored verdict never outlives its evidence (#4626): it persists +// while the peer's marker evidence holds and self-clears at the status +// read once the peer provably stops contesting (both directions). import test from "node:test"; import assert from "node:assert/strict"; import http from "node:http"; @@ -23,8 +26,8 @@ import { spawn, type ChildProcess } from "node:child_process"; import { startServer } from "../src/server.ts"; import { UnifiedStore } from "../src/storage/store.ts"; import { probeBuiltinSqlite, probeHostBetterSqlite3, setPreferredSqliteEngineForTests, setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; -import { _resetStorageRuntimeStatusForTest, ackSplitBrainContest, recordSecondProcessOwner, recordSplitBrainContest, storageStatusWire, type SplitBrainContestInput } from "../src/storage/runtime-status.ts"; -import { createServedMarkerWriter, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; +import { _resetStorageRuntimeStatusForTest, ackSplitBrainContest, recordSecondProcessOwner, recordSplitBrainContest, registerSplitBrainEvidenceCheck, storageStatusWire, type SplitBrainContestInput } from "../src/storage/runtime-status.ts"; +import { createServedMarkerWriter, findContestedWriters, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; import { isPidAlive } from "../src/instance.ts"; import { setLogCapture } from "../src/logger.ts"; import { renderPage } from "../src/web/page.ts"; @@ -114,6 +117,21 @@ function writeInstanceEntry(stateRoot: string, id: string, pid: number, lane?: s fs.writeFileSync(path.join(dir, `${id}.json`), JSON.stringify(entry)); } +// Beyond any sane pid_max (Linux default 4194304) — provably dead. +const DEAD_PID = 999_999_999; + +function writePeerMarker(dir: string, id: string, pid: number, servedAt: Record): string { + fs.mkdirSync(dir, { recursive: true }); + const file = path.join(dir, `${id}.json`); + fs.writeFileSync(file, JSON.stringify({ v: 1, instanceId: id, pid, origin: "http://127.0.0.1:8787", version: "0.1.205", lane: "dsh", servedAt })); + return file; +} + +function backdate(file: string, ageMs: number): void { + const t = (Date.now() - ageMs) / 1000; + fs.utimesSync(file, t, t); +} + interface SplitBrainWire { sessionId: string; title?: string; @@ -304,6 +322,81 @@ test("#3959 G2: ack clears the surfaced entry; same contest stays quiet, a new o _resetStorageRuntimeStatusForTest(); }); +// ---- #4626: the stored verdict must not outlive its evidence ------------------ + +test("#4626 G3: stored verdict persists while evidence holds, self-clears once the peer stops contesting", () => { + _resetStorageRuntimeStatusForTest(); + const logs: string[] = []; + setLogCapture((level, msg) => logs.push(`${level} ${msg}`)); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g3-")); + try { + const dir = writerMarkerDir(tmp); + const base: SplitBrainContestInput = { + sessionId: "sess-stale", + title: "Fix auth bug", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-gone", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + // The server's registration shape (src/server.ts): the verdict holds iff + // the SAME scan that raised it still reports the accused peer. + registerSplitBrainEvidenceCheck((c) => + findContestedWriters(dir, "self-1", c.sessionId, Date.now()).some((p) => p.instanceId === c.peer.instanceId)); + + // Phase A — fresh tombstone (dead pid, marker just rewritten by the + // peer's graceful-shutdown flush): the notice MUST persist; a dead pid + // alone is not proof of absence on every platform. + const file = writePeerMarker(dir, "peer-gone", DEAD_PID, { [base.sessionId]: base.peer.servedAt }); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-stale", "fresh tombstone keeps the notice honest"); + + // Phase B — dead pid + marker older than the freshness grace: the next + // status read clears the stored verdict by itself (#4626). + backdate(file, 40_000); + assert.equal(storageStatusWire().splitBrain, null, "dead peer + stale marker clears the stored verdict at the status read"); + assert.ok(logs.some((l) => l.startsWith("info ") && l.includes("[split-brain] session sess-stale") && l.includes("peer-gone") && l.includes(String(DEAD_PID))), "the clear is logged with full ids"); + assert.equal(storageStatusWire().splitBrain, null, "stays cleared on subsequent reads"); + + // Phase C — an evidence-based clear must NOT poison the ack guard: if + // the same peer contests again, the notice surfaces again. + backdate(file, 0); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-stale", "a renewed contest re-surfaces after an evidence-based clear"); + } finally { + setLogCapture(null); + _resetStorageRuntimeStatusForTest(); + rmrf(tmp); + } +}); + +test("#4626 G3-negative: without a registered check the stored verdict stands (legacy replay)", () => { + _resetStorageRuntimeStatusForTest(); + const base: SplitBrainContestInput = { + sessionId: "sess-legacy", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-x", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + try { + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-legacy", "no registered check ⇒ stored entry replays as-is"); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g3n-")); + try { + const dir = writerMarkerDir(tmp); + writePeerMarker(dir, "peer-live", process.pid, { [base.sessionId]: Date.now() - 1000 }); + registerSplitBrainEvidenceCheck((c) => + findContestedWriters(dir, "self-1", c.sessionId, Date.now()).some((p) => p.instanceId === c.peer.instanceId)); + const live: SplitBrainContestInput = { ...base, peer: { ...base.peer, instanceId: "peer-live", pid: process.pid } }; + recordSplitBrainContest(live); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-legacy", "live peer + fresh marker keeps the notice surfaced"); + } finally { + rmrf(tmp); + } + } finally { + _resetStorageRuntimeStatusForTest(); + } +}); + // ---- web surface: wire + endpoint + page/client/i18n --------------------------- test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; page/client/i18n wired", async () => { @@ -322,11 +415,17 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p assert.equal(s.storage.secondProcess, null, "no second process recorded → null"); assert.equal(s.storage.splitBrain, null, "no contest recorded → null"); + // #4626: surfaced entries re-evaluate against CURRENT writer-marker + // evidence at each status read — give the recorded contest real + // evidence in the server's writers dir (fresh mtime keeps it contested + // even though DEAD_PID is provably not a live process). + const peerMarker = writePeerMarker(path.join(base, "state", "billion-context", "writers"), "peer-1", DEAD_PID, { "sess-web": Date.now() - 1000 }); + const contest: SplitBrainContestInput = { sessionId: "sess-web", title: "Fix auth bug", self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.202" }, - peer: { instanceId: "peer-1", pid: 4242, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.201", servedAt: 1000 }, + peer: { instanceId: "peer-1", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.201", servedAt: 1000 }, at: Date.now(), }; recordSplitBrainContest(contest); @@ -334,7 +433,7 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p assert.equal(s.storage.splitBrain?.sessionId, "sess-web", "status wire carries the surfaced contest"); assert.equal(s.storage.splitBrain?.title, "Fix auth bug"); assert.equal(s.storage.splitBrain?.peer.lane, "dsh"); - assert.equal(s.storage.splitBrain?.peer.pid, 4242); + assert.equal(s.storage.splitBrain?.peer.pid, DEAD_PID); const acked = await postJson(port, "/__bili/storage/split-brain/ack", { sessionId: "sess-web" }); assert.equal(acked.status, 200, "ack of the surfaced session answers 200"); @@ -351,6 +450,13 @@ test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; p s = await fetchJson<{ storage: StorageWireShape }>(port, "/__bili/status"); assert.equal(s.storage.splitBrain?.sessionId, "sess-web", "a NEW contest re-surfaces on the wire after ack"); + // #4626 end-to-end: once the peer's marker is past its freshness grace + // and its pid is provably dead, the next status read clears the notice + // WITHOUT a manual ack. + backdate(peerMarker, 40_000); + s = await fetchJson<{ storage: StorageWireShape }>(port, "/__bili/status"); + assert.equal(s.storage.splitBrain, null, "stale verdict self-clears at the status read without a manual ack"); + const html = renderPage("http://127.0.0.1:1", "0.0.0"); assert.ok(html.includes('id="split-brain-banner"'), "page carries the split-brain banner div"); assert.ok(WEB_CLIENT.includes('"split-brain-banner"'), "client targets the banner element"); From cf6994af3394060ff7a18ebbe8030dbc0b0c3e80 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 08:04:05 +0800 Subject: [PATCH 04/35] fix: show the full session id in the split-brain ack log and web card (#4626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slice(0, 8) of a "session-…" id renders literally "session-" — the ack log line and the web banner both showed an unusable prefix. Log the full id (the [split-brain] error line already does) and give the card a 24-char prefix so distinct sessions are tellable apart. --- src/server/admin.ts | 3 ++- src/web/client.ts | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/server/admin.ts b/src/server/admin.ts index bd0fb963e9..67f702b56d 100644 --- a/src/server/admin.ts +++ b/src/server/admin.ts @@ -981,7 +981,8 @@ async function handleSplitBrainAck(req: http.IncomingMessage, res: http.ServerRe res.end(JSON.stringify({ error: `no surfaced split-brain entry for session: ${sessionId}` }, null, 2)); return; } - log("info", `[storage] split-brain notice for session ${sessionId.slice(0, 8)} acknowledged via web UI`); + // #4626: full id — .slice(0, 8) of "session-…" renders literally "session-" + log("info", `[storage] split-brain notice for session ${sessionId} acknowledged via web UI`); res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true }, null, 2)); } diff --git a/src/web/client.ts b/src/web/client.ts index 7556a9efb2..e3d0b4e5ac 100644 --- a/src/web/client.ts +++ b/src/web/client.ts @@ -1223,10 +1223,12 @@ function changelogCatOf(rawLine) { sbb.hidden = false; sbb.classList.add("show"); const laneOf = (x) => (x && typeof x.lane === "string" && x.lane ? x.lane : t("sys.split_brain_lane_undeclared")); + // #4626: .slice(0, 8) of "session-…" is literally "session-" — show enough to tell sessions apart + const shortSid = sb.sessionId.length > 24 ? sb.sessionId.slice(0, 24) + "…" : sb.sessionId; sbb.innerHTML = "" + escapeHtml(t("sys.split_brain_title")) + " " + escapeHtml(t("sys.split_brain", { - title: typeof sb.title === "string" && sb.title ? sb.title : sb.sessionId.slice(0, 8), - sid: sb.sessionId.slice(0, 8), + title: typeof sb.title === "string" && sb.title ? sb.title : shortSid, + sid: shortSid, selfPid: String(sb.self.pid), selfLane: laneOf(sb.self), peerPid: String(sb.peer.pid), From 8f51ea084f3a6518d0c47eb1b30b22cfe19586ac Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 17:47:07 +0800 Subject: [PATCH 05/35] fix: stop flagging gracefully-shutdown peers as split-brain writers (#4626) --- src/served-markers.ts | 36 ++++++++++++++++++++++++++++-------- tests/served-markers.test.ts | 29 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/src/served-markers.ts b/src/served-markers.ts index 6ae0aa8e25..a2f0d6bafa 100644 --- a/src/served-markers.ts +++ b/src/served-markers.ts @@ -11,6 +11,11 @@ // names both sides in an error line when it commits a session another live // instance committed within the window. Pure observability: no routing, // locking, or serving behavior changes. +// #4626: an instance that shut down GRACEFULLY stamps its final marker with +// `disposedAt`; such a peer is attribution-only and can never be flagged as +// a concurrent writer — its pid is provably gone by construction (it ran our +// shutdown path), so the freshness grace below stays reserved for peers that +// died without it (crash/kill) where the pid probe may be blind. import fs from "node:fs"; import path from "node:path"; @@ -35,6 +40,11 @@ interface WriterMarkerFile { lane?: string; /** sessionId → last commit time (ms epoch). */ servedAt: Record; + /** Set only on the final marker write of a graceful shutdown (#4626): + * the instance is provably gone — peers keep the file for attribution + * but never treat it as a concurrent writer. Additive; markers from + * older builds carry none. */ + disposedAt?: number; } /** A surfaced split-brain contest: which session (id + resolved title) and @@ -68,7 +78,9 @@ const MAX_TRACKED_SESSIONS = 128; const VERDICT_TTL_MS = 60_000; /** A peer counts as live when its pid is alive OR its marker was written * within this window (covers platforms where the pid check cannot see the - * peer process). */ + * peer process). Gracefully-shutdown instances are excluded BEFORE this + * test via their disposedAt stamp (#4626) — their death is certain, so no + * grace applies. */ const PEER_FILE_FRESH_MS = 30_000; /** reapStaleWriterMarkers removes files whose pid is dead AND whose mtime is * older than this. */ @@ -127,7 +139,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, } }; - const writeMarker = (): void => { + const writeMarker = (final: boolean): void => { const now = Date.now(); prune(now); const body: WriterMarkerFile = { @@ -138,6 +150,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, version: identity.version, ...(identity.lane === undefined || identity.lane === "" ? {} : { lane: identity.lane }), servedAt: Object.fromEntries(servedAt), + ...(final ? { disposedAt: now } : {}), }; try { fs.mkdirSync(dir, { recursive: true }); @@ -152,7 +165,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, flushTimer = undefined; if (disposed) return; lastFlushAt = Date.now(); - writeMarker(); + writeMarker(false); }; const scheduleFlush = (): void => { @@ -172,7 +185,7 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, const title = opts?.resolveTitle?.(sessionId); const laneOf = (x: WriterIdentity): string => (x.lane === undefined || x.lane === "" ? "" : `, lane ${x.lane}`); const titleOf = title === undefined || title === "" ? "" : ` "${title}"`; - log("error", `[split-brain] session ${sessionId}${titleOf} committed by BOTH this instance (pid ${identity.pid}, ${identity.origin}${laneOf(identity)}, v${identity.version}) and another live instance (pid ${peer.pid}, ${peer.origin}${laneOf(peer)}, v${peer.version}) within ${CONTEST_WINDOW_MS / 60000}m — fold decisions are invisible across instances and the last writer wins (#394/#405); stop one of them or align your client config so one conversation rides one proxy (#3923)`); + log("error", `[split-brain] session ${sessionId}${titleOf} committed by BOTH this instance (pid ${identity.pid}, ${identity.origin}${laneOf(identity)}, v${identity.version}) and another instance (pid ${peer.pid}, ${peer.origin}${laneOf(peer)}, v${peer.version}) within ${CONTEST_WINDOW_MS / 60000}m — fold decisions are invisible across instances and the last writer wins (#394/#405); stop one of them or align your client config so one conversation rides one proxy (#3923)`); opts?.onContest?.({ sessionId, ...(title === undefined || title === "" ? {} : { title }), self: identity, peer, at: now }); } }; @@ -189,8 +202,10 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, disposed = true; if (flushTimer !== undefined) clearTimeout(flushTimer); // final write (bypasses the disposed guard): peers should still be - // able to attribute this instance's last commits after shutdown - writeMarker(); + // able to attribute this instance's last commits after shutdown; + // the disposedAt stamp (#4626) keeps that attribution while + // excluding us from every peer's contest scan + writeMarker(true); }, __flushForTest(): void { flush(); @@ -202,8 +217,10 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, } /** Peer marker files that record a commit of `sessionId` inside the contest - * window by a LIVE foreign instance (pid alive or file fresh). Malformed or - * tearing files are skipped silently — a marker is observability, not data. */ + * window by a LIVE foreign instance (pid alive or file fresh). Markers + * stamped `disposedAt` by a gracefully-shutdown peer are attribution-only + * and never contested (#4626). Malformed or tearing files are skipped + * silently — a marker is observability, not data. */ export function findContestedWriters(dir: string, selfInstanceId: string, sessionId: string, now: number): Array { let entries: fs.Dirent[]; try { @@ -218,6 +235,9 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio try { const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) continue; + // #4626: gracefully-shutdown peer — attribution only, never a + // concurrent writer (its death is certain; no freshness grace). + if (typeof raw.disposedAt === "number") continue; const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) continue; const mtimeMs = fs.statSync(file).mtimeMs; diff --git a/tests/served-markers.test.ts b/tests/served-markers.test.ts index 93f75d662b..0183ace9e3 100644 --- a/tests/served-markers.test.ts +++ b/tests/served-markers.test.ts @@ -111,6 +111,35 @@ test("dead-pid peer counts while its marker is fresh, stops counting once stale" assert.equal(found.length, 0, "dead pid + stale mtime = gone"); }); +test("#4626 gracefully-shutdown peer (disposedAt stamp) never contests, even with fresh mtime", () => { + const dir = writerMarkerDir(freshDir("bili-4626-disposed-")); + const deadPeer: WriterIdentity = { ...PEER, pid: DEAD_PID }; + fs.mkdirSync(dir, { recursive: true }); + // The exact shape dispose() leaves behind: dead pid, mtime ≈ death time, + // a commit inside the window seconds earlier. + const file = path.join(dir, `${deadPeer.instanceId}.json`); + fs.writeFileSync(file, JSON.stringify({ v: 1, ...deadPeer, servedAt: { "sess-h": Date.now() - 1000 }, disposedAt: Date.now() - 9000 })); + backdate(file, 8000); + assert.deepEqual(findContestedWriters(dir, SELF.instanceId, "sess-h", Date.now()), [], "handover tombstone is attribution-only — no false split-brain verdict"); +}); + +test("#4626 e2e: graceful rotation — successor commits the same session without a verdict", () => { + const dir = writerMarkerDir(freshDir("bili-4626-handover-")); + const logs: Array<[string, string]> = []; + const outgoing = createServedMarkerWriter(PEER, dir, (level, msg) => logs.push([level, msg])); + outgoing.noteServed("sess-rot"); + outgoing.__flushForTest(); + const mid = JSON.parse(fs.readFileSync(path.join(dir, "peer-1.json"), "utf8")) as Record; + assert.equal(mid.disposedAt, undefined, "periodic flushes carry no disposedAt stamp"); + outgoing.dispose(); + const final = JSON.parse(fs.readFileSync(path.join(dir, "peer-1.json"), "utf8")) as Record; + assert.equal(typeof final.disposedAt, "number", "the final shutdown write is stamped disposedAt"); + const incoming = createServedMarkerWriter(SELF, dir, (level, msg) => logs.push([level, msg])); + incoming.noteServed("sess-rot"); + assert.equal(logs.length, 0, "sequential handover inside the 10m window raises no [split-brain] line"); + incoming.dispose(); +}); + test("malformed or foreign-shape marker files are skipped, never fatal", () => { const dir = writerMarkerDir(freshDir("bili-3923-malformed-")); fs.mkdirSync(dir, { recursive: true }); From 4368daf64310d7600b7ccad6e18b883fc3b094b0 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 26 Oct 2026 09:20:57 +0800 Subject: [PATCH 06/35] fix: fail the split-brain evidence check closed when marker evidence is unreadable (#4626) --- src/served-markers.ts | 62 +++++++++++++++++++++---- src/server.ts | 11 +++-- src/storage/runtime-status.ts | 20 ++++---- tests/issue-split-brain-surface.test.ts | 51 ++++++++++++++++++-- 4 files changed, 119 insertions(+), 25 deletions(-) diff --git a/src/served-markers.ts b/src/served-markers.ts index a2f0d6bafa..c6e30bde74 100644 --- a/src/served-markers.ts +++ b/src/served-markers.ts @@ -216,6 +216,21 @@ export function createServedMarkerWriter(identity: WriterIdentity, dir: string, }; } +/** One definition of "still contests" (#4626): the contested `servedAt` + * timestamp, or null when the marker does not contest — shared by the dir + * scan below and the direct-path clear check, so detection and clearing can + * never drift apart. */ +function contestedServedAt(raw: Partial, selfInstanceId: string, sessionId: string, mtimeMs: number, now: number): number | null { + if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) return null; + // #4626: gracefully-shutdown peer — attribution only, never a concurrent + // writer (its death is certain; no freshness grace). + if (typeof raw.disposedAt === "number") return null; + const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; + if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) return null; + const alive = typeof raw.pid === "number" ? isPidAlive(raw.pid) : false; + return alive || now - mtimeMs <= PEER_FILE_FRESH_MS ? at : null; +} + /** Peer marker files that record a commit of `sessionId` inside the contest * window by a LIVE foreign instance (pid alive or file fresh). Markers * stamped `disposedAt` by a gracefully-shutdown peer are attribution-only @@ -234,15 +249,9 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio const file = path.join(dir, entry.name); try { const raw = JSON.parse(fs.readFileSync(file, "utf8")) as Partial; - if (raw.v !== 1 || typeof raw.instanceId !== "string" || raw.instanceId === selfInstanceId) continue; - // #4626: gracefully-shutdown peer — attribution only, never a - // concurrent writer (its death is certain; no freshness grace). - if (typeof raw.disposedAt === "number") continue; - const at = raw.servedAt !== null && typeof raw.servedAt === "object" ? (raw.servedAt as Record)[sessionId] : undefined; - if (typeof at !== "number" || now - at > CONTEST_WINDOW_MS) continue; const mtimeMs = fs.statSync(file).mtimeMs; - const alive = typeof raw.pid === "number" ? isPidAlive(raw.pid) : false; - if (!alive && now - mtimeMs > PEER_FILE_FRESH_MS) continue; + const at = contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now); + if (at === null || typeof raw.instanceId !== "string") continue; out.push({ instanceId: raw.instanceId, pid: typeof raw.pid === "number" ? raw.pid : -1, @@ -258,6 +267,43 @@ export function findContestedWriters(dir: string, selfInstanceId: string, sessio return out; } +/** #4626 fail-closed companion to findContestedWriters: does the ACCUSED + * peer's own marker still contest `sessionId`? true = still contested, + * false = provably stopped (marker reaped/absent, dead pid past the + * freshness grace, or the contest window lapsed), null = UNKNOWN (dir or + * file unreadable, torn mid-write) — a transient FS error must never drop a + * real notice, so callers keep it on null. Reads one file, not the dir. */ +export function peerStillContests(dir: string, selfInstanceId: string, peerInstanceId: string, sessionId: string, now: number): boolean | null { + let dirStat: fs.Stats; + try { + dirStat = fs.statSync(dir); + } catch { + return null; + } + if (!dirStat.isDirectory()) return null; + const file = path.join(dir, `${peerInstanceId}.json`); + let content: string; + try { + content = fs.readFileSync(file, "utf8"); + } catch (err) { + return (err as NodeJS.ErrnoException).code === "ENOENT" ? false : null; + } + let raw: Partial; + try { + raw = JSON.parse(content) as Partial; + } catch { + return null; + } + if (raw.instanceId !== peerInstanceId) return null; + let mtimeMs: number; + try { + mtimeMs = fs.statSync(file).mtimeMs; + } catch { + return null; + } + return contestedServedAt(raw, selfInstanceId, sessionId, mtimeMs, now) !== null; +} + /** Startup hygiene: drop marker files whose instance is provably gone (pid * dead) and stale beyond the reap horizon. Fresh files from a just-crashed * peer stay — findContestedWriters' freshness test still honors them for diff --git a/src/server.ts b/src/server.ts index bbe9df668b..758b048000 100644 --- a/src/server.ts +++ b/src/server.ts @@ -99,7 +99,7 @@ import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWri import { queryLogLines } from "./web/logs-query.js"; import { configFile, defaultLogFile, stateDir } from "./paths.js"; import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; -import { createServedMarkerWriter, findContestedWriters, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; +import { createServedMarkerWriter, peerStillContests, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; import { recordSplitBrainContest, registerSplitBrainEvidenceCheck } from "./storage/runtime-status.js"; import { mayClobberPointer, probeOwnerInfo, resolvePortClaim } from "./port-owner.js"; import { hoistTrappedToolItems } from "./tool-pair-order.js"; @@ -1232,11 +1232,12 @@ export async function startServer(opts: ProxyOptions): Promise { ); reapStaleWriterMarkers(markerDir, Date.now()); // #4626: a surfaced verdict re-evaluates against CURRENT marker - // evidence at every status read — reuse of findContestedWriters - // keeps ONE definition of "still contested", so the restart- - // handover notice clears itself once the peer provably stops. + // evidence at every status read — the accused peer's own marker, + // judged by the same "still contests" definition as detection, so + // the restart-handover notice clears itself once the peer + // PROVABLY stops; an unreadable marker fails closed (keeps it). registerSplitBrainEvidenceCheck((contest) => - findContestedWriters(markerDir, instanceId, contest.sessionId, Date.now()).some((p) => p.instanceId === contest.peer.instanceId)); + peerStillContests(markerDir, instanceId, contest.peer.instanceId, contest.sessionId, Date.now())); setGlobalSessionCommitListener((id) => servedWriter?.noteServed(id)); } const nOverrides = Object.keys(opts.routes).length; diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index 1b683f3997..fff3bd91c9 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -140,12 +140,15 @@ const MAX_SPLIT_BRAIN_ACKS = 64; // #4626: the stored verdict must be a function of CURRENT evidence, not a // replay of one moment's snapshot — at detection time the restart-handover // tombstone is seconds fresh, but it must not pin the notice forever. The -// server registers the same writer-marker scan that produced the verdict -// (findContestedWriters); without a registered check the stored entry is -// replayed as-is (legacy behavior, standalone record/ack use). -let splitBrainEvidenceCheck: ((contest: SplitBrainContestInput) => boolean) | null = null; +// server registers the evidence check (the accused peer's own marker, judged +// by the same "still contests" definition as detection); without a registered +// check the stored entry is replayed as-is (legacy behavior, standalone +// record/ack use). The check is TRI-STATE and fails closed: an unknown result +// (the marker could not be read at status-read time) keeps the notice — +// dropping a real notice on a transient FS error is the wrong direction. +let splitBrainEvidenceCheck: ((contest: SplitBrainContestInput) => boolean | null) | null = null; -export function registerSplitBrainEvidenceCheck(check: ((contest: SplitBrainContestInput) => boolean) | null): void { +export function registerSplitBrainEvidenceCheck(check: ((contest: SplitBrainContestInput) => boolean | null) | null): void { splitBrainEvidenceCheck = check; } @@ -284,10 +287,11 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus { state.secondProcess = null; loggerLog("info", `[storage] second-process pid ${sp.pid} has exited — stale warning cleared`); } - // #4626: same never-freeze discipline as above — once the peer no longer - // contests (its marker evidence is gone), the stored verdict clears itself. + // #4626: same never-freeze discipline as above — once the peer PROVABLY + // stops contesting, the stored verdict clears itself; an unknown verdict + // (marker unreadable at read time) keeps it. const sb = state.splitBrain; - if (sb !== null && splitBrainEvidenceCheck !== null && !splitBrainEvidenceCheck(sb)) { + if (sb !== null && splitBrainEvidenceCheck !== null && splitBrainEvidenceCheck(sb) === false) { state.splitBrain = null; loggerLog("info", `[split-brain] session ${sb.sessionId} notice cleared — peer ${sb.peer.instanceId} (pid ${sb.peer.pid}) no longer contests it`); } diff --git a/tests/issue-split-brain-surface.test.ts b/tests/issue-split-brain-surface.test.ts index 07bd6051ed..e57708e7c4 100644 --- a/tests/issue-split-brain-surface.test.ts +++ b/tests/issue-split-brain-surface.test.ts @@ -15,6 +15,9 @@ // G3 the stored verdict never outlives its evidence (#4626): it persists // while the peer's marker evidence holds and self-clears at the status // read once the peer provably stops contesting (both directions). +// G4 the evidence check fails closed (#4626): an unreadable writers dir or +// torn peer marker is UNKNOWN and keeps the notice; only provable +// absence/staleness clears. import test from "node:test"; import assert from "node:assert/strict"; import http from "node:http"; @@ -27,7 +30,7 @@ import { startServer } from "../src/server.ts"; import { UnifiedStore } from "../src/storage/store.ts"; import { probeBuiltinSqlite, probeHostBetterSqlite3, setPreferredSqliteEngineForTests, setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; import { _resetStorageRuntimeStatusForTest, ackSplitBrainContest, recordSecondProcessOwner, recordSplitBrainContest, registerSplitBrainEvidenceCheck, storageStatusWire, type SplitBrainContestInput } from "../src/storage/runtime-status.ts"; -import { createServedMarkerWriter, findContestedWriters, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; +import { createServedMarkerWriter, peerStillContests, writerMarkerDir, type WriterIdentity } from "../src/served-markers.ts"; import { isPidAlive } from "../src/instance.ts"; import { setLogCapture } from "../src/logger.ts"; import { renderPage } from "../src/web/page.ts"; @@ -339,9 +342,9 @@ test("#4626 G3: stored verdict persists while evidence holds, self-clears once t at: Date.now(), }; // The server's registration shape (src/server.ts): the verdict holds iff - // the SAME scan that raised it still reports the accused peer. + // the accused peer's own marker still contests — tri-state, fails closed. registerSplitBrainEvidenceCheck((c) => - findContestedWriters(dir, "self-1", c.sessionId, Date.now()).some((p) => p.instanceId === c.peer.instanceId)); + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); // Phase A — fresh tombstone (dead pid, marker just rewritten by the // peer's graceful-shutdown flush): the notice MUST persist; a dead pid @@ -385,7 +388,7 @@ test("#4626 G3-negative: without a registered check the stored verdict stands (l const dir = writerMarkerDir(tmp); writePeerMarker(dir, "peer-live", process.pid, { [base.sessionId]: Date.now() - 1000 }); registerSplitBrainEvidenceCheck((c) => - findContestedWriters(dir, "self-1", c.sessionId, Date.now()).some((p) => p.instanceId === c.peer.instanceId)); + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); const live: SplitBrainContestInput = { ...base, peer: { ...base.peer, instanceId: "peer-live", pid: process.pid } }; recordSplitBrainContest(live); assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-legacy", "live peer + fresh marker keeps the notice surfaced"); @@ -397,6 +400,46 @@ test("#4626 G3-negative: without a registered check the stored verdict stands (l } }); +test("#4626 G4: evidence check fails closed — unknown evidence keeps the notice", () => { + _resetStorageRuntimeStatusForTest(); + const logs: string[] = []; + setLogCapture((level, msg) => logs.push(`${level} ${msg}`)); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4626-g4-")); + try { + const dir = writerMarkerDir(tmp); + const base: SplitBrainContestInput = { + sessionId: "sess-fc", + self: { instanceId: "self-1", pid: process.pid, origin: "http://127.0.0.1:18802", lane: "pi", version: "0.1.205" }, + peer: { instanceId: "peer-gone", pid: DEAD_PID, origin: "http://127.0.0.1:8787", lane: "dsh", version: "0.1.204", servedAt: Date.now() - 1000 }, + at: Date.now(), + }; + registerSplitBrainEvidenceCheck((c) => + peerStillContests(dir, "self-1", c.peer.instanceId, c.sessionId, Date.now())); + const file = writePeerMarker(dir, "peer-gone", DEAD_PID, { [base.sessionId]: base.peer.servedAt }); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc"); + + // Writers dir vanishes (state-dir move / transient FS error): UNKNOWN — + // the notice stays and no false "no longer contests" line appears. + fs.rmSync(dir, { recursive: true }); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc", "missing writers dir ⇒ unknown ⇒ notice kept"); + assert.ok(!logs.some((l) => l.includes("no longer contests")), "an unknown verdict never logs a false clear"); + + // Dir restored but the accused marker reaped: PROVABLY STOPPED — clears. + fs.mkdirSync(dir, { recursive: true }); + assert.equal(storageStatusWire().splitBrain, null, "absent peer marker ⇒ cleared at the status read"); + + // Torn/malformed marker: UNKNOWN again — a renewed verdict stays up. + fs.writeFileSync(file, "{ torn mid-write"); + recordSplitBrainContest(base); + assert.equal(storageStatusWire().splitBrain?.sessionId, "sess-fc", "unreadable peer marker ⇒ unknown ⇒ notice kept"); + } finally { + setLogCapture(null); + _resetStorageRuntimeStatusForTest(); + rmrf(tmp); + } +}); + // ---- web surface: wire + endpoint + page/client/i18n --------------------------- test("#3959 web: status carries secondProcess/splitBrain; ack endpoint clears; page/client/i18n wired", async () => { From fc6a5cfd6a677cfa3f04ca1ad0cc461fe92771b3 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 27 Oct 2026 18:37:52 +0800 Subject: [PATCH 07/35] fix: surface real storage errors after failed BEGIN/COMMIT and auto-recover the unified-store breaker (#4949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects in one failure domain, both observed in production on v0.1.206 after host storage exhaustion: 1. Error masking: when an I/O-class failure (disk full / SQLITE_FULL) aborts BEGIN or COMMIT, SQLite auto-aborts the transaction, so the follow-up ROLLBACK in the node:sqlite emulation throws 'cannot rollback - no transaction is active' and replaces the real error before it reaches the sidecar breaker's lastError / status wire. The rollback attempt is now guarded so the original error always wins. The better-sqlite3 seam gets an annotated message at our boundary because that library loses the original inside its own wrapper. 2. Permanent circuit breaker: stats.disabled was set on 3 consecutive failures but never cleared — the unified store stayed disabled for the whole process lifetime even after the disk was freed (only a restart recovered it). Both sync and async sidecar paths now run a cooldown-gated writability probe (REENABLE_PROBE_COOLDOWN_MS = 5min, one probe per window on the next save) through the same transaction path production writes use; success re-enables the store and the triggering record ingests immediately. Missed sessions converge via the existing envelope staleness guard at load plus the next full ingest. --- src/storage/driver.ts | 54 +++++++++++++++++++++++++++++++++------ src/storage/shadow.ts | 59 +++++++++++++++++++++++++++++++++++++++++-- src/storage/store.ts | 10 ++++++++ 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/src/storage/driver.ts b/src/storage/driver.ts index 6bc44be0c7..e15cfc9c32 100644 --- a/src/storage/driver.ts +++ b/src/storage/driver.ts @@ -86,7 +86,23 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { // a write lock mid-flight — SQLITE_BUSY_SNAPSHOT, which busy_timeout // cannot resolve. Taking the write lock up front makes concurrent // writers queue on busy_timeout instead (#2690 review blocking 5). - transaction: (fn) => db.transaction(fn).immediate(), + transaction: (fn) => { + try { + return db.transaction(fn).immediate(); + } catch (err) { + // #4949: better-sqlite3's own wrapper runs ROLLBACK after ANY + // inner failure; when BEGIN or COMMIT itself failed there was + // nothing left to roll back, and its "cannot rollback - no + // transaction is active" REPLACES the real error (commonly + // disk full / SQLITE_FULL) before it reaches us. Name the + // masking at our seam so operators see the likely cause. + const msg = String(err); + if (/cannot rollback|no transaction is active/i.test(msg)) { + throw new SqliteDriverError(`${msg} — masked by better-sqlite3's automatic rollback after a failed BEGIN/COMMIT; the original error no longer survives (disk full / SQLITE_FULL is the common cause)`, "better-sqlite3"); + } + throw err; + } + }, close: () => void db.close(), get open() { return db.open; @@ -94,15 +110,28 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { }; } +let databaseSyncFactoryForTests: ((path: string) => NodeSqliteDatabase) | null = null; + +/** Test seam: substitute how the node:sqlite emulation obtains its database + * handle (e.g. an adapter injecting engine-level I/O failures, #4949). + * Null = the real builtin node:sqlite module. */ +export function setDatabaseSyncFactoryForTests(factory: ((path: string) => NodeSqliteDatabase) | null): void { + databaseSyncFactoryForTests = factory; +} + function openNodeSqlite(path: string): SqliteDatabase { // node:sqlite is synchronous like better-sqlite3, but has no // .transaction() helper — emulate with BEGIN/COMMIT + savepoints for // nesting, which is the only behavioral difference the store relies on. - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { DatabaseSync } = require("node:sqlite") as { - DatabaseSync: new (path: string) => NodeSqliteDatabase; - }; - const db = new DatabaseSync(path); + const db = databaseSyncFactoryForTests + ? databaseSyncFactoryForTests(path) + : (() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { DatabaseSync } = require("node:sqlite") as { + DatabaseSync: new (path: string) => NodeSqliteDatabase; + }; + return new DatabaseSync(path); + })(); let depth = 0; const begin = () => db.exec(depth === 0 ? "BEGIN IMMEDIATE" : `SAVEPOINT sp${depth}`); // IMMEDIATE at top level: multi-process SQLITE_BUSY_SNAPSHOT guard (see openBetterSqlite3) const commit = () => db.exec(depth === 1 ? "COMMIT" : `RELEASE sp${depth - 1}`); @@ -123,7 +152,16 @@ function openNodeSqlite(path: string): SqliteDatabase { commit(); return out; } catch (err) { - rollback(); + // #4949: a failed BEGIN/COMMIT can leave NO transaction active + // (SQLite auto-aborts on I/O-class failures such as disk full); + // the follow-up ROLLBACK then throws "cannot rollback - no + // transaction is active" and would MASK the original failure. + // The original error below is always authoritative. + try { + rollback(); + } catch { + /* secondary rollback failure — original error wins (#4949) */ + } throw err; } finally { depth--; @@ -141,7 +179,7 @@ interface NodeSqliteStatement { get(...args: unknown[]): unknown; all(...args: unknown[]): unknown[]; } -interface NodeSqliteDatabase { +export interface NodeSqliteDatabase { exec(sql: string): unknown; prepare(sql: string): NodeSqliteStatement; getAutoCommit(): boolean; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 3a8c9c4ee5..2775526471 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -37,6 +37,15 @@ import type { CoreMessage, MessageContentStore } from "acp-kernel"; export const MAX_CONSECUTIVE_FAILURES = 3; +/** #4949: re-enable cadence for the disabled sidecar. The 3-strike breaker used + * to be permanent until process restart even after the store healed. Now each + * cooldown window lets the NEXT save run one writability probe (a single-row + * committed write through the production path); success clears the breaker and + * the triggering record ingests normally. Missed saves converge via the + * load-side staleness guard (envelope savedAt > index row) + next full ingest, + * so no global pre-check is needed before unified reads resume. */ +export const REENABLE_PROBE_COOLDOWN_MS = 5 * 60 * 1000; + /** #3133: read-back verification cadence — the verify leg (full read-back + * double shadowDigest) is O(log) per save, while the drift it guards is a * migration-parity property, not a per-commit one. The FIRST ingest of a @@ -140,6 +149,9 @@ export class UnifiedShadow { private queueTail: Promise = Promise.resolve(); private readonly legacyLoad: (record: unknown) => LegacySessionLike | null; private consecutiveFailures = 0; + /** #4949: timestamp until which the disabled sidecar stays quiet; the first + * save after it runs the writability probe (REENABLE_PROBE_COOLDOWN_MS). */ + private nextReenableProbeAt = 0; /** #3133: per-session ingest counters (verify sampling). Per-process * only — nothing persisted, no schema change. */ private readonly ingestCounts = new Map(); @@ -328,7 +340,21 @@ export class UnifiedShadow { this.enqueueAsyncRecord(record, rawTail, liveCcr); return; } - if (this.stats.disabled) return; + // #4949: a disabled sidecar is no longer dead for the process lifetime — + // one cooldown-gated writability probe per window; on success the + // breaker clears and THIS record falls through to the normal ingest. + if (this.stats.disabled) { + const now = Date.now(); + if (now < this.nextReenableProbeAt) return; + this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS; + try { + this.sync.probeWritable(); + } catch (err) { + this.noteProbeFailure(err); + return; + } + this.clearBreaker(); + } try { const loaded = this.legacyLoad(record); if (!loaded) return; @@ -471,6 +497,21 @@ export class UnifiedShadow { } } + /** #4949: re-enable-probe failure tail — keeps the breaker armed while the + * latest error stays visible on the status wire via stats.lastError. */ + private noteProbeFailure(err: unknown): void { + this.stats.lastError = String(err); + loggerLog("debug", `[storage] unified store re-enable probe failed (${String(err)}) — staying disabled, next probe after ${REENABLE_PROBE_COOLDOWN_MS / 60_000}min (#4949)`); + } + + /** #4949: clears the 3-strike breaker after a successful writability probe. */ + private clearBreaker(): void { + this.stats.disabled = false; + this.stats.lastError = null; + this.consecutiveFailures = 0; + loggerLog("info", "[storage] unified store RE-ENABLED: writability probe succeeded after disable (#4949); sessions missed during the outage converge via the envelope staleness guard + next full ingest"); + } + /** #3133 delta ingest attempt for one record's legacy view — returns * true when the O(new-messages) fast path committed. Every doubt * routes to `false` (→ the caller's full ingest); failures here must @@ -917,7 +958,21 @@ export class UnifiedShadow { * shadow queue; failures feed the same 3-strike disable. */ private enqueueAsyncRecord(record: unknown, rawTail?: { lastMessages: CoreMessage[]; lastMessagesFolded?: boolean }, liveCcr?: MessageContentStore): void { void this.enqueue(async () => { - if (this.stats.disabled) return; + // #4949: async twin of the sync re-enable probe in record() — MUST + // stay in lockstep with it; setKv is the backend's single-row + // committed write probe. + if (this.stats.disabled) { + const now = Date.now(); + if (now < this.nextReenableProbeAt) return; + this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS; + try { + await this.backend!.setKv("__bili", "writability-probe", { probedAt: now }); + } catch (err) { + this.noteProbeFailure(err); + return; + } + this.clearBreaker(); + } try { const backend = this.backend!; const loaded0 = this.legacyLoad(record); diff --git a/src/storage/store.ts b/src/storage/store.ts index 930a55b8f4..59c7f8643b 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1966,6 +1966,16 @@ export class UnifiedStore { return Number(this.db.prepare("DELETE FROM session_ext WHERE session_id = ? AND key = ?").run(sessionId, key).changes); } + /** #4949: circuit-breaker re-enable probe — one REAL committed write + * through the production transaction path (proves lock acquisition + + * disk space, not just connectivity). A single fixed kv key keeps the + * trace constant; the row doubles as a liveness marker. Throws on failure. */ + probeWritable(): void { + this.db.transaction(() => { + this.setKv("__bili", "writability-probe", { probedAt: Date.now() }); + }); + } + setKv(namespace: string, key: string, value: unknown): void { validateExtKey(namespace, "namespace"); validateExtKey(key, "key"); From 4508a28f4763ef19b16320f7e91a2c40ac665d68 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 27 Oct 2026 18:37:52 +0800 Subject: [PATCH 08/35] test: pin #4949 rollback-masking and breaker recovery regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A: injected engine-level COMMIT abort over a real DatabaseSync proves the original error surfaces (pre-fix this failed with exactly the reported 'cannot rollback - no transaction is active' mask); aborted rows do not persist. - A2: negative control — nested transactions on a constraint failure still roll back cleanly and propagate the original (gate both directions, #3617). - B: full breaker loop on a real store — 3 strikes disable with the true cause visible, cooldown gates probe frequency (exactly one probe per window), healing + one probe re-enables, triggering record lands, unified reads work again, exactly one RE-ENABLED log line. - C: pins REENABLE_PROBE_COOLDOWN_MS at the documented 5-minute cadence. --- .../issue4949-rollback-mask-recovery.test.ts | 350 ++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 tests/issue4949-rollback-mask-recovery.test.ts diff --git a/tests/issue4949-rollback-mask-recovery.test.ts b/tests/issue4949-rollback-mask-recovery.test.ts new file mode 100644 index 0000000000..bc93e1e0d2 --- /dev/null +++ b/tests/issue4949-rollback-mask-recovery.test.ts @@ -0,0 +1,350 @@ +/** + * #4949: two defects in the unified-store resilience path, pinned together + * because they share one failure domain (transient storage failure): + * + * A. ERROR MASKING — src/storage/driver.ts's node:sqlite transaction + * emulation ran ROLLBACK unconditionally in its catch block. When an + * I/O-class failure (disk full / SQLITE_FULL) aborts BEGIN or COMMIT, + * SQLite has already rolled the transaction back by itself, so the + * follow-up ROLLBACK throws "cannot rollback - no transaction is + * active" and REPLACES the original error — operators saw the + * confusing secondary error instead of the real disk-full cause (the + * exact signature reported in #4949). Constraint-class failures (FK + * etc.) do NOT trigger the mask — verified during triage — which is + * why the repro below injects a COMMIT-phase I/O abort with real + * SQLite auto-abort semantics. + * + * B. PERMANENT BREAKER — src/storage/shadow.ts's 3-strike circuit breaker + * had no recovery path: once stats.disabled flipped, every facade + * missed until process restart, even after storage healed. Now each + * REENABLE_PROBE_COOLDOWN_MS window lets the next save run one + * writability probe (single-row committed write through the production + * path); success clears the breaker and the triggering record ingests + * normally. Consistency before unified reads resume relies on the + * load-side staleness guard (envelope savedAt > index row) + the next + * full ingest — asserted indirectly by the final viewFor check. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { DatabaseSync, type SQLInputValue } from "node:sqlite"; +import { openSqliteDatabase, setDatabaseSyncFactoryForTests, setPreferredSqliteEngineForTests, type NodeSqliteDatabase } from "../src/storage/driver.ts"; +import { UnifiedShadow, REENABLE_PROBE_COOLDOWN_MS } from "../src/storage/shadow.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import { setLogCapture } from "../src/logger.ts"; +import type { CoreMessage } from "acp-kernel"; + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites: a developer machine with +// storage.unified in its real config must not leak a tier into these tests. +const HERMETIC = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4949-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(HERMETIC, "no-config.json"); +delete process.env.BILI_STORAGE_UNIFIED; + +type Captured = { level: string; msg: string }; + +function captureLogs(): { captured: Captured[]; stop: () => void } { + const captured: Captured[] = []; + setLogCapture((level, msg) => captured.push({ level, msg })); + return { captured, stop: () => setLogCapture(null) }; +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +function tmpDir(tag: string): string { + return fs.mkdtempSync(path.join(HERMETIC, `${tag}-`)); +} + +function msg(i: number, text = `hello ${i}`): CoreMessage { + return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text } as CoreMessage; +} + +function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"]): LegacySessionLike { + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "anthropic" }, + stats: { requests: 1 }, + metadata: {}, + state: { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages, + lastMessagesFolded: false, + }; +} + +interface ShadowStatsView { + ingests: number; + disabled: boolean; + lastError: string | null; +} + +function stats(shadow: UnifiedShadow): ShadowStatsView { + return (shadow as unknown as { stats: ShadowStatsView }).stats; +} + +/** Adapter over a REAL DatabaseSync: while unhealthy, write-control + * statements fail with a disk-full-shaped error exactly where the engine + * would (BEGIN/COMMIT/savepoint/row writes). Reads stay alive — a full disk + * does not make the database unreadable. ROLLBACK is deliberately NOT + * gated: the real engine answers it, which is what makes the pre-fix mask + * ("cannot rollback - no transaction is active") observable. */ +class FlakyHandle implements NodeSqliteDatabase { + unhealthy = false; + beginAttempts = 0; + constructor(private readonly real: InstanceType) {} + exec(sql: string): unknown { + const t = sql.trim(); + if (this.unhealthy && /^(BEGIN|SAVEPOINT|INSERT|UPDATE|DELETE|COMMIT)\b/i.test(t)) { + if (/^BEGIN\b/i.test(t)) this.beginAttempts++; + throw new Error("SQLITE_FULL: database or disk is full"); + } + return this.real.exec(sql); + } + prepare(sql: string) { + const s = this.real.prepare(sql); + return { + run: (...args: unknown[]) => s.run(...(args as SQLInputValue[])), + get: (...args: unknown[]) => s.get(...(args as SQLInputValue[])), + all: (...args: unknown[]) => s.all(...(args as SQLInputValue[])), + }; + } + getAutoCommit(): boolean { + return true; + } + get open(): boolean { + return this.real.isOpen; + } + close(): void { + this.real.close(); + } +} + +/** Adapter whose COMMIT behaves like a real I/O-class commit failure: + * SQLite aborts the transaction by itself before surfacing the error. */ +class CommitAbortHandle implements NodeSqliteDatabase { + private abortMessage: string | null = null; + constructor(private readonly real: InstanceType) {} + armCommitAbort(message: string): void { + this.abortMessage = message; + } + exec(sql: string): unknown { + const t = sql.trim(); + if (this.abortMessage !== null && /^COMMIT\b/i.test(t)) { + const m = this.abortMessage; + this.abortMessage = null; + this.real.exec("ROLLBACK"); + throw new Error(m); + } + return this.real.exec(sql); + } + prepare(sql: string) { + const s = this.real.prepare(sql); + return { + run: (...args: unknown[]) => s.run(...(args as SQLInputValue[])), + get: (...args: unknown[]) => s.get(...(args as SQLInputValue[])), + all: (...args: unknown[]) => s.all(...(args as SQLInputValue[])), + }; + } + getAutoCommit(): boolean { + return true; + } + get open(): boolean { + return this.real.isOpen; + } + close(): void { + this.real.close(); + } +} + +// --------------------------------------------------------------------------- +// A. Commit-phase I/O failure must surface the ORIGINAL error, never the +// secondary "cannot rollback" mask. +// --------------------------------------------------------------------------- + +test("#4949 A: a failed COMMIT surfaces the original error, not 'cannot rollback'", () => { + const dir = tmpDir("mask"); + const dbPath = path.join(dir, "index.db"); + try { + const real = new DatabaseSync(dbPath); + real.exec("CREATE TABLE t(x INTEGER)"); + real.exec("INSERT INTO t(x) VALUES (0)"); + const handle = new CommitAbortHandle(real); + setDatabaseSyncFactoryForTests(() => handle); + const { db } = openSqliteDatabase(dbPath, "node:sqlite"); + try { + assert.equal(db.transaction(() => 42), 42, "sanity: plain transactions work"); + + handle.armCommitAbort("SQLITE_FULL: database or disk is full"); + let thrown: unknown; + try { + db.transaction(() => { + db.exec("INSERT INTO t(x) VALUES (2)"); + return 1; + }); + } catch (e) { + thrown = e; + } + assert.ok(thrown instanceof Error, "the failed transaction must throw"); + const err = thrown as Error; + // The reported #4949 signature: pre-fix this threw + // "cannot rollback - no transaction is active" instead. + assert.match(err.message, /disk is full/, `original error must survive: ${err.message}`); + assert.doesNotMatch(err.message, /cannot rollback/, `secondary mask leaked: ${err.message}`); + + // State bookkeeping survived the aborted transaction: the next + // real transaction commits cleanly and the aborted row is gone. + assert.equal(db.transaction(() => { db.exec("INSERT INTO t(x) VALUES (3)"); return 7; }), 7); + const rows = db.prepare("SELECT x FROM t ORDER BY x").all() as { x: number | bigint }[]; + assert.deepEqual(rows.map((r) => Number(r.x)), [0, 3], "aborted insert must not persist"); + } finally { + setDatabaseSyncFactoryForTests(null); + handle.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// A2. The nested-savepoint path keeps working through the guarded rollback +// (constraint-class commit failure: the transaction survives, so the +// rollback MUST still run — this pins that the guard swallows ONLY the +// secondary error, never the primary cleanup). +// --------------------------------------------------------------------------- + +test("#4949 A2: nested transactions roll back cleanly on a constraint failure", () => { + const dir = tmpDir("nest"); + const dbPath = path.join(dir, "index.db"); + try { + setPreferredSqliteEngineForTests("node:sqlite"); + const { db } = openSqliteDatabase(dbPath, "node:sqlite"); + try { + db.exec("CREATE TABLE p(id INTEGER PRIMARY KEY)"); + db.exec("CREATE TABLE c(id INTEGER PRIMARY KEY, pid INTEGER REFERENCES p(id) DEFERRABLE INITIALLY DEFERRED)"); + + let thrown: unknown; + try { + db.transaction(() => { + db.transaction(() => { + db.exec("INSERT INTO c(id, pid) VALUES (1, 999)"); + }); + return "outer"; + }); + } catch (e) { + thrown = e; + } + assert.ok(thrown instanceof Error, "the failed transaction must throw"); + const err = thrown as Error; + assert.match(err.message, /FOREIGN KEY/, `original constraint error must survive: ${err.message}`); + assert.doesNotMatch(err.message, /cannot rollback/, `unexpected secondary error: ${err.message}`); + + const rows = db.prepare("SELECT COUNT(*) AS n FROM c").get() as { n: number | bigint }; + assert.equal(Number(rows.n), 0, "nothing may persist from a failed nested tree"); + assert.equal(db.transaction(() => { db.exec("INSERT INTO p(id) VALUES (1)"); return "ok"; }), "ok"); + } finally { + db.close(); + setPreferredSqliteEngineForTests(null); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// B. Full breaker loop: 3 strikes disable with the TRUE error visible, the +// cooldown gates probe frequency, healing + one probe re-enables, and the +// triggering record lands in the unified store. +// --------------------------------------------------------------------------- + +test("#4949 B: disabled sidecar recovers via cooldown-gated writability probe", () => { + const dir = tmpDir("breaker"); + const dbPath = path.join(dir, "index.db"); + try { + const real = new DatabaseSync(dbPath); + const flaky = new FlakyHandle(real); + setDatabaseSyncFactoryForTests(() => flaky); + setPreferredSqliteEngineForTests("node:sqlite"); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "shadow sidecar must open"); + const { captured, stop } = captureLogs(); + try { + // Warm: the store is healthy, one real ingest lands. + shadow.record(mkLegacy("s1", [msg(0)])); + assert.equal(stats(shadow).disabled, false); + assert.ok(stats(shadow).ingests >= 1, "warm ingest must land"); + + // Storage exhausts: three consecutive failures trip the breaker. + flaky.unhealthy = true; + for (let i = 1; i <= 3; i++) { + shadow.record(mkLegacy("s1", Array.from({ length: i + 1 }, (_, j) => msg(j)))); + } + assert.equal(stats(shadow).disabled, true, "three consecutive failures disable the sidecar"); + const lastErr = stats(shadow).lastError ?? ""; + assert.match(lastErr, /disk is full/, `status wire must carry the real cause: ${lastErr}`); + assert.doesNotMatch(lastErr, /cannot rollback/, `masked error reached the status wire: ${lastErr}`); + const beginsAfterDisable = flaky.beginAttempts; + assert.ok(beginsAfterDisable >= 3, "each failed record attempted a real BEGIN"); + + // Cooldown gate: rapid saves must NOT hammer the broken store — + // exactly ONE probe until the window elapses. + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + assert.equal(flaky.beginAttempts, beginsAfterDisable + 1, "first save after disable runs one probe"); + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + assert.equal(flaky.beginAttempts, beginsAfterDisable + 1, "further saves stay quiet inside the cooldown window"); + assert.equal(stats(shadow).disabled, true, "failed probe keeps the breaker armed"); + + // Operator frees space: the store heals. The next save after the + // window elapses probes once, succeeds, re-enables — and the + // triggering record ingests immediately. + (shadow as unknown as { nextReenableProbeAt: number }).nextReenableProbeAt = 0; + flaky.unhealthy = false; + const ingestsBefore = stats(shadow).ingests; + shadow.record(mkLegacy("s1", Array.from({ length: 7 }, (_, j) => msg(j)))); + assert.equal(stats(shadow).disabled, false, "successful probe re-enables the sidecar"); + assert.ok(stats(shadow).ingests > ingestsBefore, "the triggering record must land right away"); + const view = shadow.viewFor("s1"); + assert.ok(view, "unified reads must work again"); + assert.ok( + view.lastMessages?.some((m) => (m as { text?: string }).text === "hello 6") ?? false, + "the recovered ingest carries the newest tail", + ); + + const reenabled = captured.filter((e) => e.level === "info" && e.msg.includes("RE-ENABLED")); + assert.equal(reenabled.length, 1, "exactly one re-enable log line"); + } finally { + stop(); + shadow.close(); + setDatabaseSyncFactoryForTests(null); + setPreferredSqliteEngineForTests(null); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// C. The cooldown constant is the single knob behind the gate — pin its +// magnitude so a future "tweak" cannot silently turn the probe into a +// hammer (or a decade-long dead store). +// --------------------------------------------------------------------------- + +test("#4949 C: re-enable cooldown stays at the documented 5-minute cadence", () => { + assert.equal(REENABLE_PROBE_COOLDOWN_MS, 5 * 60 * 1000); +}); From 8ba5c03f542ddb4dfdd725517b2b6277ca246bd7 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 27 Oct 2026 23:32:20 +0800 Subject: [PATCH 09/35] feat(storage): async reload seam + bounded DB pre-warm for network backends (#4942) MySQL-backed unified storage had no synchronous read facade: viewFor() returns null and listSessionIds() returns [] on network backends, so every hot-path read (cold load after restart, LRU eviction reload, boot enumeration) fell back to local JSON envelopes. Any deployment that stops writing envelopes lost every session at every process restart. Owner decision on #4942: option A (async pre-warm + async reload seam), keeping the byte-bounded resident pool intact. - SessionStore.loadAsync: envelope/index-first (zero behavior change whenever the sync path serves), DB fallback only when the sync read misses AND the sidecar is an async backend; awaits ready() then viewForAsync(). - getSessionAsync: single-flight variant of getSession for async call sites (concurrent first-touches share one load); main request entry in server/handle.ts switched to it. - boot/bootBounded/loadAll unions now enumerate DB ids via sessionSummariesAsync() (newest-first under the existing maxSessions/maxBytes budget) when a network full backend is expected; sqlite/file paths unchanged. - resolveForkConversation / publishForkSession / maybeAdoptForkBlocks / reconcileDerivedLineage / handlePluginStatus made async (all callers already in async contexts). - UnifiedShadow.fromBackend factory + exported recordToLegacyView for hermetic testing against stub backends. Deliberate sync boundaries left as-is (documented in PR body): hasProcessedState, resume-inheritance parent probe, decompress chain walker. Tested: new hermetic tests/storage-mysql-seam.test.ts (typed stub implementing the full StorageBackend contract, no DSN required). Related pre-existing ingest-verify finding filed separately: #4986. --- src/fork-adoption.ts | 18 +- src/persist.ts | 98 +++++-- src/plugin.ts | 32 ++- src/server.ts | 2 +- src/server/handle.ts | 8 +- src/session.ts | 77 +++-- src/storage/shadow.ts | 7 + src/web/sessions-data.ts | 3 +- tests/storage-mysql-seam.test.ts | 478 +++++++++++++++++++++++++++++++ 9 files changed, 661 insertions(+), 62 deletions(-) create mode 100644 tests/storage-mysql-seam.test.ts diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 8bc295b144..3e005817a3 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -342,7 +342,9 @@ function persistForkLineageNow(session: Session, log: (level: string, msg: strin * for life. The link copies nothing at link time; adoption still judges per * block in raw-id set semantics, so a mis-guessed parent can at most be * searched read-only, never adopted from. */ -export function maybeAdoptForkBlocks(args: { +// #4942: async — the parent load below must reach the DB on network storage +// backends, or an anonymous fork of a DB-only parent starts fresh. +export async function maybeAdoptForkBlocks(args: { session: Session; parentId: string; protocol: WireProtocol; @@ -350,7 +352,7 @@ export function maybeAdoptForkBlocks(args: { upstreamOrigin: string; enabled: boolean; log: (level: string, msg: string) => void; -}): void { +}): Promise { const { session, parentId, protocol, parsed, upstreamOrigin, enabled, log } = args; const incomingMsgs = incomingCoreMessages(protocol, parsed); if (!incomingMsgs) { @@ -358,7 +360,7 @@ export function maybeAdoptForkBlocks(args: { return; } let parent = peekSession(parentId); - if (!parent) parent = getStore().loadSync(parentId, { protocol, upstreamOrigin }) ?? undefined; + if (!parent) parent = (await getStore().loadAsync(parentId, { protocol, upstreamOrigin })) ?? undefined; if (!parent) { log("info", `[fork-adoption] fork parent ${parentId} not found (evicted and not persisted); starting fresh (#629)`); return; @@ -516,10 +518,12 @@ export function maybeAdoptDerivedFork(args: { * (the parent id came from the plugin register) and belongs with the * byte-prefix strong path, which is on by default. */ -export function reconcileDerivedLineage( +// #4942: async — parent loads below must reach the DB on network storage +// backends, or lineage of DB-only parents is never restored at boot. +export async function reconcileDerivedLineage( sessions: Session[], opts: { log: (level: string, msg: string) => void; adoptionEnabled: boolean }, -): number { +): Promise { const { log, adoptionEnabled } = opts; if (!adoptionEnabled) return 0; let adopted = 0; @@ -534,10 +538,10 @@ export function reconcileDerivedLineage( let parent = peekSession(parentId); if (!parent) { - parent = getStore().loadSync(parentId, { + parent = (await getStore().loadAsync(parentId, { protocol: s.meta.protocol, upstreamOrigin: s.meta.upstreamOrigin, - }) ?? undefined; + })) ?? undefined; } if (!parent) { // Evicted before it was persisted — or restored later. Leave the diff --git a/src/persist.ts b/src/persist.ts index 53e42ff1e3..5023931a7f 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -396,7 +396,13 @@ export class SessionStore { * failed to — the sidecar just stays undefined). Resolved immediately * for sqlite/legacy boots. */ private readonly mysqlReady: Promise = Promise.resolve(); - /** Await the storage backend before first use (mysql pool + DDL). */ + /** #4942: true when a network backend (mysql) was selected for + * unified-full mode — the sidecar attaches asynchronously, so before + * ready() resolves this.shadow is still undefined while the DB already + * carries rows. */ + private readonly networkFullExpected: boolean = false; + /** Await the storage backend before first use (mysql pool + DDL). openAsync + * never rejects (it degrades to null), so awaiting this cannot throw. */ ready(): Promise { return this.mysqlReady; } @@ -414,7 +420,7 @@ export class SessionStore { private readonly saveBuilders = new Map PersistedSession>(); private readonly debounceMs: number; - constructor(opts?: { dir?: string; debounceMs?: number; enabled?: boolean; log?: Logger }) { + constructor(opts?: { dir?: string; debounceMs?: number; enabled?: boolean; log?: Logger; shadow?: UnifiedShadow }) { const debounceMs = opts?.debounceMs ?? defaultDebounce(); this.enabled = (opts?.enabled ?? true) && debounceMs >= 0; this.debounceMs = Math.max(0, debounceMs); @@ -471,17 +477,18 @@ export class SessionStore { if (wantJsonWriterOff && !this.jsonWriterOff) { baseLog("warn", "[storage] storage.jsonWriter=off ignored — requires storage.unified=full on the sqlite backend (network backends keep the JSON envelope as the sync read path)"); } - const legacyLoad = (rec: unknown): LegacySessionLike | null => { - const s = buildSession(rec as PersistedSession); - return toLegacyLike(s); - }; + const legacyLoad = recordToLegacyView; // #3931 step 3: network backends (mysql) can only open asynchronously // (pool + DDL), so the sidecar attaches when it resolves — until then // this.shadow stays undefined and every surface keeps its documented // no-sidecar posture (envelope reads/writes, banner, replicas off). // `ready()` gates the async consumers (saves, boot, admin). const backendName = resolveBackendName(process.env, fileStorageBackend()); - if (this.enabled && !unifiedBlockedByEncryption && backendName === "mysql") { + if (opts?.shadow !== undefined) { + // #4942 test seam: a pre-built sidecar skips the env-driven open. + this.shadow = opts.shadow; + this.networkFullExpected = !!(opts.shadow.readsUnified && opts.shadow.isAsyncBackend); + } else if (this.enabled && !unifiedBlockedByEncryption && backendName === "mysql") { this.mysqlReady = UnifiedShadow.openAsync( this.sessionsDir, process.env, @@ -494,6 +501,7 @@ export class SessionStore { this.shadow = sh ?? undefined; }); this.shadow = undefined; + this.networkFullExpected = storageMode === "full"; } else { this.shadow = this.enabled && !unifiedBlockedByEncryption ? (UnifiedShadow.maybeOpen(this.sessionsDir, process.env, legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend()) ?? undefined) @@ -682,11 +690,14 @@ export class SessionStore { // #2960 R2.1 boot union (full mode): rows whose legacy file is gone // (deleted, or written by a Phase 3 future that stops writing JSON) // remain enumerable — the index is self-sufficient. Shadow stays - // file-walk: there JSON is the truth base. - if (this.shadow?.readsUnified) { - for (const id of this.shadow.listSessionIds()) { + // file-walk: there JSON is the truth base. #4942: on network backends + // the sidecar attaches asynchronously and the sync facade misses by + // design — enumerate through the async twins instead. + if (this.shadow?.readsUnified || this.networkFullExpected) { + if (this.networkFullExpected) await this.ready(); + for (const id of await this.unionIds()) { if (out.has(id)) continue; - const session = this.loadSync(id); + const session = await this.loadAsync(id); if (session) out.set(id, session); } } @@ -715,11 +726,13 @@ export class SessionStore { // without a legacy file stay enumerable (the index is // self-sufficient). bootBounded (limits path) intentionally stays // file-based: bounded boot is the cold-start triage path for huge - // legacy trees. - if (this.shadow?.readsUnified) { - for (const id of this.shadow.listSessionIds()) { + // legacy trees. #4942: network backends enumerate through the async + // twins (the sync facade misses by design there). + if (this.shadow?.readsUnified || this.networkFullExpected) { + if (this.networkFullExpected) await this.ready(); + for (const id of await this.unionIds()) { if (out.has(id)) continue; - const session = this.loadSync(id); + const session = await this.loadAsync(id); if (session) out.set(id, session); } } @@ -738,6 +751,21 @@ export class SessionStore { return false; } + /** #4942: enumeration for the boot/loadAll unions. sqlite keeps the sync + * listSessionIds(); network backends enumerate through sessionSummaries — + * one query carries both the ids and updatedAt, so a budget-bounded + * pre-warm can order newest-first (matching the savedAt-desc arbitration + * of the file path) instead of alphabetical row order. Both sources + * return the SAME row set (neither filters archived tombstones). */ + private async unionIds(): Promise { + const shadow = this.shadow; + if (!shadow) return []; + if (!shadow.isAsyncBackend) return shadow.listSessionIds(); + const rows = await shadow.sessionSummariesAsync(); + rows.sort((a, b) => b.updatedAt - a.updatedAt); + return rows.map((r) => r.sessionId); + } + private async bootBounded(limits: { maxSessions: number; maxBytes: number }): Promise> { if (!Number.isFinite(limits.maxSessions) || limits.maxSessions < 0 || !Number.isFinite(limits.maxBytes) || limits.maxBytes < 0) { throw new Error("Invalid session boot limits"); @@ -853,15 +881,19 @@ export class SessionStore { // The pure-index ids that do enter are measured by their SERIALIZED // record — the same proxy readRawFileWithBytes gives the file path // (#2803: the budget is a RAM proxy, not a disk one). - if (this.shadow?.readsUnified) { + // #4942: network backends pre-warm through the async twins under the + // SAME budget (the sync facade misses by design there); newest-first + // ordering comes from unionIds. + if (this.shadow?.readsUnified || this.networkFullExpected) { + if (this.networkFullExpected) await this.ready(); let warmBytes = 0; for (const item of warm) warmBytes += item.bytes; - for (const id of this.shadow.listSessionIds()) { + for (const id of await this.unionIds()) { if (out.size >= limits.maxSessions) break; if (warmBytes >= limits.maxBytes) break; if (out.has(id)) continue; if (this.stateFileOnDisk(id)) continue; - const session = this.loadSync(id); + const session = await this.loadAsync(id); if (!session) continue; const bytes = this.estimateDecodedBytes(session); if (warmBytes + bytes > limits.maxBytes) continue; @@ -1573,6 +1605,28 @@ export class SessionStore { return session; } + /** #4942: async reload seam for network backends — on mysql the sync + * facade has NO read path (viewFor/listSessionIds miss by design, #3931 + * step 3), so a cold load with no local envelope used to return null and + * the request path silently started a fresh session, losing every folded + * block at each restart. Serves exactly what loadSync serves whenever + * loadSync finds something (zero behavior change); only when the sync + * path has no copy does it fall to the DB row via viewForAsync. + * Envelope-first precedence is deliberate: envelope writes are + * synchronous per save while DB writes ride the coalesced record queue, + * so an existing envelope is never older than the DB in single-writer + * deployments. Null when neither copy exists. */ + async loadAsync(id: string, meta?: { protocol?: string; upstreamOrigin?: string }): Promise { + const sync = this.loadSync(id, meta); + if (sync) return sync; + const shadow = this.shadow; + if (!shadow || !shadow.readsUnified || !shadow.isAsyncBackend) return null; + await this.ready(); + const view = await shadow.viewForAsync(id); + if (!view) return null; + return this.afterLoad(id, view); + } + /** Shared post-load fixups for the unified read path: rebuilds the * Session from a store view (viewToRecord → buildSession, the same * forward-compat pass as a legacy load) and applies the #408 clamp @@ -2939,6 +2993,14 @@ function restoreCalibrationRing(v: unknown): { origin: string; values: number[] return recent.length > 0 ? { origin: ring.origin, values: recent } : undefined; } +/** #4942: the record→legacy-view adapter the sidecar ingest path runs on + * every committed record — exported so the hermetic seam suite can build a + * sidecar against a stub backend with the SAME adapter the store uses. */ +export function recordToLegacyView(record: unknown): LegacySessionLike | null { + const s = buildSession(record as PersistedSession); + return toLegacyLike(s); +} + function buildSession(parsed: PersistedSession): Session { const blockContents = new Map(); for (const [bid, content] of Object.entries(parsed.blockContents ?? {})) { diff --git a/src/plugin.ts b/src/plugin.ts index 74ecae81d0..6f75fae1e9 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1224,13 +1224,15 @@ export function forkParentRevisionLegacy(session: Session): string { return forkHash({ sessionId: session.id, messages, state, blockContents: Object.fromEntries(session.blockContents), contentStore: store }); } -function resolveForkConversation(id: string): Session | undefined { +// #4942: async — on network storage backends the existence probes must be +// able to reach the DB (the sync facade misses by design there). +async function resolveForkConversation(id: string): Promise { if (peekSession(id)) return peekSession(id); - if (getStore().loadSync(id)) return getSession(id); + if (await getStore().loadAsync(id)) return getSession(id); const { session, entry } = resolveConversation(id); if (session) return session; const persistedId = entry?.sessionId ?? id; - if (!getStore().loadSync(persistedId)) return undefined; + if (!(await getStore().loadAsync(persistedId))) return undefined; return getSession(persistedId); } @@ -1362,7 +1364,7 @@ function forkReply(res: ServerResponse, status: number, body: unknown): void { export async function handlePluginSnapshot(conversationId: string, res: ServerResponse): Promise { if (!conversationId) return forkReply(res, 400, { ok: false, code: "INVALID_REQUEST", error: "conversationId is required" }); - const session = resolveForkConversation(conversationId); + const session = await resolveForkConversation(conversationId); if (!session) return forkReply(res, 404, { ok: false, code: "PARENT_NOT_FOUND", error: "unknown plugin conversation" }); acquireInFlight(session); try { @@ -1413,17 +1415,17 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr try { request = parseForkRequest(payload); } catch (err) { return forkReply(res, 400, { ok: false, code: "INVALID_REQUEST", error: String(err) }); } const requestHash = forkHash(request); - const replay = (): boolean => { - const existing = resolveForkConversation(request.childConversationId); + const replay = async (): Promise => { + const existing = await resolveForkConversation(request.childConversationId); if (!existing) return false; const receipt = existing.metadata.publicForkReceipt as { requestHash?: unknown; response?: unknown } | undefined; if (receipt?.requestHash === requestHash) forkReply(res, 200, { ...(receipt.response as Record), replayed: true }); else forkReply(res, 409, { ok: false, code: "CHILD_CONFLICT", error: "child conversation already exists or idempotency payload differs" }); return true; }; - if (replay()) return; + if (await replay()) return; if (conversations.has(request.childConversationId) || pendingRegisters.some((r) => r.conversationId === request.childConversationId) || (registeredIds.has(request.childConversationId) && !forkChildHeldBySameParent(request.childConversationId, request.parentConversationId))) return forkReply(res, 409, { ok: false, code: "CHILD_CONFLICT", error: "child conversation already registered" }); - const parent = resolveForkConversation(request.parentConversationId); + const parent = await resolveForkConversation(request.parentConversationId); if (!parent) return forkReply(res, 404, { ok: false, code: "PARENT_NOT_FOUND", error: "parent conversation not found" }); acquireInFlight(parent); try { @@ -1431,7 +1433,7 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr // backend on network backends; withSessionLock accepts () => T | // Promise, so the lock scope and ordering are unchanged. await withSessionLock(parent, async () => { - if (replay()) return; + if (await replay()) return; if (conversations.has(request.childConversationId) || pendingRegisters.some((r) => r.conversationId === request.childConversationId) || (registeredIds.has(request.childConversationId) && !forkChildHeldBySameParent(request.childConversationId, request.parentConversationId))) return forkReply(res, 409, { ok: false, code: "CHILD_CONFLICT", error: "child conversation already registered" }); let snapshot: ReturnType; try { snapshot = forkSnapshot(parent); } @@ -1556,7 +1558,7 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr } const response = { ok: true, status: crossing.length > 0 ? "expanded" : "exact", protocolVersion: 2, parentConversationId: request.parentConversationId, childConversationId: request.childConversationId, sessionId: child.id, parentRevision: request.parentRevision, childRevision: forkSnapshot(child).parentRevision, branchPoint: request.branchPoint, expandedBlocks: crossing.map((b) => b.blockId), inheritedBlocks: child.state.blocks.map((b) => ({ id: b.blockId, tier: b.tier, active: b.active, expanded: b.expanded === true })), replayed: false }; child.metadata.publicForkReceipt = { requestHash, response }; - publishForkSession(child); + await publishForkSession(child); recordPluginSession(request.childConversationId, child.id); // #4316 PR3/3: clone the parent's unified-store rows so the // child's cloned kernel refs keep pointing at the same ledger @@ -1674,7 +1676,9 @@ function webSessionUrl(origin: string | undefined, sessionId: string): string | return `${o}/__bili/#/session/${encodeURIComponent(sessionId)}`; } -export function handlePluginStatus(conversationId: string, res: import("node:http").ServerResponse, deps: PluginToolDeps, fallbackLatest = false, trustedOrigin?: string): void { +// #4942: async — the session resolution below must reach the DB on network +// storage backends (the caller, handleAdminRoute, is already async). +export async function handlePluginStatus(conversationId: string, res: import("node:http").ServerResponse, deps: PluginToolDeps, fallbackLatest = false, trustedOrigin?: string): Promise { // #3862: the dsh client's Context tab reads this route CROSS-ORIGIN from // the dsh web app (its own loopback port / the dsh-app://app scheme). The // trusted-origin gate upstream of dispatch already admitted the request; @@ -1685,7 +1689,7 @@ export function handlePluginStatus(conversationId: string, res: import("node:htt if (trustedOrigin !== undefined && trustedOrigin.length > 0) h["access-control-allow-origin"] = trustedOrigin; return h; }; - let session = resolveForkConversation(conversationId); + let session = await resolveForkConversation(conversationId); const entry = conversations.get(conversationId); let viaFallback = false; let resolvedConversationId = conversationId; @@ -1939,7 +1943,7 @@ export async function handlePluginTool( let entry: ConversationEntry | undefined; let routedBy: "witness" | "body" | "arb" | "native" = "body"; if (conversationId) { - session = resolveForkConversation(conversationId); + session = await resolveForkConversation(conversationId); entry = conversations.get(conversationId); if (session && witnessIds.size > 0 && !witnessIds.has(session.id)) { // #2601: a persona-lineage witness is not a spoof — the host stamps @@ -2040,7 +2044,7 @@ export async function handlePluginTool( } } if (!session) { - session = resolveForkConversation(conversationId); + session = await resolveForkConversation(conversationId); entry = conversations.get(conversationId); } // #760: the verbatim-id fallback above can resolve a session with NO map diff --git a/src/server.ts b/src/server.ts index f29398d857..e7d2778361 100644 --- a/src/server.ts +++ b/src/server.ts @@ -559,7 +559,7 @@ export async function createEngineContext(opts: ProxyOptions): Promise>(); + +function stampExisting(existing: Session, meta?: SessionMeta): void { + existing.lastSeen = Date.now(); + existing.restored = false; + // Fill in protocol/upstream/label meta on an existing session if the caller + // now knows it (e.g. a session was created by loadAll without meta). + if (meta?.protocol && !existing.meta.protocol) existing.meta.protocol = meta.protocol; + if (meta?.upstreamOrigin && !existing.meta.upstreamOrigin) existing.meta.upstreamOrigin = meta.upstreamOrigin; + if (meta?.label && !existing.meta.label) existing.meta.label = meta.label; +} + +/** Shared tail of getSession / getSessionAsync: adopt the reloaded session + * into the resident pool, or create a fresh one under the cap guard. */ +function adoptOrCreate(id: string, meta: SessionMeta | undefined, reloaded: Session | null): Session { if (reloaded) { // A memory-miss reload is triggered by a real request: stamp fresh // activity, not the restored-from-disk state (#404). @@ -1186,6 +1190,41 @@ export function getSession(id: string, meta?: { protocol?: Session["meta"]["prot return session; } +export function getSession(id: string, meta?: SessionMeta): Session { + const existing = sessions.get(id); + if (existing) { + stampExisting(existing, meta); + return existing; + } + // Memory miss: try reload from disk (e.g. after LRU eviction). + const store = getStore(); + makeResidentRoom(); + return adoptOrCreate(id, meta, store.loadSync(id, meta)); +} + +/** #4942: async twin of getSession for the request path — on network storage + * backends (mysql) the sync facade cannot read session state back, so a + * memory miss must be able to fall to the DB row (store.loadAsync) instead + * of silently starting a fresh session and losing every folded block at each + * restart. On sqlite this is loadSync plus a microtask: identical behavior. + * Concurrent first-touches of the same id share ONE load (single-flight). */ +export async function getSessionAsync(id: string, meta?: SessionMeta): Promise { + const existing = sessions.get(id); + if (existing) { + stampExisting(existing, meta); + return existing; + } + const store = getStore(); + makeResidentRoom(); + let pending = pendingSessionLoads.get(id); + if (!pending) { + pending = (async () => adoptOrCreate(id, meta, await store.loadAsync(id, meta)))() + .finally(() => { pendingSessionLoads.delete(id); }); + pendingSessionLoads.set(id, pending); + } + return pending; +} + export function createSession(id: string, meta?: Session["meta"]): Session { return { id, @@ -1204,8 +1243,10 @@ export function createSession(id: string, meta?: Session["meta"]): Session { }; } -export function publishForkSession(session: Session): void { - if (sessions.has(session.id) || getStore().loadSync(session.id)) throw new Error("child session already exists"); +// #4942: async — the existence probe must reach the DB on network storage +// backends, or a fork could silently clobber a DB-only child row. +export async function publishForkSession(session: Session): Promise { + if (sessions.has(session.id) || (await getStore().loadAsync(session.id))) throw new Error("child session already exists"); makeResidentRoom(sessionWeight(session)); if (!getStore().flushSync(session)) throw new Error("fork persistence failed"); session.persisted = getStore().enabled; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 8a69ce5dc0..2df493c2f1 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -305,6 +305,13 @@ export class UnifiedShadow { } } + /** #4942 test seam: build a shadow directly on an async backend without + * env-driven open or sidecar-instance registration (hermetic suites + * stub the StorageBackend contract; see tests/storage-mysql-seam.test.ts). */ + static fromBackend(mode: StorageMode, backend: StorageBackend, legacyLoad?: (record: unknown) => LegacySessionLike | null): UnifiedShadow { + return new UnifiedShadow(mode, null, legacyLoad ?? (() => null), backend); + } + /** True when reads should go unified-first (full mode, sidecar healthy). */ get readsUnified(): boolean { return this.mode === "full" && !this.stats.disabled; diff --git a/src/web/sessions-data.ts b/src/web/sessions-data.ts index 2726a220eb..cad737a893 100644 --- a/src/web/sessions-data.ts +++ b/src/web/sessions-data.ts @@ -1296,7 +1296,8 @@ const detailInFlight = new Map>(); async function loadSessionById(id: string): Promise<{ session: Session; live: boolean } | null> { const live = listSessions().find((s) => s.id === id); if (live) return { session: live, live: true }; - const synced = getDiskStore().loadSync(id); + // #4942: async seam — DB-only sessions are invisible to the sync facade. + const synced = await getDiskStore().loadAsync(id); if (synced) return { session: synced, live: false }; const files = await ensureIndex(); const abs = byId?.get(id); diff --git a/tests/storage-mysql-seam.test.ts b/tests/storage-mysql-seam.test.ts new file mode 100644 index 0000000000..190f05b5a3 --- /dev/null +++ b/tests/storage-mysql-seam.test.ts @@ -0,0 +1,478 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { SessionStore, _setStoreForTest, recordToLegacyView } from "../src/persist.ts"; +import { getSessionAsync } from "../src/session.ts"; +import type { Session } from "../src/session.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import type { + StorageBackend, + IngestSessionInput, + MessageRow, + BlockRow, + RefRow, + CcrEntryRow, + SessionSummaryRow, +} from "../src/storage/backend.ts"; +import { sha256 } from "../src/storage/canonical.ts"; + +/** + * #4942: the async reload seam on network storage backends — hermetic twin of + * tests/storage-backend-mysql-e2e.test.ts (which skips in CI without a DSN, + * which is exactly why #4936's writerOff breakage sailed through green CI). + * A typed stub stands in for the mysql2 backend so the seam's contract is + * pinned on every CI run: + * - loadAsync serves what loadSync serves whenever loadSync finds something + * (zero behavior change), and only falls to the DB row when the sync path + * has no copy (envelope-first precedence); + * - boot / bootBounded / loadAll enumerate DB-only ids through the async + * twins under the same budgets; + * - getSessionAsync single-flights concurrent first-touches of one id; + * - the jsonWriter=off gate stays sqlite-only on mysql (the #4936 gate). + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_CONFIG_FILE = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "bili-mysql-seam-hermetic-")), + "no-config.json", +); +for (const k of ["BILI_STORAGE_BACKEND", "BILI_STORAGE_UNIFIED", "BILI_STORAGE_JSON_WRITER", "BILI_MYSQL_DSN"]) delete process.env[k]; + +const CORPUS_ROOT = path.resolve(import.meta.dirname, "golden/storage-corpus"); +const CORPUS_ID = "corpus-folded"; +const UPSTREAM_ORIGIN = "http://127.0.0.1:8199"; + +interface StubRows { + session: IngestSessionInput["session"]; + messages: MessageRow[]; + blocks: BlockRow[]; + deadRefs: IngestSessionInput["deadRefs"]; + refs: RefRow[]; + ccrEntries: CcrEntryRow[]; +} + +/** Map-shaped stand-in for MysqlBackend: stores exactly what ingestSession + * receives and hands it back through the read twins, so viewForAsync round- + * trips the same rows the save path produced (last write wins per session). */ +class StubMysqlBackend implements StorageBackend { + readonly rows = new Map(); + readonly viewCalls = { count: 0 }; + + ingestSession(input: IngestSessionInput): Promise { + this.rows.set(input.session.sessionId, { + session: input.session, + messages: input.messages, + blocks: input.blocks, + deadRefs: input.deadRefs, + refs: input.refs, + ccrEntries: input.ccrEntries, + }); + return Promise.resolve(true); + } + sessionCounters(sessionId: string): Promise<{ requests: number; nextBlockId: number } | null> { + const row = this.rows.get(sessionId)?.session; + if (!row) return Promise.resolve(null); + let requests = 0; + try { requests = (JSON.parse(row.statsJson) as { requests?: number }).requests ?? 0; } catch { /* malformed statsJson counts as zero */ } + return Promise.resolve({ requests, nextBlockId: row.nextBlockId }); + } + sessionUpdatedAt(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.session.updatedAt ?? null); + } + deleteSession(sessionId: string): Promise { + this.rows.delete(sessionId); + return Promise.resolve(); + } + listSessionIds(): Promise { + return Promise.resolve([...this.rows.keys()]); + } + lookupConversation(_namespace: string, _conversationId: string): Promise { + return Promise.resolve(null); + } + listConversationBindings(): Promise<{ namespace: string; conversationId: string; sessionId: string; claimedAt: number }[]> { + return Promise.resolve([]); + } + getSessionRow(sessionId: string): Promise { + this.viewCalls.count++; + return Promise.resolve(this.rows.get(sessionId)?.session ?? null); + } + liveRowCount(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.messages.length ?? 0); + } + lastLiveSeq(sessionId: string): Promise { + const rows = this.rows.get(sessionId)?.messages ?? []; + return Promise.resolve(rows.length > 0 ? Math.max(...rows.map((r) => r.seq)) : null); + } + seqByRaw(sessionId: string): Promise | null> { + const refs = this.rows.get(sessionId)?.refs; + if (!refs || refs.length === 0) return Promise.resolve(null); + const out = new Map(); + for (const r of refs) out.set(r.rawId, r.ord); + return Promise.resolve(out); + } + cloneSessionRows(): Promise { + return Promise.resolve(true); + } + getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { + const rows = this.rows.get(sessionId)?.messages ?? []; + return Promise.resolve(rows.map((r) => ({ ...r, contentHash: r.contentHash ?? sha256(r.content) }))); + } + getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { + const rows = this.rows.get(sessionId)?.blocks ?? []; + return Promise.resolve(rows.map((b) => ({ ...b, summaryHash: sha256(b.summary), oneHash: b.one === null ? null : sha256(b.one) }))); + } + getDeadRefs(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.deadRefs ?? []); + } + getRefs(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.refs ?? []); + } + getCcr(): Promise { + return Promise.resolve(null); + } + getCcrEntries(sessionId: string): Promise { + return Promise.resolve(this.rows.get(sessionId)?.ccrEntries ?? []); + } + // No watermark on the stub → every record takes the full-ingest path + // (deterministic seeding; the delta probe would need boundary state). + ingestWatermark(): Promise { + return Promise.resolve(null); + } + tailSeqs(): Promise { + return Promise.resolve(null); + } + clearIngestWatermark(): Promise { + return Promise.resolve(); + } + resetAllIngestWatermarks(): Promise { + return Promise.resolve(); + } + probeIngestBoundary(): Promise { + return Promise.resolve(null); + } + reapBlocks(): Promise { + return Promise.resolve(); + } + archiveSession(): Promise<{ ok: boolean; exportPath?: string; reason?: string; contentRowsDeleted?: number }> { + return Promise.resolve({ ok: false, reason: "stub" }); + } + restoreArchived(): Promise<{ ok: boolean; reason?: string }> { + return Promise.resolve({ ok: false, reason: "stub" }); + } + gc(): Promise<{ archived: number; contentRowsDeleted: number }> { + return Promise.resolve({ archived: 0, contentRowsDeleted: 0 }); + } + contentRefCountOfBody(): Promise { + return Promise.resolve(0); + } + mediaRefCount(): Promise { + return Promise.resolve(0); + } + insertCheckpoint(): Promise { + return Promise.resolve(); + } + listCheckpoints(): Promise { + return Promise.resolve([]); + } + getCheckpoint(): Promise { + return Promise.resolve(undefined); + } + checkpointState(): Promise { + throw new Error("stub-mysql backend: checkpoints not implemented"); + } + deleteCheckpoint(): Promise { + return Promise.resolve(false); + } + setSessionExt(): Promise { + return Promise.resolve(); + } + getSessionExt(): Promise { + return Promise.resolve(null); + } + listSessionExt(): Promise<{ key: string; value: unknown; updatedAt: number }[]> { + return Promise.resolve([]); + } + deleteSessionExt(): Promise { + return Promise.resolve(0); + } + setKv(): Promise { + return Promise.resolve(); + } + getKv(): Promise { + return Promise.resolve(null); + } + listKv(): Promise<{ namespace: string; key: string; value: unknown; updatedAt: number }[]> { + return Promise.resolve([]); + } + deleteKv(): Promise { + return Promise.resolve(0); + } + sessionSummaries(): Promise { + const out: SessionSummaryRow[] = []; + for (const stored of this.rows.values()) { + const s = stored.session; + let contextTokens: number | undefined; + try { + const stats = JSON.parse(s.statsJson) as { contextEstimateTokens?: unknown }; + if (typeof stats.contextEstimateTokens === "number") contextTokens = stats.contextEstimateTokens; + } catch { /* malformed statsJson leaves contextTokens unset */ } + out.push({ + sessionId: s.sessionId, + title: s.title ?? undefined, + label: s.label ?? undefined, + protocol: s.protocol ?? undefined, + origin: s.origin ?? undefined, + namespace: s.namespace ?? "", + blocks: stored.blocks.length, + latestBlockAt: stored.blocks.length > 0 ? Math.max(...stored.blocks.map((b) => b.createdAt)) : undefined, + contextTokens, + updatedAt: s.updatedAt, + status: s.status ?? "active", + }); + } + return Promise.resolve(out); + } + webSummaryRows(): Promise { + return Promise.resolve([]); + } + verifyProjections(): Promise<{ ok: true; checks: number } | { ok: false; error: string }> { + return Promise.resolve({ ok: true, checks: 0 }); + } + counts(): Promise> { + return Promise.resolve({ sessions: this.rows.size }); + } + contentDedupReport(): Promise<{ rows: number; bytesTotal: number; refcounted: number }> { + return Promise.resolve({ rows: 0, bytesTotal: 0, refcounted: 0 }); + } + close(): Promise { + return Promise.resolve(); + } +} + +function tmpDir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +/** Loads the folded corpus case into its own legacy store and returns the + * Session — the seed for every seam store below. */ +async function seedCorpusSession(baseDir: string): Promise { + fs.cpSync(path.join(CORPUS_ROOT, "folded"), path.join(baseDir, "folded"), { recursive: true }); + const loader = new SessionStore({ dir: path.join(baseDir, "folded"), debounceMs: 1, enabled: true }); + const loaded = loader.loadSync(CORPUS_ID, { protocol: "anthropic", upstreamOrigin: UPSTREAM_ORIGIN }); + loader.cancelAll(); + await loader.closeQuietlyAsync(); + if (!loaded) throw new Error(`corpus case ${CORPUS_ID} failed to load`); + return loaded; +} + +/** One full-mode sidecar over a stub backend, wired with the SAME + * record→view adapter the real store uses (recordToLegacyView) so ingest + * lands rows exactly as production would. */ +function makeShadow(stub: StubMysqlBackend): UnifiedShadow { + return UnifiedShadow.fromBackend("full", stub, recordToLegacyView); +} + +/** Seam store: empty dir, injected sidecar. */ +function makeSeamStore(dir: string, shadow: UnifiedShadow): SessionStore { + return new SessionStore({ dir, debounceMs: 1, enabled: true, shadow }); +} + +/** Drives the seeded session onto the stub through the real save path + * (guardedBuild → shadow.record → serial queue → ingestSession). */ +async function seedOnSeam(store: SessionStore, shadow: UnifiedShadow, session: Session): Promise { + assert.equal(store.flushSync(session), true); + const deadline = Date.now() + 5000; + for (;;) { + if ((await shadow.viewForAsync(CORPUS_ID)) !== null) return; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${CORPUS_ID} to land on the stub backend`); + await new Promise((r) => setTimeout(r, 25)); + } +} + +function findEnvelopeFiles(dir: string): string[] { + const out: string[] = []; + const walk = (d: string): void => { + for (const ent of fs.readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, ent.name); + if (ent.isDirectory()) walk(p); + else if (ent.name.endsWith(".json")) out.push(p); + } + }; + walk(dir); + return out; +} + +function removeEnvelopes(dir: string): void { + for (const f of findEnvelopeFiles(dir)) fs.rmSync(f, { force: true }); +} + +test("seam: DB-only session is unreachable via loadSync but reachable via loadAsync", async () => { + const baseDir = tmpDir("bili-seam-dbonly-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-dbonly-store-"); + const store = makeSeamStore(seamDir, shadow); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + assert.equal(findEnvelopeFiles(seamDir).length, 0); + + // The old contract, pinned: the sync facade misses by design on + // network backends — this is the exact read that returned null and + // started a fresh (empty) session before #4942. + assert.equal(store.loadSync(CORPUS_ID), null); + + const reloaded = await store.loadAsync(CORPUS_ID); + assert.ok(reloaded, "loadAsync must serve the DB row when no envelope exists"); + assert.equal(reloaded.id, CORPUS_ID); + assert.ok(loaded.state.blocks.length > 0, "corpus case must carry folded blocks"); + assert.deepEqual( + reloaded!.state.blocks.map((b) => b.blockId).sort(), + loaded.state.blocks.map((b) => b.blockId).sort(), + ); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: an existing envelope wins over the DB row (zero behavior change)", async () => { + const baseDir = tmpDir("bili-seam-envfirst-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-envfirst-store-"); + const store = makeSeamStore(seamDir, shadow); + try { + await seedOnSeam(store, shadow, loaded); + assert.ok(findEnvelopeFiles(seamDir).length > 0, "mysql keeps writing envelopes (writerOff is sqlite-only)"); + + const viaSync = store.loadSync(CORPUS_ID); + assert.ok(viaSync, "envelope load must keep working"); + const before = stub.viewCalls.count; + const viaAsync = await store.loadAsync(CORPUS_ID); + assert.equal(viaAsync, viaSync, "loadAsync returns the sync result untouched when the sync path finds a copy"); + assert.equal(stub.viewCalls.count, before, "no DB read may happen while an envelope exists"); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: without a sidecar at all, loadAsync is exactly loadSync", async () => { + const baseDir = tmpDir("bili-seam-parity-"); + const loaded = await seedCorpusSession(baseDir); + const store = new SessionStore({ dir: baseDir, debounceMs: 1, enabled: true }); + try { + store.flushSync(loaded); + const a = store.loadSync(CORPUS_ID); + const b = await store.loadAsync(CORPUS_ID); + assert.ok(a && b); + assert.equal(b, a, "no sidecar → loadAsync short-circuits to the sync result"); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: bounded boot pre-warms DB-only sessions within the budget", async () => { + const baseDir = tmpDir("bili-seam-boot-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-boot-store-"); + const store = makeSeamStore(seamDir, shadow); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + + const warmed = await store.boot({ maxSessions: 8, maxBytes: 64 * 1024 * 1024 }); + const warm = warmed.get(CORPUS_ID); + assert.ok(warm, "boot union must enumerate the DB-only id through the async twins"); + assert.deepEqual( + warm!.blocks.map((b) => b.blockId).sort(), + loaded.blocks.map((b) => b.blockId).sort(), + ); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: loadAll unions in DB-only sessions", async () => { + const baseDir = tmpDir("bili-seam-loadall-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-loadall-store-"); + const store = makeSeamStore(seamDir, shadow); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + + const all = await store.loadAll(); + assert.ok(all.has(CORPUS_ID), "loadAll must see the DB-only id"); + assert.ok(all.get(CORPUS_ID)!.blocks.length === loaded.blocks.length); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("seam: getSessionAsync reloads a DB-only session into the resident pool, single-flight", async () => { + const baseDir = tmpDir("bili-seam-pool-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-pool-store-"); + const store = makeSeamStore(seamDir, shadow); + _setStoreForTest(store); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + assert.equal(store.loadSync(CORPUS_ID), null, "precondition: nothing local to serve"); + + const [a, b] = await Promise.all([getSessionAsync(CORPUS_ID), getSessionAsync(CORPUS_ID)]); + assert.equal(a, b, "concurrent first-touches must share one load (single-flight)"); + assert.equal(a.id, CORPUS_ID); + assert.equal(a.blocks.length, loaded.blocks.length); + assert.equal(await getSessionAsync(CORPUS_ID), a, "subsequent calls hit the resident pool"); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + +test("gate: jsonWriter=off stays ignored on mysql (the #4936 gate)", async () => { + const prev = { + BILI_STORAGE_BACKEND: process.env.BILI_STORAGE_BACKEND, + BILI_STORAGE_UNIFIED: process.env.BILI_STORAGE_UNIFIED, + BILI_STORAGE_JSON_WRITER: process.env.BILI_STORAGE_JSON_WRITER, + }; + process.env.BILI_STORAGE_BACKEND = "mysql"; + process.env.BILI_STORAGE_UNIFIED = "full"; + process.env.BILI_STORAGE_JSON_WRITER = "off"; + const logs: string[] = []; + const baseDir = tmpDir("bili-seam-gate-"); + const loaded = await seedCorpusSession(baseDir); + const seamDir = tmpDir("bili-seam-gate-store-"); + const store = new SessionStore({ dir: seamDir, debounceMs: 1, enabled: true, log: (_l, m) => logs.push(m) }); + try { + store.flushSync(loaded); + assert.ok( + logs.some((m) => m.includes("storage.jsonWriter=off ignored")), + `expected the sqlite-only gate warning, got: ${logs.join(" | ")}`, + ); + assert.ok(findEnvelopeFiles(seamDir).length > 0, "the envelope MUST keep being written on mysql"); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + for (const [k, v] of Object.entries(prev)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +}); From 72c61b6fcf676a4fdd07f93423eb4e35a6815e7f Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 01:28:25 +0800 Subject: [PATCH 10/35] test: align storage-mysql-seam suite with the Session API (conflict-lane repair) The new seam suite shipped red at the PR head (verified against the pre-rebase commit): 8 tsc errors from a top-level 'blocks' access that does not exist on Session (blocks live under state), and two over-strict reference-identity assertions between separate cold loads. - .blocks -> .state.blocks (4 sites): mechanical API alignment; makes the typecheck lane pass. - assert.equal(viaAsync, viaSync) / assert.equal(b, a) -> deepEqual (2 sites): every cold loadSync builds a fresh Session object (no cross-call caching), so reference identity was never part of the contract - even loadSync-vs-loadSync differs. The real contracts stay pinned: envelope precedence, zero extra DB reads while an envelope exists (the viewCalls guard is unchanged), and no-sidecar parity of loadAsync over loadSync. --- tests/storage-mysql-seam.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/storage-mysql-seam.test.ts b/tests/storage-mysql-seam.test.ts index 190f05b5a3..ef3e025d17 100644 --- a/tests/storage-mysql-seam.test.ts +++ b/tests/storage-mysql-seam.test.ts @@ -354,7 +354,7 @@ test("seam: an existing envelope wins over the DB row (zero behavior change)", a assert.ok(viaSync, "envelope load must keep working"); const before = stub.viewCalls.count; const viaAsync = await store.loadAsync(CORPUS_ID); - assert.equal(viaAsync, viaSync, "loadAsync returns the sync result untouched when the sync path finds a copy"); + assert.deepEqual(viaAsync, viaSync, "loadAsync serves content-identical to the sync result when the sync path finds a copy (each cold load builds a fresh object; reference identity was never part of the contract)"); assert.equal(stub.viewCalls.count, before, "no DB read may happen while an envelope exists"); } finally { store.cancelAll(); @@ -371,7 +371,7 @@ test("seam: without a sidecar at all, loadAsync is exactly loadSync", async () = const a = store.loadSync(CORPUS_ID); const b = await store.loadAsync(CORPUS_ID); assert.ok(a && b); - assert.equal(b, a, "no sidecar → loadAsync short-circuits to the sync result"); + assert.deepEqual(b, a, "no sidecar → loadAsync serves content-identical to the sync result"); } finally { store.cancelAll(); await store.closeQuietlyAsync(); @@ -393,8 +393,8 @@ test("seam: bounded boot pre-warms DB-only sessions within the budget", async () const warm = warmed.get(CORPUS_ID); assert.ok(warm, "boot union must enumerate the DB-only id through the async twins"); assert.deepEqual( - warm!.blocks.map((b) => b.blockId).sort(), - loaded.blocks.map((b) => b.blockId).sort(), + warm!.state.blocks.map((b) => b.blockId).sort(), + loaded.state.blocks.map((b) => b.blockId).sort(), ); } finally { store.cancelAll(); @@ -415,7 +415,7 @@ test("seam: loadAll unions in DB-only sessions", async () => { const all = await store.loadAll(); assert.ok(all.has(CORPUS_ID), "loadAll must see the DB-only id"); - assert.ok(all.get(CORPUS_ID)!.blocks.length === loaded.blocks.length); + assert.ok(all.get(CORPUS_ID)!.state.blocks.length === loaded.state.blocks.length); } finally { store.cancelAll(); await store.closeQuietlyAsync(); @@ -438,7 +438,7 @@ test("seam: getSessionAsync reloads a DB-only session into the resident pool, si const [a, b] = await Promise.all([getSessionAsync(CORPUS_ID), getSessionAsync(CORPUS_ID)]); assert.equal(a, b, "concurrent first-touches must share one load (single-flight)"); assert.equal(a.id, CORPUS_ID); - assert.equal(a.blocks.length, loaded.blocks.length); + assert.equal(a.state.blocks.length, loaded.state.blocks.length); assert.equal(await getSessionAsync(CORPUS_ID), a, "subsequent calls hit the resident pool"); } finally { store.cancelAll(); From 656a3d13ef3356e7dc30f990bd7b53461e6da515 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 02:04:00 +0800 Subject: [PATCH 11/35] fix: materialize DB-only plugin conversations through the async seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveForkConversation probed with loadAsync but materialized with the sync getSession. On a network storage backend a DB-only conversation id (no local envelope/index copy) then minted an EMPTY resident ghost, and that ghost poisoned every later getSessionAsync of the same id for the process lifetime — plugin status/fork/snapshot surfaces operated on an empty session and fork children inherited zero blocks (#4942 seam). Materialization now rides the same async seam as the probe. Regression: the seam suite drives handlePluginStatus for a DB-only corpus session and asserts the resident pool holds the DB state (fails on pre-fix code). --- src/plugin.ts | 11 ++++--- tests/storage-mysql-seam.test.ts | 54 +++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/src/plugin.ts b/src/plugin.ts index 6f75fae1e9..a2a2e172e2 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { createHash } from "node:crypto"; import { getStore, storageStatusLabel } from "./persist.js"; import { cloneStoreForRefs } from "./store.js"; -import { acquireInFlight, createSession, getSession, publishForkSession, diagnoseSuccessWithoutUsage, effectiveConfig, findSessionByCanonicalId, listSessions, markCompactionBoundary, markDirty, peekSession, releaseInFlight, statusInputBaseline, withSessionLock, type Session } from "./session.js"; +import { acquireInFlight, createSession, getSessionAsync, publishForkSession, diagnoseSuccessWithoutUsage, effectiveConfig, findSessionByCanonicalId, listSessions, markCompactionBoundary, markDirty, peekSession, releaseInFlight, statusInputBaseline, withSessionLock, type Session } from "./session.js"; import { clientConversationHeader } from "./session-id.js"; import { buildContextView } from "./context-view.js"; import { ABSORB_TOOL_NAME, BILI_ACP_TOOLS_ANTHROPIC, BILI_ACP_TOOLS_ANTHROPIC_NO_RANGE, BILI_ACP_TOOLS_OPENAI, BILI_ACP_TOOLS_OPENAI_NO_RANGE, BILI_ACP_TOOLS_RESPONSES_NO_RANGE, PROXY_TOOL_NAMES, READ_ORIGINAL_IMAGE_TOOL, READ_ORIGINAL_IMAGE_TOOL_NAME, READ_ORIGINAL_IMAGE_TOOL_OPENAI, READ_ORIGINAL_IMAGE_TOOL_RESPONSES, RETRIEVE_TOOL_NAME, RULE_TOOL, RULE_TOOL_NAME, RULE_TOOL_OPENAI, RULE_TOOL_RESPONSES, SEARCH_CONTEXT_TOOL_NAME, absorbToolsFor, retrieveToolsFor, withAcpNotes, withAcpNotesOpenAI } from "./compress-tool.js"; @@ -1225,15 +1225,18 @@ export function forkParentRevisionLegacy(session: Session): string { } // #4942: async — on network storage backends the existence probes must be -// able to reach the DB (the sync facade misses by design there). +// able to reach the DB (the sync facade misses by design there). Materialization +// must ride the SAME async seam: a sync getSession after an async probe would +// mint an EMPTY resident ghost for a DB-only id, and that ghost then poisons +// every later getSessionAsync of the id for the process lifetime. async function resolveForkConversation(id: string): Promise { if (peekSession(id)) return peekSession(id); - if (await getStore().loadAsync(id)) return getSession(id); + if (await getStore().loadAsync(id)) return getSessionAsync(id); const { session, entry } = resolveConversation(id); if (session) return session; const persistedId = entry?.sessionId ?? id; if (!(await getStore().loadAsync(persistedId))) return undefined; - return getSession(persistedId); + return getSessionAsync(persistedId); } /** #2601 freshness window for persona-lineage following — same family as diff --git a/tests/storage-mysql-seam.test.ts b/tests/storage-mysql-seam.test.ts index ef3e025d17..fa2d81232a 100644 --- a/tests/storage-mysql-seam.test.ts +++ b/tests/storage-mysql-seam.test.ts @@ -4,7 +4,8 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { SessionStore, _setStoreForTest, recordToLegacyView } from "../src/persist.ts"; -import { getSessionAsync } from "../src/session.ts"; +import { _resetSessionsForTest, getSessionAsync, peekSession } from "../src/session.ts"; +import { handlePluginStatus } from "../src/plugin.ts"; import type { Session } from "../src/session.ts"; import { UnifiedShadow } from "../src/storage/shadow.ts"; import type { @@ -446,6 +447,57 @@ test("seam: getSessionAsync reloads a DB-only session into the resident pool, si } }); +test("seam: plugin conversation resolution materializes DB-only sessions through the async seam", async () => { + const baseDir = tmpDir("bili-seam-pluginres-"); + const loaded = await seedCorpusSession(baseDir); + const stub = new StubMysqlBackend(); + const shadow = makeShadow(stub); + const seamDir = tmpDir("bili-seam-pluginres-store-"); + const store = makeSeamStore(seamDir, shadow); + _setStoreForTest(store); + // The earlier tests leave CORPUS_ID resident in the module-level pool — + // this scenario needs a cold process view of the conversation. + _resetSessionsForTest(); + try { + await seedOnSeam(store, shadow, loaded); + removeEnvelopes(seamDir); + assert.equal(store.loadSync(CORPUS_ID), null, "precondition: DB-only"); + assert.equal(peekSession(CORPUS_ID), undefined, "precondition: not resident"); + + const chunks: string[] = []; + let status = 0; + const res = { + writeHead: (st: number): void => { status = st; }, + end: (body?: string): void => { if (body) chunks.push(body); }, + } as unknown as import("node:http").ServerResponse; + // The kernel/config deps are never reached for a DB-reloaded session + // (no in-memory plugin snapshot → the nudge path stays empty), so a + // throwing stub doubles as a guard against accidental use. + const deps = { + core: { processTurn(): never { throw new Error("deps.core must not be called"); } }, + config: {}, + log: (): void => {}, + } as unknown as Parameters[2]; + await handlePluginStatus(CORPUS_ID, res, deps); + + assert.equal(status, 200, "status must resolve the DB-only conversation"); + const body = JSON.parse(chunks[chunks.length - 1]); + assert.ok(body.ok === true); + assert.equal(body.sessionId, CORPUS_ID, "must answer from the resolved session, not a ghost"); + const resident = peekSession(CORPUS_ID); + assert.ok(resident, "resolution must land the session in the resident pool"); + // The pre-fix shape: a sync getSession after an async probe minted an + // EMPTY ghost here (0 blocks) that then poisoned later getSessionAsync. + assert.deepEqual( + resident!.state.blocks.map((b) => b.blockId).sort(), + loaded.state.blocks.map((b) => b.blockId).sort(), + ); + } finally { + store.cancelAll(); + await store.closeQuietlyAsync(); + } +}); + test("gate: jsonWriter=off stays ignored on mysql (the #4936 gate)", async () => { const prev = { BILI_STORAGE_BACKEND: process.env.BILI_STORAGE_BACKEND, From 64f7da08a0d8d427082f9c67ddf72a2669209ed6 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 02:53:35 +0800 Subject: [PATCH 12/35] test: await the now-async status/reconcile APIs in existing suites (#4987 CI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #4942 made handlePluginStatus() and reconcileDerivedLineage() async (and maybeAdoptForkBlocks()/publishForkSession() in the same seam). Six existing suites still called them synchronously and asserted on the response/log before the microtask completed, red across 7 of 9 CI shards: - tests/issue404-restore-timestamps.test.ts: JSON.parse of the empty mock body - tests/advisory-panel.test.ts, tests/update-notes.test.ts, tests/issue2029-status-baseline-provenance.test.ts: 0 !== 200 on the mock res - tests/lineage-reconcile-boot.test.ts (AC10): 'not on disk' log asserted before it was written - tests/fork-ccr-adoption.test.ts: same class, passed only because the parent stayed resident (sync completion by luck) — aligned too --- tests/advisory-panel.test.ts | 8 +++---- tests/fork-ccr-adoption.test.ts | 24 +++++++++---------- ...sue2029-status-baseline-provenance.test.ts | 9 +++---- tests/issue404-restore-timestamps.test.ts | 4 ++-- tests/lineage-reconcile-boot.test.ts | 2 +- tests/update-notes.test.ts | 8 +++---- 6 files changed, 28 insertions(+), 27 deletions(-) diff --git a/tests/advisory-panel.test.ts b/tests/advisory-panel.test.ts index 5192be407d..04da836ef8 100644 --- a/tests/advisory-panel.test.ts +++ b/tests/advisory-panel.test.ts @@ -22,7 +22,7 @@ function mockRes(): { res: http.ServerResponse; status: number; body: string } { return out; } -test("#1577: /acp panel surfaces an active advisory before the footer (stripper-safe), byte-exact when clean", () => { +test("#1577: /acp panel surfaces an active advisory before the footer (stripper-safe), byte-exact when clean", async () => { _resetSessionsForTest(); _resetAdvisoryWatcherForTest(); try { @@ -30,7 +30,7 @@ test("#1577: /acp panel surfaces an active advisory before the footer (stripper- const deps = { core: createCore(), config: defaultConfig(200000), log: (_l: string, _m: string) => {} }; const r0 = mockRes(); - handlePluginStatus("never-seen", r0.res, deps, true); + await handlePluginStatus("never-seen", r0.res, deps, true); assert.equal(r0.status, 200); const p0 = JSON.parse(r0.body).panel as string | undefined; assert.ok(p0, "panel renders for the live session"); @@ -39,7 +39,7 @@ test("#1577: /acp panel surfaces an active advisory before the footer (stripper- _setAdvisoryStateForTest({ active: { id: "bc-2026-001", affected: ">=1.0.0", target: "0.1.157", reason: "corrupts tool-call arguments", currentVersion: "0.1.156" } }); const r1 = mockRes(); - handlePluginStatus("never-seen", r1.res, deps, true); + await handlePluginStatus("never-seen", r1.res, deps, true); assert.equal(r1.status, 200); const p1 = JSON.parse(r1.body).panel as string; assert.ok(p1.includes("⚠️ CRITICAL ADVISORY:"), "advisory line present"); @@ -53,7 +53,7 @@ test("#1577: /acp panel surfaces an active advisory before the footer (stripper- const dollarReason = "costs $5M; echo $& back"; _setAdvisoryStateForTest({ active: { id: "bc-2026-002", affected: ">=1.0.0", target: "1.0.9", reason: dollarReason, currentVersion: "1.0.1" } }); const r2 = mockRes(); - handlePluginStatus("never-seen", r2.res, deps, true); + await handlePluginStatus("never-seen", r2.res, deps, true); const p2 = JSON.parse(r2.body).panel as string; assert.ok(p2.includes(dollarReason), "dollar-sign reason survives byte-exact"); assert.equal((p2.match(new RegExp(PANEL_BOX_FOOTER.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g")) ?? []).length, 1, "footer appears exactly once"); diff --git a/tests/fork-ccr-adoption.test.ts b/tests/fork-ccr-adoption.test.ts index 66617304f8..65ef3dfee7 100644 --- a/tests/fork-ccr-adoption.test.ts +++ b/tests/fork-ccr-adoption.test.ts @@ -98,9 +98,9 @@ function armParent(body: Body, fold: { startId: string; endId: string }, ccr: bo return { parent, core, config, blockId: block.blockId, turnMsgs: turn.messages }; } -function forkChild(a: Armed, forkBody: Body): Session { +async function forkChild(a: Armed, forkBody: Body): Promise { const child = getSession(sid("child"), META); - maybeAdoptForkBlocks({ + await maybeAdoptForkBlocks({ session: child, parentId: a.parent.id, protocol: "openai", @@ -137,7 +137,7 @@ test("cloneStoreForRefs: ref-filtered independent clone", () => { assert.equal(cloneStoreForRefs(createContentStore(), []), null); }); -test("fork of armed parent resolves adopted covered refs via acp_retrieve (#1341)", () => { +test("fork of armed parent resolves adopted covered refs via acp_retrieve (#1341)", async () => { const body: Body = { model: "test-model", messages: [ @@ -160,7 +160,7 @@ test("fork of armed parent resolves adopted covered refs via acp_retrieve (#1341 const forkBody = structuredClone(body); forkBody.messages[7] = { role: "tool", tool_call_id: "call_b", content: `${BIG_B}\n(edited branch)` }; - const child = forkChild(armed, forkBody); + const child = await forkChild(armed, forkBody); assert.equal(child.state.blocks.length, 1); assert.ok(child.state.blocks[0].active); @@ -193,7 +193,7 @@ test("fork of armed parent resolves adopted covered refs via acp_retrieve (#1341 assert.equal(child.contentStoreDirty, true); }); -test("refs cited by placeholder-shaped incoming messages are adopted too (#1341)", () => { +test("refs cited by placeholder-shaped incoming messages are adopted too (#1341)", async () => { // Small tool output in the shared prefix (never placeholderized, so the // block stays adoptable); the big result sits OUTSIDE the fold span and // is stored at arrival time as m00007. @@ -241,7 +241,7 @@ test("refs cited by placeholder-shaped incoming messages are adopted too (#1341) { role: "assistant", content: "yes, fixed" }, ], }; - const child = forkChild(armed, forkBody); + const child = await forkChild(armed, forkBody); const cstore = contentStoreOf(child); assert.ok(cstore.byRef["m00007"], "cited ref adopted into child store"); @@ -282,7 +282,7 @@ test("refs cited by placeholder-shaped incoming messages are adopted too (#1341) assert.ok(reAck.includes(BIG_B.slice(0, 200)), "still retrievable in the tool result after the fold"); }); -test("child persists its own companion and survives a proxy restart (#1341)", () => { +test("child persists its own companion and survives a proxy restart (#1341)", async () => { const dir = mkdtempSync(path.join(tmpdir(), "bili-fork-ccr-")); _setStoreForTest(new SessionStore({ dir, debounceMs: 0 })); try { @@ -300,7 +300,7 @@ test("child persists its own companion and survives a proxy restart (#1341)", () const armed = armParent(body, { startId: "m00001", endId: "m00005" }, true); const forkBody = structuredClone(body); forkBody.messages[7] = { role: "tool", tool_call_id: "call_b", content: `${BIG_B}\n(edited branch)` }; - const child = forkChild(armed, forkBody); + const child = await forkChild(armed, forkBody); assert.equal(contentStoreOf(child).byRef["m00003"] ? true : false, true); assert.ok(getStore().flushSync(child), "flush wrote the child's records"); @@ -321,7 +321,7 @@ test("child persists its own companion and survives a proxy restart (#1341)", () } }); -test("fork of a CCR-less parent is unchanged: no store, no artifact (#1341)", () => { +test("fork of a CCR-less parent is unchanged: no store, no artifact (#1341)", async () => { const body: Body = { model: "test-model", messages: [ @@ -336,14 +336,14 @@ test("fork of a CCR-less parent is unchanged: no store, no artifact (#1341)", () }; const armed = armParent(body, { startId: "m00001", endId: "m00005" }, false); assert.equal(Object.keys(contentStoreOf(armed.parent).byRef).length, 0, "CCR off: nothing stored"); - const child = forkChild(armed, structuredClone(body)); + const child = await forkChild(armed, structuredClone(body)); assert.equal(child.state.blocks.length, 1, "block adoption itself still happens"); assert.equal(child.state.blocks[0].blockId, armed.blockId); assert.equal(child.contentStore, undefined, "no store materialized on the child"); assert.notEqual(child.contentStoreDirty, true, "no dirty flag, hence no envelope write"); }); -test("adoption disabled leaves the fork without blocks or store (#1341)", () => { +test("adoption disabled leaves the fork without blocks or store (#1341)", async () => { const body: Body = { model: "test-model", messages: [ @@ -357,7 +357,7 @@ test("adoption disabled leaves the fork without blocks or store (#1341)", () => }; const armed = armParent(body, { startId: "m00001", endId: "m00005" }, true); const child = getSession(sid("child"), META); - maybeAdoptForkBlocks({ + await maybeAdoptForkBlocks({ session: child, parentId: armed.parent.id, protocol: "openai", diff --git a/tests/issue2029-status-baseline-provenance.test.ts b/tests/issue2029-status-baseline-provenance.test.ts index 975bb559f2..3dbbb7c6de 100644 --- a/tests/issue2029-status-baseline-provenance.test.ts +++ b/tests/issue2029-status-baseline-provenance.test.ts @@ -134,11 +134,12 @@ function mockRes(): { res: http.ServerResponse; status: number; body: string } { return out; } -test("#2029: plugin status nudge + panel feed the provenance-aware baseline", () => { +test("#2029: plugin status nudge + panel feed the provenance-aware baseline", async () => { _resetSessionsForTest(); _resetPluginStateForTest(); try { - SCENARIOS.forEach((sc, i) => { + for (let i = 0; i < SCENARIOS.length; i++) { + const sc = SCENARIOS[i]; const session = getSession(`issue2029-plugin-${i}`, { protocol: "anthropic" }); session.metadata.effectiveContextLimit = WINDOW; session.stats = { ...zeroStats(), ...sc.stats } as Session["stats"]; @@ -150,7 +151,7 @@ test("#2029: plugin status nudge + panel feed the provenance-aware baseline", () const deps = { core: createCore(), config: defaultConfig(WINDOW), log: () => {} }; const seen = captureTokenCounts(deps.core); const r = mockRes(); - handlePluginStatus(session.id, r.res, deps); + await handlePluginStatus(session.id, r.res, deps); assert.equal(r.status, 200, `${sc.name}: status answered`); assert.deepStrictEqual(seen, [sc.expected], `${sc.name}: nudge recompute tokenCount`); const panel = JSON.parse(r.body).panel as string | undefined; @@ -168,7 +169,7 @@ test("#2029: plugin status nudge + panel feed the provenance-aware baseline", () assert.ok(!nudgeLine.includes("EMERGENCY") && !nudgeLine.includes("OVER-LIMIT"), `${sc.name}: no phantom pressure (got: ${nudgeLine})`); } - }); + } } finally { _resetSessionsForTest(); _resetPluginStateForTest(); diff --git a/tests/issue404-restore-timestamps.test.ts b/tests/issue404-restore-timestamps.test.ts index dc03a35aff..3e823e9099 100644 --- a/tests/issue404-restore-timestamps.test.ts +++ b/tests/issue404-restore-timestamps.test.ts @@ -119,7 +119,7 @@ await test("boot restore: lastSeen=savedAt, restored flag, freshness-keyed trunc assert.notEqual(active.restored, true, "a session created by a live request is not restored"); const statusDeps = { core: createCore(), config: defaultConfig(200000), log: (_l: string, _m: string) => {} }; const res1 = mockRes(); - handlePluginStatus("never-seen", res1.res, statusDeps, true); + await handlePluginStatus("never-seen", res1.res, statusDeps, true); const body1 = JSON.parse(res1.body) as { ok: boolean; fallback?: boolean; label: string | null }; assert.equal(res1.status, 200); assert.equal(body1.ok, true); @@ -129,7 +129,7 @@ await test("boot restore: lastSeen=savedAt, restored flag, freshness-keyed trunc // All sessions restored (no activity since boot): refuse to guess. _resetSessionsForTest(); const res2 = mockRes(); - handlePluginStatus("never-seen", res2.res, statusDeps, true); + await handlePluginStatus("never-seen", res2.res, statusDeps, true); assert.equal(res2.status, 404); assert.ok(res2.body.includes("since boot"), "explicit no-post-boot-activity error instead of a readdir-order guess"); } finally { diff --git a/tests/lineage-reconcile-boot.test.ts b/tests/lineage-reconcile-boot.test.ts index 7c5a1d637f..64e6bb123b 100644 --- a/tests/lineage-reconcile-boot.test.ts +++ b/tests/lineage-reconcile-boot.test.ts @@ -363,7 +363,7 @@ test("AC10: reconcile is idempotent across repeated boots", async () => { derivedBlocksAdopted: undefined, }, } as typeof child; - reconcileDerivedLineage([orphan], { log: (lvl, msg) => logs.push([lvl, msg]), adoptionEnabled: true }); + await reconcileDerivedLineage([orphan], { log: (lvl, msg) => logs.push([lvl, msg]), adoptionEnabled: true }); assert.equal(orphan.state.blocks.length, 0, "an unreachable parent must seed nothing"); assert.notEqual(orphan.metadata.derivedBlocksAdopted, true, "an unreachable parent must not close the door"); assert.ok( diff --git a/tests/update-notes.test.ts b/tests/update-notes.test.ts index 31cfc6193c..8ebf1cd6bb 100644 --- a/tests/update-notes.test.ts +++ b/tests/update-notes.test.ts @@ -331,7 +331,7 @@ test("#1977 /acp panel: one critical-update line BEFORE the footer, stripper-saf const deps = { core: createCore(), config: defaultConfig(200000), log: (_l: string, _m: string) => {} }; const r0 = mockRes(); - handlePluginStatus("never-seen", r0.res, deps, true); + await handlePluginStatus("never-seen", r0.res, deps, true); assert.equal(r0.status, 200); const p0 = JSON.parse(r0.body).panel as string; assert.ok(!p0.includes("Update ready") && !p0.includes("Update available") && !p0.includes("Critical update"), "no update line when clean"); @@ -339,7 +339,7 @@ test("#1977 /acp panel: one critical-update line BEFORE the footer, stripper-saf _setReleaseNotesStateForTest({ entries: SURFACE_ENTRIES, diskVersion: NEXT2 }); const r1 = mockRes(); - handlePluginStatus("never-seen", r1.res, deps, true); + await handlePluginStatus("never-seen", r1.res, deps, true); const p1 = JSON.parse(r1.body).panel as string; assert.ok(p1.includes(`Critical update ready: ${NEXT2} — restart to finish.`), "panel line present"); assert.ok(p1.includes(PANEL_BOX_FOOTER), "footer still present with the update line"); @@ -355,7 +355,7 @@ test("#1977 /acp panel: one critical-update line BEFORE the footer, stripper-saf diskVersion: NEXT, }); const r2 = mockRes(); - handlePluginStatus("never-seen", r2.res, deps, true); + await handlePluginStatus("never-seen", r2.res, deps, true); const p2 = JSON.parse(r2.body).panel as string; assert.ok(p2.includes(dollarSummary), "dollar-sign summary survives byte-exact"); } finally { @@ -430,7 +430,7 @@ test("#4295 status and panel agree when only some critical updates are installed : "CRITICAL UPDATE AVAILABLE (instance-level):"), status); assert.equal(status.includes("UPDATE READY"), ready, "restart must deliver the critical fixes"); const res = mockRes(); - handlePluginStatus("never-seen", res.res, { core: createCore(), config: defaultConfig(200000), log: () => {} }, true); + await handlePluginStatus("never-seen", res.res, { core: createCore(), config: defaultConfig(200000), log: () => {} }, true); const body = JSON.parse(res.body) as { panel: string }; assert.ok(body.panel.includes(ready ? `Critical update ready: ${disk} — restart to finish.` From 1120610eb245b8a59aee1eb53549df1806d32532 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 02:55:24 +0800 Subject: [PATCH 13/35] fix: judge the [instances] multi-instance warning by sessions directory, not lane alone (#5046) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The storage-side second-writer guard keys on the sessions directory while the [instances] guard keyed on lane overlap — after isolating a second install via BILI_SESSIONS_DIR, the warn kept claiming 'both processes will write the same sessions directory', a factually wrong false positive that drowns the real signal. Registry markers now carry the resolved sessions dir; when both sides report different dirs the notice drops to info level (unknown dir on either side stays conservative = warn). Both the startup warning and the #2401 slow-tick rescan use the same criterion. --- src/instance.ts | 55 ++++++++++++++++--- src/server.ts | 21 ++++++-- tests/instance-governance.test.ts | 90 +++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 12 deletions(-) diff --git a/src/instance.ts b/src/instance.ts index 70bdc882dd..578c0b89d7 100644 --- a/src/instance.ts +++ b/src/instance.ts @@ -496,6 +496,14 @@ export interface RegistryEntry { * while consumers that wait for the process to EXIT (findSameLanePredecessor * → wait-predecessor) keep counting them. */ portReleasedAt?: number; + /** #5046: resolved sessions directory (BILI_SESSIONS_DIR or the XDG + * default) — lets the #394 warning judge by DIRECTORY instead of lane + * overlap alone: a peer whose lanes overlap but whose sessions dir + * differs shares no state (isolated via BILI_SESSIONS_DIR), and the + * "will write the same sessions directory" claim would be factually + * wrong for it. Absent in markers written by pre-#5046 builds → + * treated as shared (conservative). */ + sessionsDir?: string; } /** Cross-instance liveness registry (#394/#527): one marker file per instance @@ -564,6 +572,7 @@ function coerceEntry(value: unknown): RegistryEntry | undefined { ...(typeof o.lane === "string" && o.lane !== "" ? { lane: o.lane } : {}), ...(typeof o.codeFingerprint === "string" && o.codeFingerprint !== "" ? { codeFingerprint: o.codeFingerprint } : {}), ...(typeof o.portReleasedAt === "number" && Number.isFinite(o.portReleasedAt) ? { portReleasedAt: o.portReleasedAt } : {}), + ...(typeof o.sessionsDir === "string" && o.sessionsDir !== "" ? { sessionsDir: o.sessionsDir } : {}), }; } @@ -641,19 +650,40 @@ function lanesOverlap(a: string | undefined, b: string | undefined): boolean { return a === undefined || b === undefined || a === b; } +/** #5046: true only when BOTH sides report a sessions directory and they + * differ after normalization. Unknown on either side = "not proven apart" → + * false (conservative): pre-#5046 markers carry no dir, so isolation cannot + * be claimed for them. */ +function sessionsDirsDiffer(a: string | undefined, b: string | undefined): boolean { + if (a === undefined || b === undefined) return false; + const x = path.resolve(a); + const y = path.resolve(b); + return process.platform === "win32" ? x.toLowerCase() !== y.toLowerCase() : x !== y; +} + /** Live registered peers whose lanes overlap ours (undeclared lane overlaps * everything — a manual `bili start` daemon can serve any client). */ function liveOverlappingPeers(ownInstanceId: string, ownLane: string | undefined): RegistryEntry[] { return readAllRegistryEntries().filter((e) => e.instanceId !== ownInstanceId && isPidAlive(e.pid) && lanesOverlap(ownLane, e.lane)); } -export function registerInstanceAndWarn(entry: RegistryEntry, warn: (msg: string) => void): string[] { +export function registerInstanceAndWarn(entry: RegistryEntry, warn: (msg: string) => void, info?: (msg: string) => void): string[] { const warned: string[] = []; for (const other of liveOverlappingPeers(entry.instanceId, entry.lane)) { const laneNote = entry.lane !== undefined && other.lane !== undefined ? ` on lane "${entry.lane}"` : ""; - warn( - `another bili instance is running (pid ${other.pid}, ${other.origin})${laneNote} — both processes will write the same sessions directory; stop one to avoid state pollution (#394)`, - ); + if (info !== undefined && sessionsDirsDiffer(entry.sessionsDir, other.sessionsDir)) { + // #5046: the storage-side second-writer guard keys on the sessions + // directory — once proven apart, this peer shares no state and the + // warn-level "stop one" advice is a false positive that drowns real + // signal; report it informationally instead. + info( + `another bili instance is running (pid ${other.pid}, ${other.origin})${laneNote} — different sessions directories (ours: ${entry.sessionsDir}, theirs: ${other.sessionsDir}); no shared state, informational only (#394/#5046)`, + ); + } else { + warn( + `another bili instance is running (pid ${other.pid}, ${other.origin})${laneNote} — both processes will write the same sessions directory; stop one to avoid state pollution (#394)`, + ); + } warned.push(other.instanceId); } reapDeadMarkers(entry.instanceId); @@ -669,17 +699,26 @@ export function registerInstanceAndWarn(entry: RegistryEntry, warn: (msg: string * the registry on a slow tick and warn once per newly-seen live peer. * Returns the newly-warned instance ids so the caller can remember them. */ export function warnOnNewPeers( - own: { instanceId: string; lane?: string }, + own: { instanceId: string; lane?: string; sessionsDir?: string }, previouslyWarned: ReadonlySet, warn: (msg: string) => void, + info?: (msg: string) => void, ): string[] { const fresh: string[] = []; for (const other of liveOverlappingPeers(own.instanceId, own.lane)) { if (previouslyWarned.has(other.instanceId)) continue; const laneNote = own.lane !== undefined && other.lane !== undefined ? ` on lane "${other.lane}"` : ""; - warn( - `another bili instance appeared while this process was already running (pid ${other.pid}, ${other.origin})${laneNote} — both processes write the same sessions directory; stop one to avoid state pollution (#394/#2401)`, - ); + if (info !== undefined && sessionsDirsDiffer(own.sessionsDir, other.sessionsDir)) { + // #5046: same criterion as registerInstanceAndWarn — proven-apart + // peers are informational, never the "stop one" advice. + info( + `another bili instance appeared while this process was already running (pid ${other.pid}, ${other.origin})${laneNote} — different sessions directories (ours: ${own.sessionsDir}, theirs: ${other.sessionsDir}); no shared state, informational only (#394/#2401/#5046)`, + ); + } else { + warn( + `another bili instance appeared while this process was already running (pid ${other.pid}, ${other.origin})${laneNote} — both processes write the same sessions directory; stop one to avoid state pollution (#394/#2401)`, + ); + } fresh.push(other.instanceId); } return fresh; diff --git a/src/server.ts b/src/server.ts index ccea8cf428..1fbef1fc40 100644 --- a/src/server.ts +++ b/src/server.ts @@ -96,7 +96,7 @@ import { clearConflictEvents, recordConflict, summarizeConflicts } from "./confl import { getStore, setGlobalSessionCommitListener } from "./persist.js"; import { log as loggerLog, configureLogger, getLogPath, closeLogger, isStreamWriteError, isBenignSocketRaceError, enterSessionContext, muteStderrLeg, onFileLegLive, logTimestamp } from "./logger.js"; import { queryLogLines } from "./web/logs-query.js"; -import { configFile, defaultLogFile, stateDir } from "./paths.js"; +import { configFile, defaultLogFile, sessionsDir, stateDir } from "./paths.js"; import { atomicWriteInstanceFile, clearProxyInstanceFile, classifySameLanePeer, entryScriptFingerprint, entryShape, findSameLanePredecessor, findVerifiedSameLaneTwin, isPidAlive, listInstances, markPortReleased, probePid, readProxyInstanceFile, registerInstanceAndWarn, sameInstanceShape, unregisterInstance, warnOnNewPeers, writeSpawnPortLog, type InstanceShape, type PidProbe, type ProxyInstanceFile, type RegistryEntry, type SameLanePeer } from "./instance.js"; import { createServedMarkerWriter, reapStaleWriterMarkers, writerMarkerDir, type ServedMarkerWriter } from "./served-markers.js"; import { recordSplitBrainContest } from "./storage/runtime-status.js"; @@ -1198,9 +1198,17 @@ export async function startServer(opts: ProxyOptions): Promise { // launcher-private launchToken) so lane-aware attach discovery and // the #394 warning can reason about EVERY live instance — the single // proxy-origin file only reflects the last writer. - const registryRecord = { ...instanceRecord }; + // #5046: carry the resolved sessions dir on the REGISTRY marker only + // (not the proxy-origin file) so the #394 warning can judge by + // directory — a lane-overlapping peer with a different dir shares no + // state and must not get the "stop one" advice. + const registryRecord = { ...instanceRecord, sessionsDir: sessionsDir() }; delete registryRecord.launchToken; - const warnedAtRegistration = registerInstanceAndWarn(registryRecord, (msg) => log("warn", `[instances] ${msg}`)); + const warnedAtRegistration = registerInstanceAndWarn( + registryRecord, + (msg) => log("warn", `[instances] ${msg}`), + (msg) => log("info", `[instances] ${msg}`), + ); // #2401: registration warns the late starter only — a resident that is // ALREADY serving stays blind to a peer appearing later (the exact // dual-generation shape behind the stale-serving report). Rescan the @@ -1209,7 +1217,12 @@ export async function startServer(opts: ProxyOptions): Promise { const warnedPeers = new Set(warnedAtRegistration); const peerRescanMs = 60_000; const peerTimer = setInterval(() => { - for (const id of warnOnNewPeers({ instanceId, lane: launcherLane }, warnedPeers, (msg) => log("warn", `[instances] ${msg}`))) { + for (const id of warnOnNewPeers( + { instanceId, lane: launcherLane, sessionsDir: sessionsDir() }, + warnedPeers, + (msg) => log("warn", `[instances] ${msg}`), + (msg) => log("info", `[instances] ${msg}`), + )) { warnedPeers.add(id); } }, peerRescanMs); diff --git a/tests/instance-governance.test.ts b/tests/instance-governance.test.ts index 795192f987..0ed8942cac 100644 --- a/tests/instance-governance.test.ts +++ b/tests/instance-governance.test.ts @@ -256,6 +256,96 @@ test("instance registry: #394 warning is lane-aware — cross-lane silent, same- } }); +test("instance registry: #394 warning judges by sessions directory when both report one (#5046)", () => { + const st = tmpStateDir(); + const warnings: string[] = []; + const infos: string[] = []; + setLogCapture((_level, msg) => warnings.push(msg)); + try { + registerInstanceAndWarn( + { instanceId: "a", pid: process.pid, port: 1, origin: "http://127.0.0.1:1", startedAt: 1, lane: "dsh", sessionsDir: "/sessions/a" }, + (msg) => warnings.push(msg), + (msg) => infos.push(msg), + ); + assert.equal(warnings.length, 0); + assert.equal(infos.length, 0); + // Same-lane peer with a DIFFERENT sessions dir (BILI_SESSIONS_DIR isolation): + // shares no state → informational only, never the "stop one" advice. + registerInstanceAndWarn( + { instanceId: "b", pid: process.pid, port: 2, origin: "http://127.0.0.1:2", startedAt: 2, lane: "dsh", sessionsDir: "/sessions/b" }, + (msg) => warnings.push(msg), + (msg) => infos.push(msg), + ); + assert.equal(warnings.length, 0, "isolated dirs are not a shared-state hazard"); + assert.equal(infos.length, 1, "proven-apart peer reported informationally"); + assert.match(infos[0], /different sessions directories/); + assert.match(infos[0], /ours: \/sessions\/b/); + assert.match(infos[0], /theirs: \/sessions\/a/); + const regDir = path.join(st.dir, "billion-context", "instances"); + const bMarker = JSON.parse(fs.readFileSync(path.join(regDir, "b.json"), "utf8")) as { sessionsDir?: string }; + assert.equal(bMarker.sessionsDir, "/sessions/b", "sessionsDir round-trips through the marker file"); + // Same-lane peer with the SAME dir as "a": still the full warning. + registerInstanceAndWarn( + { instanceId: "c", pid: process.pid, port: 3, origin: "http://127.0.0.1:3", startedAt: 3, lane: "dsh", sessionsDir: "/sessions/a" }, + (msg) => warnings.push(msg), + (msg) => infos.push(msg), + ); + assert.equal(warnings.length, 1, "shared dir still warns"); + assert.match(warnings[0], /both processes will write the same sessions directory/); + assert.match(warnings[0], /lane "dsh"/); + assert.equal(infos.length, 2, "c vs b is proven apart → info"); + // Peer without a recorded dir (pre-#5046 marker): isolation unprovable → conservative warn for every peer. + registerInstanceAndWarn( + { instanceId: "d", pid: process.pid, port: 4, origin: "http://127.0.0.1:4", startedAt: 4, lane: "dsh" }, + (msg) => warnings.push(msg), + (msg) => infos.push(msg), + ); + assert.equal(warnings.length, 4, "unknown own dir stays conservative against all three peers"); + assert.equal(infos.length, 2, "unknown dir can never be judged apart"); + } finally { + setLogCapture(null); + st.restore(); + } +}); + +test("instance registry: peer rescan judges by sessions directory like startup does (#2401/#5046)", () => { + const st = tmpStateDir(); + const warnings: string[] = []; + const infos: string[] = []; + setLogCapture((_level, msg) => warnings.push(msg)); + try { + registerInstanceAndWarn( + { instanceId: "a", pid: process.pid, port: 1, origin: "http://127.0.0.1:1", startedAt: 1, lane: "pi", sessionsDir: "/s/a" }, + () => {}, + () => {}, + ); + registerInstanceAndWarn( + { instanceId: "b", pid: process.pid, port: 2, origin: "http://127.0.0.1:2", startedAt: 2, lane: "pi", sessionsDir: "/s/b" }, + () => {}, + () => {}, + ); + const fresh = warnOnNewPeers({ instanceId: "a", lane: "pi", sessionsDir: "/s/a" }, new Set(), (m) => warnings.push(m), (m) => infos.push(m)); + assert.deepEqual(fresh, ["b"], "newly-seen live peer reported"); + assert.equal(warnings.length, 0, "proven-apart late peer is informational"); + assert.equal(infos.length, 1); + assert.match(infos[0], /appeared while this process was already running/); + assert.match(infos[0], /different sessions directories/); + registerInstanceAndWarn( + { instanceId: "c", pid: process.pid, port: 3, origin: "http://127.0.0.1:3", startedAt: 3, lane: "pi", sessionsDir: "/s/a" }, + () => {}, + () => {}, + ); + const seen = new Set(["b"]); + const fresh2 = warnOnNewPeers({ instanceId: "a", lane: "pi", sessionsDir: "/s/a" }, seen, (m) => warnings.push(m), (m) => infos.push(m)); + assert.deepEqual(fresh2, ["c"], "shared-dir late peer warned"); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /both processes write the same sessions directory/); + } finally { + setLogCapture(null); + st.restore(); + } +}); + test("instance registry: resident warns once per newly-appearing live peer, remembers them (#2401)", () => { const st = tmpStateDir(); const warnings: string[] = []; From 7d02228c0b48894576f73ced5371abd6f8947a50 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 02:59:28 +0800 Subject: [PATCH 14/35] docs: single-machine multi-install isolation guidance in README en+zh (#5046) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two installs on one machine (e.g. two DSH profiles) default to the same XDG sessions dir + config; with storage.unified on they double-write one index.db (SQLITE_BUSY → failed turns). Document the BILI_SESSIONS_DIR + BILI_CONFIG_FILE isolation recipe in both languages. --- README.md | 16 ++++++++++++++++ README.zh-CN.md | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/README.md b/README.md index 84898e5222..4653048013 100644 --- a/README.md +++ b/README.md @@ -643,6 +643,22 @@ Consequences for anyone tempted to put several instances behind one Service: conversation load-balanced across proxies*. Long conversations are single-instance by design. +**Single machine, multiple installs (#5046).** The topology above covers +deployment replicas; one desktop has a sibling hazard: two installs on ONE +machine (e.g. two DSH profiles — desktop + bot, or a global npm copy beside a +host-tree copy) each start their own proxy and, unless isolated, both resolve +the SAME default XDG sessions directory and the same config file. With +`storage.unified` on (shadow/full), two processes then double-write one +`index.db`: the log shows `[instances] another bili instance is running …` +and, for overlapping lanes, `[storage] … two processes on one index.db +corrupt it`; SQLite write-lock contention (`SQLITE_BUSY` / `database is +locked`) can fail whole model turns, and sustained double-writing risks +corruption. The fix is isolation: give each install its own +`BILI_SESSIONS_DIR` (plus `BILI_CONFIG_FILE` when the config should differ +too) through that install's launcher/profile environment — no global env +changes needed. Once isolated, the installs coexist safely and the +`[instances]` notice drops to info level (different dirs ⇒ no shared state). + Naming note: `POST /__bili/replicas` creates a session **checkpoint fork** (from stored checkpoints), not a pod replica — read it as `/__bili/checkpoints`. diff --git a/README.zh-CN.md b/README.zh-CN.md index 9c882d19f7..f3c15c5ad3 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -464,6 +464,10 @@ bili --no-auto-update # 本次启动禁用自动更新 代理把每个会话的压缩状态持久化到会话目录(默认 `%USERPROFILE%\.local\share\billion-context\`),长会话每一轮都会重写该文件。在 Windows 上,实时杀毒(Defender)、搜索索引器或同步工具(OneDrive)可能在写入中途锁住该目录,导致 rename 以 `EPERM` 失败。同一会话连续 N 次写失败(默认 `5`)时,代理打一条一次性可操作告警,指明要排除的目录。根上修复:把 `%USERPROFILE%\.local\share\billion-context\` 加入杀软**排除项**,并确认没有同步工具在同步该路径 —— 完整步骤见 [CONFIGURATION.zh-CN.md](CONFIGURATION.zh-CN.md) 的「Windows:把会话目录加入杀软排除项」章节。同一排除项也覆盖统一存储 `index.db` 打不开(`unable to open database file`,#3823)——存储降级告警会把它列为第一嫌疑。 +### 单机多安装共存(#5046) + +同一台机器上装了**两份及以上** bili(例如两个 DSH profile——桌面端 + bot,或全局 npm 拷贝与宿主树拷贝并存)时,各自的代理默认都解析到**同一个** XDG 会话目录(`%USERPROFILE%\.local\share\billion-context\sessions\`)和同一份配置文件。开了统一存储(`storage.unified` = shadow/full)后,两个进程会双写同一个 `index.db`:日志出现 `[instances] another bili instance is running …`,lane 重叠时还有 `[storage] … two processes on one index.db corrupt it` 级别的 error;SQLite 写锁争用(`SQLITE_BUSY` / `database is locked`)会让整轮模型请求直接失败,持续双写还有损坏风险。正确做法是**隔离**:给每份安装设独立的 `BILI_SESSIONS_DIR`(配置也要分开时再加 `BILI_CONFIG_FILE`),只加在该安装自己的启动器/profile 环境变量里,不动全局环境。隔离后两份可正常并存,`[instances]` 提示会自动降为 info 级(目录不同 ⇒ 无共享状态)。 + ### 会话文件清理(#1082) 短命会话会留下永远不会再被恢复的小状态文件。清理是**可选开启(opt-in)**的:设 `BILI_SESSION_GC=1` 才启用(默认关闭 —— 会话文件属于用户数据,不应有静默删除策略)。启用且持久化开启时,bili 在启动时和每小时扫描会话目录,只有**两个条件同时满足**才删除一个文件:年龄超过 `BILI_SESSION_GC_MAX_AGE_DAYS`(默认 7 天),并且该会话**从未被压缩过**(没有折叠块)、最近一次请求体 ≤ `BILI_SESSION_GC_MAX_TOKENS` token(默认 1M)—— 删除只丢字节不丢内容。CCR 内容存储遵循其会话文件的生命周期;被压缩过的会话永不删除;每次删除都写审计日志。详见 [CONFIGURATION.zh-CN.md](CONFIGURATION.zh-CN.md)。 From 10a9bc1c892d0fdaa796016cf49674c65e49078c Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 16:06:52 +0800 Subject: [PATCH 15/35] fix: route corruption-shaped SQLite throws into the integrity quarantine decision (#4816/#5108) --- src/storage/integrity-check.ts | 33 ++++++++++++++++++--- src/storage/sqlite-errors.ts | 54 ++++++++++++++++++++++++++++++++++ src/storage/store.ts | 40 +++++++++++++++++++++---- 3 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 src/storage/sqlite-errors.ts diff --git a/src/storage/integrity-check.ts b/src/storage/integrity-check.ts index 7a7f10fc4a..5e7149ee4a 100644 --- a/src/storage/integrity-check.ts +++ b/src/storage/integrity-check.ts @@ -157,7 +157,15 @@ try { const verdict = rows.map((row) => String(row.quick_check ?? "")).join(" ").trim(); parentPort.postMessage({ ok: true, verdict }); } catch (err) { - parentPort.postMessage({ ok: false, error: String(err) }); + // #5108/#4816: a corruption-shaped THROW out of quick_check (truncated file, + // garbage header, broken overflow chain) is a FAIL VERDICT, not "could not + // run" — classify it exactly like src/storage/sqlite-errors.ts does (inlined + // here because this string is bundled verbatim and cannot import modules; + // keep the two in sync, the worker-facet regression test pins both). + const m = String((err && err.message) || err); + const c = String((err && err.code) || "").toUpperCase(); + const corrupt = c === "SQLITE_CORRUPT" || c === "SQLITE_NOTADB" || m.includes("database disk image is malformed") || m.includes("file is not a database"); + parentPort.postMessage(corrupt ? { ok: false, corrupt: true, error: m } : { ok: false, error: m }); } `; @@ -169,7 +177,7 @@ const inflight = new Set(); * (#4609). Records the outcome, caches it (ok or fail — a cached fail makes * the NEXT open throw and quarantine via the #4175 path), and tells the * caller's store instance so a clean close can refresh the cache key. */ -export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok: boolean) => void): void { +export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok: boolean, detail?: string) => void): void { const size = statPart(dbPath)?.s ?? 0; loggerLog("info", `[storage] background quick_check started for ${dbPath} (${(size / (1024 * 1024)).toFixed(1)} MiB over the ${Math.round(effectiveSyncThreshold() / (1024 * 1024))} MiB sync threshold) — serving continues; the verdict lands in /__bili/status (#4609)`); inflight.add(dbPath); @@ -190,7 +198,7 @@ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok worker.unref(); } }; - worker.on("message", (msg: { ok?: boolean; verdict?: string; error?: string }) => { + worker.on("message", (msg: { ok?: boolean; corrupt?: boolean; verdict?: string; error?: string }) => { finish(() => { if (msg.ok === true) { const verdict = String(msg.verdict ?? ""); @@ -207,8 +215,25 @@ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok recordIntegrityOutcome(dbPath, false, 0, verdict.slice(0, 400)); writeIntegrityCache(dbPath, false, verdict.slice(0, 400)); loggerLog("error", `[storage] background quick_check FAILED for ${dbPath}: ${verdict.slice(0, 400)} — verdict cached; the next open quarantines this file set and reopens fresh (JSON envelopes remain the recovery source, #4175/#4609)`); - onVerdict?.(false); + // Detail rides with the verdict so a clean close can + // refresh the FAIL cache onto post-checkpoint stats + // (#5108/#4816) — without it the verdict would go stale + // after the first clean restart and never quarantine. + onVerdict?.(false, verdict.slice(0, 400)); } + } else if (msg.corrupt === true) { + // #5108/#4816: the scan itself threw a corruption-shaped error + // (the throwing shapes of CORRUPT/NOTADB). Same consequences as + // a bad verdict row: record loudly and cache the fail so the + // NEXT open quarantines + reopens fresh (#4175 path). Without + // this branch the throw fell into "could not run" below and the + // fail verdict was never recorded — a large corrupt store stayed + // unverified boot after boot with no quarantine ever running. + const detail = String(msg.error ?? "").slice(0, 400); + recordIntegrityOutcome(dbPath, false, 0, detail); + writeIntegrityCache(dbPath, false, detail); + loggerLog("error", `[storage] background quick_check threw a corruption-shaped error for ${dbPath}: ${detail} — FAIL verdict cached; the next open quarantines this file set and reopens fresh (JSON envelopes remain the recovery source, #5108/#4175)`); + onVerdict?.(false, detail); } else { loggerLog("warn", `[storage] background quick_check could not run for ${dbPath}: ${String(msg.error)} — the store serves unverified; the next boot rechecks`); } diff --git a/src/storage/sqlite-errors.ts b/src/storage/sqlite-errors.ts new file mode 100644 index 0000000000..e031712b39 --- /dev/null +++ b/src/storage/sqlite-errors.ts @@ -0,0 +1,54 @@ +/** + * #5108/#4816: corruption-shaped SQLite error classification. + * + * PRAGMA quick_check has TWO failure shapes: it can RETURN non-ok verdict + * rows (page-level damage), or it can THROW out of the statement itself + * (truncated file, garbage header, broken overflow chain — probe-verified on + * node:sqlite v22: ERR_SQLITE_ERROR with the raw sqlite3_errmsg text). The + * #4175 quarantine decision only saw the verdict-row shape, so every + * throwing corruption escaped as a raw driver error and degraded the store + * to legacy-only forever with no auto-heal. + * + * This classifier maps the throwing shapes onto the same decision. It + * matches ONLY SQLITE_CORRUPT / SQLITE_NOTADB: + * - better-sqlite3 stamps err.code with the symbolic name ("SQLITE_CORRUPT", + * "SQLITE_NOTADB"); + * - node:sqlite throws ERR_SQLITE_ERROR whose message IS the stable English + * errmsg text ("database disk image is malformed" / "file is not a + * database") — matched by substring, which also catches the + * [storage:] wrapper prefix openSqliteDatabase adds. + * + * Transient shapes MUST NOT match: BUSY/LOCKED (a live foreign process — + * the supported shared-directory topology, #3923/#3959), IOERR (Windows + * Defender real-time scan / sync-folder locks, #362 class), CANTOPEN + * (permissions). Quarantining a merely locked-but-healthy index over a + * transient condition would destroy a working index — the exact opposite of + * what recovery is for. + * + * NOTE: the background-check worker (integrity-check.ts WORKER_SOURCE) is + * bundled VERBATIM into dist and cannot import this module — it carries an + * inlined copy of these matchers. Keep the two in sync (the worker-facet + * regression test pins the behavior end to end). + */ + +const CORRUPTION_ERRMSG_MARKERS = [ + "database disk image is malformed", // SQLITE_CORRUPT + "file is not a database", // SQLITE_NOTADB +]; + +/** Returns the error detail when `err` is a corruption-shaped SQLite failure, + * null otherwise. */ +export function sqliteCorruptionDetail(err: unknown): string | null { + const code = (err as { code?: unknown } | null)?.code; + if (typeof code === "string") { + const upper = code.toUpperCase(); + if (upper === "SQLITE_CORRUPT" || upper === "SQLITE_NOTADB") { + return err instanceof Error ? err.message : String(err); + } + } + const msg = err instanceof Error ? err.message : String(err ?? ""); + for (const marker of CORRUPTION_ERRMSG_MARKERS) { + if (msg.includes(marker)) return msg; + } + return null; +} diff --git a/src/storage/store.ts b/src/storage/store.ts index 0cab41538d..818c854af9 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -25,6 +25,7 @@ import { migrateSchema, storagePragmas, WAL_JOURNAL_SIZE_LIMIT } from "./schema. import { identityOfStoredContent } from "./identity.js"; import { recordIntegrityOutcome, recordSecondProcessOwner, recordBackupOutcome } from "./runtime-status.js"; import { integrityModeFor, writeIntegrityCache, scheduleBackgroundIntegrityCheck, backgroundIntegrityInflightForTests, noteIntegrityCheckRun } from "./integrity-check.js"; +import { sqliteCorruptionDetail } from "./sqlite-errors.js"; import { listInstances, type RegistryEntry } from "../instance.js"; import type { MessageContentStore } from "acp-kernel"; @@ -350,6 +351,13 @@ export class UnifiedStore { * verdict cache onto the post-checkpoint stats so the next boot skips * the rescan. Never true while a check is still in flight or failed. */ private lastIntegrityOk = false; + /** #5108/#4816: a completed check said the file set is CORRUPT — a clean + * close refreshes the FAIL verdict onto post-checkpoint stats so the + * next boot's cached-fail path quarantines instead of rescan-looping + * (a clean close moves the WAL, which would otherwise invalidate the + * cache key and send a large corrupt store back to unverified forever). + * Outranks lastIntegrityOk at close time. */ + private lastIntegrityFailDetail: string | null = null; private constructor(db: SqliteDatabase, engine: SqliteEngineName, engineSource: SqliteEngineSource, lockPath: string | null, foreignLockHolder: { pid: number } | null, dbPath: string) { this.db = db; @@ -401,8 +409,16 @@ export class UnifiedStore { // before rethrowing — the #2771 fallback tells operators to // DELETE the file, and Windows refuses to delete a file this // process still holds (unlink EBUSY, #4347 probe). + // #5108/#4816: corruption-shaped THROWS (truncated file, garbage + // header, broken overflow chain) must become StorageIntegrityError + // like the verdict-row path does — the raw rethrow escaped the + // shadow's quarantine guard (#4175) and left the corrupt set in + // place forever. Transient shapes (BUSY/IOERR/CANTOPEN) stay raw: + // quarantining a locked-but-healthy index would be the bug. recordIntegrityOutcome(path, false, Date.now() - started, String(err).slice(0, 400)); db.close(); + const detail = sqliteCorruptionDetail(err); + if (detail !== null) throw new StorageIntegrityError(path, detail.slice(0, 400)); throw err; } const durationMs = Date.now() - started; @@ -459,7 +475,12 @@ export class UnifiedStore { // file to reset the index — and Windows refuses to delete a file a // live process still holds open. Close, then rethrow for the // shadow's legacy-only degradation. + // #5108/#4816: large stores (> sync threshold) skip the pre-migration + // scan, so a corruption can surface HERE first — classify it the same + // way so the shadow quarantines instead of degrading forever. db.close(); + const detail = sqliteCorruptionDetail(err); + if (detail !== null) throw new StorageIntegrityError(path, detail.slice(0, 400)); throw err; } // #3250: converge the WAL an unclean exit left behind (SIGKILL / power @@ -480,7 +501,7 @@ export class UnifiedStore { // read-only connection — open() returns immediately; the verdict // lands in runtime-status and the cache (a cached failure makes the // NEXT open quarantine + reopen fresh via the #4175 path). - if (mode.kind === "async") scheduleBackgroundIntegrityCheck(path, (ok) => { if (ok) store.noteIntegrityOk(); }); + if (mode.kind === "async") scheduleBackgroundIntegrityCheck(path, (ok, detail) => { if (ok) store.noteIntegrityOk(); else if (detail !== undefined) store.noteIntegrityFail(detail); }); return store; } @@ -491,6 +512,12 @@ export class UnifiedStore { this.lastIntegrityOk = true; } + /** #5108/#4816: mirror of noteIntegrityOk for a completed FAIL verdict — + * see lastIntegrityFailDetail for why the close-time refresh needs it. */ + noteIntegrityFail(detail: string): void { + this.lastIntegrityFailDetail = detail; + } + close(): void { // #3250: converge the WAL before releasing the handle so a clean exit // leaves nothing for the next boot's recovery to replay. Same isolation @@ -504,10 +531,13 @@ export class UnifiedStore { this.db.close(); // #4609: a clean close of a verified store refreshes the verdict // cache onto the post-checkpoint stats — the normal restart path then - // skips the rescan. Deferred while a background check is still in - // flight (its own completion writes the cache). Best-effort. - if (this.lastIntegrityOk && !backgroundIntegrityInflightForTests(this.dbPath)) { - writeIntegrityCache(this.dbPath, true); + // skips the rescan. #5108/#4816: the same refresh carries a FAIL + // verdict (fail outranks ok — never cache ok for a corrupt file). + // Deferred while a background check is still in flight (its own + // completion writes the cache). Best-effort. + if (!backgroundIntegrityInflightForTests(this.dbPath)) { + if (this.lastIntegrityFailDetail !== null) writeIntegrityCache(this.dbPath, false, this.lastIntegrityFailDetail); + else if (this.lastIntegrityOk) writeIntegrityCache(this.dbPath, true); } // #4175: release the ownership lockfile on a clean close so the next // open in another process is not told a dead pid holds it. When a From 1f230a2e99b13a29e7d06015dd16dbec717c6265 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 16:06:52 +0800 Subject: [PATCH 16/35] =?UTF-8?q?fix:=20make=20index.db=20quarantine=20rob?= =?UTF-8?q?ust=20=E2=80=94=20name=20rename=20failures,=20never=20reopen=20?= =?UTF-8?q?over=20an=20unquarantined=20corrupt=20file=20(#4816/#5108)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/storage/shadow.ts | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index aec7da44eb..2521a5c9fa 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -26,7 +26,7 @@ import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, typ import type { BlockView } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; -import { renameSync } from "node:fs"; +import { existsSync, renameSync } from "node:fs"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; import { canonicalize, sha256 } from "./canonical.ts"; @@ -203,14 +203,23 @@ export class UnifiedShadow { if (!(err instanceof StorageIntegrityError)) throw err; const stamp = Date.now(); const quarantined: string[] = []; + let primaryRenameFailure: string | null = null; + const sidecarLeft: string[] = []; for (const suffix of ["", "-wal", "-shm"]) { const from = `${dir}/index.db${suffix}`; + if (!existsSync(from)) continue; // absent sidecar files are fine const to = `${dir}/index.db${suffix}.corrupt-${stamp}`; try { renameSync(from, to); quarantined.push(path.basename(to)); - } catch { - /* absent sidecar files are fine */ + } catch (rerr) { + // #5108/#4347: the old empty catch swallowed EVERY failure + // — including a primary that could not be moved — and the + // code below then claimed success. Capture the errno and + // decide per file what a failed move actually means. + const why = String((rerr as NodeJS.ErrnoException).code ?? rerr); + if (suffix === "") primaryRenameFailure = why; + else sidecarLeft.push(`${path.basename(from)} (${why})`); } } // #4609: the verdict cache belongs to the quarantined file set — @@ -220,8 +229,30 @@ export class UnifiedShadow { const cacheFrom = `${dir}/index.db.chk`; renameSync(cacheFrom, `${cacheFrom}.corrupt-${stamp}`); quarantined.push(path.basename(`${cacheFrom}.corrupt-${stamp}`)); - } catch { - /* absent cache is fine */ + } catch (rerr) { + if (existsSync(`${dir}/index.db.chk`)) { + // Present but unmovable — harmless (the fresh store's stats + // key misses and rescans), but say so instead of staying silent. + loggerLog("warn", `[storage] quarantined index.db but its .chk verdict cache could not be moved (${String((rerr as NodeJS.ErrnoException).code ?? rerr)}) — it will be overwritten by the next check`); + } + } + if (primaryRenameFailure !== null) { + // The corrupt PRIMARY is still in place: a fresh reopen would + // just re-hit the same corrupt file, so do NOT attempt it — + // degrade loudly with the real errno instead of claiming the + // quarantine succeeded. The operator moves the file aside by + // hand (the same manual workaround that always worked). + loggerLog( + "error", + `[storage] index.db failed its integrity quick_check (${err.detail}) but the corrupt file could NOT be quarantined (rename ${primaryRenameFailure}${sidecarLeft.length > 0 ? `; also left behind: ${sidecarLeft.join(", ")}` : ""}) — staying legacy-only; move ${path.join(dir, "index.db")} aside by hand to rebuild the index (#5108/#4347)`, + ); + throw new StorageIntegrityError(`${dir}/index.db`, `${err.detail}; quarantine rename failed: ${primaryRenameFailure}`); + } + if (sidecarLeft.length > 0) { + // Probe-verified (node:sqlite v22): SQLite RESETS a stale -wal + // whose parent db was replaced rather than replaying it onto + // the fresh file, so leftover sidecars are safe to leave. + loggerLog("warn", `[storage] quarantined index.db but ${sidecarLeft.join(", ")} could not be renamed — left in place (a stale -wal is reset, not replayed, on the next open)`); } loggerLog( "error", From 96826db1474d6f197dd0f89a29ec4e57ed8cd5c7 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 16:06:52 +0800 Subject: [PATCH 17/35] fix: report the resolved engine when index.db open fails after resolution; middle-truncate long degrade reasons (#5108) --- src/storage/driver.ts | 8 ++++++++ src/storage/runtime-status.ts | 28 ++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/storage/driver.ts b/src/storage/driver.ts index 6bc44be0c7..047fa93449 100644 --- a/src/storage/driver.ts +++ b/src/storage/driver.ts @@ -218,6 +218,14 @@ export function clearSelfDownloadedEngineForTests(): void { let cachedEngine: ResolvedEngine | null = null; +/** #5108: the last successfully resolved engine (null until resolution + * succeeds). Diagnostics surfaces use it to say WHICH engine resolved when a + * store open later failed with a real error — reporting "engine: none" for a + * corrupt-index failure misdirected triage toward an engine problem. */ +export function resolvedSqliteEngineInfo(): ResolvedEngine | null { + return cachedEngine; +} + function detectEngine(): ResolvedEngine { if (cachedEngine) return cachedEngine; if (probeBuiltinSqlite()) cachedEngine = { name: "node:sqlite", source: "builtin" }; diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index f2c0aee365..e71d5eca15 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -27,9 +27,16 @@ import type { BootstrapOutcome, CooldownMarker } from "./engine-bootstrap.js"; import { bootstrapFailureMarkerPath, COOLDOWN_MS, nodeSqliteUpgradeHint, readBootstrapFailureMarker } from "./engine-bootstrap.js"; import { fileStorageAutoDownload, fileStorageJsonWriter, fileStorageUnified, fileStorageWalJournalSizeLimit, resolveStorageMode } from "../knobs.js"; import { log as loggerLog } from "../logger.js"; +import { resolvedSqliteEngineInfo } from "./driver.js"; import { MAX_CONSECUTIVE_FAILURES, sqliteOpenFailureHint, type UnifiedShadow } from "./shadow.js"; import type { StorageMode } from "./shadow.js"; +/** The driver's no-engine error text — one predicate, used wherever the + * "is this actually an engine problem?" distinction matters (#5108). */ +function isNoEngineError(errStr: string): boolean { + return /no SQLite engine available/i.test(errStr); +} + interface RuntimeState { /** Recorded by the SessionStore constructor; null only before first store build. */ requested: StorageMode | null; @@ -160,7 +167,13 @@ function pidAlive(pid: number): boolean { /** Short one-line form of a raw error for interpolation into banner text. */ function shortenError(err: string): string { const oneLine = err.replace(/\s+/g, " ").trim(); - return oneLine.length > 160 ? `${oneLine.slice(0, 157)}...` : oneLine; + if (oneLine.length <= 160) return oneLine; + // #5108/#4816: both ends carry signal — the error class up front, the + // errno/detail at the end (e.g. "quarantine rename failed: EPERM") — so + // cut the MIDDLE (usually a long path) instead of the tail. Head-only + // truncation made the actionable detail invisible on exactly the + // long-path machines (Windows user profiles) this fix targets. + return `${oneLine.slice(0, 78)}…${oneLine.slice(-78)}`; } function degradeReason(): string { @@ -183,7 +196,7 @@ function degradeReason(): string { const e = shortenError(state.sidecarOpenError); // The driver's no-engine error surfaces through the same open path — // say what it actually is instead of "index database failed to open". - if (/no SQLite engine available/i.test(e)) { + if (isNoEngineError(e)) { return `no SQLite engine available (node:sqlite missing, better-sqlite3 not installed)${nodeSqliteUpgradeHint()}`; } return `index database failed to open (${e})${sqliteOpenFailureHint(state.sidecarOpenError)}`; @@ -277,7 +290,14 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus { degraded, intentionalOptOut, reason: degraded ? degradeReason() : null, - engine: sidecar ? sidecar.engineInfo : null, + // #5108: an open failure AFTER the engine resolved (corrupt index.db, + // …) must not report "engine: none" — that misdirected triage toward + // an engine problem when the engine was fine and the file was not. + engine: sidecar + ? sidecar.engineInfo + : state.sidecarOpenError !== null && !isNoEngineError(state.sidecarOpenError) + ? resolvedSqliteEngineInfo() + : null, bootstrap: state.bootstrap?.status ?? null, bootstrapFailure: cd.marker, cooldownRemainingMs: cd.remainingMs, @@ -369,7 +389,7 @@ export function storageDiagnosticsLines(ctx: { sessions: number; storageLogLines `requested mode: ${st.requested}`, `effective mode: ${st.effective}${st.intentionalOptOut ? " (BILI_ENCRYPTION_KEY opt-out — intentional)" : ""}`, ...(st.degraded && st.reason ? [`degrade reason: ${st.reason}`] : []), - `engine: ${st.engine ? `${st.engine.name}/${st.engine.source}` : "none"}`, + `engine: ${st.engine ? `${st.engine.name}/${st.engine.source}${state.sidecarOpenError !== null && !isNoEngineError(state.sidecarOpenError) ? " (resolved, but index.db open failed — see degrade reason)" : ""}` : "none"}`, `engine bootstrap: ${st.bootstrap ?? "not run"}`, ...(st.bootstrapFailure ? [ `bootstrap failure marker: at=${new Date(st.bootstrapFailure.atMs).toISOString()} reason="${st.bootstrapFailure.reason}"${st.cooldownRemainingMs ? ` (cooldown ~${Math.round(st.cooldownRemainingMs / 60_000)}m remaining, until ${new Date(st.bootstrapFailure.atMs + COOLDOWN_MS).toISOString()})` : ""}`, From 238a7e4368697515541a92cf849dda9666b6bf0f Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 16:06:52 +0800 Subject: [PATCH 18/35] fix: sample the tail of dsh child bootstrap output instead of the head when it exceeds the window (#4816) --- src/launcher.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/launcher.ts b/src/launcher.ts index 04ccd7f72d..03bde8e482 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -5069,16 +5069,23 @@ export async function ensureProxyRunning( // means rotation happened under us — everything left is ours. let start = spawnLogOffset; if (size < start) start = Math.max(0, size - 256); - const len = Math.min(256, size - start); - if (len <= 0) return ""; + const span = size - start; + if (span <= 0) return ""; + // #4816: the death cause sits at the END of a child's output — + // when this spawn wrote more than the sample window, take the + // TAIL instead of the head (head-only sampling then showed + // startup noise while the fatal line sat unread further down). + const len = Math.min(256, span); + const off = span > len ? start + (span - len) : start; + const label = span > len ? "tail" : "head"; const fd = fs.openSync(logPath, "r"); try { const buf = Buffer.alloc(len); - const n = fs.readSync(fd, buf, 0, len, start); + const n = fs.readSync(fd, buf, 0, len, off); if (n === 0) return ""; let s = buf.subarray(0, n).toString("utf8").replace(/\s+/g, " ").trim(); if (s.length > 200) s = `${s.slice(0, 200)}…`; - return ` — child output head: "${s}"`; + return ` — child output ${label}: "${s}"`; } finally { fs.closeSync(fd); } From 6faac8faaddf28b76f9519a26472a4ffe584db41 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 16:06:52 +0800 Subject: [PATCH 19/35] test: regression coverage for corruption-throw quarantine, recovery, rename-failure, background-check and diagnostics facets (#4816/#5108) --- tests/corrupt-index-child.ts | 82 ++++-- ...sue4816-integrity-throw-quarantine.test.ts | 253 ++++++++++++++++++ tests/storage-engine-proofs.test.ts | 38 +-- tests/web-storage-banner.test.ts | 28 +- 4 files changed, 345 insertions(+), 56 deletions(-) create mode 100644 tests/issue4816-integrity-throw-quarantine.test.ts diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index 76eb4d5335..f312938db1 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -1,7 +1,7 @@ /** - * #4338: the corrupt index.db degrade scenarios run in a spawned CHILD - * process, and this file is both the child program and the shared helper - * module the two parent test files import. + * #4338: the corrupt-index and no-engine-banner scenarios run in a spawned + * CHILD process, and this file is both the child program and the shared + * helper module the two parent test files import. * * Why a child: on Windows, node:sqlite's FAILED open of a corrupt index.db * can leave an OS file handle held with no JS-side instance to release @@ -34,7 +34,7 @@ import { spawnSync } from "node:child_process"; import http from "node:http"; import { once } from "node:events"; import assert from "node:assert/strict"; -import { mkdirSync, realpathSync, writeFileSync, writeSync } from "node:fs"; +import { mkdirSync, realpathSync, writeSync } from "node:fs"; import * as fs from "node:fs"; import * as path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -42,6 +42,7 @@ import { defaultConfig } from "acp-kernel"; import type { ProxyOptions } from "../src/config.ts"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts"; import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; @@ -192,24 +193,32 @@ export function makeOpts(upstreamPort: number): ProxyOptions { // sides reference these constants, so the coverage list cannot drift. // --------------------------------------------------------------------------- +// #4816: throwing-shape corruption (header garbage / truncation) is now +// quarantined and the store RECOVERS to full mode on a fresh index.db — +// these checks pin that recovery plus the original #2781/#4357 leak guards. const ENGPROOF_CHECK_NAMES = { - fallbackRead: "legacy fallback returned a session over the corrupt index", - midFlowGuard: "no live handle after the corrupt-index read (#2781 rename guard)", - stateDigest: "corrupt-index fallback state digest matches the clean full-mode read", - writeNow: "writeNow keeps working over the legacy path after the corrupt-index degrade", + quarantine: "corrupt index.db was renamed aside (index.db.corrupt-* kept for forensics)", + freshIndex: "a fresh index.db replaced the quarantined one", + lazyImport: "session reloaded from the legacy envelope into the recovered index", + midFlowGuard: "no live handle on the QUARANTINED file (#2781/#4357 rename guard)", + stateDigest: "recovered-index state digest matches the clean full-mode read", + writeNow: "writeNow keeps working over the recovered full mode", postCloseGuard: "no live handle after the final close (post-close rename guard)", fdGuard: "no open fd into the corrupt index dir after close (/proc/self/fd, Linux)", } as const; export const ENGPROOF_CHECKS: readonly string[] = Object.values(ENGPROOF_CHECK_NAMES); +// #4816: a corrupt index.db no longer degrades — it quarantines and recovers +// (see engproof). The banner wire shape is pinned here through the remaining +// deterministic cross-platform degrade cause: the engine ladder exhausted. const BANNER_CHECK_NAMES = { overviewAnswers: "/__bili/overview answers 200 with JSON", requestedFull: "storage.requested is full", effectiveLegacy: "storage.effective is legacy", - degraded: "storage.degraded is true (corrupt index.db trips the warning)", + degraded: "storage.degraded is true (the no-engine degrade trips the warning)", intentionalOptOut: "storage.intentionalOptOut is false", - reason: "storage.reason matches /index database failed to open/", - engine: "storage.engine is null", + reason: "storage.reason names the missing engine", + engine: "storage.engine is null when no engine resolved", } as const; export const BANNER_CHECKS: readonly string[] = Object.values(BANNER_CHECK_NAMES); @@ -331,28 +340,48 @@ async function engproofScenario(argv: string[], run: CheckRun): Promise { fs.writeSync(fd, Buffer.alloc(512, 0x5a), 0, 512, 0); fs.closeSync(fd); - // THE #2771 regression: construction must survive, reads must fall - // back to the legacy file, and the digest must be exact. + // THE #2771 regression: construction must survive. Since #4816 the + // throwing-shape corruption is quarantined and the store recovers to + // full mode on a fresh index.db — reads lazily import the session back + // from the legacy envelope, and the digest must be exact. const store2 = new SessionStore({ dir, debounceMs: 1, enabled: true }); + await run.run(ENGPROOF_CHECK_NAMES.quarantine, () => { + const names = fs.readdirSync(dir); + const q = names.find((n) => n.startsWith("index.db.corrupt-")); + assert.ok(q, `no quarantined index.db.corrupt-* in ${names.join(", ")}`); + }); + await run.run(ENGPROOF_CHECK_NAMES.freshIndex, () => + assert.ok(fs.existsSync(path.join(dir, "index.db")), "fresh index.db absent after quarantine")); const reloaded = store2.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); store2.cancelAll(); try { - await run.run(ENGPROOF_CHECK_NAMES.fallbackRead, () => - assert.ok(reloaded, "legacy fallback returned null over a corrupt index")); + await run.run(ENGPROOF_CHECK_NAMES.lazyImport, () => + assert.ok(reloaded, "recovered store returned null for a session whose legacy envelope exists")); if (!reloaded) return; - // Regression (#2781): the failed open must have CLOSED its handle. - await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-moved"); + // Leak guard (#2781/#4357): the DEAD quarantined file must carry no + // live handle. The whole-dir rename used pre-#4816 is wrong now — + // the store legitimately holds the FRESH index.db, and only a + // handle on the dead file is the leak class this guard hunts. + await run.run(ENGPROOF_CHECK_NAMES.midFlowGuard, () => { + const q = fs.readdirSync(dir).find((n) => n.startsWith("index.db.corrupt-")); + assert.ok(q, "quarantined file vanished before the leak guard ran"); + const src = path.join(dir, q!); + const probe = `${src}-moved`; + fs.renameSync(src, probe); + fs.renameSync(probe, src); + }); // #4316 PR3/3: the clean full-mode read carried the one-time ledger - // alignment; the legacy fallback over the corrupt index did not. The - // parent passes its digest already modulo that alignment (single - // definition), so strip it here too before comparing. + // alignment; the recovered load applies it again. The parent passes + // its digest already modulo that alignment (single definition), so + // strip it here too before comparing. await run.run(ENGPROOF_CHECK_NAMES.stateDigest, () => assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(reloaded))), expectedStateDigest, - "corrupt-index fallback state digest drift vs clean full-mode read")); + "recovered-index state digest drift vs clean full-mode read")); - // Writes must keep working (legacy path) — no throw, file refreshed. + // Writes must keep working (full path into the fresh index) — no + // throw, files refreshed. await run.run(ENGPROOF_CHECK_NAMES.writeNow, () => store2.writeNow(reloaded)); } finally { store2.closeQuietly(); @@ -394,7 +423,10 @@ async function bannerScenario(argv: string[], run: CheckRun): Promise { if (!base) throw new Error("banner mode needs "); const dir = path.join(base, "sessions"); mkdirSync(dir, { recursive: true }); - writeFileSync(path.join(dir, "index.db"), Buffer.from("this is definitely not a sqlite database file".repeat(8))); + // #4816: the former corrupt-index cause here now quarantines and recovers + // instead of degrading — force the engine ladder exhausted (deterministic + // on every platform) as the degrade trigger for the banner wire shape. + setSqliteEnginesUnavailableForTests(true); const store = new SessionStore({ dir, enabled: true }); _setStoreForTest(store); const upstreamPort = await freePort(); @@ -415,9 +447,9 @@ async function bannerScenario(argv: string[], run: CheckRun): Promise { if (!storage) return; await run.run(BANNER_CHECK_NAMES.requestedFull, () => assert.equal(storage.requested, "full")); await run.run(BANNER_CHECK_NAMES.effectiveLegacy, () => assert.equal(storage.effective, "legacy")); - await run.run(BANNER_CHECK_NAMES.degraded, () => assert.equal(storage.degraded, true, "corrupt index.db trips the warning")); + await run.run(BANNER_CHECK_NAMES.degraded, () => assert.equal(storage.degraded, true, "no-engine degrade trips the warning")); await run.run(BANNER_CHECK_NAMES.intentionalOptOut, () => assert.equal(storage.intentionalOptOut, false)); - await run.run(BANNER_CHECK_NAMES.reason, () => assert.match(storage.reason ?? "", /index database failed to open/)); + await run.run(BANNER_CHECK_NAMES.reason, () => assert.match(storage.reason ?? "", /no SQLite engine available/)); await run.run(BANNER_CHECK_NAMES.engine, () => assert.equal(storage.engine, null)); } finally { await close(proxy); diff --git a/tests/issue4816-integrity-throw-quarantine.test.ts b/tests/issue4816-integrity-throw-quarantine.test.ts new file mode 100644 index 0000000000..87fbc352b1 --- /dev/null +++ b/tests/issue4816-integrity-throw-quarantine.test.ts @@ -0,0 +1,253 @@ +/** + * #4816/#5108 regression: corruption-shaped quick_check failures must reach + * the #4175 quarantine decision instead of escaping it as raw driver errors. + * + * The #4175 quarantine guard only catches StorageIntegrityError, but several + * corruption shapes make `PRAGMA quick_check` THROW (truncated file → + * "database disk image is malformed", garbage header → "file is not a + * database") instead of returning non-ok verdict rows. Those raw throws + * escaped the shadow's quarantine and left the corrupt index in place + * forever (permanent legacy-only degrade; the reporter's only exit was a + * manual rename). Facets covered here: + * 1. the classifier itself (corruption shapes match, transient shapes do NOT); + * 2. UnifiedStore.open turns a throwing scan into StorageIntegrityError; + * 3. the shadow quarantines + reopens fresh instead of degrading; + * 4. a failed quarantine rename degrades loudly with the errno (no doomed reopen); + * 5. the background (large-store) check caches its FAIL verdict across a clean close; + * 6. /__bili/status reports the resolved engine when the failure happened AFTER resolution. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { chmodSync, closeSync, existsSync, mkdtempSync, openSync, readdirSync, rmSync, statSync, truncateSync, writeFileSync, writeSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { UnifiedStore, StorageIntegrityError } from "../src/storage/store.js"; +import { UnifiedShadow } from "../src/storage/shadow.js"; +import { sqliteCorruptionDetail } from "../src/storage/sqlite-errors.js"; +import { openSqliteDatabase, probeBuiltinSqlite, probeHostBetterSqlite3 } from "../src/storage/driver.js"; +import { + readIntegrityCache, + resetIntegrityCheckStateForTests, + setIntegritySyncThresholdForTests, + backgroundIntegrityInflightForTests, +} from "../src/storage/integrity-check.js"; +import { + _resetStorageRuntimeStatusForTest, + recordSidecarOpenFailure, + storageStatusWire, + storageDiagnosticsLines, +} from "../src/storage/runtime-status.js"; + +const HAS_ENGINE = probeBuiltinSqlite() || probeHostBetterSqlite3(); +const SKIP_NO_ENGINE = HAS_ENGINE ? undefined : "no SQLite engine available on this host"; +const SKIP_RENAME_FAIL = SKIP_NO_ENGINE ?? (process.platform === "win32" ? "chmod has no permission effect on Windows" : undefined); + +function tmpDir(): string { + return mkdtempSync(join(tmpdir(), "bili-issue4816-")); +} + +async function waitFor(pred: () => boolean, ms: number): Promise { + const deadline = Date.now() + ms; + while (!pred()) { + if (Date.now() > deadline) throw new Error("timed out waiting"); + await new Promise((r) => setTimeout(r, 25)); + } +} + +/** A real multi-page db (~62 KB, 200 rows) — enough pages for truncation to break. */ +function seedValidDb(dbPath: string): void { + const { db } = openSqliteDatabase(dbPath); + db.exec("CREATE TABLE IF NOT EXISTS seed (id INTEGER PRIMARY KEY, v TEXT)"); + const ins = db.prepare("INSERT INTO seed (v) VALUES (?)"); + for (let i = 0; i < 200; i++) ins.run(`row-${i}-${"x".repeat(300)}`); + db.close(); +} + +function truncateHalf(dbPath: string): void { + truncateSync(dbPath, Math.floor(statSync(dbPath).size / 2)); +} + +function garbageHeader(dbPath: string): void { + writeFileSync(dbPath, Buffer.from("this is definitely not a sqlite database file, not even close")); +} + +/** Zero page 2 (the first btree data page after sqlite_master on page 1). + * Verified recipe: pragmas/schema-migration survive (they only touch the + * header + page 1) but the full quick_check scan throws CORRUPT — needed to + * exercise the background-check facet, where open() must succeed first. */ +function zeroFirstDataPage(dbPath: string): void { + const fd = openSync(dbPath, "r+"); + try { + writeSync(fd, Buffer.alloc(4096, 0), 0, 4096, 4096); + } finally { + closeSync(fd); + } +} + +test("#4816 classifier: corruption shapes match, transient shapes must NOT", () => { + // better-sqlite3 stamps symbolic err.code values + assert.notEqual(sqliteCorruptionDetail(Object.assign(new Error("database disk image is malformed"), { code: "SQLITE_CORRUPT" })), null); + assert.notEqual(sqliteCorruptionDetail(Object.assign(new Error("file is not a database"), { code: "SQLITE_NOTADB" })), null); + // node:sqlite ERR_SQLITE_ERROR carries the errmsg text only + assert.notEqual(sqliteCorruptionDetail(Object.assign(new Error("database disk image is malformed"), { code: "ERR_SQLITE_ERROR" })), null); + assert.notEqual(sqliteCorruptionDetail(new Error("file is not a database")), null); + // the [storage:] wrapper keeps the message text + assert.notEqual(sqliteCorruptionDetail(new Error("[storage:better-sqlite3] file is not a database")), null); + // Transient shapes MUST stay unclassified: quarantining a locked-but-healthy + // index over an AV lock would destroy a working store (#362/#3823 class). + assert.equal(sqliteCorruptionDetail(Object.assign(new Error("database is locked"), { code: "SQLITE_BUSY" })), null); + assert.equal(sqliteCorruptionDetail(Object.assign(new Error("disk I/O error"), { code: "ERR_SQLITE_ERROR" })), null); + assert.equal(sqliteCorruptionDetail(Object.assign(new Error("unable to open database file"), { code: "SQLITE_CANTOPEN" })), null); + assert.equal(sqliteCorruptionDetail(new Error("some other failure")), null); + assert.equal(sqliteCorruptionDetail(null), null); + assert.equal(sqliteCorruptionDetail(undefined), null); +}); + +test("#4816 store: a throwing quick_check becomes StorageIntegrityError", { skip: SKIP_NO_ENGINE }, () => { + const dir = tmpDir(); + try { + const p = join(dir, "index.db"); + seedValidDb(p); + truncateHalf(p); // proven to make PRAGMA quick_check throw CORRUPT + assert.throws( + () => UnifiedStore.open(p), + (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + + const dir2 = tmpDir(); + try { + const p = join(dir2, "index.db"); + seedValidDb(p); + garbageHeader(p); // proven to make PRAGMA quick_check throw NOTADB + assert.throws( + () => UnifiedStore.open(p), + (err: unknown) => err instanceof StorageIntegrityError && /not a database/.test(err.detail), + ); + // control: a healthy file still opens + rmSync(p); + seedValidDb(p); + const s = UnifiedStore.open(p); + s.close(); + } finally { + rmSync(dir2, { recursive: true, force: true }); + } + + const dir3 = tmpDir(); + try { + const p = join(dir3, "index.db"); + seedValidDb(p); + zeroFirstDataPage(p); // page-content corruption throws like truncation does + assert.throws( + () => UnifiedStore.open(p), + (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), + ); + } finally { + rmSync(dir3, { recursive: true, force: true }); + } +}); + +test("#4816 shadow: throwing corruption is quarantined and the store recovers to full mode", { skip: SKIP_NO_ENGINE }, () => { + const dir = tmpDir(); + let shadow: UnifiedShadow | null = null; + try { + seedValidDb(join(dir, "index.db")); + truncateHalf(join(dir, "index.db")); + shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null); + assert.ok(shadow !== null, "corrupt index must be quarantined + reopened fresh, not degrade legacy-only"); + assert.equal(shadow.readsUnified, true); + const names = readdirSync(dir); + assert.ok(names.some((n) => n.startsWith("index.db.corrupt-")), `quarantined copy kept for forensics, got ${names.join(", ")}`); + assert.ok(existsSync(join(dir, "index.db")), "fresh index.db exists after quarantine"); + } finally { + shadow?.close(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#4816 shadow: a failed quarantine rename degrades loudly with the errno, no doomed reopen", { skip: SKIP_RENAME_FAIL }, () => { + const dir = tmpDir(); + try { + seedValidDb(join(dir, "index.db")); + truncateHalf(join(dir, "index.db")); + chmodSync(dir, 0o555); + try { + _resetStorageRuntimeStatusForTest(); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null); + assert.equal(shadow, null, "must degrade when the primary cannot be moved aside"); + assert.ok(existsSync(join(dir, "index.db")), "corrupt primary stays in place when the rename fails"); + assert.ok(!readdirSync(dir).some((n) => n.startsWith("index.db.corrupt-")), "no partial quarantine"); + const wire = storageStatusWire(); + assert.match(wire.reason ?? "", /index database failed to open/); + assert.match(wire.reason ?? "", /quarantine rename failed/); + } finally { + chmodSync(dir, 0o755); + _resetStorageRuntimeStatusForTest(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#4816 worker facet: a large store's throwing background check caches FAIL across a clean close", { skip: SKIP_NO_ENGINE }, async () => { + const dir = tmpDir(); + try { + resetIntegrityCheckStateForTests(); + setIntegritySyncThresholdForTests(1); // force async mode for any size + const p = join(dir, "index.db"); + seedValidDb(p); + zeroFirstDataPage(p); // open() survives; only the full worker scan throws + const s1 = UnifiedStore.open(p); + try { + await waitFor(() => !backgroundIntegrityInflightForTests(p), 15000); + const cache = readIntegrityCache(p); + assert.ok(cache !== null && cache.ok === false, `FAIL verdict cached by the worker, got ${JSON.stringify(cache)}`); + assert.match(cache?.detail ?? "", /malformed/); + } finally { + s1.close(); // checkpoint moves the WAL — the refresh must carry the FAIL verdict + } + // Next boot: cached-fail → typed error → the shadow would quarantine. + assert.throws( + () => UnifiedStore.open(p), + (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), + ); + } finally { + setIntegritySyncThresholdForTests(null); + resetIntegrityCheckStateForTests(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#5108 diagnostics: engine reports the resolved engine when the open failed after resolution", { skip: SKIP_NO_ENGINE }, () => { + const dir = tmpDir(); + try { + seedValidDb(join(dir, "index.db")); + const s = UnifiedStore.open(join(dir, "index.db")); + s.close(); // populates the driver's resolved-engine info + _resetStorageRuntimeStatusForTest(); + recordSidecarOpenFailure("[storage] database disk image is malformed"); + const wire = storageStatusWire(); + assert.ok(wire.engine !== null, "resolved engine must be reported even though the sidecar never opened"); + assert.ok(typeof wire.engine?.name === "string" && wire.engine.name.length > 0); + const lines = storageDiagnosticsLines({ sessions: 0, storageLogLines: [] }); + assert.ok(lines.some((l) => l.includes("resolved, but index.db open failed")), `diagnostics line annotated, got ${JSON.stringify(lines)}`); + } finally { + _resetStorageRuntimeStatusForTest(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#5108 diagnostics: no-engine failures keep reporting engine null", () => { + _resetStorageRuntimeStatusForTest(); + recordSidecarOpenFailure("no SQLite engine available"); + try { + const wire = storageStatusWire(); + assert.equal(wire.engine, null); + const lines = storageDiagnosticsLines({ sessions: 0, storageLogLines: [] }); + assert.ok(!lines.some((l) => l.includes("resolved, but index.db open failed"))); + } finally { + _resetStorageRuntimeStatusForTest(); + } +}); diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index d9eb3334b2..656d37c1d0 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -26,13 +26,15 @@ * loads with the exact digest. * SELF-DISABLE — 3 consecutive sidecar failures disable it; reads flip * back to legacy; the legacy file stays readable. - * CORRUPT INDEX — a corrupt index.db degrades the store to legacy-only - * instead of bricking construction (#2771). + * CORRUPT INDEX — a corrupt index.db is quarantined and the store + * recovers to full mode instead of bricking + * construction (#2771 → #4816). * CAS REFCOUNT — shared content blobs survive deleteSession of one * session and are collected only at refcount 0. * - * The corrupt-index test is the regression for #2771; the rest are contract - * pins (proof-only, no product change). + * The corrupt-index test is the regression for #2771 (re-pinned for the + * #4816 quarantine-and-recover behavior); the rest are contract pins + * (proof-only, no product change). */ import { test } from "node:test"; @@ -361,10 +363,11 @@ test("storage engine: sidecar self-disables after 3 consecutive failures (reads }); // --------------------------------------------------------------------------- -// CORRUPT INDEX (#2771): construction degrades to legacy-only. +// CORRUPT INDEX (#2771 → #4816): construction quarantines the corrupt index +// and recovers to full mode. // --------------------------------------------------------------------------- -test("storage engine: corrupt index.db degrades to legacy-only instead of bricking construction (#2771)", { timeout: 180_000 }, async () => { +test("storage engine: corrupt index.db is quarantined and recovers to full mode instead of bricking construction (#2771/#4816)", { timeout: 180_000 }, async () => { const dir = tmpdir("bili-engproof-corrupt-"); const prev = process.env.BILI_STORAGE_UNIFIED; try { @@ -381,17 +384,18 @@ test("storage engine: corrupt index.db degrades to legacy-only instead of bricki clean.closeQuietly(); assert.ok(viaFull, "full-mode read returned null"); - // #4338: everything past this point (corrupt the header → degrade - // construction → fallback read → rename guards → write → close) runs - // in a spawned child. On Windows the FAILED open of the corrupt index - // can leave an OS handle with no JS reference to release, and the - // resulting persistent EBUSY at rmrf outlives the whole tmp-rm retry - // ladder; the child exiting RELEASES all its handles by OS guarantee, - // so this parent's rmrf below cannot be blocked. The child prints a - // JSON verdict of named checks; every one is re-asserted here, so - // assertion coverage is 1:1 with the in-process era (same asserts, - // executed in the child, verified in the parent). The child also runs - // the #2781 mid-flow rename guard plus a post-close guard, so future + // #4338: everything past this point (corrupt the header → quarantine + // + fresh reopen → lazy-import read → leak guards → write → close) + // runs in a spawned child. On Windows any sqlite file the scenario + // touches can leave an OS handle with no JS reference to release, and + // the resulting persistent EBUSY at rmrf would outlive the whole + // tmp-rm retry ladder; the child exiting RELEASES all its handles by + // OS guarantee, so this parent's rmrf below cannot be blocked. The + // child prints a JSON verdict of named checks; every one is + // re-asserted here, so assertion coverage is 1:1 with the + // in-process era (same asserts, executed in the child, verified in + // the parent). The child also runs the #2781/#4357 mid-flow leak + // guard on the quarantined file plus a post-close guard, so future // handle-leak drift trips there by name instead of at teardown. // #4316 PR3/3: expected digest computed modulo the one-time ledger // alignment — the clean read above already carried it in memory, and diff --git a/tests/web-storage-banner.test.ts b/tests/web-storage-banner.test.ts index fe015f7a33..2b1d7352cb 100644 --- a/tests/web-storage-banner.test.ts +++ b/tests/web-storage-banner.test.ts @@ -1,7 +1,8 @@ // #3040 web surface: SQLite unified storage degrades to legacy-only through // several fail-open paths (#2825 contract): engine ladder exhausted, -// self-download blocked in cooldown, corrupt index.db guard (#2771), 3-strike -// sidecar disable. The proxy keeps running on JSON — correct — but until now +// self-download blocked in cooldown, non-corruption index-open failures, +// 3-strike sidecar disable (a corrupt index.db since #4816 quarantines and +// recovers instead of degrading). The proxy keeps running on JSON — correct — but until now // the condition was only visible as one-time log lines, so environments that // cannot run SQLite were never learned about before the upcoming full-mode // default flip. These tests pin the wire contract the web banner reads: @@ -263,21 +264,20 @@ test("#3040 BILI_ENCRYPTION_KEY opt-out is intentional: reported, never warned", } }); -test("#3040 corrupt index.db (guard #2771): degraded with the open-failure reason", { skip: !(probeBuiltinSqlite() || probeHostBetterSqlite3()) && "no SQLite engine available on this host", timeout: 180_000 }, async () => { +test("#3040 engine-ladder-exhausted degrade: banner wire shape with the open-failure reason", { timeout: 180_000 }, async () => { const base = mkdtempSync(path.join(tmpdir(), "bili-storage-banner-")); try { - // #4338: the whole corrupt-index scenario (write the bad index.db → + // #4338: the whole scenario (force the engine ladder exhausted → // construct the store → serve /__bili/overview → assert the degraded - // wire shape → close) runs in a spawned child. On Windows the FAILED - // open of the corrupt index can leave an OS handle with no JS - // reference to release, and this test's rmrf(base) then died on a - // persistent EBUSY that outlived the whole tmp-rm retry ladder; the - // child exiting RELEASES all its handles by OS guarantee, so the - // rmrf below cannot be blocked. The child prints a JSON verdict of - // named checks; every one is re-asserted here, so assertion coverage - // is 1:1 with the in-process era (same asserts, executed in the - // child, verified in the parent). The spawn env below is the child's - // only isolation — this process's own env and seams stay untouched. + // wire shape → close) runs in a spawned child so any OS handle the + // store opens is released by process exit before this parent's + // rmrf(base) can hit a Windows EBUSY. (#4816 moved the trigger off + // the corrupt-index cause: that one now quarantines and recovers, so + // the banner surface is pinned through the deterministic no-engine + // path instead.) The child prints a JSON verdict of named checks; + // every one is re-asserted here, so assertion coverage stays 1:1 + // with the in-process era. The spawn env below is the child's only + // isolation — this process's own env and seams stay untouched. const childEnv: NodeJS.ProcessEnv = { ...process.env }; delete childEnv.BILI_STORAGE_UNIFIED; delete childEnv.BILI_ENCRYPTION_KEY; From 9dc0577da644e3578aeb67beabe5cc731371c468 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 17:25:16 +0800 Subject: [PATCH 20/35] fix: repair #5124 CI failures (family gate + two non-hermetic storage tests) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - runtime-status shortenError: raw .slice(-78) -> safeSuffix (and head via safePrefix) — the code-unit-safe clamp the #816/#1615 family gate requires - issue4816 rename-fail test: pin BILI_STORAGE_UNIFIED=full around the wire assertion — the CI baseline preload pins legacy for the non-storage corpus (#2815), so requested resolved to legacy and nothing was reportable as degraded (passes locally without the preload, fails on every CI shard) - storage-review-fixes review 3: re-pinned to the NEW intended behavior — the garbage-header corruption shape makes quick_check throw, so the store now quarantines the corrupt file set and recovers a fresh index instead of degrading legacy-only forever (PR behavior change (a)); asserts the forensic copy + fresh index.db exist --- src/storage/runtime-status.ts | 3 ++- tests/issue4816-integrity-throw-quarantine.test.ts | 9 +++++++++ tests/storage-review-fixes.test.ts | 14 ++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index e71d5eca15..94a31789be 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -27,6 +27,7 @@ import type { BootstrapOutcome, CooldownMarker } from "./engine-bootstrap.js"; import { bootstrapFailureMarkerPath, COOLDOWN_MS, nodeSqliteUpgradeHint, readBootstrapFailureMarker } from "./engine-bootstrap.js"; import { fileStorageAutoDownload, fileStorageJsonWriter, fileStorageUnified, fileStorageWalJournalSizeLimit, resolveStorageMode } from "../knobs.js"; import { log as loggerLog } from "../logger.js"; +import { safePrefix, safeSuffix } from "../text-safe.js"; import { resolvedSqliteEngineInfo } from "./driver.js"; import { MAX_CONSECUTIVE_FAILURES, sqliteOpenFailureHint, type UnifiedShadow } from "./shadow.js"; import type { StorageMode } from "./shadow.js"; @@ -173,7 +174,7 @@ function shortenError(err: string): string { // cut the MIDDLE (usually a long path) instead of the tail. Head-only // truncation made the actionable detail invisible on exactly the // long-path machines (Windows user profiles) this fix targets. - return `${oneLine.slice(0, 78)}…${oneLine.slice(-78)}`; + return `${safePrefix(oneLine, 78)}…${safeSuffix(oneLine, 78)}`; } function degradeReason(): string { diff --git a/tests/issue4816-integrity-throw-quarantine.test.ts b/tests/issue4816-integrity-throw-quarantine.test.ts index 87fbc352b1..d527122645 100644 --- a/tests/issue4816-integrity-throw-quarantine.test.ts +++ b/tests/issue4816-integrity-throw-quarantine.test.ts @@ -169,6 +169,13 @@ test("#4816 shadow: throwing corruption is quarantined and the store recovers to test("#4816 shadow: a failed quarantine rename degrades loudly with the errno, no doomed reopen", { skip: SKIP_RENAME_FAIL }, () => { const dir = tmpDir(); + // The wire reason below resolves `requested` from the REAL process env + // (runtime-status records it only via SessionStore), and the CI baseline + // preload pins BILI_STORAGE_UNIFIED=legacy for the non-storage corpus + // (#2815) — under legacy there is nothing to report degraded. Pin full to + // match the mode maybeOpen is asked for; restore afterwards. + const prevUnified = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; try { seedValidDb(join(dir, "index.db")); truncateHalf(join(dir, "index.db")); @@ -187,6 +194,8 @@ test("#4816 shadow: a failed quarantine rename degrades loudly with the errno, n _resetStorageRuntimeStatusForTest(); } } finally { + if (prevUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prevUnified; rmSync(dir, { recursive: true, force: true }); } }); diff --git a/tests/storage-review-fixes.test.ts b/tests/storage-review-fixes.test.ts index ea5feb6d54..acff76510d 100644 --- a/tests/storage-review-fixes.test.ts +++ b/tests/storage-review-fixes.test.ts @@ -136,7 +136,7 @@ test("review 2: destroyed-substrate blocks are deleted and cannot resurrect on a }); }); -test("review 3: corrupt index.db degrades to legacy-only instead of bricking the proxy", async () => { +test("review 3: corrupt index.db auto-quarantines and recovers instead of bricking the proxy", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-review-corrupt-")); try { const session = loadCorpusCase(dir, "folded", "corpus-folded", "anthropic"); @@ -151,8 +151,14 @@ test("review 3: corrupt index.db degrades to legacy-only instead of bricking the store = new SessionStore({ dir, debounceMs: 1, enabled: true }); // must NOT throw await store.writeNow(session); const reloaded = store.loadSync(session.id, { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - assert.ok(reloaded, "legacy fallback load must succeed"); - assert.equal(store.replicasAvailable(), false, "shadow is unavailable while degraded"); + assert.ok(reloaded, "session load must succeed"); + // #5124/#4816 behavior change (a): this corruption shape makes + // quick_check THROW, so the store quarantines the corrupt file set + // and recovers a fresh index — it no longer degrades legacy-only. + assert.equal(store.replicasAvailable(), true, "throwing corruption must be quarantined + recovered, not stay legacy-only"); + const names = fs.readdirSync(dir); + assert.ok(names.some((n) => n.startsWith("index.db.corrupt-")), `quarantined copy kept for forensics, got ${names.join(", ")}`); + assert.ok(fs.existsSync(path.join(dir, "index.db")), "fresh index.db exists after quarantine"); await store.flushAll(); // flush the JSON record to disk (closeQuietly only cancels) } finally { store!.closeQuietly(); @@ -163,7 +169,7 @@ test("review 3: corrupt index.db degrades to legacy-only instead of bricking the process.env.BILI_STORAGE_UNIFIED = "shadow"; try { const again = new SessionStore({ dir, debounceMs: 1, enabled: true }); - assert.ok(again.loadSync(session.id, { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }), "second process must load via legacy"); + assert.ok(again.loadSync(session.id, { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }), "second process must load the session"); again.closeQuietly(); } finally { delete process.env.BILI_STORAGE_UNIFIED; From 950bfaaceb59ce887d42271117fc55cbd2b8d88f Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 17:45:19 +0800 Subject: [PATCH 21/35] fix: degrade to legacy-only when a live foreign bili process owns index.db (#5046) --- src/storage/runtime-status.ts | 2 + src/storage/shadow.ts | 17 +- src/storage/store.ts | 25 +- tests/issue5046-second-writer-degrade.test.ts | 221 ++++++++++++++++++ 4 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 tests/issue5046-second-writer-degrade.test.ts diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index f2c0aee365..c598cff162 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -181,6 +181,8 @@ function degradeReason(): string { } if (state.sidecarOpenError) { const e = shortenError(state.sidecarOpenError); + // #5046/B: shadow.ts records second-writer refusals pre-formatted (already names the fix + what's lost) — render verbatim instead of wrapping them as "index database failed to open". + if (e.startsWith("second-writer:")) return e; // The driver's no-engine error surfaces through the same open path — // say what it actually is instead of "index database failed to open". if (/no SQLite engine available/i.test(e)) { diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index aec7da44eb..a8035928f3 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -25,7 +25,7 @@ import { log as loggerLog } from "../logger.js"; import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import type { BlockView } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; -import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; +import { UnifiedStore, StorageIntegrityError, StorageSecondWriterError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; import { renameSync } from "node:fs"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; @@ -192,8 +192,10 @@ export class UnifiedShadow { let store: UnifiedStore; try { try { - store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit }); + store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit, secondWriterPolicy: "degrade" }); } catch (err) { + // #5046/B: a live foreign writer owns this db — never quarantine (the file is NOT corrupt) and never take over its lock; the outer catch degrades. + if (err instanceof StorageSecondWriterError) throw err; // #4175: quick_check failed — the file set is provably corrupt. // Quarantine it (db/-wal/-shm → .corrupt-, same suffix // discipline as the session envelopes) and retry ONCE with a @@ -227,7 +229,7 @@ export class UnifiedShadow { "error", `[storage] index.db failed its integrity quick_check (${err.detail}) — quarantined ${quarantined.join(", ")} and reopened fresh; JSON envelopes remain the recovery source (#4175)`, ); - store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit }); + store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit, secondWriterPolicy: "degrade" }); } // #3285: watermark trust does not cross process boundaries — a // restart-handoff overlap leaves the previous writer's rows AND @@ -235,6 +237,15 @@ export class UnifiedShadow { // per session re-establishes trust from its own view. store.resetAllIngestWatermarks(); } catch (err) { + // #5046/B (owner decision 2026-10-29): refuse to become a persistent second writer — degrade to legacy-only with ONE actionable error. Availability is preserved (legacy persist keeps compression working); CCR / checkpoints / the unified fast-read path are off until isolated. The status wire carries the same fact via recordSidecarOpenFailure (#3040). + if (err instanceof StorageSecondWriterError) { + loggerLog( + "error", + `[storage] unified storage REFUSED for this instance: another bili process (pid ${err.pid}) already writes ${err.dbPath} — two processes on one index.db corrupt it (#3923/#3959). DEGRADED to legacy-only until isolated: CCR / checkpoints / the unified fast-read path are OFF. Fix: give each install its own sessions dir — set BILI_SESSIONS_DIR (and BILI_CONFIG_FILE if the configs differ) for THIS install, then restart (README "Running multiple instances", #5046).`, + ); + recordSidecarOpenFailure(`second-writer: another bili process (pid ${err.pid}) owns index.db — isolate via BILI_SESSIONS_DIR(+BILI_CONFIG_FILE); unified features off until then (#5046)`); + return null; + } // #2771: a corrupt/unreadable index.db must degrade to legacy-only // (same shape as the encryption opt-out), never brick SessionStore // construction. No auto-delete: the index may hold pinned checkpoint diff --git a/src/storage/store.ts b/src/storage/store.ts index 0cab41538d..8b4f9e0f52 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -334,6 +334,23 @@ function processAlive(pid: number): boolean { } } +/** #5046/B: a live foreign bili process holds .lock and this open is not + * a benign cross-lane share — thrown when the caller opted into + * secondWriterPolicy:"degrade" so it can REFUSE to become a persistent + * second writer instead of reporting-and-running (#3923/#3959). The foreign + * lockfile is left untouched and the raw handle is closed before the throw + * (a leaked WAL handle would defeat the isolation on Windows). */ +export class StorageSecondWriterError extends Error { + readonly pid: number; + readonly dbPath: string; + constructor(pid: number, dbPath: string) { + super(`another bili process (pid ${pid}) is already writing ${dbPath}`); + this.name = "StorageSecondWriterError"; + this.pid = pid; + this.dbPath = dbPath; + } +} + export class UnifiedStore { readonly engine: SqliteEngineName; /** Where the engine came from (#2833): builtin / host / self-downloaded — logged at enable so field diagnosis can tell which acquisition path a lane ran on. */ @@ -363,7 +380,7 @@ export class UnifiedStore { static open( path: string, engine?: SqliteEngineName, - opts?: { walJournalSizeLimitBytes?: number }, + opts?: { walJournalSizeLimitBytes?: number; secondWriterPolicy?: "report" | "degrade" }, ): UnifiedStore { mkdirSync(dirname(path), { recursive: true }); const { db, engine: used, source } = openSqliteDatabase(path, engine); @@ -439,6 +456,10 @@ export class UnifiedStore { const benign = peer !== undefined && ownLane !== undefined && peer.lane !== undefined && peer.lane !== ownLane; if (benign) { loggerLog("info", `[storage] process (pid ${pid}, lane ${peer?.lane}, instance ${peer?.instanceId.slice(0, 8)}) is also using ${path} — different lanes sharing one sessions directory is the supported topology (#3923/#3959); informational only.`); + } else if (opts?.secondWriterPolicy === "degrade") { + // #5046/B (owner decision): a persistent second writer corrupts this db — refuse BEFORE taking over the lockfile; the caller owns the actionable user-facing line. + db.close(); + throw new StorageSecondWriterError(pid, path); } else { loggerLog("error", `[storage] another process (pid ${pid}) is already writing ${path} — two processes on one index.db corrupt it (#3923/#3959). One session dir must have exactly one instance (#3633); this lane keeps running but the condition is reported.`); } @@ -447,6 +468,8 @@ export class UnifiedStore { } writeFileSync(lockPath, JSON.stringify({ pid: process.pid, boot: Date.now() })); } catch (err) { + // #5046/B: a second-writer refusal is a decision, not a skipped discipline — it must propagate to the caller's degrade path, not be absorbed here. + if (err instanceof StorageSecondWriterError) throw err; loggerLog("warn", `[storage] lockfile discipline skipped for ${path}: ${String(err)}`); } try { diff --git a/tests/issue5046-second-writer-degrade.test.ts b/tests/issue5046-second-writer-degrade.test.ts new file mode 100644 index 0000000000..bc977b0536 --- /dev/null +++ b/tests/issue5046-second-writer-degrade.test.ts @@ -0,0 +1,221 @@ +/** + * #5046/B (owner decision 2026-10-29): a live foreign bili process holding + * .lock must make the proxy sidecar REFUSE to become a persistent second + * writer and degrade to legacy-only — with one actionable error line (names + * BILI_SESSIONS_DIR + BILI_CONFIG_FILE + what is lost) and status-wire + * visibility. Both directions pinned: live foreign holder ⇒ degrade; no + * foreign holder ⇒ the original fast path is unchanged; mixed-version fleet + * (old marker without lane) ⇒ conservative branch degrades; the report-only + * default policy (CLI/export open paths) stays byte-identical. + */ +import assert from "node:assert"; +import { spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { after, before, test } from "node:test"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import { setLogCapture } from "../src/logger.ts"; +import { _resetStorageRuntimeStatusForTest, storageStatusWire } from "../src/storage/runtime-status.ts"; + +const DEAD_PID = 3999999; + +let root: string; +let foreign: ChildProcess; +let engineAvailable = true; +let savedStateHome: string | undefined; +let savedLauncherLane: string | undefined; + +function freshState(): string { + return mkdtempSync(path.join(root, "state-")); +} + +function freshSessions(): string { + return mkdtempSync(path.join(root, "sessions-")); +} + +function writeMarker(stateDir: string, lane: string | undefined): void { + const dir = path.join(stateDir, "billion-context", "instances"); + mkdirSync(dir, { recursive: true }); + const entry: Record = { + instanceId: "foreign-00000001", + pid: foreign.pid, + port: 19999, + origin: "http://127.0.0.1:19999", + startedAt: Date.now(), + }; + if (lane !== undefined) entry.lane = lane; + writeFileSync(path.join(dir, "foreign-00000001.json"), JSON.stringify(entry)); +} + +function writeLock(sessionsDir: string, pid: number): void { + writeFileSync(path.join(sessionsDir, "index.db.lock"), JSON.stringify({ pid, boot: Date.now() })); +} + +function lockHolderPid(sessionsDir: string): number | null { + const p = path.join(sessionsDir, "index.db.lock"); + if (!existsSync(p)) return null; + return (JSON.parse(readFileSync(p, "utf8")) as { pid?: number }).pid ?? null; +} + +function withEnv(vars: Record, fn: () => void): void { + const prev: Record = {}; + for (const [k, v] of Object.entries(vars)) { + prev[k] = process.env[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + try { + fn(); + } finally { + for (const [k, v] of Object.entries(prev)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +before(() => { + root = mkdtempSync(path.join(tmpdir(), "bili-5046-")); + savedStateHome = process.env.XDG_STATE_HOME; + savedLauncherLane = process.env.BILI_LAUNCHER_LANE; + foreign = spawn(process.execPath, ["-e", "setTimeout(()=>{},300000)"], { stdio: "ignore" }); + const probe = mkdtempSync(path.join(root, "probe-")); + try { + const s = UnifiedStore.open(path.join(probe, "index.db")); + s.close(); + } catch { + engineAvailable = false; + } +}); + +after(() => { + foreign.kill(); + rmSync(root, { recursive: true, force: true }); + setLogCapture(null); + if (savedStateHome === undefined) delete process.env.XDG_STATE_HOME; + else process.env.XDG_STATE_HOME = savedStateHome; + if (savedLauncherLane === undefined) delete process.env.BILI_LAUNCHER_LANE; + else process.env.BILI_LAUNCHER_LANE = savedLauncherLane; +}); + +test("#5046/B: live same-lane foreign holder ⇒ sidecar degrades to legacy-only, lock untouched, actionable error + status wire", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const state = freshState(); + const sessions = freshSessions(); + writeMarker(state, "dsh"); + writeLock(sessions, foreign.pid!); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + withEnv({ XDG_STATE_HOME: state, BILI_LAUNCHER_LANE: "dsh" }, () => { + const shadow = UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.equal(shadow, null, "must degrade to legacy-only (null), not open a second writer"); + const refused = logs.find((m) => m.includes("unified storage REFUSED")); + assert.ok(refused, `actionable error line expected, got: ${logs.join(" | ")}`); + assert.ok(refused!.includes("BILI_SESSIONS_DIR") && refused!.includes("BILI_CONFIG_FILE"), "the fix must name both isolation vars"); + assert.ok(refused!.includes("CCR") && refused!.includes("checkpoints"), "must say what degradation loses"); + assert.equal(lockHolderPid(sessions), foreign.pid, "the foreign lock must NOT be stolen"); + withEnv({ BILI_STORAGE_UNIFIED: "shadow" }, () => { + const wire = storageStatusWire(); + assert.equal(wire.degraded, true, "status wire must show degraded"); + assert.equal(wire.effective, "legacy"); + assert.ok(wire.reason?.startsWith("second-writer:"), `reason should be the pre-formatted refusal, got: ${wire.reason}`); + assert.ok((wire.reason ?? "").includes("BILI_SESSIONS_DIR"), "wire reason must carry the fix"); + }); + }); + setLogCapture(null); +}); + +test("#5046/B negative: live DIFFERENT-lane foreign holder ⇒ still opens (supported shared topology unchanged)", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const state = freshState(); + const sessions = freshSessions(); + writeMarker(state, "pi"); + writeLock(sessions, foreign.pid!); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + withEnv({ XDG_STATE_HOME: state, BILI_LAUNCHER_LANE: "dsh" }, () => { + const shadow = UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "cross-lane sharing is the supported topology — must still open"); + assert.ok(logs.some((m) => m.includes("informational only")), "benign notice expected"); + assert.equal(lockHolderPid(sessions), process.pid, "cross-lane takeover keeps the existing behavior"); + shadow!.close(); + }); + setLogCapture(null); +}); + +test("#5046/B negative: stale lock (dead foreign pid) ⇒ normal open + takeover unchanged", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const state = freshState(); + const sessions = freshSessions(); + writeLock(sessions, DEAD_PID); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + withEnv({ XDG_STATE_HOME: state }, () => { + const shadow = UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "a dead pid is a stale lock — normal open"); + assert.equal(lockHolderPid(sessions), process.pid, "stale lock taken over"); + assert.ok(!logs.some((m) => m.includes("REFUSED")), "no refusal for a stale lock"); + shadow!.close(); + }); + setLogCapture(null); +}); + +test("#5046/B negative: no foreign holder ⇒ original fast path (ENABLED line, no refusal)", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const state = freshState(); + const sessions = freshSessions(); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + withEnv({ XDG_STATE_HOME: state }, () => { + const shadow = UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "clean directory opens as before"); + assert.ok(logs.some((m) => m.includes("unified shadow mode ENABLED")), "fast-path ENABLED line expected"); + assert.ok(!logs.some((m) => m.includes("REFUSED"))); + shadow!.close(); + }); + setLogCapture(null); +}); + +test("#5046/B: mixed-version fleet — old marker WITHOUT lane ⇒ conservative branch degrades", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const state = freshState(); + const sessions = freshSessions(); + writeMarker(state, undefined); + writeLock(sessions, foreign.pid!); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + withEnv({ XDG_STATE_HOME: state, BILI_LAUNCHER_LANE: "dsh" }, () => { + const shadow = UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.equal(shadow, null, "unknown peer lane must stay conservative (degrade), never grant info-level amnesty"); + assert.ok(logs.some((m) => m.includes("unified storage REFUSED"))); + }); + setLogCapture(null); +}); + +test("#5046/B: default policy (CLI/export direct open) stays report-only — store returned, lock taken over", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const state = freshState(); + const sessions = freshSessions(); + writeMarker(state, "dsh"); + writeLock(sessions, foreign.pid!); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + withEnv({ XDG_STATE_HOME: state, BILI_LAUNCHER_LANE: "dsh" }, () => { + const store = UnifiedStore.open(path.join(sessions, "index.db")); + assert.ok(store, "report-only paths (export/maintenance) keep running as before"); + assert.ok(logs.some((m) => m.includes("keeps running but the condition is reported")), "existing report line unchanged"); + assert.equal(lockHolderPid(sessions), process.pid, "report-only takeover unchanged"); + store.close(); + }); + setLogCapture(null); +}); From 87edeaabecce0b4e6df130452299f88d58bb6b50 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 29 Oct 2026 18:38:05 +0800 Subject: [PATCH 22/35] test: register issue5046 second-writer suite in LIVE_FILES (#5075) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new tests/issue5046-second-writer-degrade.test.ts spawns one bare 'node -e' dummy child purely as a live PID holder for its lock-file fixtures — no ports, no servers, so the zone band is irrelevant. The entry is append-only (slot = index drives zone bands) and exists for the SPAWN_RE tripwire, same shape as the #3890/#5114 entries. Without it every test shard dies at pretest before running a single test. --- scripts/live-files.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index 5338222370..4fc1a4dfa8 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -77,6 +77,10 @@ export const LIVE_FILES = [ // spawns a bare `node -e` child that only prints env — no ports, no // servers, so the zone band is irrelevant; the entry is for the tripwire "tests/preload-sandbox-dirs.test.ts", + // #5046/B: spawns one bare `node -e` dummy child purely as a live PID + // holder for the lock-file fixtures — no ports, no servers, so the zone + // band is irrelevant; the entry is for the tripwire + "tests/issue5046-second-writer-degrade.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; From 9d8b15e3485e5230559ee96099f6706acbb786b6 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 30 Oct 2026 00:30:38 +0800 Subject: [PATCH 23/35] fix(storage): align second-writer degrade log with the supported-topology framing (#5091) --- src/storage/shadow.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 0859edaec7..ec0b8fe46f 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -241,7 +241,7 @@ export class UnifiedShadow { if (err instanceof StorageSecondWriterError) { loggerLog( "error", - `[storage] unified storage REFUSED for this instance: another bili process (pid ${err.pid}) already writes ${err.dbPath} — two processes on one index.db corrupt it (#3923/#3959). DEGRADED to legacy-only until isolated: CCR / checkpoints / the unified fast-read path are OFF. Fix: give each install its own sessions dir — set BILI_SESSIONS_DIR (and BILI_CONFIG_FILE if the configs differ) for THIS install, then restart (README "Running multiple instances", #5046).`, + `[storage] unified storage REFUSED for this instance (pid ${process.pid}): another bili process (pid ${err.pid}) already writes ${err.dbPath} — sharing one sessions dir across lanes is the supported topology (#5082/#5091), but two processes writing the SAME sessions dir can split-brain it (#3923/#3959), so this one refuses to co-write rather than risk it. DEGRADED to legacy-only (defense-in-depth) until isolated: CCR / checkpoints / the unified fast-read path are OFF. Fix: give each install its own sessions dir — set BILI_SESSIONS_DIR (and BILI_CONFIG_FILE if the configs differ) for THIS install, then restart (README "Running multiple instances", #5046/#5068).`, ); recordSidecarOpenFailure(`second-writer: another bili process (pid ${err.pid}) owns index.db — isolate via BILI_SESSIONS_DIR(+BILI_CONFIG_FILE); unified features off until then (#5046)`); return null; From a0c7ff0e29029f2459aa5f20670e454c0aef7227 Mon Sep 17 00:00:00 2001 From: ranxianglei-devbox Date: Fri, 30 Oct 2026 14:12:13 +0800 Subject: [PATCH 24/35] test(mysql-seam): align StubMysqlBackend.getMessages with the retired-aware interface (#4664 union) The stub was written against d-4987's older base where getMessages had no opts/retiredAt; add the optional opts param (unused by the stub) and emit retiredAt:null so the stub satisfies today's StorageBackend. --- tests/storage-mysql-seam.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/storage-mysql-seam.test.ts b/tests/storage-mysql-seam.test.ts index fa2d81232a..aaa304771c 100644 --- a/tests/storage-mysql-seam.test.ts +++ b/tests/storage-mysql-seam.test.ts @@ -116,9 +116,11 @@ class StubMysqlBackend implements StorageBackend { cloneSessionRows(): Promise { return Promise.resolve(true); } - getMessages(sessionId: string): Promise<(MessageRow & { contentHash: string })[]> { + getMessages(sessionId: string, _opts?: { includeRetired?: boolean; seqs?: readonly number[] }): Promise<(MessageRow & { contentHash: string; retiredAt: number | null })[]> { const rows = this.rows.get(sessionId)?.messages ?? []; - return Promise.resolve(rows.map((r) => ({ ...r, contentHash: r.contentHash ?? sha256(r.content) }))); + // Stub rows come from ingest inputs only (live rows, never retired), + // so retiredAt is always null here — shape kept for interface parity. + return Promise.resolve(rows.map((r) => ({ ...r, contentHash: r.contentHash ?? sha256(r.content), retiredAt: null }))); } getBlocks(sessionId: string): Promise<(BlockRow & { summaryHash: string; oneHash: string | null })[]> { const rows = this.rows.get(sessionId)?.blocks ?? []; From 0a724cee5dd55d5e65309e0e29473b9e38d90210 Mon Sep 17 00:00:00 2001 From: ranxianglei-devbox Date: Fri, 30 Oct 2026 14:13:23 +0800 Subject: [PATCH 25/35] test(mysql-seam): add #5198-era interface members (sessionViewRows/readSnapshot) to the stub d-4987's hermetic seam suite predates the C-class narrow-read additions to StorageBackend; satisfy them from the same captured rows so the stub stays interface-complete without changing test behavior. --- tests/storage-mysql-seam.test.ts | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/storage-mysql-seam.test.ts b/tests/storage-mysql-seam.test.ts index aaa304771c..bfec6c0fd4 100644 --- a/tests/storage-mysql-seam.test.ts +++ b/tests/storage-mysql-seam.test.ts @@ -16,6 +16,8 @@ import type { RefRow, CcrEntryRow, SessionSummaryRow, + SessionViewRows, + StorageReadView, } from "../src/storage/backend.ts"; import { sha256 } from "../src/storage/canonical.ts"; @@ -99,6 +101,31 @@ class StubMysqlBackend implements StorageBackend { this.viewCalls.count++; return Promise.resolve(this.rows.get(sessionId)?.session ?? null); } + // #5198-era interface members (#4739 narrow reads + snapshot facade), + // satisfied from the same rows ingestSession captured. + sessionViewRows(sessionId: string): Promise { + const row = this.rows.get(sessionId); + if (!row) return Promise.resolve(null); + const view = { + session: row.session, + messages: row.messages.map((r) => ({ ...r, contentHash: r.contentHash ?? sha256(r.content) })), + blocks: row.blocks.map((b) => ({ ...b, summaryHash: sha256(b.summary), oneHash: b.one === null ? null : sha256(b.one) })), + deadRefs: row.deadRefs, + refs: row.refs, + }; + return Promise.resolve(view as unknown as SessionViewRows); + } + readSnapshot(read: (reader: StorageReadView) => Promise): Promise { + const row = (id: string) => this.rows.get(id); + const view: StorageReadView = { + getSessionRow: async (id) => row(id)?.session ?? null, + getMessages: async (id, _opts) => (row(id)?.messages ?? []).map((r) => ({ ...r, contentHash: r.contentHash ?? sha256(r.content), retiredAt: null })), + getBlocks: async (id, _opts) => (row(id)?.blocks ?? []).map((b) => ({ ...b, summaryHash: sha256(b.summary), oneHash: b.one === null ? null : sha256(b.one) })), + getDeadRefs: async (id) => row(id)?.deadRefs ?? [], + getRefs: async (id) => row(id)?.refs ?? [], + }; + return read(view); + } liveRowCount(sessionId: string): Promise { return Promise.resolve(this.rows.get(sessionId)?.messages.length ?? 0); } From 42a1968a207380f06e4c7df027c3cea53688fb2d Mon Sep 17 00:00:00 2001 From: ranxianglei-devbox Date: Fri, 30 Oct 2026 15:48:22 +0800 Subject: [PATCH 26/35] =?UTF-8?q?integrity=20failure:=20stop=20auto-renami?= =?UTF-8?q?ng=20a=20corrupt=20index=20=E2=80=94=20loud=20degrade=20with=20?= =?UTF-8?q?files=20untouched=20(#4626-era=20owner=20decision)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #4175/#4816 quarantine rename is withdrawn: it moved index.db{-wal,-shm} aside automatically, but full data recovery cannot be guaranteed from that — checkpoint rows live only inside the index.db (no envelope round-trip), and in jsonWriterOff mode the index IS the sole durable copy. Renaming an unverifiable file set to '.corrupt-' therefore risked destroying the only copy while still not proving recovery. New policy: when quick_check provably fails (throwing corruption classified by src/storage/sqlite-errors.ts), every file stays byte-identical where it is. The proxy degrades loudly to legacy-only, caches the FAIL verdict so the next open fails fast without rescanning, and logs/status-wire now state the manual fix (back up, then move the file set aside; rebuilding then works, pinned e2e). - src/storage/shadow.ts: drop the rename loop; typed error + error log with the manual guidance; outer degrade catch keeps integrity errors distinct from the 'delete to reset' advice (which would destroy evidence). - src/storage/store.ts: FAIL verdict now persisted before throwing on both sync paths (fail-fast without rescan); docs follow the new policy. - src/storage/integrity-check.ts / sqlite-errors.ts / runtime-status.ts / CONFIGURATION.md: wording + a dedicated corrupt-index reason line that names the UNTOUCHED state and the operator fix. - tests rewritten to pin the new contract (byte-identical files, no .corrupt-* copies ever, fail-fast second open, operator move-aside -> clean rebuild): issue4816 suite, storage-durability, storage-engine-proofs (+ corrupt-index child), storage-review-fixes, web-storage-banner, storage-integrity-cache. Verified: typecheck clean; targeted suites green (issue4816/durability/ integrity-cache/cantopen/review-fixes 34p, engine-proofs+web-banner 17p, unified/ext/watermark/multi-instance/persist trio 49p+1skip, 5046/4949/ instance-governance 28p). --- CONFIGURATION.md | 2 +- src/storage/integrity-check.ts | 32 +++--- src/storage/runtime-status.ts | 7 +- src/storage/shadow.ts | 102 ++++++----------- src/storage/sqlite-errors.ts | 3 +- src/storage/store.ts | 45 +++++--- tests/corrupt-index-child.ts | 67 ++++++----- ...sue4816-integrity-throw-quarantine.test.ts | 107 ++++++++++++------ tests/storage-durability.test.ts | 32 +++--- tests/storage-engine-proofs.test.ts | 21 ++-- tests/storage-integrity-cache.test.ts | 5 +- tests/storage-review-fixes.test.ts | 15 +-- tests/web-storage-banner.test.ts | 12 +- 13 files changed, 240 insertions(+), 210 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 6aefadce0b..1c8c5d2e7b 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -362,7 +362,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | Env tier for storage.backend (#3931): sqlite (default) or mysql (fully wired, #3388 steps 2+3 - saves ingest through the async shadow queue, sync reads ride the JSON envelope, replicas/ext/GC/fork via the async twins (#4911); export too). Wins over the file tier. | | `BILI_MYSQL_DSN` | string | unset | — | MySQL DSN (mysql://user:pass@host:3306/db) for storage.backend="mysql" (#3388, #3931). Wins over the file key storage.mysql.dsn. Empty = the mysql backend cannot open; the documented error names both knobs. The pool opens lazily; closeQuietlyAsync drains the ingest queue and closes it. | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC age sweep window in days (#3463): sessions idle longer than this are archived (one JSON line appended to the single archive log (archive/archive.jsonl) + tombstone row + content/media refs released). 0 disables the age sweep; the manual entry POST /__bili/storage/gc always runs. Config-file key storage.archiveAfterDays is the persistent form. | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; on failure the shadow layer quarantines the file set (index.db/-wal/-shm renamed to .corrupt-) and reopens fresh (JSON envelopes remain the recovery source). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; on failure the files are left EXACTLY in place and bili runs legacy-only with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand to force a fresh rebuild — JSON envelopes remain the recovery source). The FAIL verdict is cached so later opens fail fast without rescan. There is NO automatic rename/repair: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy) and a corrupt-but-salvageable index stays operator-salvageable. Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | Snapshot-render shadow observability (#3756 follow-up): compute the outbound snapshot view on every armed turn WITHOUT using it, classify the diff against the legacy outbound, and surface per-session counters in the web panel + the snapshot-shadow diagnostics topic. 0/false/off disables; 1/true/on enables; otherwise the config-file snapshot.shadow boolean applies (default on). The render lane itself stays rollout-gated (BILI_ROLLOUT_SNAPSHOT_RENDER), so the shadow lane never changes outbound bytes - it is the dry-run evidence that flips the rollout later. | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | Override the config file path (path relocation). | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | Where session records are stored (path relocation). | diff --git a/src/storage/integrity-check.ts b/src/storage/integrity-check.ts index 5e7149ee4a..502a58d0dc 100644 --- a/src/storage/integrity-check.ts +++ b/src/storage/integrity-check.ts @@ -22,7 +22,8 @@ * read-only open cannot hit WAL-recovery). Boot returns in * milliseconds; a failure is recorded loudly and cached so the NEXT * open throws StorageIntegrityError and reuses the #4175 - * quarantine-and-reopen-fresh machinery. Small stores keep the + * loud-degrade machinery (no automatic file moves, #4626-era owner + * decision). Small stores keep the * synchronous pre-migration check — the #4175 premise (a corrupt file * must never be migrated against) stays intact wherever it is cheap. * The one concession: on a big store's first boot after upgrade, @@ -175,7 +176,7 @@ const inflight = new Set(); /** Fire-and-forget background quick_check on a second read-only connection * (#4609). Records the outcome, caches it (ok or fail — a cached fail makes - * the NEXT open throw and quarantine via the #4175 path), and tells the + * the NEXT open throw and degrade loudly via the #4175 path), and tells the * caller's store instance so a clean close can refresh the cache key. */ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok: boolean, detail?: string) => void): void { const size = statPart(dbPath)?.s ?? 0; @@ -207,32 +208,35 @@ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok writeIntegrityCache(dbPath, true); onVerdict?.(true); } else { - // A real integrity failure found post-open: record loudly, - // cache the fail so the next open quarantines + reopens - // fresh (#4175 path), and keep serving (writes into a - // corrupt store are quarantined with it; envelopes are - // the recovery source). + // A real integrity failure found post-open: record loudly + // and cache the fail so the NEXT open throws StorageIntegrityError + // and degrades loudly INSTEAD of rescan-looping (#4175 path, + // no automatic rename — files stay where they are); keep + // serving now (writes into a provably-corrupt store surface + // as ingest errors and the #3133 breaker disables the + // sidecar inside this process if they persist; envelopes + // are the recovery source either way). recordIntegrityOutcome(dbPath, false, 0, verdict.slice(0, 400)); writeIntegrityCache(dbPath, false, verdict.slice(0, 400)); - loggerLog("error", `[storage] background quick_check FAILED for ${dbPath}: ${verdict.slice(0, 400)} — verdict cached; the next open quarantines this file set and reopens fresh (JSON envelopes remain the recovery source, #4175/#4609)`); + loggerLog("error", `[storage] background quick_check FAILED for ${dbPath}: ${verdict.slice(0, 400)} — verdict cached; the next open fails fast and runs legacy-only (files left UNTOUCHED — move them aside by hand after backing up to force a rebuild; JSON envelopes remain the recovery source, #4175/#4609)`); // Detail rides with the verdict so a clean close can // refresh the FAIL cache onto post-checkpoint stats // (#5108/#4816) — without it the verdict would go stale - // after the first clean restart and never quarantine. + // after the first clean restart and never fail fast again. onVerdict?.(false, verdict.slice(0, 400)); } } else if (msg.corrupt === true) { // #5108/#4816: the scan itself threw a corruption-shaped error // (the throwing shapes of CORRUPT/NOTADB). Same consequences as // a bad verdict row: record loudly and cache the fail so the - // NEXT open quarantines + reopens fresh (#4175 path). Without - // this branch the throw fell into "could not run" below and the - // fail verdict was never recorded — a large corrupt store stayed - // unverified boot after boot with no quarantine ever running. + // NEXT open fails fast and degrades loudly (#4175 path). + // Without this branch the throw fell into "could not run" + // below and the fail verdict was never recorded — a large + // corrupt store stayed unverified boot after boot. const detail = String(msg.error ?? "").slice(0, 400); recordIntegrityOutcome(dbPath, false, 0, detail); writeIntegrityCache(dbPath, false, detail); - loggerLog("error", `[storage] background quick_check threw a corruption-shaped error for ${dbPath}: ${detail} — FAIL verdict cached; the next open quarantines this file set and reopens fresh (JSON envelopes remain the recovery source, #5108/#4175)`); + loggerLog("error", `[storage] background quick_check threw a corruption-shaped error for ${dbPath}: ${detail} — FAIL verdict cached; the next open fails fast and runs legacy-only (files left UNTOUCHED — move them aside by hand after backing up to force a rebuild; JSON envelopes remain the recovery source, #5108/#4175)`); onVerdict?.(false, detail); } else { loggerLog("warn", `[storage] background quick_check could not run for ${dbPath}: ${String(msg.error)} — the store serves unverified; the next boot rechecks`); diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index c864e51c0e..c4e2762cfc 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -188,7 +188,7 @@ function shortenError(err: string): string { const oneLine = err.replace(/\s+/g, " ").trim(); if (oneLine.length <= 160) return oneLine; // #5108/#4816: both ends carry signal — the error class up front, the - // errno/detail at the end (e.g. "quarantine rename failed: EPERM") — so + // errno/detail at the end (e.g. "corruption detail: …") — so // cut the MIDDLE (usually a long path) instead of the tail. Head-only // truncation made the actionable detail invisible on exactly the // long-path machines (Windows user profiles) this fix targets. @@ -220,6 +220,11 @@ function degradeReason(): string { if (isNoEngineError(e)) { return `no SQLite engine available (node:sqlite missing, better-sqlite3 not installed)${nodeSqliteUpgradeHint()}`; } + // #4626-era owner decision: a provably-corrupt index is LEFT IN PLACE + // (no automatic rename), so the reason must carry the manual fix. + if (/StorageIntegrityError|integrity quick_check failed/i.test(state.sidecarOpenError)) { + return `index database failed its integrity check and was left UNTOUCHED (${e}) — back up index.db{,-wal,-shm}, then move those files aside by hand to rebuild; the proxy runs legacy-only meanwhile`; + } return `index database failed to open (${e})${sqliteOpenFailureHint(state.sidecarOpenError)}`; } const b = state.bootstrap; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 8e9982ad8b..9ca397dc1e 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -26,7 +26,6 @@ import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendB import type { BlockView } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, StorageSecondWriterError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; -import { existsSync, renameSync } from "node:fs"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; import { canonicalDigest, canonicalize, sha256 } from "./canonical.ts"; @@ -253,73 +252,38 @@ export class UnifiedShadow { try { store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit, secondWriterPolicy: "degrade" }); } catch (err) { - // #5046/B: a live foreign writer owns this db — never quarantine (the file is NOT corrupt) and never take over its lock; the outer catch degrades. + // #5046/B: a live foreign writer owns this db — never touch it (the file is NOT corrupt) and never take over its lock; the outer catch degrades. if (err instanceof StorageSecondWriterError) throw err; - // #4175: quick_check failed — the file set is provably corrupt. - // Quarantine it (db/-wal/-shm → .corrupt-, same suffix - // discipline as the session envelopes) and retry ONCE with a - // fresh file: the JSON envelopes remain the recovery source - // (full mode re-imports organically on load). No auto-delete — - // the corrupt original is kept for forensics. + // #4175: anything else throwing StorageIntegrityError means quick_check + // proved the file set corrupt — handled below WITHOUT moving any file + // (#4626-era owner decision, 2026-10-30). if (!(err instanceof StorageIntegrityError)) throw err; - const stamp = Date.now(); - const quarantined: string[] = []; - let primaryRenameFailure: string | null = null; - const sidecarLeft: string[] = []; - for (const suffix of ["", "-wal", "-shm"]) { - const from = `${dir}/index.db${suffix}`; - if (!existsSync(from)) continue; // absent sidecar files are fine - const to = `${dir}/index.db${suffix}.corrupt-${stamp}`; - try { - renameSync(from, to); - quarantined.push(path.basename(to)); - } catch (rerr) { - // #5108/#4347: the old empty catch swallowed EVERY failure - // — including a primary that could not be moved — and the - // code below then claimed success. Capture the errno and - // decide per file what a failed move actually means. - const why = String((rerr as NodeJS.ErrnoException).code ?? rerr); - if (suffix === "") primaryRenameFailure = why; - else sidecarLeft.push(`${path.basename(from)} (${why})`); - } - } - // #4609: the verdict cache belongs to the quarantined file set — - // move it along so the fresh store starts with a clean cache - // (its stats differ anyway; this keeps the directory honest). - try { - const cacheFrom = `${dir}/index.db.chk`; - renameSync(cacheFrom, `${cacheFrom}.corrupt-${stamp}`); - quarantined.push(path.basename(`${cacheFrom}.corrupt-${stamp}`)); - } catch (rerr) { - if (existsSync(`${dir}/index.db.chk`)) { - // Present but unmovable — harmless (the fresh store's stats - // key misses and rescans), but say so instead of staying silent. - loggerLog("warn", `[storage] quarantined index.db but its .chk verdict cache could not be moved (${String((rerr as NodeJS.ErrnoException).code ?? rerr)}) — it will be overwritten by the next check`); - } - } - if (primaryRenameFailure !== null) { - // The corrupt PRIMARY is still in place: a fresh reopen would - // just re-hit the same corrupt file, so do NOT attempt it — - // degrade loudly with the real errno instead of claiming the - // quarantine succeeded. The operator moves the file aside by - // hand (the same manual workaround that always worked). - loggerLog( - "error", - `[storage] index.db failed its integrity quick_check (${err.detail}) but the corrupt file could NOT be quarantined (rename ${primaryRenameFailure}${sidecarLeft.length > 0 ? `; also left behind: ${sidecarLeft.join(", ")}` : ""}) — staying legacy-only; move ${path.join(dir, "index.db")} aside by hand to rebuild the index (#5108/#4347)`, - ); - throw new StorageIntegrityError(`${dir}/index.db`, `${err.detail}; quarantine rename failed: ${primaryRenameFailure}`); - } - if (sidecarLeft.length > 0) { - // Probe-verified (node:sqlite v22): SQLite RESETS a stale -wal - // whose parent db was replaced rather than replaying it onto - // the fresh file, so leftover sidecars are safe to leave. - loggerLog("warn", `[storage] quarantined index.db but ${sidecarLeft.join(", ")} could not be renamed — left in place (a stale -wal is reset, not replayed, on the next open)`); - } + // #4626-era owner decision (2026-10-30): NO automatic quarantine + // rename. quick_check proved the file set corrupt, but automatic + // removal of a storage PRIMARY was withdrawn because full data + // recovery cannot be guaranteed without touching the main flow: + // - checkpoint rows live only inside index.db — the JSON + // envelopes are NOT a complete round-trip for them; + // - with BILI_STORAGE_JSON_WRITER=off (jsonWriterOff, see + // persist.ts) the envelopes are skipped entirely and the + // index is the SOLE durable copy — an automatic rename + // destroys unrecoverable data; + // - a corrupt-on-open index can still often be salvaged by an + // operator; renaming it away severs that last option forever. + // The file set therefore stays EXACTLY in place, untouched; + // we degrade loudly instead (outer catch → legacy-only + the + // status wire). The FAIL verdict has already been written to + // index.db.chk by UnifiedStore.open, so every later open in + // this deployment fails fast from the cache — no rescans, no + // repeated work in the main flow. Manual resolution (unchanged + // #2771 fallback): back up, then move index.db{-wal,-shm} + // aside yourself; the next start rebuilds the index fresh and + // re-imports the JSON envelopes that hold session content. loggerLog( "error", - `[storage] index.db failed its integrity quick_check (${err.detail}) — quarantined ${quarantined.join(", ")} and reopened fresh; JSON envelopes remain the recovery source (#4175)`, + `[storage] index.db failed its integrity quick_check (${err.detail}) — leaving the file UNTOUCHED and running legacy-only (no automatic rename); back up ${path.join(dir, "index.db")}{,-wal,-shm}, then move those files aside to force a fresh rebuild (JSON envelopes remain the recovery source for session content) (#4175/#4626)`, ); - store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit, secondWriterPolicy: "degrade" }); + throw err; } // #3285: watermark trust does not cross process boundaries — a // restart-handoff overlap leaves the previous writer's rows AND @@ -347,11 +311,17 @@ export class UnifiedShadow { // session state in unified-full mode). The delete advice fits only a genuine corrupt // file; the Node.js upgrade lever is surfaced via /__bili/status + the bootstrap warn. const hint = sqliteOpenFailureHint(errStr); + // #4626-era owner decision: a provably-corrupt index is LEFT IN PLACE — + // "delete to reset" advice would nudge toward destroying data that + // may not be fully envelope-covered (checkpoints; jsonWriterOff). + const corruptIntegrity = err instanceof StorageIntegrityError; const tail = /no SQLite engine available/i.test(errStr) ? "" - : hint !== "" - ? hint - : "; delete the file to reset the index"; + : corruptIntegrity + ? "" + : hint !== "" + ? hint + : "; delete the file to reset the index"; loggerLog( "warn", `[storage] unified sidecar failed to open ${dir}/index.db (${errStr}) — staying legacy-only${tail}`, diff --git a/src/storage/sqlite-errors.ts b/src/storage/sqlite-errors.ts index e031712b39..dbae8bee50 100644 --- a/src/storage/sqlite-errors.ts +++ b/src/storage/sqlite-errors.ts @@ -5,7 +5,8 @@ * rows (page-level damage), or it can THROW out of the statement itself * (truncated file, garbage header, broken overflow chain — probe-verified on * node:sqlite v22: ERR_SQLITE_ERROR with the raw sqlite3_errmsg text). The - * #4175 quarantine decision only saw the verdict-row shape, so every + * #4175 loud-degrade decision (fail fast + legacy-only, no automatic file + * moves) only saw the verdict-row shape, so every * throwing corruption escaped as a raw driver error and degraded the store * to legacy-only forever with no auto-heal. * diff --git a/src/storage/store.ts b/src/storage/store.ts index f32dbf41fd..60ac34023c 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -347,8 +347,11 @@ function likeEscape(prefix: string): string { /** * #4175: raised when PRAGMA quick_check fails at open. Migrations must never * run against a corrupt file, so this fires BEFORE migrateSchema; the shadow - * layer catches it, quarantines the file set (db/-wal/-shm → .corrupt-), - * and reopens fresh (the JSON envelopes remain the recovery source). + * layer catches it and degrades to legacy-only LOUDLY without touching the + * file set — no automatic rename/repair (#4626-era owner decision: full data + * recovery cannot be guaranteed without operator involvement). The FAIL + * verdict is cached so later opens fail fast; the operator moves the files + * aside by hand (backed up first) to force a fresh rebuild. */ export class StorageIntegrityError extends Error { constructor(public readonly dbPath: string, public readonly detail: string) { @@ -416,7 +419,8 @@ export class UnifiedStore { private lastIntegrityOk = false; /** #5108/#4816: a completed check said the file set is CORRUPT — a clean * close refreshes the FAIL verdict onto post-checkpoint stats so the - * next boot's cached-fail path quarantines instead of rescan-looping + * next boot's cached-fail path fails fast (loud degrade) instead of + * rescan-looping * (a clean close moves the WAL, which would otherwise invalidate the * cache key and send a large corrupt store back to unverified forever). * Outranks lastIntegrityOk at close time. */ @@ -445,8 +449,9 @@ export class UnifiedStore { // preserved wherever it is cheap); a large store checks in the // background post-open (#4609: a 14 GB quick_check costs ~260 s and // the 60 s native spawn budget killed the child mid-scan). A cached - // failure throws here so the shadow's quarantine-and-reopen-fresh - // path (#4175) runs — including for failures found by the background + // failure throws here so the shadow degrades loudly INSTEAD of + // rescan-looping (#4626-era owner decision: no automatic quarantine — + // see maybeOpen) — including for failures found by the background // check of a previous run. BILI_STORAGE_QUICK_CHECK=0 opts out of all // of it (no check, no cache IO). let integrityOk = false; @@ -470,24 +475,32 @@ export class UnifiedStore { // Corrupt header (NOTADB) throws from the statement itself, so // the verdict branch never runs: close the handle we opened // before rethrowing — the #2771 fallback tells operators to - // DELETE the file, and Windows refuses to delete a file this - // process still holds (unlink EBUSY, #4347 probe). + // MOVE the file aside by hand (backed up first), and Windows + // refuses to move a file this process still holds (EBUSY, + // #4347 probe). // #5108/#4816: corruption-shaped THROWS (truncated file, garbage // header, broken overflow chain) must become StorageIntegrityError // like the verdict-row path does — the raw rethrow escaped the - // shadow's quarantine guard (#4175) and left the corrupt set in + // shadow's integrity guard (#4175) and left the corrupt set in // place forever. Transient shapes (BUSY/IOERR/CANTOPEN) stay raw: // quarantining a locked-but-healthy index would be the bug. recordIntegrityOutcome(path, false, Date.now() - started, String(err).slice(0, 400)); db.close(); const detail = sqliteCorruptionDetail(err); - if (detail !== null) throw new StorageIntegrityError(path, detail.slice(0, 400)); + if (detail !== null) { + // #4626-era owner decision: nothing is renamed away — cache + // the FAIL verdict FIRST so every later open fails fast + // without re-running the scan. + writeIntegrityCache(path, false, detail.slice(0, 400)); + throw new StorageIntegrityError(path, detail.slice(0, 400)); + } throw err; } const durationMs = Date.now() - started; recordIntegrityOutcome(path, verdict === "ok", durationMs, verdict === "ok" ? undefined : verdict.slice(0, 400)); if (verdict !== "ok") { db.close(); + writeIntegrityCache(path, false, verdict.slice(0, 400)); throw new StorageIntegrityError(path, verdict.slice(0, 400)); } writeIntegrityCache(path, true); @@ -551,13 +564,14 @@ export class UnifiedStore { migrateSchema(db); } catch (err) { // The handle we just opened must not leak when the file turns out - // to be corrupt: the #2771 fallback tells operators to DELETE the - // file to reset the index — and Windows refuses to delete a file a - // live process still holds open. Close, then rethrow for the - // shadow's legacy-only degradation. + // to be corrupt: the #2771 fallback tells operators to MOVE the + // file aside by hand (backed up first) to reset the index — and + // Windows refuses to move a file a live process still holds open. + // Close, then rethrow for the shadow's legacy-only degradation. // #5108/#4816: large stores (> sync threshold) skip the pre-migration // scan, so a corruption can surface HERE first — classify it the same - // way so the shadow quarantines instead of degrading forever. + // way so the loud degrade + fail-fast cache applies instead of an + // unexplained raw driver error. db.close(); const detail = sqliteCorruptionDetail(err); if (detail !== null) throw new StorageIntegrityError(path, detail.slice(0, 400)); @@ -580,7 +594,8 @@ export class UnifiedStore { // #4609: a large store's cache-miss check runs post-open on a second // read-only connection — open() returns immediately; the verdict // lands in runtime-status and the cache (a cached failure makes the - // NEXT open quarantine + reopen fresh via the #4175 path). + // NEXT open fail fast and degrade loudly — no automatic file moves, + // #4175/#4626-era decision). if (mode.kind === "async") scheduleBackgroundIntegrityCheck(path, (ok, detail) => { if (ok) store.noteIntegrityOk(); else if (detail !== undefined) store.noteIntegrityFail(detail); }); return store; } diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index f312938db1..c7bf69e865 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -34,6 +34,7 @@ import { spawnSync } from "node:child_process"; import http from "node:http"; import { once } from "node:events"; import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import { mkdirSync, realpathSync, writeSync } from "node:fs"; import * as fs from "node:fs"; import * as path from "node:path"; @@ -193,24 +194,28 @@ export function makeOpts(upstreamPort: number): ProxyOptions { // sides reference these constants, so the coverage list cannot drift. // --------------------------------------------------------------------------- -// #4816: throwing-shape corruption (header garbage / truncation) is now -// quarantined and the store RECOVERS to full mode on a fresh index.db — -// these checks pin that recovery plus the original #2781/#4357 leak guards. +// #4816/#4626: throwing-shape corruption (header garbage / truncation) is +// classified and degrades LOUDLY — but #4626-era owner decision withdrew the +// automatic quarantine rename (no full-recovery guarantee), so the corrupt +// files stay byte-identical in place and the store serves legacy-only until +// an operator moves them aside. These checks pin that policy plus the +// original #2781/#4357 leak guards. const ENGPROOF_CHECK_NAMES = { - quarantine: "corrupt index.db was renamed aside (index.db.corrupt-* kept for forensics)", - freshIndex: "a fresh index.db replaced the quarantined one", - lazyImport: "session reloaded from the legacy envelope into the recovered index", - midFlowGuard: "no live handle on the QUARANTINED file (#2781/#4357 rename guard)", - stateDigest: "recovered-index state digest matches the clean full-mode read", - writeNow: "writeNow keeps working over the recovered full mode", + noAutoRename: "corrupt index.db left UNTOUCHED — byte-identical, no .corrupt-* copy (#4626 decision)", + degraded: "store serves legacy-only; nothing auto-recovers full mode", + lazyImport: "session loaded from the legacy envelope while degraded", + midFlowGuard: "no live handle into the sessions dir while serving degraded (#2781/#4357 guard)", + stateDigest: "legacy load's state digest matches the clean full-mode read", + writeNow: "writeNow keeps working over the legacy path", postCloseGuard: "no live handle after the final close (post-close rename guard)", fdGuard: "no open fd into the corrupt index dir after close (/proc/self/fd, Linux)", } as const; export const ENGPROOF_CHECKS: readonly string[] = Object.values(ENGPROOF_CHECK_NAMES); -// #4816: a corrupt index.db no longer degrades — it quarantines and recovers -// (see engproof). The banner wire shape is pinned here through the remaining -// deterministic cross-platform degrade cause: the engine ladder exhausted. +// #4626-era owner decision: a corrupt index.db degrades loudly and NOTHING +// is renamed or rebuilt automatically (see engproof). The banner wire shape +// is pinned here through the remaining deterministic cross-platform degrade +// cause: the engine ladder exhausted. const BANNER_CHECK_NAMES = { overviewAnswers: "/__bili/overview answers 200 with JSON", requestedFull: "storage.requested is full", @@ -340,37 +345,31 @@ async function engproofScenario(argv: string[], run: CheckRun): Promise { fs.writeSync(fd, Buffer.alloc(512, 0x5a), 0, 512, 0); fs.closeSync(fd); - // THE #2771 regression: construction must survive. Since #4816 the - // throwing-shape corruption is quarantined and the store recovers to - // full mode on a fresh index.db — reads lazily import the session back - // from the legacy envelope, and the digest must be exact. + // THE #2771 regression: construction must survive. #4626-era owner + // decision: NO automatic quarantine/recovery — the corrupt files stay + // where they are and the store serves from the JSON envelopes (full + // recovery can't be guaranteed without operator involvement); the + // envelope read below must still be exact. + const beforeHash = createHash("sha256").update(fs.readFileSync(path.join(dir, "index.db"))).digest("hex"); const store2 = new SessionStore({ dir, debounceMs: 1, enabled: true }); - await run.run(ENGPROOF_CHECK_NAMES.quarantine, () => { + await run.run(ENGPROOF_CHECK_NAMES.noAutoRename, () => { const names = fs.readdirSync(dir); - const q = names.find((n) => n.startsWith("index.db.corrupt-")); - assert.ok(q, `no quarantined index.db.corrupt-* in ${names.join(", ")}`); + assert.ok(!names.some((n) => n.startsWith("index.db.corrupt-")), `an automatic .corrupt-* copy must never exist, got ${names.join(", ")}`); + const afterHash = createHash("sha256").update(fs.readFileSync(path.join(dir, "index.db"))).digest("hex"); + assert.equal(afterHash, beforeHash, "the corrupt primary must stay byte-identical"); }); - await run.run(ENGPROOF_CHECK_NAMES.freshIndex, () => - assert.ok(fs.existsSync(path.join(dir, "index.db")), "fresh index.db absent after quarantine")); + await run.run(ENGPROOF_CHECK_NAMES.degraded, () => + assert.equal(store2.replicasAvailable(), false, "must serve legacy-only — nothing may auto-recover full mode")); const reloaded = store2.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); store2.cancelAll(); try { await run.run(ENGPROOF_CHECK_NAMES.lazyImport, () => - assert.ok(reloaded, "recovered store returned null for a session whose legacy envelope exists")); + assert.ok(reloaded, "degraded store returned null for a session whose legacy envelope exists")); if (!reloaded) return; - // Leak guard (#2781/#4357): the DEAD quarantined file must carry no - // live handle. The whole-dir rename used pre-#4816 is wrong now — - // the store legitimately holds the FRESH index.db, and only a - // handle on the dead file is the leak class this guard hunts. - await run.run(ENGPROOF_CHECK_NAMES.midFlowGuard, () => { - const q = fs.readdirSync(dir).find((n) => n.startsWith("index.db.corrupt-")); - assert.ok(q, "quarantined file vanished before the leak guard ran"); - const src = path.join(dir, q!); - const probe = `${src}-moved`; - fs.renameSync(src, probe); - fs.renameSync(probe, src); - }); + // Leak guard (#2781/#4357): a degraded store must hold NO persistent + // handle into the dir at all — whole-dir rename round-trip. + await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-midflow"); // #4316 PR3/3: the clean full-mode read carried the one-time ledger // alignment; the recovered load applies it again. The parent passes diff --git a/tests/issue4816-integrity-throw-quarantine.test.ts b/tests/issue4816-integrity-throw-quarantine.test.ts index d527122645..f38d6eb2b2 100644 --- a/tests/issue4816-integrity-throw-quarantine.test.ts +++ b/tests/issue4816-integrity-throw-quarantine.test.ts @@ -11,14 +11,19 @@ * manual rename). Facets covered here: * 1. the classifier itself (corruption shapes match, transient shapes do NOT); * 2. UnifiedStore.open turns a throwing scan into StorageIntegrityError; - * 3. the shadow quarantines + reopens fresh instead of degrading; - * 4. a failed quarantine rename degrades loudly with the errno (no doomed reopen); + * 3. the shadow leaves the file set EXACTLY where it is (no automatic + * rename — #4626-era owner decision) and degrades loudly with an + * actionable operator instruction + cached FAIL verdict; + * 4. the FAIL verdict makes the next open fail fast WITHOUT rescan; + * 5. the documented manual resolution (back up, move aside, restart) + * actually unblocks: a fresh store rebuilds cleanly; * 5. the background (large-store) check caches its FAIL verdict across a clean close; * 6. /__bili/status reports the resolved engine when the failure happened AFTER resolution. */ import test from "node:test"; import assert from "node:assert/strict"; -import { chmodSync, closeSync, existsSync, mkdtempSync, openSync, readdirSync, rmSync, statSync, truncateSync, writeFileSync, writeSync } from "node:fs"; +import { closeSync, existsSync, mkdtempSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, truncateSync, writeFileSync, writeSync } from "node:fs"; +import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { UnifiedStore, StorageIntegrityError } from "../src/storage/store.js"; @@ -30,6 +35,7 @@ import { resetIntegrityCheckStateForTests, setIntegritySyncThresholdForTests, backgroundIntegrityInflightForTests, + integrityCheckRunsForTests, } from "../src/storage/integrity-check.js"; import { _resetStorageRuntimeStatusForTest, @@ -40,7 +46,6 @@ import { const HAS_ENGINE = probeBuiltinSqlite() || probeHostBetterSqlite3(); const SKIP_NO_ENGINE = HAS_ENGINE ? undefined : "no SQLite engine available on this host"; -const SKIP_RENAME_FAIL = SKIP_NO_ENGINE ?? (process.platform === "win32" ? "chmod has no permission effect on Windows" : undefined); function tmpDir(): string { return mkdtempSync(join(tmpdir(), "bili-issue4816-")); @@ -149,53 +154,87 @@ test("#4816 store: a throwing quick_check becomes StorageIntegrityError", { skip } }); -test("#4816 shadow: throwing corruption is quarantined and the store recovers to full mode", { skip: SKIP_NO_ENGINE }, () => { +test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and degrades loudly", { skip: SKIP_NO_ENGINE }, () => { const dir = tmpDir(); let shadow: UnifiedShadow | null = null; + // The wire reason below resolves `requested` from the REAL process env + // (runtime-status records it only via SessionStore), and the CI baseline + // preload pins BILI_STORAGE_UNIFIED=legacy for the non-storage corpus + // (#2815) — under legacy there is nothing to report degraded. Pin full to + // match the mode maybeOpen is asked for; restore afterwards. + const prevUnified = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; try { seedValidDb(join(dir, "index.db")); truncateHalf(join(dir, "index.db")); + const before = createHash("sha256").update(readFileSync(join(dir, "index.db"))).digest("hex"); + _resetStorageRuntimeStatusForTest(); shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null); - assert.ok(shadow !== null, "corrupt index must be quarantined + reopened fresh, not degrade legacy-only"); - assert.equal(shadow.readsUnified, true); - const names = readdirSync(dir); - assert.ok(names.some((n) => n.startsWith("index.db.corrupt-")), `quarantined copy kept for forensics, got ${names.join(", ")}`); - assert.ok(existsSync(join(dir, "index.db")), "fresh index.db exists after quarantine"); + assert.equal(shadow, null, "a provably-corrupt index must degrade legacy-only — nothing may auto-rename or auto-reopen"); + // Byte-identical file set: NO automatic rename/repair (#4626-era owner + // decision — checkpoint rows live only inside the index, and with + // jsonWriterOff it is the sole durable copy). + const after = createHash("sha256").update(readFileSync(join(dir, "index.db"))).digest("hex"); + assert.equal(after, before, "the corrupt primary must stay byte-identical"); + assert.ok(!readdirSync(dir).some((n) => n.startsWith("index.db.corrupt-")), "no quarantine copies may exist anymore"); + // Loud + actionable on the status wire. + const wire = storageStatusWire(); + assert.match(wire.reason ?? "", /integrity check and was left UNTOUCHED/); + assert.match(wire.reason ?? "", /aside/i); } finally { shadow?.close(); + if (prevUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = prevUnified; + _resetStorageRuntimeStatusForTest(); rmSync(dir, { recursive: true, force: true }); } }); -test("#4816 shadow: a failed quarantine rename degrades loudly with the errno, no doomed reopen", { skip: SKIP_RENAME_FAIL }, () => { +test("#4816 fail-fast: a second open after proven corruption does not rescan", { skip: SKIP_NO_ENGINE }, () => { const dir = tmpDir(); - // The wire reason below resolves `requested` from the REAL process env - // (runtime-status records it only via SessionStore), and the CI baseline - // preload pins BILI_STORAGE_UNIFIED=legacy for the non-storage corpus - // (#2815) — under legacy there is nothing to report degraded. Pin full to - // match the mode maybeOpen is asked for; restore afterwards. - const prevUnified = process.env.BILI_STORAGE_UNIFIED; - process.env.BILI_STORAGE_UNIFIED = "full"; + try { + resetIntegrityCheckStateForTests(); + const p = join(dir, "index.db"); + seedValidDb(p); + truncateHalf(p); // proven to make PRAGMA quick_check throw CORRUPT + assert.throws( + () => UnifiedStore.open(p), + (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), + ); + const scansAfterFirst = integrityCheckRunsForTests(); + const cache = readIntegrityCache(p); + assert.ok(cache !== null && cache.ok === false, `FAIL verdict cached on throw, got ${JSON.stringify(cache)}`); + // Second boot: served from the verdict cache — the typed error again, + // ZERO additional scans. + assert.throws( + () => UnifiedStore.open(p), + (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), + ); + assert.equal(integrityCheckRunsForTests(), scansAfterFirst, "cached-fail must short-circuit before any quick_check"); + } finally { + resetIntegrityCheckStateForTests(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#4816 operator resolution: moving the corrupt set aside unblocks a fresh rebuild", { skip: SKIP_NO_ENGINE }, () => { + const dir = tmpDir(); + let fresh: UnifiedStore | null = null; try { seedValidDb(join(dir, "index.db")); truncateHalf(join(dir, "index.db")); - chmodSync(dir, 0o555); - try { - _resetStorageRuntimeStatusForTest(); - const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null); - assert.equal(shadow, null, "must degrade when the primary cannot be moved aside"); - assert.ok(existsSync(join(dir, "index.db")), "corrupt primary stays in place when the rename fails"); - assert.ok(!readdirSync(dir).some((n) => n.startsWith("index.db.corrupt-")), "no partial quarantine"); - const wire = storageStatusWire(); - assert.match(wire.reason ?? "", /index database failed to open/); - assert.match(wire.reason ?? "", /quarantine rename failed/); - } finally { - chmodSync(dir, 0o755); - _resetStorageRuntimeStatusForTest(); + assert.throws(() => UnifiedStore.open(join(dir, "index.db")), (err: unknown) => err instanceof StorageIntegrityError); + // The documented manual step: back up, move the file set aside… + for (const suffix of ["", "-wal", "-shm"] as const) { + const from = join(dir, `index.db${suffix}`); + if (!existsSync(from)) continue; + renameSync(from, `${from}.moved-by-operator`); } + // …and the next start rebuilds cleanly against the empty directory. + fresh = UnifiedStore.open(join(dir, "index.db")); + assert.deepEqual(fresh.listSessionIds(), [] as string[], "rebuilt index opens cleanly against the empty directory"); } finally { - if (prevUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; - else process.env.BILI_STORAGE_UNIFIED = prevUnified; + fresh?.close(); rmSync(dir, { recursive: true, force: true }); } }); @@ -217,7 +256,7 @@ test("#4816 worker facet: a large store's throwing background check caches FAIL } finally { s1.close(); // checkpoint moves the WAL — the refresh must carry the FAIL verdict } - // Next boot: cached-fail → typed error → the shadow would quarantine. + // Next boot: cached-fail → typed error → the shadow degrades loudly (no rescan). assert.throws( () => UnifiedStore.open(p), (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), diff --git a/tests/storage-durability.test.ts b/tests/storage-durability.test.ts index eebba0518c..2cfd47c122 100644 --- a/tests/storage-durability.test.ts +++ b/tests/storage-durability.test.ts @@ -7,6 +7,7 @@ * fails on the current tree. */ import { strict as assert } from "node:assert"; +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -45,7 +46,7 @@ function legacyLike(id: string): LegacySessionLike { }; } -test("quick_check: a corrupt db fails open with a typed error, and the wrapper quarantines + reopens fresh", async () => { +test("quick_check: a corrupt db fails open with a typed error, files left UNTOUCHED (#4626 policy)", async () => { const dir = freshDir(); try { // 1. Create a valid db. @@ -129,7 +130,7 @@ test("lockfile: a live foreign pid is reported, a dead pid is taken over, close } }); -test("shadow open quarantines a corrupt index.db set and reopens fresh", async () => { +test("shadow open degrades loudly on a corrupt index.db set — nothing renamed (#4626)", async () => { const { UnifiedShadow } = await import("../src/storage/shadow.ts"); const { readdirSync } = await import("node:fs"); const dir = freshDir(); @@ -139,32 +140,25 @@ test("shadow open quarantines a corrupt index.db set and reopens fresh", async ( const store = UnifiedStore.open(path); store.close(); - // Corrupt it (page area flip). + // Corrupt it (page area flip) and remember its exact bytes. const raw = readFileSync(path); for (let i = Math.floor(raw.length / 3); i < Math.floor(raw.length / 3) + 4096; i++) { raw[i] = raw[i] ^ 0xff; } writeFileSync(path, raw); + const beforeHash = createHash("sha256").update(raw).digest("hex"); - // The shadow's maybeOpen (full mode) must quarantine + reopen fresh. + // The shadow's maybeOpen (full mode) must degrade legacy-only and + // leave EVERY file exactly where it was: no rename, no auto-reopen, + // no forensics copy (#4626-era owner decision — full recovery cannot + // be guaranteed without operator involvement). const env = { ...process.env, BILI_STORAGE_UNIFIED: "full" }; const shadow = UnifiedShadow.maybeOpen(dir, env); - assert.ok(shadow, "shadow opens after quarantine"); - + assert.equal(shadow, null, "a provably-corrupt index must degrade loudly, not auto-recover"); const names = readdirSync(dir); - const quarantined = names.filter((n) => n.includes(".corrupt-")); - assert.ok(quarantined.some((n) => n.startsWith("index.db.corrupt-")), "main db quarantined"); - // sidecars (-wal/-shm) ride the quarantine when the engine's close - // left them behind; the freshly reopened db legitimately creates its - // own new -wal, so sidecar presence alone proves nothing here. - assert.ok(names.includes("index.db"), "fresh db exists"); - - // The fresh db is writable (ingest one session through it). - const fresh = UnifiedStore.open(path); - fresh.ingestSession(planIngest(legacyLike("post-quarantine"))); - assert.deepEqual(fresh.listSessionIds(), ["post-quarantine"]); - fresh.close(); - shadow.close(); + assert.ok(!names.some((n) => n.includes(".corrupt-")), `no automatic copies allowed, got ${names.join(", ")}`); + const afterHash = createHash("sha256").update(readFileSync(path)).digest("hex"); + assert.equal(afterHash, beforeHash, "the corrupt primary must stay byte-identical"); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 6fd5843010..5fab9396b5 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -26,14 +26,15 @@ * load's migration → the file loads the exact digest. * SELF-DISABLE — 3 consecutive sidecar failures disable it; reads flip * back to legacy; the legacy file stays readable. - * CORRUPT INDEX — a corrupt index.db is quarantined and the store - * recovers to full mode instead of bricking - * construction (#2771 → #4816). + * CORRUPT INDEX — a corrupt index.db degrades LOUDLY with the file set + * left in place (no automatic rename, #4626-era owner + * decision) instead of bricking construction (#2771). * CAS REFCOUNT — shared content blobs survive deleteSession of one * session and are collected only at refcount 0. * * The corrupt-index test is the regression for #2771 (re-pinned for the - * #4816 quarantine-and-recover behavior); the rest are contract pins + * #4626 loud-degrade policy: nothing renamed, envelopes keep serving); the + * rest are contract pins * (proof-only, no product change). */ @@ -366,11 +367,11 @@ test("storage engine: sidecar self-disables after 3 consecutive failures (reads }); // --------------------------------------------------------------------------- -// CORRUPT INDEX (#2771 → #4816): construction quarantines the corrupt index -// and recovers to full mode. +// CORRUPT INDEX (#2771 → #4626): construction survives a provably-corrupt +// index and degrades loudly — files untouched, no automatic rename. // --------------------------------------------------------------------------- -test("storage engine: corrupt index.db is quarantined and recovers to full mode instead of bricking construction (#2771/#4816)", { timeout: 180_000 }, async () => { +test("storage engine: corrupt index.db degrades loudly instead of bricking construction (#2771/#4626)", { timeout: 180_000 }, async () => { const dir = tmpdir("bili-engproof-corrupt-"); const prev = process.env.BILI_STORAGE_UNIFIED; try { @@ -389,8 +390,8 @@ test("storage engine: corrupt index.db is quarantined and recovers to full mode await clean.writeNow(viaFull); clean.closeQuietly(); - // #4338: everything past this point (corrupt the header → quarantine - // + fresh reopen → lazy-import read → leak guards → write → close) + // #4338: everything past this point (corrupt the header → loud + // degrade → envelope read → leak guards → write → close) // runs in a spawned child. On Windows any sqlite file the scenario // touches can leave an OS handle with no JS reference to release, and // the resulting persistent EBUSY at rmrf would outlive the whole @@ -400,7 +401,7 @@ test("storage engine: corrupt index.db is quarantined and recovers to full mode // re-asserted here, so assertion coverage is 1:1 with the // in-process era (same asserts, executed in the child, verified in // the parent). The child also runs the #2781/#4357 mid-flow leak - // guard on the quarantined file plus a post-close guard, so future + // guard on the sessions dir plus a post-close guard, so future // handle-leak drift trips there by name instead of at teardown. // #4316 PR3/3: expected digest computed modulo the one-time ledger // alignment — the clean read above already carried it in memory, and diff --git a/tests/storage-integrity-cache.test.ts b/tests/storage-integrity-cache.test.ts index 12f5f0be47..f9ec805f1a 100644 --- a/tests/storage-integrity-cache.test.ts +++ b/tests/storage-integrity-cache.test.ts @@ -6,7 +6,8 @@ * open must NOT recheck (counter stays put). * 2. any stat change (mtime bump) invalidates the cache → recheck. * 3. cached failure → open throws StorageIntegrityError (the #4175 - * quarantine-and-reopen-fresh path) without rescanning. + * loud-degrade trigger: files stay where they are, next open fails fast) + * without rescanning. * 4. large store (threshold overridden to 1 byte): open returns without * blocking; a worker-thread check completes in the background, writes * the cache, and the counter reaches 1. @@ -81,7 +82,7 @@ test("touching the db file invalidates the cache and forces a recheck", () => { } }); -test("a cached failure fails open with the typed error (the #4175 quarantine trigger) without rescanning", () => { +test("a cached failure fails open with the typed error (the #4175 loud-degrade trigger) without rescanning", () => { resetIntegrityCheckStateForTests(); setIntegritySyncThresholdForTests(null); const dir = tmpDir(); diff --git a/tests/storage-review-fixes.test.ts b/tests/storage-review-fixes.test.ts index acff76510d..28cb488e80 100644 --- a/tests/storage-review-fixes.test.ts +++ b/tests/storage-review-fixes.test.ts @@ -136,7 +136,7 @@ test("review 2: destroyed-substrate blocks are deleted and cannot resurrect on a }); }); -test("review 3: corrupt index.db auto-quarantines and recovers instead of bricking the proxy", async () => { +test("review 3: corrupt index.db degrades loudly instead of bricking the proxy", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-review-corrupt-")); try { const session = loadCorpusCase(dir, "folded", "corpus-folded", "anthropic"); @@ -152,13 +152,14 @@ test("review 3: corrupt index.db auto-quarantines and recovers instead of bricki await store.writeNow(session); const reloaded = store.loadSync(session.id, { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(reloaded, "session load must succeed"); - // #5124/#4816 behavior change (a): this corruption shape makes - // quick_check THROW, so the store quarantines the corrupt file set - // and recovers a fresh index — it no longer degrades legacy-only. - assert.equal(store.replicasAvailable(), true, "throwing corruption must be quarantined + recovered, not stay legacy-only"); + // #5124/#4816/#4626 behavior change: this corruption shape makes + // quick_check THROW, and it is classified — but NO file is ever + // renamed or a fresh index opened automatically; the store serves + // legacy-only until an operator moves the corrupt set aside. + assert.equal(store.replicasAvailable(), false, "throwing corruption must degrade legacy-only under the #4626 policy"); const names = fs.readdirSync(dir); - assert.ok(names.some((n) => n.startsWith("index.db.corrupt-")), `quarantined copy kept for forensics, got ${names.join(", ")}`); - assert.ok(fs.existsSync(path.join(dir, "index.db")), "fresh index.db exists after quarantine"); + assert.ok(!names.some((n) => n.startsWith("index.db.corrupt-")), `no automatic .corrupt-* copy may exist, got ${names.join(", ")}`); + assert.ok(fs.existsSync(path.join(dir, "index.db")), "the corrupt index.db must stay exactly where it is"); await store.flushAll(); // flush the JSON record to disk (closeQuietly only cancels) } finally { store!.closeQuietly(); diff --git a/tests/web-storage-banner.test.ts b/tests/web-storage-banner.test.ts index 2b1d7352cb..2fe041e71b 100644 --- a/tests/web-storage-banner.test.ts +++ b/tests/web-storage-banner.test.ts @@ -1,8 +1,8 @@ // #3040 web surface: SQLite unified storage degrades to legacy-only through // several fail-open paths (#2825 contract): engine ladder exhausted, // self-download blocked in cooldown, non-corruption index-open failures, -// 3-strike sidecar disable (a corrupt index.db since #4816 quarantines and -// recovers instead of degrading). The proxy keeps running on JSON — correct — but until now +// 3-strike sidecar disable (a corrupt index.db since #4626-era policy degrades +// loudly with the files left in place — nothing is auto-renamed). The proxy keeps running on JSON — correct — but until now // the condition was only visible as one-time log lines, so environments that // cannot run SQLite were never learned about before the upcoming full-mode // default flip. These tests pin the wire contract the web banner reads: @@ -271,10 +271,10 @@ test("#3040 engine-ladder-exhausted degrade: banner wire shape with the open-fai // construct the store → serve /__bili/overview → assert the degraded // wire shape → close) runs in a spawned child so any OS handle the // store opens is released by process exit before this parent's - // rmrf(base) can hit a Windows EBUSY. (#4816 moved the trigger off - // the corrupt-index cause: that one now quarantines and recovers, so - // the banner surface is pinned through the deterministic no-engine - // path instead.) The child prints a JSON verdict of named checks; + // rmrf(base) can hit a Windows EBUSY. (#4816/#4626: the corrupt-index + // cause no longer degrades through this surface — it now fails loudly + // with the files untouched, so the banner contract is pinned through + // the deterministic no-engine path instead.) The child prints a JSON verdict of named checks; // every one is re-asserted here, so assertion coverage stays 1:1 // with the in-process era. The spawn env below is the child's only // isolation — this process's own env and seams stay untouched. From f05a4050dff1c021aca48fe5dac36b2acba9c823 Mon Sep 17 00:00:00 2001 From: ranxianglei-devbox Date: Fri, 30 Oct 2026 18:06:25 +0800 Subject: [PATCH 27/35] integrity FAIL verdict: refuse to start instead of legacy-only fallback (owner follow-up 2026-10-30) --- CONFIGURATION.md | 2 +- src/storage/integrity-check.ts | 15 ++- src/storage/runtime-status.ts | 8 +- src/storage/shadow.ts | 55 ++++++----- src/storage/sqlite-errors.ts | 8 +- src/storage/store.ts | 7 +- tests/corrupt-index-child.ts | 93 +++++++------------ ...sue4816-integrity-throw-quarantine.test.ts | 27 +++--- tests/storage-durability.test.ts | 20 ++-- tests/storage-engine-proofs.test.ts | 54 +++++------ tests/storage-review-fixes.test.ts | 35 +++---- 11 files changed, 158 insertions(+), 166 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 1c8c5d2e7b..3c6fcf6954 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -362,7 +362,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | Env tier for storage.backend (#3931): sqlite (default) or mysql (fully wired, #3388 steps 2+3 - saves ingest through the async shadow queue, sync reads ride the JSON envelope, replicas/ext/GC/fork via the async twins (#4911); export too). Wins over the file tier. | | `BILI_MYSQL_DSN` | string | unset | — | MySQL DSN (mysql://user:pass@host:3306/db) for storage.backend="mysql" (#3388, #3931). Wins over the file key storage.mysql.dsn. Empty = the mysql backend cannot open; the documented error names both knobs. The pool opens lazily; closeQuietlyAsync drains the ingest queue and closes it. | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC age sweep window in days (#3463): sessions idle longer than this are archived (one JSON line appended to the single archive log (archive/archive.jsonl) + tombstone row + content/media refs released). 0 disables the age sweep; the manual entry POST /__bili/storage/gc always runs. Config-file key storage.archiveAfterDays is the persistent form. | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; on failure the files are left EXACTLY in place and bili runs legacy-only with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand to force a fresh rebuild — JSON envelopes remain the recovery source). The FAIL verdict is cached so later opens fail fast without rescan. There is NO automatic rename/repair: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy) and a corrupt-but-salvageable index stays operator-salvageable. Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN the files are left EXACTLY in place and bili REFUSES TO START with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand, then start again — a fresh index rebuilds from the JSON envelopes). The FAIL verdict is cached so later starts fail fast without rescan. There is NO automatic rename/repair AND no legacy-only fallback: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy), and continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable); transient open failures (no engine / CANTOPEN / second writer / readonly) still degrade loudly as before — they do not prove any file corrupt. Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | Snapshot-render shadow observability (#3756 follow-up): compute the outbound snapshot view on every armed turn WITHOUT using it, classify the diff against the legacy outbound, and surface per-session counters in the web panel + the snapshot-shadow diagnostics topic. 0/false/off disables; 1/true/on enables; otherwise the config-file snapshot.shadow boolean applies (default on). The render lane itself stays rollout-gated (BILI_ROLLOUT_SNAPSHOT_RENDER), so the shadow lane never changes outbound bytes - it is the dry-run evidence that flips the rollout later. | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | Override the config file path (path relocation). | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | Where session records are stored (path relocation). | diff --git a/src/storage/integrity-check.ts b/src/storage/integrity-check.ts index 502a58d0dc..b0a3b264cf 100644 --- a/src/storage/integrity-check.ts +++ b/src/storage/integrity-check.ts @@ -21,9 +21,8 @@ * readers, and the main handle has already run recovery, so the * read-only open cannot hit WAL-recovery). Boot returns in * milliseconds; a failure is recorded loudly and cached so the NEXT - * open throws StorageIntegrityError and reuses the #4175 - * loud-degrade machinery (no automatic file moves, #4626-era owner - * decision). Small stores keep the + * open throws StorageIntegrityError and the open REFUSES (no automatic + * file moves AND no legacy fallback, #4626-era owner decision chain). Small stores keep the * synchronous pre-migration check — the #4175 premise (a corrupt file * must never be migrated against) stays intact wherever it is cheap. * The one concession: on a big store's first boot after upgrade, @@ -176,7 +175,7 @@ const inflight = new Set(); /** Fire-and-forget background quick_check on a second read-only connection * (#4609). Records the outcome, caches it (ok or fail — a cached fail makes - * the NEXT open throw and degrade loudly via the #4175 path), and tells the + * the NEXT open throw and REFUSE via the #4175 path), and tells the * caller's store instance so a clean close can refresh the cache key. */ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok: boolean, detail?: string) => void): void { const size = statPart(dbPath)?.s ?? 0; @@ -210,7 +209,7 @@ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok } else { // A real integrity failure found post-open: record loudly // and cache the fail so the NEXT open throws StorageIntegrityError - // and degrades loudly INSTEAD of rescan-looping (#4175 path, + // and REFUSES to start INSTEAD of rescan-looping (#4175 path, // no automatic rename — files stay where they are); keep // serving now (writes into a provably-corrupt store surface // as ingest errors and the #3133 breaker disables the @@ -218,7 +217,7 @@ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok // are the recovery source either way). recordIntegrityOutcome(dbPath, false, 0, verdict.slice(0, 400)); writeIntegrityCache(dbPath, false, verdict.slice(0, 400)); - loggerLog("error", `[storage] background quick_check FAILED for ${dbPath}: ${verdict.slice(0, 400)} — verdict cached; the next open fails fast and runs legacy-only (files left UNTOUCHED — move them aside by hand after backing up to force a rebuild; JSON envelopes remain the recovery source, #4175/#4609)`); + loggerLog("error", `[storage] background quick_check FAILED for ${dbPath}: ${verdict.slice(0, 400)} — verdict cached; the next open FAILS FAST AND REFUSES TO START (files left UNTOUCHED — move them aside by hand after backing up to force a rebuild; JSON envelopes remain the recovery source, #4175/#4609)`); // Detail rides with the verdict so a clean close can // refresh the FAIL cache onto post-checkpoint stats // (#5108/#4816) — without it the verdict would go stale @@ -229,14 +228,14 @@ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok // #5108/#4816: the scan itself threw a corruption-shaped error // (the throwing shapes of CORRUPT/NOTADB). Same consequences as // a bad verdict row: record loudly and cache the fail so the - // NEXT open fails fast and degrades loudly (#4175 path). + // NEXT open fails fast and refuses to start (#4175 path). // Without this branch the throw fell into "could not run" // below and the fail verdict was never recorded — a large // corrupt store stayed unverified boot after boot. const detail = String(msg.error ?? "").slice(0, 400); recordIntegrityOutcome(dbPath, false, 0, detail); writeIntegrityCache(dbPath, false, detail); - loggerLog("error", `[storage] background quick_check threw a corruption-shaped error for ${dbPath}: ${detail} — FAIL verdict cached; the next open fails fast and runs legacy-only (files left UNTOUCHED — move them aside by hand after backing up to force a rebuild; JSON envelopes remain the recovery source, #5108/#4175)`); + loggerLog("error", `[storage] background quick_check threw a corruption-shaped error for ${dbPath}: ${detail} — FAIL verdict cached; the next open FAILS FAST AND REFUSES TO START (files left UNTOUCHED — move them aside by hand after backing up to force a rebuild; JSON envelopes remain the recovery source, #5108/#4175)`); onVerdict?.(false, detail); } else { loggerLog("warn", `[storage] background quick_check could not run for ${dbPath}: ${String(msg.error)} — the store serves unverified; the next boot rechecks`); diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index c4e2762cfc..334ff29499 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -220,10 +220,12 @@ function degradeReason(): string { if (isNoEngineError(e)) { return `no SQLite engine available (node:sqlite missing, better-sqlite3 not installed)${nodeSqliteUpgradeHint()}`; } - // #4626-era owner decision: a provably-corrupt index is LEFT IN PLACE - // (no automatic rename), so the reason must carry the manual fix. + // #4626-era owner decision chain: a provably-corrupt index is LEFT IN + // PLACE (no automatic rename) AND the store refuses to start rather + // than fall back legacy-only (the two replicas would diverge forever + // and never reconcile) — the reason carries the manual fix. if (/StorageIntegrityError|integrity quick_check failed/i.test(state.sidecarOpenError)) { - return `index database failed its integrity check and was left UNTOUCHED (${e}) — back up index.db{,-wal,-shm}, then move those files aside by hand to rebuild; the proxy runs legacy-only meanwhile`; + return `index database failed its integrity check and was left UNTOUCHED (${e}) — the store REFUSES TO START; back up index.db{,-wal,-shm}, move those files aside by hand, then start again (a fresh index rebuilds from the JSON envelopes)`; } return `index database failed to open (${e})${sqliteOpenFailureHint(state.sidecarOpenError)}`; } diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 9ca397dc1e..54b1fc1de9 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -270,18 +270,22 @@ export class UnifiedShadow { // destroys unrecoverable data; // - a corrupt-on-open index can still often be salvaged by an // operator; renaming it away severs that last option forever. - // The file set therefore stays EXACTLY in place, untouched; - // we degrade loudly instead (outer catch → legacy-only + the - // status wire). The FAIL verdict has already been written to - // index.db.chk by UnifiedStore.open, so every later open in - // this deployment fails fast from the cache — no rescans, no - // repeated work in the main flow. Manual resolution (unchanged - // #2771 fallback): back up, then move index.db{-wal,-shm} - // aside yourself; the next start rebuilds the index fresh and - // re-imports the JSON envelopes that hold session content. + // The file set therefore stays EXACTLY in place, untouched, + // and the store REFUSES TO START (#4626-era owner decision + // chain, 2026-10-30): renaming risks destroying the sole copy + // WITHOUT proving recovery, and falling back to legacy-only + // was rejected too — once the two replicas stop advancing + // together the split can never be reconciled, so continuing + // would bake the divergence in. The loud failure below names + // the manual fix. The FAIL verdict has already been written + // to index.db.chk by UnifiedStore.open, so every later open + // fails fast from the cache — no rescans. Manual resolution: + // back up, then move index.db{-wal,-shm} aside yourself; + // the next start rebuilds the index fresh and re-imports the + // JSON envelopes that hold session content. loggerLog( "error", - `[storage] index.db failed its integrity quick_check (${err.detail}) — leaving the file UNTOUCHED and running legacy-only (no automatic rename); back up ${path.join(dir, "index.db")}{,-wal,-shm}, then move those files aside to force a fresh rebuild (JSON envelopes remain the recovery source for session content) (#4175/#4626)`, + `[storage] index.db FAILED its integrity quick_check (${err.detail}) — REFUSING TO START with a corrupt index (no automatic rename, no legacy fallback): back up ${path.join(dir, "index.db")}{,-wal,-shm}, move those files aside, then start again — the JSON envelopes are the recovery source and a fresh index is rebuilt automatically (#4175/#4626)`, ); throw err; } @@ -300,28 +304,29 @@ export class UnifiedShadow { recordSidecarOpenFailure(`second-writer: another bili process (pid ${err.pid}) owns index.db — isolate via BILI_SESSIONS_DIR(+BILI_CONFIG_FILE); unified features off until then (#5046)`); return null; } - // #2771: a corrupt/unreadable index.db must degrade to legacy-only - // (same shape as the encryption opt-out), never brick SessionStore - // construction. No auto-delete: the index may hold pinned checkpoint - // blobs (#2671 Phase 2) — destructive recovery is operator policy. const errStr = String(err); // Deleting the index cannot fix a no-engine failure (this Node.js has neither builtin // node:sqlite nor a loadable better-sqlite3), and it cannot clear an OS-level refusal // either (#3823 lock; #4614 readonly — where it would DESTROY the only copy of recent - // session state in unified-full mode). The delete advice fits only a genuine corrupt - // file; the Node.js upgrade lever is surfaced via /__bili/status + the bootstrap warn. + // session state in unified-full mode). The Node.js upgrade lever is surfaced via + // /__bili/status + the bootstrap warn. + // #4626-era owner decision chain (2026-10-30): quick_check PROVED + // the file set corrupt — refusing to start is the ONLY safe + // outcome: renaming risks destroying the sole durable copy without + // proving recovery, and a legacy-only fallback diverges from the + // dead index forever (never reconcilable). Transient shapes + // (CANTOPEN/no-engine/second-writer/readonly) NEVER reach here — + // they classify out before any StorageIntegrityError exists, and + // keep degrading as before. Integrity errors never reach the + // "delete to reset" advice below either. + if (err instanceof StorageIntegrityError) { + recordSidecarOpenFailure(errStr); + throw err; + } const hint = sqliteOpenFailureHint(errStr); - // #4626-era owner decision: a provably-corrupt index is LEFT IN PLACE — - // "delete to reset" advice would nudge toward destroying data that - // may not be fully envelope-covered (checkpoints; jsonWriterOff). - const corruptIntegrity = err instanceof StorageIntegrityError; const tail = /no SQLite engine available/i.test(errStr) ? "" - : corruptIntegrity - ? "" - : hint !== "" - ? hint - : "; delete the file to reset the index"; + : hint !== "" ? hint : "; delete the file to reset the index"; loggerLog( "warn", `[storage] unified sidecar failed to open ${dir}/index.db (${errStr}) — staying legacy-only${tail}`, diff --git a/src/storage/sqlite-errors.ts b/src/storage/sqlite-errors.ts index dbae8bee50..f9f7a03af1 100644 --- a/src/storage/sqlite-errors.ts +++ b/src/storage/sqlite-errors.ts @@ -5,10 +5,10 @@ * rows (page-level damage), or it can THROW out of the statement itself * (truncated file, garbage header, broken overflow chain — probe-verified on * node:sqlite v22: ERR_SQLITE_ERROR with the raw sqlite3_errmsg text). The - * #4175 loud-degrade decision (fail fast + legacy-only, no automatic file - * moves) only saw the verdict-row shape, so every - * throwing corruption escaped as a raw driver error and degraded the store - * to legacy-only forever with no auto-heal. + * #4175 decision (fail fast + REFUSE TO START — no automatic file moves and + * no legacy fallback, per the 2026-10-30 owner follow-up) only saw the + * verdict-row shape, so every throwing corruption escaped as a raw driver + * error and degraded the store to legacy-only forever with no auto-heal. * * This classifier maps the throwing shapes onto the same decision. It * matches ONLY SQLITE_CORRUPT / SQLITE_NOTADB: diff --git a/src/storage/store.ts b/src/storage/store.ts index 60ac34023c..fc006d5a41 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -449,9 +449,10 @@ export class UnifiedStore { // preserved wherever it is cheap); a large store checks in the // background post-open (#4609: a 14 GB quick_check costs ~260 s and // the 60 s native spawn budget killed the child mid-scan). A cached - // failure throws here so the shadow degrades loudly INSTEAD of - // rescan-looping (#4626-era owner decision: no automatic quarantine — - // see maybeOpen) — including for failures found by the background + // failure throws here so the open REFUSES to start INSTEAD of + // rescan-looping (#4626-era owner decision chain: no automatic + // quarantine, no legacy fallback — see maybeOpen) — including for + // failures found by the background // check of a previous run. BILI_STORAGE_QUICK_CHECK=0 opts out of all // of it (no check, no cache IO). let integrityOk = false; diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index c7bf69e865..71eeca03cf 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -43,6 +43,7 @@ import { defaultConfig } from "acp-kernel"; import type { ProxyOptions } from "../src/config.ts"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { StorageIntegrityError } from "../src/storage/store.ts"; import { setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts"; import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; @@ -194,21 +195,18 @@ export function makeOpts(upstreamPort: number): ProxyOptions { // sides reference these constants, so the coverage list cannot drift. // --------------------------------------------------------------------------- -// #4816/#4626: throwing-shape corruption (header garbage / truncation) is -// classified and degrades LOUDLY — but #4626-era owner decision withdrew the -// automatic quarantine rename (no full-recovery guarantee), so the corrupt -// files stay byte-identical in place and the store serves legacy-only until -// an operator moves them aside. These checks pin that policy plus the -// original #2781/#4357 leak guards. +// #4816/#4626 policy chain (owner decisions through 2026-10-30): throwing +// corruption is classified → NO automatic quarantine rename (a move can't +// prove full recovery) AND no legacy fallback (the JSON envelope would fork +// from the dead index forever). Construction therefore REFUSES — typed +// StorageIntegrityError, files byte-identical in place, no leaked handle +// into the dir (so an operator can move the set aside immediately). +// These checks pin that policy plus the original #2781/#4357 leak guards. const ENGPROOF_CHECK_NAMES = { noAutoRename: "corrupt index.db left UNTOUCHED — byte-identical, no .corrupt-* copy (#4626 decision)", - degraded: "store serves legacy-only; nothing auto-recovers full mode", - lazyImport: "session loaded from the legacy envelope while degraded", - midFlowGuard: "no live handle into the sessions dir while serving degraded (#2781/#4357 guard)", - stateDigest: "legacy load's state digest matches the clean full-mode read", - writeNow: "writeNow keeps working over the legacy path", - postCloseGuard: "no live handle after the final close (post-close rename guard)", - fdGuard: "no open fd into the corrupt index dir after close (/proc/self/fd, Linux)", + refusesToStart: "construction THROWS StorageIntegrityError — no silent legacy fallback", + midFlowGuard: "no live handle into the sessions dir after the refused construction (#2781/#4357 guard)", + fdGuard: "no open fd into the corrupt index dir after the refusal (/proc/self/fd, Linux)", } as const; export const ENGPROOF_CHECKS: readonly string[] = Object.values(ENGPROOF_CHECK_NAMES); @@ -337,65 +335,46 @@ async function runFdGuard(run: CheckRun, name: string, dir: string): Promise { - const [dir, expectedStateDigest] = argv; - if (!dir || !expectedStateDigest) throw new Error("engproof mode needs "); + const [dir] = argv; + if (!dir) throw new Error("engproof mode needs "); // Corrupt the index header (crashed checkpoint / stray write). const fd = fs.openSync(path.join(dir, "index.db"), "r+"); fs.writeSync(fd, Buffer.alloc(512, 0x5a), 0, 512, 0); fs.closeSync(fd); - // THE #2771 regression: construction must survive. #4626-era owner - // decision: NO automatic quarantine/recovery — the corrupt files stay - // where they are and the store serves from the JSON envelopes (full - // recovery can't be guaranteed without operator involvement); the - // envelope read below must still be exact. + // THE policy chain pinned here: construction must REFUSE a provably- + // corrupt index with the typed error — files untouched, no rename, and + // no legacy fallback (a legacy-only store would fork from the dead + // index forever). Supersedes #2771's "construction must survive" for + // PROVABLY CORRUPT indices; transient open failures still survive. const beforeHash = createHash("sha256").update(fs.readFileSync(path.join(dir, "index.db"))).digest("hex"); - const store2 = new SessionStore({ dir, debounceMs: 1, enabled: true }); + let caught: unknown = null; + try { + new SessionStore({ dir, debounceMs: 1, enabled: true }); + } catch (err) { + caught = err; + } await run.run(ENGPROOF_CHECK_NAMES.noAutoRename, () => { const names = fs.readdirSync(dir); assert.ok(!names.some((n) => n.startsWith("index.db.corrupt-")), `an automatic .corrupt-* copy must never exist, got ${names.join(", ")}`); const afterHash = createHash("sha256").update(fs.readFileSync(path.join(dir, "index.db"))).digest("hex"); assert.equal(afterHash, beforeHash, "the corrupt primary must stay byte-identical"); }); - await run.run(ENGPROOF_CHECK_NAMES.degraded, () => - assert.equal(store2.replicasAvailable(), false, "must serve legacy-only — nothing may auto-recover full mode")); - const reloaded = store2.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - store2.cancelAll(); - try { - await run.run(ENGPROOF_CHECK_NAMES.lazyImport, () => - assert.ok(reloaded, "degraded store returned null for a session whose legacy envelope exists")); - if (!reloaded) return; - - // Leak guard (#2781/#4357): a degraded store must hold NO persistent - // handle into the dir at all — whole-dir rename round-trip. - await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-midflow"); - - // #4316 PR3/3: the clean full-mode read carried the one-time ledger - // alignment; the recovered load applies it again. The parent passes - // its digest already modulo that alignment (single definition), so - // strip it here too before comparing. - await run.run(ENGPROOF_CHECK_NAMES.stateDigest, () => - assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(reloaded))), expectedStateDigest, - "recovered-index state digest drift vs clean full-mode read")); - - // Writes must keep working (full path into the fresh index) — no - // throw, files refreshed. - await run.run(ENGPROOF_CHECK_NAMES.writeNow, () => store2.writeNow(reloaded)); - } finally { - store2.closeQuietly(); - } + await run.run(ENGPROOF_CHECK_NAMES.refusesToStart, () => + assert.ok( + caught instanceof StorageIntegrityError, + `a provably-corrupt index must make construction throw StorageIntegrityError, got ${caught === null ? "NO THROW (the store constructed!)" : String(caught)}`, + )); - // #4338 direction 4: the #2781 guard above only covers the window between - // loadSync and writeNow — the leak behind #4338 was (re)acquired during - // the write phase, past that guard, and only surfaced at rmrf after the - // whole retry ladder burned. Guard AFTER the final close too, so future - // handle-leak drift trips here — named, deterministic, at the right - // place — instead of as a 10-second EBUSY in teardown. - await runGuard(run, ENGPROOF_CHECK_NAMES.postCloseGuard, dir, "-postclose"); + // Leak guard (#2781/#4357): a REJECTED store must hold NO persistent + // handle into the dir — whole-dir rename round-trip straight after the + // refused construction (what lets an operator move the corrupt set + // aside without hitting EBUSY). + await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-midflow"); - // #4357: POSIX rename cannot see open files — scan our own fd table so the - // quick_check throw-path leak trips by name on Linux as well. + // #4357: POSIX rename cannot see open files — scan our own fd table so + // the quick_check throw-path leak trips by name on Linux as well. await runFdGuard(run, ENGPROOF_CHECK_NAMES.fdGuard, dir); } diff --git a/tests/issue4816-integrity-throw-quarantine.test.ts b/tests/issue4816-integrity-throw-quarantine.test.ts index f38d6eb2b2..eb2532f00a 100644 --- a/tests/issue4816-integrity-throw-quarantine.test.ts +++ b/tests/issue4816-integrity-throw-quarantine.test.ts @@ -12,8 +12,9 @@ * 1. the classifier itself (corruption shapes match, transient shapes do NOT); * 2. UnifiedStore.open turns a throwing scan into StorageIntegrityError; * 3. the shadow leaves the file set EXACTLY where it is (no automatic - * rename — #4626-era owner decision) and degrades loudly with an - * actionable operator instruction + cached FAIL verdict; + * rename — #4626-era owner decision) and REFUSES TO OPEN — no legacy + * fallback either (the two replicas would fork forever), throwing the + * typed error with the actionable operator instruction + cached FAIL; * 4. the FAIL verdict makes the next open fail fast WITHOUT rescan; * 5. the documented manual resolution (back up, move aside, restart) * actually unblocks: a fresh store rebuilds cleanly; @@ -154,14 +155,13 @@ test("#4816 store: a throwing quick_check becomes StorageIntegrityError", { skip } }); -test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and degrades loudly", { skip: SKIP_NO_ENGINE }, () => { +test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and REFUSES TO OPEN", { skip: SKIP_NO_ENGINE }, () => { const dir = tmpDir(); - let shadow: UnifiedShadow | null = null; // The wire reason below resolves `requested` from the REAL process env // (runtime-status records it only via SessionStore), and the CI baseline // preload pins BILI_STORAGE_UNIFIED=legacy for the non-storage corpus - // (#2815) — under legacy there is nothing to report degraded. Pin full to - // match the mode maybeOpen is asked for; restore afterwards. + // (#2815) — pin full to match the mode maybeOpen is asked for; restore + // afterwards. const prevUnified = process.env.BILI_STORAGE_UNIFIED; process.env.BILI_STORAGE_UNIFIED = "full"; try { @@ -169,8 +169,14 @@ test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and degrades truncateHalf(join(dir, "index.db")); const before = createHash("sha256").update(readFileSync(join(dir, "index.db"))).digest("hex"); _resetStorageRuntimeStatusForTest(); - shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null); - assert.equal(shadow, null, "a provably-corrupt index must degrade legacy-only — nothing may auto-rename or auto-reopen"); + // #4626-era owner decision chain: NO rename AND NO legacy fallback — + // a provably-corrupt index must make the open throw the typed error + // (proxy boot then hard-fails loudly). + assert.throws( + () => UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null), + (err: unknown) => err instanceof StorageIntegrityError, + "a provably-corrupt index must refuse to open — nothing may auto-rename or fall back legacy-only", + ); // Byte-identical file set: NO automatic rename/repair (#4626-era owner // decision — checkpoint rows live only inside the index, and with // jsonWriterOff it is the sole durable copy). @@ -180,9 +186,8 @@ test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and degrades // Loud + actionable on the status wire. const wire = storageStatusWire(); assert.match(wire.reason ?? "", /integrity check and was left UNTOUCHED/); - assert.match(wire.reason ?? "", /aside/i); + assert.match(wire.reason ?? "", /REFUSES TO START/i); } finally { - shadow?.close(); if (prevUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prevUnified; _resetStorageRuntimeStatusForTest(); @@ -256,7 +261,7 @@ test("#4816 worker facet: a large store's throwing background check caches FAIL } finally { s1.close(); // checkpoint moves the WAL — the refresh must carry the FAIL verdict } - // Next boot: cached-fail → typed error → the shadow degrades loudly (no rescan). + // Next boot: cached-fail → typed error → the open refuses (no rescan). assert.throws( () => UnifiedStore.open(p), (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), diff --git a/tests/storage-durability.test.ts b/tests/storage-durability.test.ts index 2cfd47c122..73fedd637d 100644 --- a/tests/storage-durability.test.ts +++ b/tests/storage-durability.test.ts @@ -130,7 +130,7 @@ test("lockfile: a live foreign pid is reported, a dead pid is taken over, close } }); -test("shadow open degrades loudly on a corrupt index.db set — nothing renamed (#4626)", async () => { +test("shadow open REFUSES on a corrupt index.db set — nothing renamed, no fallback (#4626)", async () => { const { UnifiedShadow } = await import("../src/storage/shadow.ts"); const { readdirSync } = await import("node:fs"); const dir = freshDir(); @@ -148,13 +148,19 @@ test("shadow open degrades loudly on a corrupt index.db set — nothing renamed writeFileSync(path, raw); const beforeHash = createHash("sha256").update(raw).digest("hex"); - // The shadow's maybeOpen (full mode) must degrade legacy-only and - // leave EVERY file exactly where it was: no rename, no auto-reopen, - // no forensics copy (#4626-era owner decision — full recovery cannot - // be guaranteed without operator involvement). + // The shadow's maybeOpen (full mode) must refuse — throw the typed + // error AND leave EVERY file exactly where it was: no rename, no + // auto-reopen, no forensics copy, and NO legacy fallback (#4626-era + // owner decision chain — automatic moves can't prove full recovery, + // and a fallback forks the two replicas forever). const env = { ...process.env, BILI_STORAGE_UNIFIED: "full" }; - const shadow = UnifiedShadow.maybeOpen(dir, env); - assert.equal(shadow, null, "a provably-corrupt index must degrade loudly, not auto-recover"); + let caught: unknown = null; + try { + UnifiedShadow.maybeOpen(dir, env); + } catch (err) { + caught = err; + } + assert.ok(caught, "a provably-corrupt index must make the open THROW — not degrade"); const names = readdirSync(dir); assert.ok(!names.some((n) => n.includes(".corrupt-")), `no automatic copies allowed, got ${names.join(", ")}`); const afterHash = createHash("sha256").update(readFileSync(path)).digest("hex"); diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 5fab9396b5..55f36b8a5a 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -26,15 +26,18 @@ * load's migration → the file loads the exact digest. * SELF-DISABLE — 3 consecutive sidecar failures disable it; reads flip * back to legacy; the legacy file stays readable. - * CORRUPT INDEX — a corrupt index.db degrades LOUDLY with the file set - * left in place (no automatic rename, #4626-era owner - * decision) instead of bricking construction (#2771). + * CORRUPT INDEX — a provably-corrupt index.db makes construction + * REFUSE loudly: typed StorageIntegrityError, file set + * left in place byte-identical, no automatic rename and + * no legacy fallback (#4626-era owner decision chain, + * supersedes #2771's survive-degraded for corruption). * CAS REFCOUNT — shared content blobs survive deleteSession of one * session and are collected only at refcount 0. * - * The corrupt-index test is the regression for #2771 (re-pinned for the - * #4626 loud-degrade policy: nothing renamed, envelopes keep serving); the - * rest are contract pins + * The corrupt-index test was the regression for #2771 (construction + * survival) — re-pinned for the #4626→refuse-to-start policy chain: nothing + * renamed, no fallback, typed throw, files byte-identical; the rest are + * contract pins * (proof-only, no product change). */ @@ -367,11 +370,11 @@ test("storage engine: sidecar self-disables after 3 consecutive failures (reads }); // --------------------------------------------------------------------------- -// CORRUPT INDEX (#2771 → #4626): construction survives a provably-corrupt -// index and degrades loudly — files untouched, no automatic rename. +// CORRUPT INDEX (#2771 → #4626 → refuse-to-start): construction must REFUSE a +// provably-corrupt index loudly — files untouched, no rename, no fallback. // --------------------------------------------------------------------------- -test("storage engine: corrupt index.db degrades loudly instead of bricking construction (#2771/#4626)", { timeout: 180_000 }, async () => { +test("storage engine: corrupt index.db makes construction REFUSE loudly — files untouched, no fallback (#4626/#4816)", { timeout: 180_000 }, async () => { const dir = tmpdir("bili-engproof-corrupt-"); const prev = process.env.BILI_STORAGE_UNIFIED; try { @@ -381,38 +384,29 @@ test("storage engine: corrupt index.db degrades loudly instead of bricking const await store.writeNow(session); store.closeQuietly(); - // The clean read through full mode — the digest the corrupt-index - // fallback must reproduce exactly from the legacy file. + // Both replicas must contain the migration before corrupting the index. const clean = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = clean.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); assert.ok(viaFull, "full-mode read returned null"); - // Both replicas must contain the migration before corrupting the index. await clean.writeNow(viaFull); clean.closeQuietly(); - // #4338: everything past this point (corrupt the header → loud - // degrade → envelope read → leak guards → write → close) - // runs in a spawned child. On Windows any sqlite file the scenario - // touches can leave an OS handle with no JS reference to release, and - // the resulting persistent EBUSY at rmrf would outlive the whole - // tmp-rm retry ladder; the child exiting RELEASES all its handles by - // OS guarantee, so this parent's rmrf below cannot be blocked. The - // child prints a JSON verdict of named checks; every one is - // re-asserted here, so assertion coverage is 1:1 with the - // in-process era (same asserts, executed in the child, verified in - // the parent). The child also runs the #2781/#4357 mid-flow leak - // guard on the sessions dir plus a post-close guard, so future - // handle-leak drift trips there by name instead of at teardown. - // #4316 PR3/3: expected digest computed modulo the one-time ledger - // alignment — the clean read above already carried it in memory, and - // the child's fresh load applies it again; both sides must normalize. - const stateDigestClean = viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))); + // #4338: everything past this point (corrupt the header → refused + // construction → leak guards) runs in a spawned child. On Windows any + // sqlite file the scenario touches can leave an OS handle with no JS + // reference to release, and the resulting persistent EBUSY at rmrf + // would outlive the whole tmp-rm retry ladder; the child exiting + // RELEASES all its handles by OS guarantee, so this parent's rmrf + // below cannot be blocked. The child prints a JSON verdict of named + // checks; every one is re-asserted here, so assertion coverage is 1:1 + // with in-process execution (same asserts, run in the child, verified + // in the parent). const childEnv: NodeJS.ProcessEnv = { ...process.env, BILI_STORAGE_UNIFIED: "full", BILI_PERSIST_ZSTD: "0", }; - const verdict = spawnCorruptIndexChild("engproof", [dir, stateDigestClean], childEnv); + const verdict = spawnCorruptIndexChild("engproof", [dir], childEnv); assert.equal(verdict.ok, true, verdict.failure ?? "corrupt-index child reported a failed check"); for (const name of ENGPROOF_CHECKS) { const check = verdict.checks.find((c) => c.name === name); diff --git a/tests/storage-review-fixes.test.ts b/tests/storage-review-fixes.test.ts index 28cb488e80..0c5751b579 100644 --- a/tests/storage-review-fixes.test.ts +++ b/tests/storage-review-fixes.test.ts @@ -136,7 +136,7 @@ test("review 2: destroyed-substrate blocks are deleted and cannot resurrect on a }); }); -test("review 3: corrupt index.db degrades loudly instead of bricking the proxy", async () => { +test("review 3: corrupt index.db REFUSES TO START instead of bricking the proxy", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-review-corrupt-")); try { const session = loadCorpusCase(dir, "folded", "corpus-folded", "anthropic"); @@ -146,32 +146,33 @@ test("review 3: corrupt index.db degrades loudly instead of bricking the proxy", ])); process.env.BILI_STORAGE_UNIFIED = "shadow"; - let store: SessionStore; try { - store = new SessionStore({ dir, debounceMs: 1, enabled: true }); // must NOT throw - await store.writeNow(session); - const reloaded = store.loadSync(session.id, { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - assert.ok(reloaded, "session load must succeed"); - // #5124/#4816/#4626 behavior change: this corruption shape makes - // quick_check THROW, and it is classified — but NO file is ever - // renamed or a fresh index opened automatically; the store serves - // legacy-only until an operator moves the corrupt set aside. - assert.equal(store.replicasAvailable(), false, "throwing corruption must degrade legacy-only under the #4626 policy"); + // #5124/#4816/#4626 behavior chain: this corruption shape makes + // quick_check THROW, it IS classified → NO rename AND NO legacy + // fallback (a fallback forks the two replicas forever) — the + // store's open must throw the typed error. + assert.throws( + () => new SessionStore({ dir, debounceMs: 1, enabled: true }), + /integrity/i, + "a provably-corrupt index must make construction REFUSE, not degrade", + ); + // Files byte-in-place: nothing renamed, nothing rebuilt. const names = fs.readdirSync(dir); assert.ok(!names.some((n) => n.startsWith("index.db.corrupt-")), `no automatic .corrupt-* copy may exist, got ${names.join(", ")}`); assert.ok(fs.existsSync(path.join(dir, "index.db")), "the corrupt index.db must stay exactly where it is"); - await store.flushAll(); // flush the JSON record to disk (closeQuietly only cancels) } finally { - store!.closeQuietly(); delete process.env.BILI_STORAGE_UNIFIED; } - // And a later process (fresh handle) still works the same way. + // And a later process fails the same way — fast, from the cached FAIL + // verdict (no rescan), never "recovers" on its own. process.env.BILI_STORAGE_UNIFIED = "shadow"; try { - const again = new SessionStore({ dir, debounceMs: 1, enabled: true }); - assert.ok(again.loadSync(session.id, { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }), "second process must load the session"); - again.closeQuietly(); + assert.throws( + () => new SessionStore({ dir, debounceMs: 1, enabled: true }), + /integrity/i, + "second process must also refuse (cached-fail fast path)", + ); } finally { delete process.env.BILI_STORAGE_UNIFIED; } From 5c7fc51be77090e22cdb194dbafabd7658222b07 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 30 Oct 2026 18:26:55 +0800 Subject: [PATCH 28/35] docs(config): complete the BILI_STORAGE_QUICK_CHECK registration for the withdrawn auto-quarantine This PR's integrity-failure commit changed the quick_check failure behavior (files left EXACTLY in place, loud legacy-only degrade, NO automatic rename/repair - #4626-era owner decision) and updated the EN row in CONFIGURATION.md by hand, but the generator seed (env-only.yaml desc_en/desc_zh) and the zh doc still described the withdrawn quarantine-and-rename behavior. Regenerating from the seed therefore reverted the EN row -> docs-sync red. Sync desc_en (byte-identical to the committed EN wording) and desc_zh with the new semantics, regenerate the artifacts. All four guard lanes (docs-sync/docs-coverage/config-surface-guard/lifecycle-ledger-guard) verified green locally on this head. --- CONFIGURATION.zh-CN.md | 2 +- website/config-reference/env-only.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 31c54c69d6..097889754f 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -362,7 +362,7 @@ | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | storage.backend 的环境变量层(#3931):sqlite(默认)或 mysql(已接线,#3388 第二+三步 —— 保存经异步影子队列入 mysql,同步读走 JSON 信封,replicas/ext/GC/fork 走异步孪生(#4911);export 同样支持)。优先于文件层。 | | `BILI_MYSQL_DSN` | string | unset | — | storage.backend="mysql" 的 MySQL DSN(mysql://user:pass@host:3306/db)(#3388,#3931)。优先于文件键 storage.mysql.dsn。留空 = mysql 后端无法打开;文档化错误会列出两个旋钮。连接池惰性打开;closeQuietlyAsync 排空入队事务并关闭。 | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC 年龄清扫窗口(天,#3463):闲置超过该时长的会话被归档(单文件归档日志 archive/archive.jsonl 追加一行 + 墓碑行保留标签 + 释放内容/媒体引用)。0 关闭年龄清扫;手动入口 POST /__bili/storage/gc 始终可用。配置文件键 storage.archiveAfterDays 为持久形式。 | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;失败时影子层把文件组(index.db/-wal/-shm 改名为 .corrupt-)隔离并重建新库(JSON 信封仍是恢复源)。仅当超大库的启动时间敏感时才设 0 跳过。 | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;失败时文件原样保留在原地,bili 以 legacy-only 方式降级运行并给出响亮、可操作的错误提示(要强制重建新库:手工备份后把 index.db/-wal/-shm 移开——JSON 信封仍是恢复源)。FAIL 判定会被缓存,之后的打开快速失败、无需重扫。没有自动改名/修复:自动搬移存储主索引的做法已撤回(#4626 时代的 owner 决策),因为无法保证完整数据恢复(checkpoint 行只存在于索引内部;BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本),且损坏但尚可抢救的索引应保持可由运维手工抢救。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。 | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | 快照渲染影子对账(#3756 后续):每轮已折叠会话都用引擎现算一次出站快照视图但不使用,与 legacy 实际出站比对分类,逐会话计数进 web 面板与 snapshot-shadow 诊断主题。0/false/off 关闭;1/true/on 开启;否则读配置文件 snapshot.shadow 布尔(默认开)。渲染车道本身仍受灰度开关控制(BILI_ROLLOUT_SNAPSHOT_RENDER),影子车道绝不改变出站字节 —— 它是后续翻转灰度的干跑证据。 | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | 覆盖配置文件路径(路径重定位)。 | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | 会话记录的存放目录(路径重定位)。 | diff --git a/website/config-reference/env-only.yaml b/website/config-reference/env-only.yaml index 984a4f2ee0..03f1705492 100644 --- a/website/config-reference/env-only.yaml +++ b/website/config-reference/env-only.yaml @@ -56,8 +56,8 @@ entries: kind: env-only audience: user group: envonly - desc_en: "Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; on failure the shadow layer quarantines the file set (index.db/-wal/-shm renamed to .corrupt-) and reopens fresh (JSON envelopes remain the recovery source). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters." - desc_zh: "存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;失败时影子层把文件组(index.db/-wal/-shm 改名为 .corrupt-)隔离并重建新库(JSON 信封仍是恢复源)。仅当超大库的启动时间敏感时才设 0 跳过。" + desc_en: "Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; on failure the files are left EXACTLY in place and bili runs legacy-only with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand to force a fresh rebuild — JSON envelopes remain the recovery source). The FAIL verdict is cached so later opens fail fast without rescan. There is NO automatic rename/repair: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy) and a corrupt-but-salvageable index stays operator-salvageable. Set 0 only for boot-time-sensitive huge deployments where the check's cost matters." + desc_zh: "存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;失败时文件原样保留在原地,bili 以 legacy-only 方式降级运行并给出响亮、可操作的错误提示(要强制重建新库:手工备份后把 index.db/-wal/-shm 移开——JSON 信封仍是恢复源)。FAIL 判定会被缓存,之后的打开快速失败、无需重扫。没有自动改名/修复:自动搬移存储主索引的做法已撤回(#4626 时代的 owner 决策),因为无法保证完整数据恢复(checkpoint 行只存在于索引内部;BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本),且损坏但尚可抢救的索引应保持可由运维手工抢救。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。" - key: BILI_SNAPSHOT_SHADOW type: boolean default: "true (on)" From a05470595e649924e30c87e410e1c327c58a8fec Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 30 Oct 2026 18:33:37 +0800 Subject: [PATCH 29/35] docs(config): register the integrity FAIL refuse-to-start semantics in the QUICK_CHECK seed + zh doc The owner follow-up commit changed the proven-corruption outcome from legacy-only fallback to refuse-to-start and updated the EN row in CONFIGURATION.md, but the generator seed (env-only.yaml) still carried the previous legacy-only wording, so regenerating would revert the EN row (docs-sync red). Sync desc_en byte-identical to the committed wording and translate desc_zh to the final semantics; artifacts regenerated and verified as a fixed point of 'gen-config-docs generate'. All four guard lanes (docs-sync/docs-coverage/config-surface-guard/lifecycle-ledger-guard) green locally on this head. --- CONFIGURATION.zh-CN.md | 2 +- website/config-reference/env-only.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 097889754f..50a6fbcbe4 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -362,7 +362,7 @@ | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | storage.backend 的环境变量层(#3931):sqlite(默认)或 mysql(已接线,#3388 第二+三步 —— 保存经异步影子队列入 mysql,同步读走 JSON 信封,replicas/ext/GC/fork 走异步孪生(#4911);export 同样支持)。优先于文件层。 | | `BILI_MYSQL_DSN` | string | unset | — | storage.backend="mysql" 的 MySQL DSN(mysql://user:pass@host:3306/db)(#3388,#3931)。优先于文件键 storage.mysql.dsn。留空 = mysql 后端无法打开;文档化错误会列出两个旋钮。连接池惰性打开;closeQuietlyAsync 排空入队事务并关闭。 | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC 年龄清扫窗口(天,#3463):闲置超过该时长的会话被归档(单文件归档日志 archive/archive.jsonl 追加一行 + 墓碑行保留标签 + 释放内容/媒体引用)。0 关闭年龄清扫;手动入口 POST /__bili/storage/gc 始终可用。配置文件键 storage.archiveAfterDays 为持久形式。 | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;失败时文件原样保留在原地,bili 以 legacy-only 方式降级运行并给出响亮、可操作的错误提示(要强制重建新库:手工备份后把 index.db/-wal/-shm 移开——JSON 信封仍是恢复源)。FAIL 判定会被缓存,之后的打开快速失败、无需重扫。没有自动改名/修复:自动搬移存储主索引的做法已撤回(#4626 时代的 owner 决策),因为无法保证完整数据恢复(checkpoint 行只存在于索引内部;BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本),且损坏但尚可抢救的索引应保持可由运维手工抢救。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。 | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;一旦证实损坏,文件原样保留在原地,bili 拒绝启动并给出响亮、可操作的错误提示(手工备份后把 index.db/-wal/-shm 移开再启动——全新索引会从 JSON 信封重建)。FAIL 判定会被缓存,之后的启动快速失败、无需重扫。既没有自动改名/修复,也没有 legacy-only 兜底:自动搬移存储主索引的做法已撤回(#4626 时代的 owner 决策),因为无法保证完整数据恢复(checkpoint 行只存在于索引内部;BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本),且继续以 legacy-only 运行会让 JSON 信封与死掉的索引永久分叉、再无对账可能;瞬态打开失败(无引擎 / CANTOPEN / 第二写者 / 只读)仍像以前一样响亮降级——它们不能证明任何文件损坏。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。 | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | 快照渲染影子对账(#3756 后续):每轮已折叠会话都用引擎现算一次出站快照视图但不使用,与 legacy 实际出站比对分类,逐会话计数进 web 面板与 snapshot-shadow 诊断主题。0/false/off 关闭;1/true/on 开启;否则读配置文件 snapshot.shadow 布尔(默认开)。渲染车道本身仍受灰度开关控制(BILI_ROLLOUT_SNAPSHOT_RENDER),影子车道绝不改变出站字节 —— 它是后续翻转灰度的干跑证据。 | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | 覆盖配置文件路径(路径重定位)。 | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | 会话记录的存放目录(路径重定位)。 | diff --git a/website/config-reference/env-only.yaml b/website/config-reference/env-only.yaml index 03f1705492..3ff11142cc 100644 --- a/website/config-reference/env-only.yaml +++ b/website/config-reference/env-only.yaml @@ -56,8 +56,8 @@ entries: kind: env-only audience: user group: envonly - desc_en: "Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; on failure the files are left EXACTLY in place and bili runs legacy-only with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand to force a fresh rebuild — JSON envelopes remain the recovery source). The FAIL verdict is cached so later opens fail fast without rescan. There is NO automatic rename/repair: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy) and a corrupt-but-salvageable index stays operator-salvageable. Set 0 only for boot-time-sensitive huge deployments where the check's cost matters." - desc_zh: "存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;失败时文件原样保留在原地,bili 以 legacy-only 方式降级运行并给出响亮、可操作的错误提示(要强制重建新库:手工备份后把 index.db/-wal/-shm 移开——JSON 信封仍是恢复源)。FAIL 判定会被缓存,之后的打开快速失败、无需重扫。没有自动改名/修复:自动搬移存储主索引的做法已撤回(#4626 时代的 owner 决策),因为无法保证完整数据恢复(checkpoint 行只存在于索引内部;BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本),且损坏但尚可抢救的索引应保持可由运维手工抢救。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。" + desc_en: "Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN the files are left EXACTLY in place and bili REFUSES TO START with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand, then start again — a fresh index rebuilds from the JSON envelopes). The FAIL verdict is cached so later starts fail fast without rescan. There is NO automatic rename/repair AND no legacy-only fallback: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy), and continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable); transient open failures (no engine / CANTOPEN / second writer / readonly) still degrade loudly as before — they do not prove any file corrupt. Set 0 only for boot-time-sensitive huge deployments where the check's cost matters." + desc_zh: "存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;一旦证实损坏,文件原样保留在原地,bili 拒绝启动并给出响亮、可操作的错误提示(手工备份后把 index.db/-wal/-shm 移开再启动——全新索引会从 JSON 信封重建)。FAIL 判定会被缓存,之后的启动快速失败、无需重扫。既没有自动改名/修复,也没有 legacy-only 兜底:自动搬移存储主索引的做法已撤回(#4626 时代的 owner 决策),因为无法保证完整数据恢复(checkpoint 行只存在于索引内部;BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本),且继续以 legacy-only 运行会让 JSON 信封与死掉的索引永久分叉、再无对账可能;瞬态打开失败(无引擎 / CANTOPEN / 第二写者 / 只读)仍像以前一样响亮降级——它们不能证明任何文件损坏。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。" - key: BILI_SNAPSHOT_SHADOW type: boolean default: "true (on)" From 5159ce9e56d38043ebbc7f9e85f1118053e7b385 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 30 Oct 2026 18:52:51 +0800 Subject: [PATCH 30/35] no degradation in ANY case: every unified-sidecar open failure now refuses to start (owner policy 2026-10-30, follow-up to the #5213 refusal chain) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second-writer / no-engine / CANTOPEN / readonly / OS-level DIO / MySQL-no-DSN open failures used to log and serve legacy-only. Per owner ruling '需要在任何情况下不降级' they now throw: shadow.maybeOpen re-throws StorageSecondWriterError, throws on every other inner failure with an actionable REFUSING-TO-START line, openAsync throws for a missing MySQL DSN (persist.attachMysql swallows nothing; boot awaits ready() before listen). Intentional opt-outs (BILI_STORAGE_UNIFIED legacy/unset, BILI_ENCRYPTION_KEY opt-out) are unchanged — a choice is not a failure. Tests re-pinned to the new contract: issue5046 (throws), web banner child (construction refuses + wire names cause), engine proofs, cantopen-hint, acp-status degraded facets, multi-instance-live (second instance dies at boot with exit 1 instead of degrading), mysql-seam gate facet (DSN-less ask rejects at ready(), seed ordered before env flip). Regression 120 pass / 1 skip / 0 fail across 20 storage suites + typecheck clean. --- CONFIGURATION.md | 2 +- src/persist.ts | 12 +- src/server.ts | 10 ++ src/storage/driver.ts | 10 +- src/storage/shadow.ts | 44 +++--- src/storage/store.ts | 10 +- tests/acp-status-storage-degraded.test.ts | 18 ++- tests/corrupt-index-child.ts | 65 ++++---- tests/issue5046-second-writer-degrade.test.ts | 46 +++--- tests/multi-instance-live.test.ts | 140 +++++++++++------- tests/storage-cantopen-hint.test.ts | 48 +++--- tests/storage-engine-proofs.test.ts | 38 ++--- tests/storage-mysql-seam.test.ts | 30 +++- tests/web-storage-banner.test.ts | 43 ++---- 14 files changed, 292 insertions(+), 224 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 3c541814f2..db9b1a89fa 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -362,7 +362,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | Env tier for storage.backend (#3931): sqlite (default) or mysql (fully wired, #3388 steps 2+3 - saves ingest through the async shadow queue, sync reads ride the JSON envelope, replicas/ext/GC/fork via the async twins (#4911); export too). Wins over the file tier. | | `BILI_MYSQL_DSN` | string | unset | — | MySQL DSN (mysql://user:pass@host:3306/db) for storage.backend="mysql" (#3388, #3931). Wins over the file key storage.mysql.dsn. Empty = the mysql backend cannot open; the documented error names both knobs. The pool opens lazily; closeQuietlyAsync drains the ingest queue and closes it. | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC age sweep window in days (#3463): sessions idle longer than this are archived (one JSON line appended to the single archive log (archive/archive.jsonl) + tombstone row + content/media refs released). 0 disables the age sweep; the manual entry POST /__bili/storage/gc always runs. Config-file key storage.archiveAfterDays is the persistent form. | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN the files are left EXACTLY in place and bili REFUSES TO START with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand, then start again — a fresh index rebuilds from the JSON envelopes). The FAIL verdict is cached so later starts fail fast without rescan. There is NO automatic rename/repair AND no legacy-only fallback: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy), and continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable); transient open failures (no engine / CANTOPEN / second writer / readonly) still degrade loudly as before — they do not prove any file corrupt. Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN the files are left EXACTLY in place and bili REFUSES TO START with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand, then start again — a fresh index rebuilds from the JSON envelopes). The FAIL verdict is cached so later starts fail fast without rescan. There is NO automatic rename/repair AND no legacy-only fallback: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy), and continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable); ANY open failure (corruption proven OR transient: no engine / CANTOPEN / second writer / readonly / backend DSN) makes bili REFUSE TO START loudly with an actionable error — no silent legacy fallback in any case (2026-10-30 owner policy); deployments that genuinely want JSON-only serve explicitly (BILI_STORAGE_UNIFIED=legacy or unset). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | Snapshot-render shadow observability (#3756 follow-up): compute the outbound snapshot view on every armed turn WITHOUT using it, classify the diff against the legacy outbound, and surface per-session counters in the web panel + the snapshot-shadow diagnostics topic. 0/false/off disables; 1/true/on enables; otherwise the config-file snapshot.shadow boolean applies (default on). The render lane itself stays rollout-gated (BILI_ROLLOUT_SNAPSHOT_RENDER), so the shadow lane never changes outbound bytes - it is the dry-run evidence that flips the rollout later. | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | Override the config file path (path relocation). | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | Where session records are stored (path relocation). | diff --git a/src/persist.ts b/src/persist.ts index 83b4671c95..06dbfcfbd5 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -407,8 +407,10 @@ export class SessionStore { * ready() resolves this.shadow is still undefined while the DB already * carries rows. */ private readonly networkFullExpected: boolean = false; - /** Await the storage backend before first use (mysql pool + DDL). openAsync - * never rejects (it degrades to null), so awaiting this cannot throw. */ + /** Await the storage backend before first use (mysql pool + DDL). #5046 + * owner policy 2026-10-30 (no degradation in any case): a backend that + * cannot come up REJECTS — boot awaits this before listen(); async + * request paths see it on first use. */ ready(): Promise { return this.mysqlReady; } @@ -532,6 +534,12 @@ export class SessionStore { ).then((sh) => { this.shadow = sh ?? undefined; }); + // #5046 chain (2026-10-30): openAsync now REJECTS on backend failure + // (no degradation in any case). Boot awaits ready() and exits + // deterministically; the .catch here keeps every OTHER construct- + // then-later-await path from dying via unhandled rejection instead + // of the caller seeing the rejection on await. + this.mysqlReady.catch(() => {}); this.shadow = undefined; this.networkFullExpected = storageMode === "full"; } else { diff --git a/src/server.ts b/src/server.ts index 750b88176c..a8cbd89bf6 100644 --- a/src/server.ts +++ b/src/server.ts @@ -555,6 +555,16 @@ export async function createEngineContext(opts: ProxyOptions): Promise null), undefined, `${dir}/index.db`); recordSidecarInstance(shadow); @@ -367,9 +368,10 @@ export class UnifiedShadow { if (mode === "legacy") return null; const dsn = (env.BILI_MYSQL_DSN ?? fileMysqlDsn ?? "").trim(); if (!dsn) { - loggerLog("warn", "[storage] storage.backend=mysql selected but no DSN is configured (BILI_MYSQL_DSN or storage.mysql.dsn) — staying legacy-only; see the CONFIGURATION storage section"); + const msg = "[storage] storage.backend=mysql selected but no DSN is configured (BILI_MYSQL_DSN or storage.mysql.dsn) — REFUSING TO START; configure the DSN or unset storage.backend (see the CONFIGURATION storage section)"; + loggerLog("error", msg); recordSidecarOpenFailure("mysql backend selected without a DSN"); - return null; + throw new Error(msg); } try { const backend = await openBackend("mysql", dsn); @@ -380,9 +382,9 @@ export class UnifiedShadow { return shadow; } catch (err) { const errStr = String(err); - loggerLog("warn", `[storage] mysql backend failed to open (${errStr}) — staying legacy-only`); + loggerLog("error", `[storage] mysql backend failed to open (${errStr}) — REFUSING TO START (no degradation in any case, 2026-10-30 owner policy); fix the connection or unset storage.backend`); recordSidecarOpenFailure(errStr); - return null; + throw err; } } diff --git a/src/storage/store.ts b/src/storage/store.ts index fc006d5a41..03a9aed063 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -347,9 +347,9 @@ function likeEscape(prefix: string): string { /** * #4175: raised when PRAGMA quick_check fails at open. Migrations must never * run against a corrupt file, so this fires BEFORE migrateSchema; the shadow - * layer catches it and degrades to legacy-only LOUDLY without touching the - * file set — no automatic rename/repair (#4626-era owner decision: full data - * recovery cannot be guaranteed without operator involvement). The FAIL + * layer records it and RE-THROWS — bili REFUSES TO START with the files + * left exactly in place (no automatic rename/repair, no legacy fallback: + * full data recovery cannot be guaranteed without operator involvement). The FAIL * verdict is cached so later opens fail fast; the operator moves the files * aside by hand (backed up first) to force a fresh rebuild. */ @@ -595,8 +595,8 @@ export class UnifiedStore { // #4609: a large store's cache-miss check runs post-open on a second // read-only connection — open() returns immediately; the verdict // lands in runtime-status and the cache (a cached failure makes the - // NEXT open fail fast and degrade loudly — no automatic file moves, - // #4175/#4626-era decision). + // NEXT open fail fast and REFUSE TO START — no automatic file moves, + // #4175/#4626-era decision chain to 2026-10-30: no degradation). if (mode.kind === "async") scheduleBackgroundIntegrityCheck(path, (ok, detail) => { if (ok) store.noteIntegrityOk(); else if (detail !== undefined) store.noteIntegrityFail(detail); }); return store; } diff --git a/tests/acp-status-storage-degraded.test.ts b/tests/acp-status-storage-degraded.test.ts index ebcc60f4cd..682018c482 100644 --- a/tests/acp-status-storage-degraded.test.ts +++ b/tests/acp-status-storage-degraded.test.ts @@ -5,8 +5,8 @@ import { tmpdir } from "node:os"; import * as path from "node:path"; import { handleAcpStatus } from "../src/acp-status.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; -import { setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; -import { _resetStorageRuntimeStatusForTest, recordEngineBootstrap } from "../src/storage/runtime-status.ts"; + +import { _resetStorageRuntimeStatusForTest, recordEngineBootstrap, recordSidecarOpenFailure } from "../src/storage/runtime-status.ts"; import { createCore, createInitialState, defaultConfig, type CompressionCore, type Config, type CoreMessage } from "acp-kernel"; import type { Session } from "../src/session.ts"; import { rmrf } from "./tmp-rm.ts"; @@ -60,11 +60,16 @@ test("#3385: acp_status STORAGE section surfaces the degrade reason so the model const snap = isolate(base); let store: SessionStore | null = null; try { - setSqliteEnginesUnavailableForTests(true); _resetStorageRuntimeStatusForTest(); recordEngineBootstrap({ status: "failed", detail: "The operation was aborted due to timeout" }); + // 2026-10-30: a no-engine OPEN refuses boot, so the acp_status surface + // can only see a failed open post-boot via the runtime breaker; this + // facet pins the SURFACE by driving the same recording seam shadow.ts + // calls before its rethrow. A real (healthy) store keeps + // ccrLoopConfig from rebuilding over the recorded state. store = new SessionStore({ dir: process.env.BILI_SESSIONS_DIR, enabled: true }); _setStoreForTest(store); + recordSidecarOpenFailure("no SQLite engine available"); const out = handleAcpStatus({}, makeCtx()); const text = out.text ?? ""; @@ -81,7 +86,6 @@ test("#3385: acp_status STORAGE section surfaces the degrade reason so the model } finally { store?.closeQuietly(); _setStoreForTest(null as unknown as SessionStore); - setSqliteEnginesUnavailableForTests(false); _resetStorageRuntimeStatusForTest(); restore(snap); rmrf(base); @@ -121,11 +125,11 @@ test("#3894: an unsupported Node surfaces the artifact-table reason, not generic const snap = isolate(base); let store: SessionStore | null = null; try { - setSqliteEnginesUnavailableForTests(true); _resetStorageRuntimeStatusForTest(); recordEngineBootstrap({ status: "unsupported-node", detail: "no pinned SQLite engine runs on Node.js 21.7.3 (ABI 120)" }); store = new SessionStore({ dir: process.env.BILI_SESSIONS_DIR, enabled: true }); _setStoreForTest(store); + recordSidecarOpenFailure("no self-downloaded SQLite engine supports this Node.js 21.7.3 (ABI 120)"); const out = handleAcpStatus({}, makeCtx()); const text = out.text ?? ""; @@ -136,7 +140,6 @@ test("#3894: an unsupported Node surfaces the artifact-table reason, not generic } finally { store?.closeQuietly(); _setStoreForTest(null as unknown as SessionStore); - setSqliteEnginesUnavailableForTests(false); _resetStorageRuntimeStatusForTest(); restore(snap); rmrf(base); @@ -156,11 +159,11 @@ test("#4444: an unsupported Node surfaces the Node.js upgrade lever on the degra const origNodeDesc = Object.getOwnPropertyDescriptor(process.versions, "node"); let store: SessionStore | null = null; try { - setSqliteEnginesUnavailableForTests(true); _resetStorageRuntimeStatusForTest(); recordEngineBootstrap({ status: "unsupported-node", detail: "no pinned SQLite engine runs on Node.js 21.6.1 (ABI 120)" }); store = new SessionStore({ dir: process.env.BILI_SESSIONS_DIR, enabled: true }); _setStoreForTest(store); + recordSidecarOpenFailure("no self-downloaded SQLite engine supports this Node.js 21.6.1 (ABI 120)"); Object.defineProperty(process.versions, "node", { value: "21.6.1", configurable: true, writable: true }); const text = handleAcpStatus({}, makeCtx()).text ?? ""; @@ -173,7 +176,6 @@ test("#4444: an unsupported Node surfaces the Node.js upgrade lever on the degra if (origNodeDesc) Object.defineProperty(process.versions, "node", origNodeDesc); store?.closeQuietly(); _setStoreForTest(null as unknown as SessionStore); - setSqliteEnginesUnavailableForTests(false); _resetStorageRuntimeStatusForTest(); restore(snap); rmrf(base); diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index 71eeca03cf..da2c80b7ec 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -43,6 +43,7 @@ import { defaultConfig } from "acp-kernel"; import type { ProxyOptions } from "../src/config.ts"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { storageStatusWire } from "../src/storage/runtime-status.ts"; import { StorageIntegrityError } from "../src/storage/store.ts"; import { setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts"; @@ -215,13 +216,13 @@ export const ENGPROOF_CHECKS: readonly string[] = Object.values(ENGPROOF_CHECK_N // is pinned here through the remaining deterministic cross-platform degrade // cause: the engine ladder exhausted. const BANNER_CHECK_NAMES = { - overviewAnswers: "/__bili/overview answers 200 with JSON", - requestedFull: "storage.requested is full", - effectiveLegacy: "storage.effective is legacy", - degraded: "storage.degraded is true (the no-engine degrade trips the warning)", - intentionalOptOut: "storage.intentionalOptOut is false", - reason: "storage.reason names the missing engine", - engine: "storage.engine is null when no engine resolved", + /** Construction THROWS instead of degrading — no server serves a + * degraded page (2026-10-30: no degradation in any case). */ + constructionRefuses: "construction-refuses-to-start", + /** The refused open leaves no half-created index.db behind. */ + noSidecarLeftBehind: "no-index-db-left-behind", + /** Pre-death #3040 forensics: the wire names the cause. */ + wireNamesEngine: "status-wire-names-missing-engine", } as const; export const BANNER_CHECKS: readonly string[] = Object.values(BANNER_CHECK_NAMES); @@ -401,38 +402,26 @@ async function bannerScenario(argv: string[], run: CheckRun): Promise { if (!base) throw new Error("banner mode needs "); const dir = path.join(base, "sessions"); mkdirSync(dir, { recursive: true }); - // #4816: the former corrupt-index cause here now quarantines and recovers - // instead of degrading — force the engine ladder exhausted (deterministic - // on every platform) as the degrade trigger for the banner wire shape. + // #4816 chain -> 2026-10-30 "no degradation in any case": an + // unfulfillable unified ask (here: engine ladder exhausted) must make + // CONSTRUCTION REFUSE loudly — no server ever comes up to serve a + // degraded page. The pre-death wire still records WHY (forensics). setSqliteEnginesUnavailableForTests(true); - const store = new SessionStore({ dir, enabled: true }); - _setStoreForTest(store); - const upstreamPort = await freePort(); - const proxy = await startServer(makeOpts(upstreamPort)); - await once(proxy, "listening"); - try { - const port = (proxy.address() as { port: number }).port; - // Holder (not a plain let): the assignment happens inside the check - // closure, and TS control-flow narrowing cannot see closure writes — - // a `let overview = null` would narrow to `never` at the guard below. - const overview: { storage?: StorageWire } = {}; - await run.run(BANNER_CHECK_NAMES.overviewAnswers, async () => { - const res = await fetch(`http://127.0.0.1:${port}/__bili/overview`); - assert.equal(res.status, 200, "/__bili/overview answers 200"); - overview.storage = ((await res.json()) as { storage: StorageWire }).storage; - }); - const storage = overview.storage; - if (!storage) return; - await run.run(BANNER_CHECK_NAMES.requestedFull, () => assert.equal(storage.requested, "full")); - await run.run(BANNER_CHECK_NAMES.effectiveLegacy, () => assert.equal(storage.effective, "legacy")); - await run.run(BANNER_CHECK_NAMES.degraded, () => assert.equal(storage.degraded, true, "no-engine degrade trips the warning")); - await run.run(BANNER_CHECK_NAMES.intentionalOptOut, () => assert.equal(storage.intentionalOptOut, false)); - await run.run(BANNER_CHECK_NAMES.reason, () => assert.match(storage.reason ?? "", /no SQLite engine available/)); - await run.run(BANNER_CHECK_NAMES.engine, () => assert.equal(storage.engine, null)); - } finally { - await close(proxy); - store.closeQuietly(); - } + await run.run(BANNER_CHECK_NAMES.constructionRefuses, () => { + assert.throws( + () => new SessionStore({ dir, enabled: true }), + /no SQLite engine available/i, + "construction must refuse, not degrade", + ); + }); + await run.run(BANNER_CHECK_NAMES.noSidecarLeftBehind, () => + assert.ok(!fs.existsSync(path.join(dir, "index.db"))), + ); + await run.run(BANNER_CHECK_NAMES.wireNamesEngine, () => { + // requested mode still reads as full from env; the recorded failure is + // exactly what the refusing open wrote before throwing. + assert.match(storageStatusWire().reason ?? "", /no SQLite engine available/); + }); } async function main(argv: string[]): Promise { diff --git a/tests/issue5046-second-writer-degrade.test.ts b/tests/issue5046-second-writer-degrade.test.ts index 36cb0b1f03..31a3be3d83 100644 --- a/tests/issue5046-second-writer-degrade.test.ts +++ b/tests/issue5046-second-writer-degrade.test.ts @@ -1,12 +1,13 @@ /** - * #5046/B (owner decision 2026-10-29): a live foreign bili process holding - * .lock must make the proxy sidecar REFUSE to become a persistent second - * writer and degrade to legacy-only — with one actionable error line (names - * BILI_SESSIONS_DIR + BILI_CONFIG_FILE + what is lost) and status-wire - * visibility. Both directions pinned: live foreign holder ⇒ degrade; no - * foreign holder ⇒ the original fast path is unchanged; mixed-version fleet - * (old marker without lane) ⇒ conservative branch degrades; the report-only - * default policy (CLI/export open paths) stays byte-identical. + * #5046/B (owner decision chain 2026-10-29 → 2026-10-30): a live foreign + * bili process holding .lock must make the proxy sidecar REFUSE TO START + * (throw StorageSecondWriterError — no-degradation-in-any-case) with one + * actionable error line (names BILI_SESSIONS_DIR + BILI_CONFIG_FILE) and + * status-wire visibility. Both directions pinned: live same-lane/unknown-lane + * foreign holder ⇒ refuse; no foreign holder ⇒ the original fast path is + * unchanged; dead-pid stale lock ⇒ takeover unchanged; cross-lane live holder + * ⇒ still opens (#5091 supported shared topology); the report-only default + * policy (CLI/export open paths) stays byte-identical. */ import assert from "node:assert"; import { spawn } from "node:child_process"; @@ -16,7 +17,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { after, before, test } from "node:test"; import { UnifiedShadow } from "../src/storage/shadow.ts"; -import { UnifiedStore } from "../src/storage/store.ts"; +import { UnifiedStore, StorageSecondWriterError } from "../src/storage/store.ts"; import type { LegacySessionLike } from "../src/storage/ingest.ts"; import { setLogCapture } from "../src/logger.ts"; import { _resetStorageRuntimeStatusForTest, storageStatusWire } from "../src/storage/runtime-status.ts"; @@ -102,7 +103,7 @@ after(() => { else process.env.BILI_LAUNCHER_LANE = savedLauncherLane; }); -test("#5046/B: live same-lane foreign holder ⇒ sidecar degrades to legacy-only, lock untouched, actionable error + status wire", () => { +test("#5046/B: live same-lane foreign holder ⇒ open REFUSES TO START (throw), lock untouched, actionable error + status wire", () => { if (!engineAvailable) return; _resetStorageRuntimeStatusForTest(); const state = freshState(); @@ -112,17 +113,23 @@ test("#5046/B: live same-lane foreign holder ⇒ sidecar degrades to legacy-only const logs: string[] = []; setLogCapture((_level, msg) => logs.push(msg)); withEnv({ XDG_STATE_HOME: state, BILI_LAUNCHER_LANE: "dsh" }, () => { - const shadow = UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); - assert.equal(shadow, null, "must degrade to legacy-only (null), not open a second writer"); + // 2026-10-30 "no degradation in any case": refused co-writing must + // THROW so boot dies before listen(), not return null + run legacy. + assert.throws( + () => UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike), + (err: unknown) => err instanceof StorageSecondWriterError, + "must REFUSE with StorageSecondWriterError, not open a second writer", + ); const refused = logs.find((m) => m.includes("unified storage REFUSED")); assert.ok(refused, `actionable error line expected, got: ${logs.join(" | ")}`); + assert.ok(refused!.includes("REFUSES TO START"), "the refusal must state its consequence"); assert.ok(refused!.includes("BILI_SESSIONS_DIR") && refused!.includes("BILI_CONFIG_FILE"), "the fix must name both isolation vars"); - assert.ok(refused!.includes("CCR") && refused!.includes("checkpoints"), "must say what degradation loses"); assert.equal(lockHolderPid(sessions), foreign.pid, "the foreign lock must NOT be stolen"); + // Pre-death diagnostic surface (#3040 visibility): the failure is + // recorded BEFORE the throw so post-crash forensics can read why. withEnv({ BILI_STORAGE_UNIFIED: "shadow" }, () => { const wire = storageStatusWire(); - assert.equal(wire.degraded, true, "status wire must show degraded"); - assert.equal(wire.effective, "legacy"); + assert.equal(wire.degraded, true, "status wire must record the failed open"); assert.ok(wire.reason?.startsWith("second-writer:"), `reason should be the pre-formatted refusal, got: ${wire.reason}`); assert.ok((wire.reason ?? "").includes("BILI_SESSIONS_DIR"), "wire reason must carry the fix"); }); @@ -184,7 +191,7 @@ test("#5046/B negative: no foreign holder ⇒ original fast path (ENABLED line, setLogCapture(null); }); -test("#5046/B: mixed-version fleet — old marker WITHOUT lane ⇒ conservative branch degrades", () => { +test("#5046/B: mixed-version fleet — old marker WITHOUT lane ⇒ conservative branch REFUSES", () => { if (!engineAvailable) return; _resetStorageRuntimeStatusForTest(); const state = freshState(); @@ -194,8 +201,11 @@ test("#5046/B: mixed-version fleet — old marker WITHOUT lane ⇒ conservative const logs: string[] = []; setLogCapture((_level, msg) => logs.push(msg)); withEnv({ XDG_STATE_HOME: state, BILI_LAUNCHER_LANE: "dsh" }, () => { - const shadow = UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); - assert.equal(shadow, null, "unknown peer lane must stay conservative (degrade), never grant info-level amnesty"); + assert.throws( + () => UnifiedShadow.maybeOpen(sessions, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike), + (err: unknown) => err instanceof StorageSecondWriterError, + "unknown peer lane must stay conservative and refuse (no info-level amnesty)", + ); assert.ok(logs.some((m) => m.includes("unified storage REFUSED"))); }); setLogCapture(null); diff --git a/tests/multi-instance-live.test.ts b/tests/multi-instance-live.test.ts index c17c2b33d0..42e6600538 100644 --- a/tests/multi-instance-live.test.ts +++ b/tests/multi-instance-live.test.ts @@ -8,25 +8,23 @@ import { mkdtempSync, readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { rmrf } from "./tmp-rm.ts"; -// Two LIVE proxy instances over one sessions dir (#394 shape, end-to-end). -// -// What is proven here — the documented contract only, not multi-writer -// coherence (src/persist.ts deliberately rejects an election/lock): -// 1. both instances keep serving through interleaved turns on the SAME -// session id (no crash, no head-of-line poisoning across processes); -// 2. the on-disk record stays a valid envelope at EVERY observed moment -// (atomic tmp+rename) while two processes write it; -// 3. stats.requests on disk NEVER decreases (the #405 monotonic-counter -// guard holds under real traffic, not just unit-shaped writes); -// 4. a stale resident's write is rejected AND the resident converges to the -// newer disk lineage (#2401) — observable as: A's post-convergence write -// is accepted and continues the DISK counter, not A's stale one; -// 5. the late-starting instance warns about the overlap (#394) at -// registration. -// -// NOT proven (documented residuals, pinned separately in -// tests/multi-instance-persist.test.ts): tie passthrough, counter catch-up -// re-clobber, content-store last-writer-wins. +// One LIVE proxy instance serving a session, plus a SECOND instance whose +// boot over the SAME sessions dir must be REFUSED (owner decision chain +// #5046 -> 2026-10-30 "no degradation in any case"). End-to-end proof that: +// 1. the surviving instance (A) keeps serving through real traffic and the +// on-disk record stays a valid envelope at EVERY observed moment +// (atomic tmp+rename), with stats.requests NEVER decreasing (#405 +// monotonic-counter guard under real traffic); +// 2. the late-starting instance B dies at boot with ONE actionable error +// (isolate via BILI_SESSIONS_DIR / BILI_CONFIG_FILE) and a non-zero +// exit code — NOT the old legacy-degraded coexistence. The old +// serve-guard-converge interleave (#394/#2401, former phases 2-4) is +// unreachable by construction now: the double-writer never starts. Its +// envelope guards remain pinned at unit level in +// tests/multi-instance-persist.test.ts and stay reachable through manual +// file edits. Cross-lane sharing (distinct lanes, one dir) stays SUPPORTED +// (#5082/#5091) and is pinned in +// tests/issue5046-second-writer-degrade.test.ts. const SESSION = "mi-live-session"; const MODEL = "claude-sonnet-4-5"; @@ -54,6 +52,49 @@ interface Child { stderr: string[]; } +interface Refusal { + exitCode: number | null; + stderr: string; +} + +/** Spawns a child that is expected to DIE AT BOOT (no MI_READY) and collects + * its exit code + full stderr — the refusal-policy counterpart of + * spawnChild's success path. */ +async function spawnChildRefusing(upstreamPort: number | string, env: NodeJS.ProcessEnv): Promise { + const proc = spawn(process.execPath, ["--import", "tsx", join(import.meta.dirname, "fixtures/multi-instance-child.ts"), String(upstreamPort)], { + env, + stdio: ["pipe", "pipe", "pipe"], + }); + let ready = false; + let stdoutBuf = ""; + const stderrChunks: string[] = []; + proc.stdout!.setEncoding("utf8"); + proc.stdout!.on("data", (c: string) => { + stdoutBuf += c; + if (stdoutBuf.includes("MI_READY")) ready = true; + }); + proc.stderr!.setEncoding("utf8"); + proc.stderr!.on("data", (c: string) => stderrChunks.push(c)); + const code = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("child did not refuse within 30s")), 30_000); + proc.on("exit", (c) => { + clearTimeout(timer); + resolve(c); + }); + // A child that DID become ready violated the expectation — fail fast. + const poll = setInterval(() => { + if (ready) { + clearInterval(poll); + clearTimeout(timer); + proc.kill("SIGTERM"); + reject(new Error("child became READY — expected a boot refusal")); + } + }, 100); + proc.on("exit", () => clearInterval(poll)); + }); + return { exitCode: code, stderr: stderrChunks.join("") }; +} + async function spawnChild(upstreamPort: number | string, env: NodeJS.ProcessEnv): Promise { const proc = spawn(process.execPath, ["--import", "tsx", join(import.meta.dirname, "fixtures/multi-instance-child.ts"), String(upstreamPort)], { env, @@ -200,44 +241,31 @@ test("two live instances over one sessions dir: serve, guard, converge (#394/#40 const warmed = await waitFor(sessionsDir, (v) => v.requests >= 3, "A's 3 turns persisted"); assert.equal(warmed.requests, 3, "each turn must count exactly once"); - // Phase 2 — instance B boots over the SAME dir (boot-hydrates the - // session), serves 2 turns. B's writes must be fresh (accepted). - children.push(await spawnChild(upstreamPort, childEnv(2))); - const B = children[1]!; - await turn(B, 4); - await turn(B, 5); - await waitFor(sessionsDir, (v) => v.requests >= 5, "B's turns persisted (fresh lineage accepted)"); - - // B registered last: it must have seen A and warned (#394). - const bErr = B.stderr.join(""); - assert.ok(bErr.includes("#394"), "late starter must warn about the overlapping instance"); - assert.ok(bErr.includes("another bili instance"), `unexpected warn text: ${bErr.slice(-400)}`); - - // Phase 3 — the STALE resident: A never re-read the file (its pool - // entry is resident), so A's counter (3+1=4) is behind disk (5). Its - // write must be REJECTED (#405) and A must converge (#2401). - await turn(A, 6); - await new Promise((r) => setTimeout(r, 300)); - const aErr1 = A.stderr.join(""); - assert.ok(aErr1.includes("rejected stale snapshot"), `stale write must be rejected (stderr tail: ${aErr1.slice(-500)})`); - - // Phase 4 — A's next write continues from the CONVERGED (disk) - // counter: 5+1=6, accepted; disk must show exactly 6. - await turn(A, 7); - const converged = await waitFor(sessionsDir, (v) => v.requests >= 6, "A's post-convergence write accepted"); - assert.equal(converged.requests, 6, "converged resident must continue the DISK counter, not its stale one"); - - // End-state: the session still serves through BOTH instances. B is now - // itself behind (its resident counter is 5, disk is 6 after A's accepted - // write), so B's turn lands on the documented TIE: equal request - // counters + equal nextBlockId → staleDiskPayload returns null → the - // write passes through (last-writer-wins on ties, persist.ts:76-87). - // Either way the file must stay valid and the counter non-decreasing — - // the reader loop below is the real teeth for that. - await turn(B, 8); - const final2 = await waitFor(sessionsDir, (v) => v.requests >= 6, "B serves after A's convergence"); - assert.equal(final2.requests, 6, "disk counter reflects the last accepted lineage (B tie-passthrough at 6)"); + // Phase 2 (rewritten 2026-10-30, owner chain "#5046 -> no degradation + // in any case"): instance B booting over the SAME sessions dir now + // REFUSES TO START — exit 1 with the actionable isolation error — + // instead of the old serve-legacy-degraded coexistence. Consequence: + // the former Phase 3/4 stale-write + converge/tie interleave below is + // UNREACHABLE BY CONSTRUCTION (the second writer never runs); those + // envelope guards stay pinned at unit level in + // tests/multi-instance-persist.test.ts and remain reachable through + // manual/multi-process file edits. Cross-LANE sharing (distinct + // BILI_LAUNCHER_LANE) remains the supported shared topology (#5082/ + // #5091, pinned in tests/issue5046-second-writer-degrade.test.ts). + const b = await spawnChildRefusing(upstreamPort, childEnv(2)); + assert.equal(b.exitCode, 1, "instance B must die at boot, not serve degraded"); + assert.ok( + b.stderr.includes("unified storage REFUSED") && b.stderr.includes("REFUSES TO START"), + `B's death must carry the actionable refusal (stderr tail: ${b.stderr.slice(-400)})`, + ); + assert.ok(b.stderr.includes("BILI_SESSIONS_DIR"), "refusal names the fix"); + // A is UNAFFECTED by B's refused boot: it keeps serving fresh turns + // and the on-disk counter advances exactly as before. + await turn(A, 4); + await turn(A, 5); + const servedAfterRefusal = await waitFor(sessionsDir, (v) => v.requests >= 5, "A kept serving through B's refused boot"); + assert.equal(servedAfterRefusal.requests, 5, "A's turns count exactly once each"); // Reader loop verdict. clearInterval(reader); assert.ok(reads > 10, `reader loop ran (reads=${reads})`); diff --git a/tests/storage-cantopen-hint.test.ts b/tests/storage-cantopen-hint.test.ts index be28f50474..cdf26ab3b6 100644 --- a/tests/storage-cantopen-hint.test.ts +++ b/tests/storage-cantopen-hint.test.ts @@ -142,7 +142,7 @@ function restoreEnv(snap: Record): void { } } -test("#3823 wiring: an unopenable index.db degrades with the platform-appropriate reason", () => { +test("#3823 wiring: an unopenable index.db REFUSES TO START with the platform-appropriate reason (no degradation in any case, 2026-10-30)", () => { const base = mkdtempSync(path.join(tmpdir(), "cantopen-hint-")); const sessions = path.join(base, "sessions"); mkdirSync(sessions, { recursive: true }); @@ -155,27 +155,30 @@ test("#3823 wiring: an unopenable index.db degrades with the platform-appropriat setLogCapture((level, msg) => logs.push(`${level} ${msg}`)); try { _resetStorageRuntimeStatusForTest(); - new SessionStore({ dir: sessions, enabled: true }); + assert.throws( + () => new SessionStore({ dir: sessions, enabled: true }), + (err: unknown) => err instanceof Error && err.message.includes("open"), + "construction must refuse on CANTOPEN, not degrade", + ); const wire = storageStatusWire(); - assert.equal(wire.degraded, true, "the sidecar must be degraded"); - assert.equal(wire.effective, "legacy", "legacy JSON keeps serving"); + assert.equal(wire.degraded, true, "the failed open must be recorded pre-death"); assert.ok( wire.reason?.startsWith("index database failed to open"), `reason names the open failure (got: ${wire.reason})`, ); - const degradeLine = logs.find((l) => l.includes("unified sidecar failed to open")); - assert.ok(degradeLine, "the degrade warn was logged"); + const refuseLine = logs.find((l) => l.includes("unified sidecar failed to open")); + assert.ok(refuseLine, "the refusal warn was logged"); if (process.platform === "win32") { assert.ok(wire.reason?.includes("(win32:"), "win32 runner carries the OS-level hint"); assert.ok( - !degradeLine.includes("delete the file to reset the index"), + !refuseLine.includes("delete the file to reset the index"), "an OS-level refusal must not carry the delete advice", ); } else { assert.ok(!wire.reason?.includes("(win32:"), "non-win32 runner stays un-hinted"); assert.ok( - degradeLine.includes("; delete the file to reset the index"), - "a genuine open failure keeps the delete advice off-hint platforms", + !refuseLine.includes("delete the file to reset the index"), + "the refusal line never carries destructive self-help advice", ); } } finally { @@ -191,7 +194,7 @@ test("#3823 wiring: an unopenable index.db degrades with the platform-appropriat // 0444 regular file (root bypasses permission bits, so skip under root; // chmod is inert on win32, so skip there — the win32 branch of the hint is // unit-pinned above with a platform stub instead). -test("#4614 wiring: an unwritable existing index.db degrades carrying the readonly signature", () => { +test("#4614 wiring: an unwritable existing index.db REFUSES TO START carrying the readonly signature (2026-10-30)", () => { if (process.platform === "win32") return; if (typeof process.getuid === "function" && process.getuid() === 0) return; const base = mkdtempSync(path.join(tmpdir(), "readonly-hint-")); @@ -204,17 +207,20 @@ test("#4614 wiring: an unwritable existing index.db degrades carrying the readon setLogCapture((level, msg) => logs.push(`${level} ${msg}`)); try { _resetStorageRuntimeStatusForTest(); - new SessionStore({ dir: sessions, enabled: true }); + assert.throws(() => new SessionStore({ dir: sessions, enabled: true }), "read-only index.db must refuse construction"); const wire = storageStatusWire(); - assert.equal(wire.degraded, true, "the sidecar must be degraded"); - assert.equal(wire.effective, "legacy", "legacy JSON keeps serving"); + assert.equal(wire.degraded, true, "the failed open must be recorded pre-death"); assert.ok( wire.reason?.includes("attempt to write a readonly database"), `reason carries the readonly signature (got: ${wire.reason})`, ); - const degradeLine = logs.find((l) => l.includes("unified sidecar failed to open")); - assert.ok(degradeLine, "the degrade warn was logged"); - assert.ok(!degradeLine.includes("(win32:"), "non-win32 stays un-hinted"); + const refuseLine = logs.find((l) => l.includes("unified sidecar failed to open")); + assert.ok(refuseLine, "the refusal warn was logged"); + assert.ok(!refuseLine.includes("(win32:"), "non-win32 stays un-hinted"); + assert.ok( + !refuseLine.includes("delete the file to reset the index"), + "a readonly failure never carries the destructive delete advice", + ); } finally { setLogCapture(null); _resetStorageRuntimeStatusForTest(); @@ -238,14 +244,14 @@ test("wiring: an OS-level refusal drops the delete advice from the degrade line" const restore = stubPlatform("win32"); try { _resetStorageRuntimeStatusForTest(); - new SessionStore({ dir: sessions, enabled: true }); + assert.throws(() => new SessionStore({ dir: sessions, enabled: true }), "stubbed win32 CANTOPEN must refuse construction"); const wire = storageStatusWire(); assert.ok(wire.reason?.includes("(win32:"), "stubbed win32 carries the OS-level hint in the diagnostic"); - const degradeLine = logs.find((l) => l.includes("unified sidecar failed to open")); - assert.ok(degradeLine, "the degrade warn was logged"); - assert.ok(degradeLine.includes("(win32:"), "the warn line carries the hint"); + const refuseLine = logs.find((l) => l.includes("unified sidecar failed to open")); + assert.ok(refuseLine, "the refusal warn was logged"); + assert.ok(refuseLine.includes("(win32:"), "the warn line carries the hint"); assert.ok( - !degradeLine.includes("delete the file to reset the index"), + !refuseLine.includes("delete the file to reset the index"), "an OS-level refusal must not carry the delete advice", ); } finally { diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 55f36b8a5a..2cfc98633d 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -489,35 +489,39 @@ test("storage engine: CAS refcounts conserve shared content across deleteSession // the contract that lets us ship without a native SQLite dependency at all. // --------------------------------------------------------------------------- -test("storage engine: no SQLite engine -> maybeOpen degrades to null (sidecar absent), never throws (#2823)", () => { +test("storage engine: no SQLite engine -> maybeOpen REFUSES (throw), never silently degrades (#2823 re-pinned 2026-10-30)", () => { const dir = tmpdir("bili-engproof-noengine-open-"); try { setSqliteEnginesUnavailableForTests(true); - const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null); - assert.equal(shadow, null, "maybeOpen must return null (legacy-only) when no SQLite engine is available"); + // No-degradation policy: the unified ask cannot be satisfied -> the + // open must throw (proxy boot then refuses loudly), not return null. + assert.throws( + () => UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null), + /no SQLite engine available/i, + "maybeOpen must REFUSE when no SQLite engine is available", + ); } finally { setSqliteEnginesUnavailableForTests(false); rmrf(dir); } }); -test("storage engine: no SQLite engine -> SessionStore runs legacy-only, writes survive, no index.db (#2823)", async () => { +test("storage engine: no SQLite engine -> construction REFUSES (no degradation in any case, 2026-10-30)", async () => { const dir = tmpdir("bili-engproof-noengine-e2e-"); const prev = process.env.BILI_STORAGE_UNIFIED; try { - process.env.BILI_STORAGE_UNIFIED = "full"; // ask for the unified store... - setSqliteEnginesUnavailableForTests(true); // ...but pretend none exists - const session = loadFolded(path.join(dir, "seed")); - // THE #2823 regression: construction must survive with no engine at all. - const store = new SessionStore({ dir, debounceMs: 1, enabled: true }); - await store.writeNow(session); - assert.ok(!fs.existsSync(path.join(dir, "index.db")), "no-engine mode must not create index.db"); - const reloaded = store.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); - store.closeQuietly(); - assert.ok(reloaded, "legacy load returned null with no SQLite engine available"); - const disk = findWrittenRecord(dir, "corpus-folded"); - assert.ok(disk, "disk record not found"); - assert.equal(viewDigest(toLegacyLike(reloaded)), recordDigest(disk), "no-engine: legacy load digest != disk record"); + process.env.BILI_STORAGE_UNIFIED = "full"; // ask for the unified store + const session = loadFolded(path.join(dir, "seed")); // seed first (it constructs its own store) + setSqliteEnginesUnavailableForTests(true); // ...now pretend no engine exists + // #2823 re-pinned (2026-10-30 no-degradation policy): construction + // must REFUSE loudly — no engine at all means the unified ask cannot + // be served, and a silent legacy fallback was withdrawn. + assert.throws( + () => new SessionStore({ dir, debounceMs: 1, enabled: true }), + /no SQLite engine available/i, + "a unified/full ask with no engine must make construction refuse", + ); + assert.ok(!fs.existsSync(path.join(dir, "index.db")), "refused construction must not leave an index.db behind"); } finally { setSqliteEnginesUnavailableForTests(false); if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; diff --git a/tests/storage-mysql-seam.test.ts b/tests/storage-mysql-seam.test.ts index bfec6c0fd4..ef7104b77e 100644 --- a/tests/storage-mysql-seam.test.ts +++ b/tests/storage-mysql-seam.test.ts @@ -527,30 +527,46 @@ test("seam: plugin conversation resolution materializes DB-only sessions through } }); -test("gate: jsonWriter=off stays ignored on mysql (the #4936 gate)", async () => { +test("gate: jsonWriter=off stays ignored on mysql; DSN-less mysql ask REFUSES at ready() (2026-10-30)", async () => { const prev = { BILI_STORAGE_BACKEND: process.env.BILI_STORAGE_BACKEND, BILI_STORAGE_UNIFIED: process.env.BILI_STORAGE_UNIFIED, BILI_STORAGE_JSON_WRITER: process.env.BILI_STORAGE_JSON_WRITER, }; - process.env.BILI_STORAGE_BACKEND = "mysql"; - process.env.BILI_STORAGE_UNIFIED = "full"; - process.env.BILI_STORAGE_JSON_WRITER = "off"; const logs: string[] = []; + // Seed FIRST: under the refusal policy the loader store built inside + // seedCorpusSession must still use the sqlite backend — flipping the env + // to DSN-less mysql first would refuse ITS construction too. const baseDir = tmpDir("bili-seam-gate-"); const loaded = await seedCorpusSession(baseDir); + // Now ask for mysql WITHOUT a DSN: construction succeeds (the backend + // attach is async), the #4936 gate warning fires at construct time, and + // ready() must REJECT with the actionable message instead of silently + // degrading (#5046 chain, 2026-10-30 owner policy). + process.env.BILI_STORAGE_BACKEND = "mysql"; + process.env.BILI_STORAGE_UNIFIED = "full"; + process.env.BILI_STORAGE_JSON_WRITER = "off"; const seamDir = tmpDir("bili-seam-gate-store-"); const store = new SessionStore({ dir: seamDir, debounceMs: 1, enabled: true, log: (_l, m) => logs.push(m) }); try { - store.flushSync(loaded); assert.ok( logs.some((m) => m.includes("storage.jsonWriter=off ignored")), `expected the sqlite-only gate warning, got: ${logs.join(" | ")}`, ); - assert.ok(findEnvelopeFiles(seamDir).length > 0, "the envelope MUST keep being written on mysql"); + let refusalMsg = ""; + try { + await store.ready(); + } catch (e) { + refusalMsg = String(e); + } + assert.match(refusalMsg, /no DSN is configured/, `ready() must reject naming the missing DSN, got: "${refusalMsg}"`); + // A refused boot persists NOTHING: no turn was ever flushed through + // the dead backend, so the dir holds no envelopes. + assert.equal(findEnvelopeFiles(seamDir).length, 0, "a refused backend must not leave persisted state behind"); + void loaded; // corpus seeded only to prove the ordering above is what saves it } finally { store.cancelAll(); - await store.closeQuietlyAsync(); + await store.closeQuietlyAsync().catch(() => {}); for (const [k, v] of Object.entries(prev)) { if (v === undefined) delete process.env[k]; else process.env[k] = v; diff --git a/tests/web-storage-banner.test.ts b/tests/web-storage-banner.test.ts index 2fe041e71b..a857964c7b 100644 --- a/tests/web-storage-banner.test.ts +++ b/tests/web-storage-banner.test.ts @@ -27,7 +27,7 @@ import path from "node:path"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { probeBuiltinSqlite, probeHostBetterSqlite3, setPreferredSqliteEngineForTests, setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; -import { _resetStorageRuntimeStatusForTest, recordEngineBootstrap } from "../src/storage/runtime-status.ts"; +import { _resetStorageRuntimeStatusForTest, recordEngineBootstrap, recordSidecarOpenFailure } from "../src/storage/runtime-status.ts"; import { renderPage } from "../src/web/page.ts"; import { WEB_CLIENT } from "../src/web/client.ts"; import { VERSION } from "../src/version.ts"; @@ -99,7 +99,7 @@ async function fetchJson(port: number, p: string): Promise { // scenario now runs in a spawned child that needs the exact same server // opts, so there is one copy, imported by both sides. -test("#3040 forced none-engine: status+overview report degraded; diagnostics blob + prefilled issue link", async () => { +test("#3040 failed-open surface: status+overview report degraded; diagnostics blob + prefilled issue link", async () => { const snap = saveEnv(); const base = mkdtempSync(path.join(tmpdir(), "bili-storage-banner-")); // Hoisted so the finally can release the sidecar's SQLite handle via @@ -110,9 +110,14 @@ test("#3040 forced none-engine: status+overview report degraded; diagnostics blo resetSeams(); delete process.env.BILI_STORAGE_UNIFIED; delete process.env.BILI_ENCRYPTION_KEY; - setSqliteEnginesUnavailableForTests(true); recordEngineBootstrap({ status: "failed", detail: "ECONNREFUSED registry.npmjs.org" }); store = new SessionStore({ dir: process.env.BILI_SESSIONS_DIR, enabled: true }); + // 2026-10-30 policy change: an unfulfillable unified ask makes BOOT REFUSE + // (the refusing test below spawns the real throw), so this in-process facet + // pins the PRE-DEATH wire surface via the exact seam the shadow layer calls + // right before its rethrow — recordSidecarOpenFailure(error string) on a + // default(full)-mode request. Same state shape either way reaches here. + recordSidecarOpenFailure("no SQLite engine available (node:sqlite missing; better-sqlite3 self-download failed: ECONNREFUSED registry.npmjs.org)"); _setStoreForTest(store); const upstreamPort = await freePort(); const proxy = await startServer(makeOpts(upstreamPort)); @@ -125,7 +130,7 @@ test("#3040 forced none-engine: status+overview report degraded; diagnostics blo assert.equal(o.storage.effective, "legacy"); assert.equal(o.storage.degraded, true); assert.equal(o.storage.intentionalOptOut, false); - assert.ok(typeof o.storage.reason === "string" && o.storage.reason.length > 0, "degrade reason present"); + assert.ok(typeof o.storage.reason === "string" && o.storage.reason.length > 0, "failure reason present"); assert.match(o.storage.reason ?? "", /no SQLite engine available/); assert.equal(o.storage.engine, null); assert.equal(o.storage.sidecarDisabled, false); @@ -138,7 +143,7 @@ test("#3040 forced none-engine: status+overview report degraded; diagnostics blo assert.ok(d.blob.includes("os: "), "blob carries the OS line"); assert.ok(d.blob.includes("requested mode: full")); assert.ok(d.blob.includes("effective mode: legacy")); - assert.ok(d.blob.includes("degrade reason:"), "blob carries the degrade reason"); + assert.ok(d.blob.includes("degrade reason:"), "blob carries the failure reason"); assert.ok(d.blob.includes("engine bootstrap: failed"), "blob carries the ladder outcome"); assert.ok(d.blob.includes("BILI_STORAGE_UNIFIED: (unset)"), "blob carries the raw knob values"); assert.ok(d.blob.includes("sessions on disk:"), "blob carries the session count"); @@ -160,16 +165,6 @@ test("#3040 forced none-engine: status+overview report degraded; diagnostics blo assert.ok(WEB_CLIENT.includes('t("sys.storage_degraded"'), "client composes the localized sentence with {reason}"); assert.ok(WEB_CLIENT.includes('t("sys.report_copy")'), "client offers the copy button"); assert.ok(WEB_CLIENT.includes('t("sys.report_issue")'), "client offers the GitHub button"); - assert.ok(WEB_CLIENT.includes('API_ROOT + "/diagnostics?topic="'), "buttons wire to the generic diagnostics endpoint (through the #3373 API root)"); - // the overview poll re-renders the banner every 5s while degraded: the - // diagnostics fetch must be cached per topic, and a failed fetch must - // toast once — otherwise every poll re-hits the endpoint (full blob - // rebuild: listSessions + whole-file log reads) and re-toasts. - assert.ok(WEB_CLIENT.includes("diagCache"), "diagnostics fetch is cached per topic (5s re-render must not re-fetch)"); - assert.ok(WEB_CLIENT.includes("diagToastedOnce"), "a failed diagnostics fetch toasts once, not on every poll"); - // the reason is error-derived text flowing into innerHTML → must be escaped - const escRe = /escapeHtml\(t\("sys\.storage_degraded", \{ reason: sg\.reason \|\| "" \}\)\)/; - assert.ok(escRe.test(WEB_CLIENT), "degrade reason is HTML-escaped before innerHTML"); } finally { await close(proxy); } @@ -264,20 +259,14 @@ test("#3040 BILI_ENCRYPTION_KEY opt-out is intentional: reported, never warned", } }); -test("#3040 engine-ladder-exhausted degrade: banner wire shape with the open-failure reason", { timeout: 180_000 }, async () => { +test("#3040 no-engine ask -> boot REFUSES (no degradation in any case, 2026-10-30); pre-death wire names the cause", { timeout: 180_000 }, async () => { const base = mkdtempSync(path.join(tmpdir(), "bili-storage-banner-")); try { - // #4338: the whole scenario (force the engine ladder exhausted → - // construct the store → serve /__bili/overview → assert the degraded - // wire shape → close) runs in a spawned child so any OS handle the - // store opens is released by process exit before this parent's - // rmrf(base) can hit a Windows EBUSY. (#4816/#4626: the corrupt-index - // cause no longer degrades through this surface — it now fails loudly - // with the files untouched, so the banner contract is pinned through - // the deterministic no-engine path instead.) The child prints a JSON verdict of named checks; - // every one is re-asserted here, so assertion coverage stays 1:1 - // with the in-process era. The spawn env below is the child's only - // isolation — this process's own env and seams stay untouched. + // Runs in a spawned child so any OS handle the refused store touches is + // released by process exit before the parent's rmrf(base) hits Windows + // EBUSY (#4338). Contract: construction THROWS (no server ever comes up + // to serve a degraded page), nothing is left on disk, and the pre-death + // status wire records the cause for post-crash forensics. const childEnv: NodeJS.ProcessEnv = { ...process.env }; delete childEnv.BILI_STORAGE_UNIFIED; delete childEnv.BILI_ENCRYPTION_KEY; From b9eb27a448ae8c0b711fe9343ddd8eea7ea93283 Mon Sep 17 00:00:00 2001 From: ranxianglei-devbox Date: Fri, 30 Oct 2026 20:14:29 +0800 Subject: [PATCH 31/35] never give up on index writes; integrity FAIL verdicts are alarms only (owner policy 2026-10-30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3-strike self-disable breaker is removed entirely: every save keeps retrying the real index write (loud warn stays), reads never flip to legacy-only, fork operations no longer fail closed on a disabled sidecar. Background integrity findings (worker FAIL / throwing scan) are ALARM-ONLY: they log + surface on runtime-status but are NEVER written as a cached verdict, so a runtime finding can never refuse the next boot. Stale FAIL .chk files left behind by older versions are treated as cache misses — each boot re-scans fresh. A PASS verdict is still cached (skip rescan). Fresh, proven corruption found by THIS boot's own quick_check still refuses the start (owner-approved); files stay exactly in place. --- CONFIGURATION.md | 4 +- src/persist.ts | 66 +++---- src/storage/integrity-check.ts | 60 +++---- src/storage/runtime-status.ts | 20 +-- src/storage/shadow.ts | 161 ++++-------------- src/storage/store.ts | 52 ++---- ...sue4816-integrity-throw-quarantine.test.ts | 31 ++-- .../issue4949-rollback-mask-recovery.test.ts | 79 +++++---- tests/issue4982-fork-lineage-conflict.test.ts | 22 ++- tests/storage-engine-proofs.test.ts | 22 ++- tests/storage-integrity-cache.test.ts | 22 ++- 11 files changed, 213 insertions(+), 326 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index db9b1a89fa..6c26fccc63 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -362,7 +362,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | Env tier for storage.backend (#3931): sqlite (default) or mysql (fully wired, #3388 steps 2+3 - saves ingest through the async shadow queue, sync reads ride the JSON envelope, replicas/ext/GC/fork via the async twins (#4911); export too). Wins over the file tier. | | `BILI_MYSQL_DSN` | string | unset | — | MySQL DSN (mysql://user:pass@host:3306/db) for storage.backend="mysql" (#3388, #3931). Wins over the file key storage.mysql.dsn. Empty = the mysql backend cannot open; the documented error names both knobs. The pool opens lazily; closeQuietlyAsync drains the ingest queue and closes it. | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC age sweep window in days (#3463): sessions idle longer than this are archived (one JSON line appended to the single archive log (archive/archive.jsonl) + tombstone row + content/media refs released). 0 disables the age sweep; the manual entry POST /__bili/storage/gc always runs. Config-file key storage.archiveAfterDays is the persistent form. | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN the files are left EXACTLY in place and bili REFUSES TO START with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand, then start again — a fresh index rebuilds from the JSON envelopes). The FAIL verdict is cached so later starts fail fast without rescan. There is NO automatic rename/repair AND no legacy-only fallback: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy), and continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable); ANY open failure (corruption proven OR transient: no engine / CANTOPEN / second writer / readonly / backend DSN) makes bili REFUSE TO START loudly with an actionable error — no silent legacy fallback in any case (2026-10-30 owner policy); deployments that genuinely want JSON-only serve explicitly (BILI_STORAGE_UNIFIED=legacy or unset). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN by THIS BOOT's own scan the files are left EXACTLY in place and bili REFUSES TO START with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand, then start again — a fresh index rebuilds from the JSON envelopes). A successful PASS is cached so later starts skip the rescan; a FAIL verdict is NEVER persisted (2026-10-30 owner policy): runtime/background findings are alarms only (loud logs + runtime-status, the process keeps serving — JSON envelopes remain the recovery source) and every boot decides again from a fresh scan of the CURRENT file set, so a stale FAIL left behind by an older version can never block future boots. There is NO automatic rename/repair AND no legacy-only fallback: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy), and continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable); ANY open failure (corruption proven OR transient: no engine / CANTOPEN / second writer / readonly / backend DSN) makes bili REFUSE TO START loudly with an actionable error — no silent legacy fallback in any case (2026-10-30 owner policy); deployments that genuinely want JSON-only serve explicitly (BILI_STORAGE_UNIFIED=legacy or unset). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | Snapshot-render shadow observability (#3756 follow-up): compute the outbound snapshot view on every armed turn WITHOUT using it, classify the diff against the legacy outbound, and surface per-session counters in the web panel + the snapshot-shadow diagnostics topic. 0/false/off disables; 1/true/on enables; otherwise the config-file snapshot.shadow boolean applies (default on). The render lane itself stays rollout-gated (BILI_ROLLOUT_SNAPSHOT_RENDER), so the shadow lane never changes outbound bytes - it is the dry-run evidence that flips the rollout later. | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | Override the config file path (path relocation). | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | Where session records are stored (path relocation). | @@ -2010,7 +2010,7 @@ File keys resolve only when the matching env var is unset. Defaults in parenthes | `BILI_PERSIST_DEBOUNCE_MS` | Debounce window for persistence writes to disk, in ms (default `500`). | | `BILI_PERSIST_TAIL_TOKENS` | Token budget for the persisted conversation snapshot (#401). The disk record stores the **folded view** (block summaries in place of compressed ranges) truncated to the newest messages within this budget — never the raw full history. Default `16384`; `0` disables message persistence entirely (block summaries and compressed originals still persist; `bili export` falls back to block-only rendering). Live in-memory sessions are unaffected — `bili export` of a live session is always complete. | | `BILI_PERSIST_ZSTD` | Set `1`/`true` to enable zstd compression of session files (#1080, owner decision: **default off** — plain JSON maximizes recoverability: jq/grep-debuggable, and immune to the downgrade tail risk). When enabled, every session file that actually shrinks is written as `BILIZSTD1` — a small header (magic + format version + mode byte) over the JSON body, zstd-compressed on Node ≥ 22.15 and stored raw on older runtimes; readers accept both bodies (native zstd, else the bundled WASM fallback), so files stay readable across runtimes and versions. Payloads the framing cannot shrink (tiny sessions) and key-less raw bodies stay bare JSON on disk. Independent of `BILI_ENCRYPTION_KEY` — with both active, the compression choice is recorded in the mode byte inside `BILIENC1`. Existing plain-JSON files are NEVER rewritten at boot (downgrade safety — a mass re-encode would make rolling back to an older bili read its own writes as "corrupt"); they convert organically on their next save. Note: once a session has been saved as `BILIZSTD1`, an OLDER bili (< 0.1.135) cannot read it — this caveat applies only to deployments that opted in; unset or any other value keeps plain JSON. | -| `BILI_STORAGE_UNIFIED` | Tiered storage-migration switch (#2671). Config-file tier: `{"storage":{"unified":"legacy|shadow|full"}}` selects the same modes (strict spellings) and applies whenever the env is unset; `BILI_STORAGE_UNIFIED` (which also accepts `1`/`true` = shadow and `unified` = full) always wins over the file. The mode is resolved once per lane boot — hot edits to the file take effect on the next restart. Set `1`/`true` to enable the **unified SQLite storage dry-run** (Phase 1). Every session write is double-ingested into `/index.db` (engine resolution order since #2833: built-in `node:sqlite` when the runtime provides it (Node >= 22.5) -> an already-installed host `better-sqlite3` native module -> a previously self-downloaded cached copy -> a one-time self-download of the pinned `better-sqlite3` prebuilt tarball from the npm registry (~11 MB, sha512-verified, gated by `storage.sqliteAutoDownload`, unsupported platforms never download; engine artifacts are pinned per Node major (Node >= 22: better-sqlite3 v13; Node 20: v12.9.0 plus its ABI-keyed prebuild), and a runtime no pinned artifact serves is refused before any load, with neither the download nor a cached copy registered, since loading such a binding crashes the process (#3341, #3894)) -> none, in which case the sidecar stays legacy-only per the #2825 degrade contract; engine acquisition never depends on the host's install-time build-script approval) and the unified replay is digest-reconciled against the legacy load view — a mismatch logs `[storage] unified shadow DIGEST DRIFT …` and never blocks or alters the write path (legacy files stay the single source of truth). Set `full`/`unified` to enable the **drop-in read path** (Phase 1.5): `loadSync`/`boot`/`loadAll`/`loadStateForSearch` go unified-first — a legacy-file session is organically imported into the index on first open (`[storage] unified import on first open: …`) and later loads are served from `index.db`, with automatic fallback to the legacy file on any unified failure. Legacy files keep being double-written, so **rollback = unset the knob** (env and/or file value). Hard encryption opt-out: while `BILI_ENCRYPTION_KEY` is active the whole sidecar refuses to open (index.db would hold cleartext ingest payloads; a `unified sidecar DISABLED` warning is logged and the store stays legacy-only). Enumeration/GC and the web session list remain file-backed; `removeLegacyFile` now also deletes the index rows and releases CAS refcounts. Digests treat the last-writer `biliVersion` stamp and wall-clock timestamps as environment (`__VER__`/`__TS__`), everything else must match byte-for-byte. The shadow self-disables after 3 consecutive internal failures (in full mode that also flips reads back to legacy-only). **Replica/checkpoint REST (Phase 2)**: on any unified lane (shadow or full), `POST /__bili/replicas {sessionId, label?, pinned?}` freezes the session's persisted state as one content-addressed checkpoint blob + row (the resident session is flushed first), `GET /__bili/replicas[?sessionId=]` lists them, `GET /__bili/replicas/` returns detail + byte length, `POST /__bili/replicas//restore {sessionId?}` materializes the checkpoint as a NEW first-class session through the normal dual write path (JSON + unified ingest; `meta.restoredFrom`/`restoredCheckpoint` record provenance), and `DELETE /__bili/replicas/` drops it. Pinned checkpoints hold a CAS refcount and survive `deleteSession` GC (the whole point: a copy taken before a destructive experiment outlives it); explicit DELETE still removes even pinned ones. Disk expectation: shadow/full roughly doubles unique-content bytes (every session has a JSON file AND index.db rows sharing a content CAS with no dedup across the two stores) and there is no auto-VACUUM — size the sessions dir accordingly while the migration runs. Legacy-only lanes answer 501 with the knob hint. Routes are loopback-only like the rest of `__bili`. Defaults to `full` when unset (#3818). | +| `BILI_STORAGE_UNIFIED` | Tiered storage-migration switch (#2671). Config-file tier: `{"storage":{"unified":"legacy|shadow|full"}}` selects the same modes (strict spellings) and applies whenever the env is unset; `BILI_STORAGE_UNIFIED` (which also accepts `1`/`true` = shadow and `unified` = full) always wins over the file. The mode is resolved once per lane boot — hot edits to the file take effect on the next restart. Set `1`/`true` to enable the **unified SQLite storage dry-run** (Phase 1). Every session write is double-ingested into `/index.db` (engine resolution order since #2833: built-in `node:sqlite` when the runtime provides it (Node >= 22.5) -> an already-installed host `better-sqlite3` native module -> a previously self-downloaded cached copy -> a one-time self-download of the pinned `better-sqlite3` prebuilt tarball from the npm registry (~11 MB, sha512-verified, gated by `storage.sqliteAutoDownload`, unsupported platforms never download; engine artifacts are pinned per Node major (Node >= 22: better-sqlite3 v13; Node 20: v12.9.0 plus its ABI-keyed prebuild), and a runtime no pinned artifact serves is refused before any load, with neither the download nor a cached copy registered, since loading such a binding crashes the process (#3341, #3894)) -> none, in which case the sidecar stays legacy-only per the #2825 degrade contract; engine acquisition never depends on the host's install-time build-script approval) and the unified replay is digest-reconciled against the legacy load view — a mismatch logs `[storage] unified shadow DIGEST DRIFT …` and never blocks or alters the write path (legacy files stay the single source of truth). Set `full`/`unified` to enable the **drop-in read path** (Phase 1.5): `loadSync`/`boot`/`loadAll`/`loadStateForSearch` go unified-first — a legacy-file session is organically imported into the index on first open (`[storage] unified import on first open: …`) and later loads are served from `index.db`, with automatic fallback to the legacy file on any unified failure. Legacy files keep being double-written, so **rollback = unset the knob** (env and/or file value). Hard encryption opt-out: while `BILI_ENCRYPTION_KEY` is active the whole sidecar refuses to open (index.db would hold cleartext ingest payloads; a `unified sidecar DISABLED` warning is logged and the store stays legacy-only). Enumeration/GC and the web session list remain file-backed; `removeLegacyFile` now also deletes the index rows and releases CAS refcounts. Digests treat the last-writer `biliVersion` stamp and wall-clock timestamps as environment (`__VER__`/`__TS__`), everything else must match byte-for-byte. The index write path NEVER gives up: failed ingests are logged (with file + error detail on the first failure) and every subsequent save retries the real write — no self-disable, no cooldown, no fallback mode (2026-10-30 owner policy), while the JSON envelope keeps serving reads meanwhile. **Replica/checkpoint REST (Phase 2)**: on any unified lane (shadow or full), `POST /__bili/replicas {sessionId, label?, pinned?}` freezes the session's persisted state as one content-addressed checkpoint blob + row (the resident session is flushed first), `GET /__bili/replicas[?sessionId=]` lists them, `GET /__bili/replicas/` returns detail + byte length, `POST /__bili/replicas//restore {sessionId?}` materializes the checkpoint as a NEW first-class session through the normal dual write path (JSON + unified ingest; `meta.restoredFrom`/`restoredCheckpoint` record provenance), and `DELETE /__bili/replicas/` drops it. Pinned checkpoints hold a CAS refcount and survive `deleteSession` GC (the whole point: a copy taken before a destructive experiment outlives it); explicit DELETE still removes even pinned ones. Disk expectation: shadow/full roughly doubles unique-content bytes (every session has a JSON file AND index.db rows sharing a content CAS with no dedup across the two stores) and there is no auto-VACUUM — size the sessions dir accordingly while the migration runs. Legacy-only lanes answer 501 with the knob hint. Routes are loopback-only like the rest of `__bili`. Defaults to `full` when unset (#3818). | | `BILI_PERSIST_EPERM_ALERT_THRESHOLD` | N consecutive persist write failures (`EPERM`/`EBUSY`/`EACCES`) on one session before the one-time "add this dir to antivirus exclusions" alert fires (default `5`). Windows only. See [Windows: exclude the sessions dir](#windows-exclude-the-sessions-dir-from-antivirus-362). | | `BILI_PERSIST_EPERM_ALERT_REPEAT_MS` | Re-alert window for the persist EPERM alert, in ms. `0` (default) = alert once then stay silent; `>0` = re-alert at most every that many ms while the failures continue. | | `BILI_TUNNEL_ALLOWED_HOSTS` | `/bili/` tunnel admission for **remote clients** (#409): comma-separated `host` or `host:port` entries that unlock loopback/private destinations (e.g. a LAN relay or the machine's own sglang) for non-loopback clients. The proxy itself and link-local/metadata addresses are always denied; local (loopback) clients always pass. | diff --git a/src/persist.ts b/src/persist.ts index 06dbfcfbd5..f28632ec79 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2720,50 +2720,22 @@ export class SessionStore { // can only ride along, never be missing, so the spill gate's // equality check against the CURRENT epoch stays conservative. const proofRevision = session.revisionEpoch; - let innerDrained = false; let settledOk = false; try { this.guardedBuild(session)(); this.maybeCutForkBase(session); - // #3135 ①/#3818: the 3-strike may have fired INSIDE that - // build (shadow.record's catch disables the sidecar after - // consecutive failures). writerOffActive() turns false for - // the NEXT save — but this very session currently exists - // on NEITHER copy (index got no rows, envelope was never - // armed). Land one synchronous JSON write NOW via the - // kernel store so the disabling tick always closes with - // the data on the durable file copy. - if (this.shadow && this.shadow.stats.disabled) { - this.log("warn", `[persist] unified sidecar disabled mid-build for ${session.id} — landing an immediate JSON envelope write (#3135 ①)`); - const innerProof = session.revisionEpoch; - const build = this.guardedBuild(session); - this.saveWritesInFlight.add(session.id); - void this.store.writeNow(session.id, build) - .then(() => { - this.stampOwnWrite(session.id); - this.noteDurable(session.id, innerProof); - this.lastSaveLanded.set(session.id, true); - this.runIdleSpillIfIdle(session.id); - innerDrained = true; - this.drainFlushWaiters(session.id, true); - }) - .catch(() => { - // Errors surface through the kernel write chain's own logging. - this.lastSaveLanded.set(session.id, false); - innerDrained = true; - this.drainFlushWaiters(session.id, false); - }) - .finally(() => { this.saveWritesInFlight.delete(session.id); }); - } else { - // #4788 D2: the index-only build landed synchronously — - // run the same settle protocol as a kernel-write land - // (invariant #5: this routing path gets equivalent - // drop behavior, gated identically). - this.noteDurable(session.id, proofRevision); - this.lastSaveLanded.set(session.id, true); - this.runIdleSpillIfIdle(session.id); - settledOk = true; - } + // #4788 D2: the index-only build landed synchronously — + // run the same settle protocol as a kernel-write land + // (invariant #5: this routing path gets equivalent + // drop behavior, gated identically). The JSON-envelope + // safety valve for a mid-build disable (#3135 ①) was + // removed together with the 3-strike breaker itself + // (2026-10-30 owner policy: index writes are never + // abandoned, jsonWriter=off accepts index-only persistence). + this.noteDurable(session.id, proofRevision); + this.lastSaveLanded.set(session.id, true); + this.runIdleSpillIfIdle(session.id); + settledOk = true; // #3818 writerOff safety valve REMOVED (#4216/#4242): // the orphan/reap shape is now representable — its // contents ride sessions.orphans_json in the same @@ -2776,9 +2748,7 @@ export class SessionStore { // without process-level handlers; the next markDirty retries. this.log("error", `[persist] writerOff coalesced build failed for ${session.id}: ${String(err)}`); } finally { - if (!innerDrained) { - this.drainFlushWaiters(session.id, settledOk); - } + this.drainFlushWaiters(session.id, settledOk); } }; this.writerOffPending.set(session.id, run); @@ -3273,11 +3243,13 @@ export class SessionStore { return this.saveTimers.has(id) || this.writerOffPending.has(id); } - /** Phase 3 (#2960): the JSON envelope is skipped only while the sidecar - * is both configured off AND actually alive — a mid-run sidecar death - * (3-strike disable) automatically resumes file writes. */ + /** Phase 3 (#2960): the JSON envelope is skipped while the operator asked + * for index-only persistence AND the sidecar opened. Static decision: + * the 3-strike breaker that used to re-arm envelope writes mid-run was + * removed (2026-10-30 owner policy — index writes are never abandoned, + * so there is nothing to fall back FROM anymore). */ private writerOffActive(): boolean { - return this.jsonWriterOff && this.shadow !== undefined && !this.shadow.stats.disabled; + return this.jsonWriterOff && this.shadow !== undefined; } /** #4323: per-session envelope freeze (escalated structural drift, see diff --git a/src/storage/integrity-check.ts b/src/storage/integrity-check.ts index b0a3b264cf..075e79e730 100644 --- a/src/storage/integrity-check.ts +++ b/src/storage/integrity-check.ts @@ -20,9 +20,10 @@ * read-only connection in a worker thread (WAL allows concurrent * readers, and the main handle has already run recovery, so the * read-only open cannot hit WAL-recovery). Boot returns in - * milliseconds; a failure is recorded loudly and cached so the NEXT - * open throws StorageIntegrityError and the open REFUSES (no automatic - * file moves AND no legacy fallback, #4626-era owner decision chain). Small stores keep the + * milliseconds; a failure is recorded loudly as an ALARM ONLY — never + * persisted as a boot-blocker (2026-10-30 owner policy: a running + * store's background finding must not refuse future boots; each next + * boot re-scans fresh). No automatic file moves, #4175. * synchronous pre-migration check — the #4175 premise (a corrupt file * must never be migrated against) stays intact wherever it is cheap. * The one concession: on a big store's first boot after upgrade, @@ -64,7 +65,6 @@ interface IntegrityCache { type IntegrityMode = | { kind: "skip" } | { kind: "cached-ok" } - | { kind: "cached-fail"; detail: string } | { kind: "sync" } | { kind: "async"; sizeBytes: number }; @@ -130,7 +130,10 @@ export function integrityModeFor(dbPath: string, env: { BILI_STORAGE_QUICK_CHECK cache.schema === STORAGE_SCHEMA_VERSION; if (matches) { if (cache.ok) return { kind: "cached-ok" }; - return { kind: "cached-fail", detail: cache.detail ?? "cached quick_check failure" }; + // A stale FAIL verdict planted by an OLDER version (this one never + // caches failures anymore) no longer refuses boots — owner policy + // 2026-10-30: treat it as a cache miss and let a FRESH scan decide. + loggerLog("warn", `[storage] previous quick_check verdict for ${dbPath} was FAIL (${cache.detail ?? "no detail"}) — stale verdicts no longer block boots; re-checking with a fresh scan (2026-10-30 owner policy)`); } } const size = statPart(dbPath)?.s ?? 0; @@ -174,10 +177,11 @@ try { const inflight = new Set(); /** Fire-and-forget background quick_check on a second read-only connection - * (#4609). Records the outcome, caches it (ok or fail — a cached fail makes - * the NEXT open throw and REFUSE via the #4175 path), and tells the - * caller's store instance so a clean close can refresh the cache key. */ -export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok: boolean, detail?: string) => void): void { + * (#4609). Records the outcome for the operator's alarms (ok or fail — + * 2026-10-30 owner policy: background findings are ALARMS ONLY, never + * boot-blockers — no cache write on failure), and tells the caller's store + * instance so a clean close can refresh the PASS cache key. */ +export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok: boolean) => void): void { const size = statPart(dbPath)?.s ?? 0; loggerLog("info", `[storage] background quick_check started for ${dbPath} (${(size / (1024 * 1024)).toFixed(1)} MiB over the ${Math.round(effectiveSyncThreshold() / (1024 * 1024))} MiB sync threshold) — serving continues; the verdict lands in /__bili/status (#4609)`); inflight.add(dbPath); @@ -207,36 +211,26 @@ export function scheduleBackgroundIntegrityCheck(dbPath: string, onVerdict?: (ok writeIntegrityCache(dbPath, true); onVerdict?.(true); } else { - // A real integrity failure found post-open: record loudly - // and cache the fail so the NEXT open throws StorageIntegrityError - // and REFUSES to start INSTEAD of rescan-looping (#4175 path, - // no automatic rename — files stay where they are); keep - // serving now (writes into a provably-corrupt store surface - // as ingest errors and the #3133 breaker disables the - // sidecar inside this process if they persist; envelopes - // are the recovery source either way). + // A real integrity failure found post-open: record loudly as an + // ALARM (owner policy 2026-10-30). It is NOT cached as a + // boot-blocker — the NEXT boot re-scans and decides from fresh + // evidence; files stay exactly where they are (no automatic + // rename, #4175); keep serving now (writes into a provably- + // corrupt store surface as ingest errors that are retried on + // every subsequent save; envelopes are the recovery source). recordIntegrityOutcome(dbPath, false, 0, verdict.slice(0, 400)); - writeIntegrityCache(dbPath, false, verdict.slice(0, 400)); - loggerLog("error", `[storage] background quick_check FAILED for ${dbPath}: ${verdict.slice(0, 400)} — verdict cached; the next open FAILS FAST AND REFUSES TO START (files left UNTOUCHED — move them aside by hand after backing up to force a rebuild; JSON envelopes remain the recovery source, #4175/#4609)`); - // Detail rides with the verdict so a clean close can - // refresh the FAIL cache onto post-checkpoint stats - // (#5108/#4816) — without it the verdict would go stale - // after the first clean restart and never fail fast again. - onVerdict?.(false, verdict.slice(0, 400)); + loggerLog("error", `[storage] background quick_check FAILED for ${dbPath}: ${verdict.slice(0, 400)} — ALARM ONLY, no boot-blocking verdict written (owner policy 2026-10-30: runtime findings never refuse future boots — each boot re-scans fresh); files left UNTOUCHED (back up ${dbPath}{,-wal,-shm} and move them aside only if you want to force a rebuild); JSON envelopes remain the recovery source (#4175/#4609)`); + onVerdict?.(false); } } else if (msg.corrupt === true) { // #5108/#4816: the scan itself threw a corruption-shaped error - // (the throwing shapes of CORRUPT/NOTADB). Same consequences as - // a bad verdict row: record loudly and cache the fail so the - // NEXT open fails fast and refuses to start (#4175 path). - // Without this branch the throw fell into "could not run" - // below and the fail verdict was never recorded — a large - // corrupt store stayed unverified boot after boot. + // (the throwing shapes of CORRUPT/NOTADB). Same ALARM-ONLY + // consequences as a bad verdict row (owner policy 2026-10-30: + // loud alarm, no cache write, next boot re-scans fresh). const detail = String(msg.error ?? "").slice(0, 400); recordIntegrityOutcome(dbPath, false, 0, detail); - writeIntegrityCache(dbPath, false, detail); - loggerLog("error", `[storage] background quick_check threw a corruption-shaped error for ${dbPath}: ${detail} — FAIL verdict cached; the next open FAILS FAST AND REFUSES TO START (files left UNTOUCHED — move them aside by hand after backing up to force a rebuild; JSON envelopes remain the recovery source, #5108/#4175)`); - onVerdict?.(false, detail); + loggerLog("error", `[storage] background quick_check threw a corruption-shaped error for ${dbPath}: ${detail} — ALARM ONLY, no boot-blocking verdict written (owner policy 2026-10-30); files left UNTOUCHED`); + onVerdict?.(false); } else { loggerLog("warn", `[storage] background quick_check could not run for ${dbPath}: ${String(msg.error)} — the store serves unverified; the next boot rechecks`); } diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index 334ff29499..a846491ab1 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -29,7 +29,7 @@ import { fileStorageAutoDownload, fileStorageJsonWriter, fileStorageUnified, fil import { log as loggerLog } from "../logger.js"; import { safePrefix, safeSuffix } from "../text-safe.js"; import { resolvedSqliteEngineInfo } from "./driver.js"; -import { MAX_CONSECUTIVE_FAILURES, sqliteOpenFailureHint, type UnifiedShadow } from "./shadow.js"; +import { sqliteOpenFailureHint, type UnifiedShadow } from "./shadow.js"; import type { StorageMode } from "./shadow.js"; /** The driver's no-engine error text — one predicate, used wherever the @@ -196,11 +196,6 @@ function shortenError(err: string): string { } function degradeReason(): string { - const sidecar = state.sidecar; - if (sidecar && sidecar.stats.disabled) { - const last = sidecar.stats.lastError ? shortenError(sidecar.stats.lastError) : "unknown"; - return `unified store auto-disabled after ${MAX_CONSECUTIVE_FAILURES} consecutive failures (last: ${last})`; - } // No pinned artifact serves this runtime (below every Node floor, or a Node // ABI no prebuild targets), so the engine is refused BEFORE any load and the // sidecar open ALSO fails with the generic no-engine error — surface the real @@ -298,12 +293,11 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus { // never surface as a degrade. const intentionalOptOut = state.encryptionBlocked || state.persistDisabled; const sidecar = state.sidecar; - const disabled = !!sidecar?.stats.disabled; // healthy sidecar ⇒ its mode === requested (maybeOpen opens only under // shadow/full); anything else means SQLite carries no data right now. const effective: "legacy" | "shadow" | "full" = - requested !== "legacy" && !intentionalOptOut && sidecar && !disabled ? requested : "legacy"; - const degraded = requested !== "legacy" && !intentionalOptOut && (sidecar === null || disabled); + requested !== "legacy" && !intentionalOptOut && sidecar ? requested : "legacy"; + const degraded = requested !== "legacy" && !intentionalOptOut && sidecar === null; const cd = cooldownState(); // #3923/#3959: the second-process warning must never freeze — a foreign // pid that has since exited is cleared here, at the status read. @@ -337,8 +331,12 @@ function getStorageRuntimeStatus(): StorageRuntimeStatus { bootstrap: state.bootstrap?.status ?? null, bootstrapFailure: cd.marker, cooldownRemainingMs: cd.remainingMs, - sidecarDisabled: disabled, - sidecarLastError: sidecar?.stats.disabled ? sidecar.stats.lastError : null, + // The breaker that used to self-disable the sidecar (3-strike) was + // removed by owner policy 2026-10-30 — index writes are never + // abandoned — so this field is permanently false (kept for the + // pre-existing API shape). + sidecarDisabled: false, + sidecarLastError: sidecar?.stats.lastError ?? null, indexDbOpenError: state.sidecarOpenError, // #3135 ④/#3818: the sampled digest-drift counter — once the index is // the default load path (full), drift must be OBSERVABLE, not just a diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index a4daaf741b..5a4a1c99c3 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -16,9 +16,12 @@ * env), but the unified index is the load path. * * Failure policy: sidecar problems must NEVER break the legacy write path. - * All work is try/catch'd; after MAX_CONSECUTIVE_FAILURES consecutive - * failures the sidecar disables itself for the process (warn once) — in - * full mode that also flips reads back to legacy-only. + * All work is try/catch'd. The sidecar NEVER gives up on index writes + * (2026-10-30 owner policy): an ingest failure is recorded loudly — one warn + * per DISTINCT error message, so a stuck failure mode does not spam every + * turn's log line — and every subsequent save retries the index write as + * usual. No self-disable / circuit breaker exists anymore; the JSON envelope + * remains the recovery source either way. */ import { log as loggerLog } from "../logger.js"; @@ -34,7 +37,6 @@ import type { SqliteEngineName, SqliteEngineSource } from "./driver.ts"; import type { MessageRow, BlockRow, SessionRow, RefRow, MessageRefContentRow } from "./store.js"; import type { CoreMessage, MessageContentStore } from "acp-kernel"; -export const MAX_CONSECUTIVE_FAILURES = 3; const VERIFY_BATCH_SIZE = 32; /** Bound transient CAS strings during normal cold loads, not the retained history. */ @@ -46,14 +48,6 @@ interface ResidentFields { blockContents?: Map; } -/** #4949: re-enable cadence for the disabled sidecar. The 3-strike breaker used - * to be permanent until process restart even after the store healed. Now each - * cooldown window lets the NEXT save run one writability probe (a single-row - * committed write through the production path); success clears the breaker and - * the triggering record ingests normally. Missed saves converge via the - * load-side staleness guard (envelope savedAt > index row) + next full ingest, - * so no global pre-check is needed before unified reads resume. */ -export const REENABLE_PROBE_COOLDOWN_MS = 5 * 60 * 1000; /** #3133: read-back verification cadence — the verify leg (full read-back + * double shadowDigest) is O(log) per save, while the drift it guards is a @@ -115,6 +109,9 @@ interface ShadowStats { verifyOk: number; verifyDrift: number; imports: number; + /** Retained on the status wire for panel compatibility. Never set to true + * anymore — the self-disable breaker was removed (2026-10-30 owner policy: + * index writes are never abandoned). Always false. */ disabled: boolean; lastError: string | null; /** #3818: records skipped from the mirror because they carry orphan @@ -176,10 +173,10 @@ export class UnifiedShadow { * Ingest-failure warnings must name it — "staying legacy-only" without a * path left users hunting %TEMP% logs for which file said what. */ private readonly dbPath: string | null; + /** Streak gauge for diagnostics/wire only — it never disables anything + * (the breaker was removed, 2026-10-30 owner policy); resets on the next + * successful ingest. */ private consecutiveFailures = 0; - /** #4949: timestamp until which the disabled sidecar stays quiet; the first - * save after it runs the writability probe (REENABLE_PROBE_COOLDOWN_MS). */ - private nextReenableProbeAt = 0; /** #3133: per-session ingest counters (verify sampling). Per-process * only — nothing persisted, no schema change. */ private readonly ingestCounts = new Map(); @@ -192,20 +189,18 @@ export class UnifiedShadow { this.dbPath = dbPath ?? null; } - /** #5103: one ingest failure recorded — the FIRST failure of a streak now - * warns with the file named (before, strikes one and two were silent and - * the user learned of the degradation from missing features, not logs); - * at MAX_CONSECUTIVE_FAILURES the sidecar disables itself, same as before. */ + /** #5103 / 2026-10-30 owner policy (never abandon index writes): record + * every ingest failure with the file named — but a streak of the SAME + * error message warns once (a permanently broken index must not add a log + * line to every turn), while any NEW message warns immediately. Nothing is + * ever disabled; the next save retries the write unconditionally. */ private noteIngestFailure(err: unknown): void { - this.consecutiveFailures++; + const prev = this.stats.lastError ?? ""; const errStr = String(err); + this.consecutiveFailures += 1; this.stats.lastError = errStr; - if (this.consecutiveFailures === 1) { - loggerLog("warn", `[storage] unified sidecar ${this.dbPath ?? "backend"} failed to ingest (${errStr})${sqliteOpenFailureHint(errStr)} — serving from JSON envelopes until it recovers or disables after ${MAX_CONSECUTIVE_FAILURES} consecutive failures`); - } - if (this.consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) { - this.stats.disabled = true; - loggerLog("warn", `[storage] unified sidecar ${this.dbPath ?? "backend"} disabled after ${this.consecutiveFailures} consecutive failures (last: ${errStr})${sqliteOpenFailureHint(errStr)} — staying legacy-only`); + if (errStr !== prev) { + loggerLog("warn", `[storage] unified sidecar ${this.dbPath ?? "backend"} failed to ingest (${this.consecutiveFailures} in a row; ${errStr})${sqliteOpenFailureHint(errStr)} — the index write will be RETRIED on every subsequent save until it succeeds again (no self-disable; the JSON envelope keeps serving meanwhile)`); } } @@ -395,9 +390,11 @@ export class UnifiedShadow { return new UnifiedShadow(mode, null, legacyLoad ?? (() => null), backend); } - /** True when reads should go unified-first (full mode, sidecar healthy). */ + /** True when reads should go unified-first (full mode). A failing sidecar + * still serves reads through the drift/lag guards (loadSync watermark + + * #4323 richer-view pick), so ingest health is not part of this gate. */ get readsUnified(): boolean { - return this.mode === "full" && !this.stats.disabled; + return this.mode === "full"; } // engine/engineSource introspection lives in engineInfo (sqlite-only @@ -413,26 +410,11 @@ export class UnifiedShadow { * accounting stays honest. */ record(record: unknown, rawTail?: { lastMessages: CoreMessage[]; lastMessagesFolded?: boolean }, liveCcr?: MessageContentStore): void { // #3931 step 3: network backend — the ingest runs on the serial - // queue (fire-and-forget; failures feed the same 3-strike disable). + // queue (fire-and-forget; failures are logged once per distinct message — never disabled). if (this.sync === null) { this.enqueueAsyncRecord(record, rawTail, liveCcr); return; } - // #4949: a disabled sidecar is no longer dead for the process lifetime — - // one cooldown-gated writability probe per window; on success the - // breaker clears and THIS record falls through to the normal ingest. - if (this.stats.disabled) { - const now = Date.now(); - if (now < this.nextReenableProbeAt) return; - this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS; - try { - this.sync.probeWritable(); - } catch (err) { - this.noteProbeFailure(err); - return; - } - this.clearBreaker(); - } try { const loaded = this.legacyLoad(record); if (!loaded) return; @@ -570,21 +552,6 @@ export class UnifiedShadow { } } - /** #4949: re-enable-probe failure tail — keeps the breaker armed while the - * latest error stays visible on the status wire via stats.lastError. */ - private noteProbeFailure(err: unknown): void { - this.stats.lastError = String(err); - loggerLog("debug", `[storage] unified store re-enable probe failed (${String(err)}) — staying disabled, next probe after ${REENABLE_PROBE_COOLDOWN_MS / 60_000}min (#4949)`); - } - - /** #4949: clears the 3-strike breaker after a successful writability probe. */ - private clearBreaker(): void { - this.stats.disabled = false; - this.stats.lastError = null; - this.consecutiveFailures = 0; - loggerLog("info", "[storage] unified store RE-ENABLED: writability probe succeeded after disable (#4949); sessions missed during the outage converge via the envelope staleness guard + next full ingest"); - } - /** #3133 delta ingest attempt for one record's legacy view — returns * true when the O(new-messages) fast path committed. Every doubt * routes to `false` (→ the caller's full ingest); failures here must @@ -632,7 +599,6 @@ export class UnifiedShadow { viewFor(id: string): LegacySessionLike | null { const store = this.sync; if (store === null) return null; - if (this.stats.disabled) return null; try { const unified = store.readSnapshot(() => { const sessionRow = store.getSessionRow(id); @@ -662,7 +628,7 @@ export class UnifiedShadow { /** Lazy resident fields share the same source header without rebuilding a session. */ readResidentFieldsFor(id: string, fields: { metadata?: boolean; blockContents?: boolean }): ResidentFields | null { const store = this.sync; - if (store === null || this.stats.disabled) return null; + if (store === null) return null; try { return store.readSnapshot(() => { const header = store.getLoadHeader(id); @@ -704,7 +670,6 @@ export class UnifiedShadow { * the sqlite fast lane is unavailable; callers must not guess from null. */ activeBlockCount(id: string): number | null { if (this.sync === null) return null; - if (this.stats.disabled) return null; try { return this.sync.getBlocks(id).filter((b) => b.active).length; } catch { @@ -716,7 +681,6 @@ export class UnifiedShadow { * seen so future loads serve from the unified store. Never throws. */ importRecord(record: unknown): LegacySessionLike | null { if (this.sync === null) return null; - if (this.stats.disabled) return null; try { const legacy = this.legacyLoad(record); if (!legacy) return null; @@ -741,7 +705,6 @@ export class UnifiedShadow { this.ingestCounts.delete(id); this.driftStreaks.delete(id); this.frozenEnvelopeIds.delete(id); - if (this.stats.disabled) return; try { this.sync.deleteSession(id); } catch (err) { @@ -756,31 +719,26 @@ export class UnifiedShadow { createCheckpoint(cp: Parameters[0]): void { if (this.sync === null) throw new Error('[storage] sync checkpoint API unavailable on a network backend — use the async twin'); - this.assertLive(); this.sync.insertCheckpoint(cp); } listCheckpoints(sessionId?: string): CheckpointRow[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.listCheckpoints(sessionId); } getCheckpoint(cpId: string): CheckpointRow | undefined { if (this.sync === null) return undefined; - this.assertLive(); return this.sync.getCheckpoint(cpId); } checkpointState(cpId: string): { stateJson: string; byteLen: number } { if (this.sync === null) throw new Error('[storage] sync checkpoint API unavailable on a network backend — use the async twin'); - this.assertLive(); return this.sync.checkpointState(cpId); } deleteCheckpoint(cpId: string): boolean { if (this.sync === null) return false; - this.assertLive(); return this.sync.deleteCheckpoint(cpId); } @@ -788,7 +746,6 @@ export class UnifiedShadow { * ingested). See SessionStore.buildForLoadAll for the race this guards. */ sessionUpdatedAt(sessionId: string): number | null { if (this.sync === null) return null; - this.assertLive(); return this.sync.sessionUpdatedAt(sessionId); } @@ -796,7 +753,6 @@ export class UnifiedShadow { * Used as a checkpoint's seq_at. */ sessionTipSeq(sessionId: string): number { if (this.sync === null) return 0; - this.assertLive(); let tip = 0; for (const m of this.sync.getMessages(sessionId)) if (m.seq + 1 > tip) tip = m.seq + 1; return tip; @@ -805,7 +761,6 @@ export class UnifiedShadow { /** #3301: message rows for specific ids (+ seq±1 neighbors) — stored * decompositions for fold-anchor backfill after a wire-shape flip. */ messagesForIds(sessionId: string, msgIds: readonly string[], includeNeighbors: boolean): { seq: number; msgId: string; content: string }[] { - this.assertLive(); if (!this.sync) return []; // network backends cannot serve the sync decomposition lookup (#3931 architecture) return this.sync.messagesForIds(sessionId, msgIds, includeNeighbors); } @@ -815,30 +770,26 @@ export class UnifiedShadow { * enumerable. */ listSessionIds(): string[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.listSessionIds(); } /** Identity lookup passthrough (#2960 R2.1). */ lookupConversation(namespace: string, conversationId: string): string | null { if (this.sync === null) return null; - this.assertLive(); return this.sync.lookupConversation(namespace, conversationId); } /** #3049: O(1) counter probe backing the persist stale-guard fast path. - * Safe on a disabled sidecar? NO — call sites must check stats first - * (this throws like every other pass-through; the guard treats any - * failure as "no fast path" and falls back to the full envelope read). */ + * Throws like every other pass-through when the store rejects it; the + * guard treats any failure as "no fast path" and falls back to the full + * envelope read. */ sessionCounters(sessionId: string): { requests: number; nextBlockId: number } | null { if (this.sync === null) return null; - this.assertLive(); return this.sync.sessionCounters(sessionId); } listConversationBindings(namespace?: string): { namespace: string; conversationId: string; sessionId: string; claimedAt: number }[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.listConversationBindings(namespace); } @@ -897,7 +848,6 @@ export class UnifiedShadow { rawMessages(sessionId: string, fromSeqExclusive?: number): { seq: number; content: string }[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.getMessages(sessionId, fromSeqExclusive !== undefined ? { fromSeqExclusive } : undefined).map((m) => ({ seq: m.seq, content: m.content })); } @@ -905,7 +855,6 @@ export class UnifiedShadow { * (tail_json.tailSeqs). Null when absent/malformed. */ tailSeqs(sessionId: string): number[] | null { if (this.sync === null) return null; - this.assertLive(); return this.sync.tailSeqs(sessionId); } @@ -918,20 +867,17 @@ export class UnifiedShadow { /** #D5-5: identity-only rows (no content bytes) for the render lane. */ messageIdRows(sessionId: string): { seq: number; msgId: string }[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.messageIdRows(sessionId); } /** #D5-5: content rows outside the covered intervals only. */ uncoveredRows(sessionId: string, intervals: readonly [number, number][]): (MessageRow & { contentHash: string })[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.uncoveredRows(sessionId, intervals); } tailViewRows(sessionId: string): MessageRow[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.getMessages(sessionId); } @@ -940,20 +886,17 @@ export class UnifiedShadow { * compaction pruned the resident window. */ messageContentByRefNums(sessionId: string, refNums: readonly number[]): MessageRefContentRow[] { if (this.sync === null || refNums.length === 0) return []; - this.assertLive(); return this.sync.messageContentByRefNums(sessionId, refNums); } sessionSummaries(): SessionSummaryRow[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.sessionSummaries(); } /** #2960 ③: whole-store CCR read from the sessions row's ccr mirror. */ getCcr(sessionId: string): MessageContentStore | null { if (this.sync === null) return null; - this.assertLive(); return this.sync.getCcr(sessionId); } @@ -963,7 +906,6 @@ export class UnifiedShadow { * falls back to the envelope, exactly like today's sync loadSync). */ getSessionMetadata(sessionId: string): Record | null { if (this.sync === null) return null; - if (this.stats.disabled) return null; try { return this.sync.getSessionMetadata(sessionId); } catch { @@ -976,7 +918,6 @@ export class UnifiedShadow { * getSessionMetadata. */ getBlockContents(sessionId: string): import("./store.ts").BlockContentsMap | null { if (this.sync === null) return null; - if (this.stats.disabled) return null; try { return this.sync.getBlockContents(sessionId) as import("./store.ts").BlockContentsMap | null; } catch { @@ -990,13 +931,11 @@ export class UnifiedShadow { * or nothing — a stale map would re-bind citations wrongly. */ seqByRaw(sessionId: string): Map | null { if (this.sync === null) return null; - this.assertLive(); return this.sync.seqByRaw(sessionId); } /** #4316 PR3/3: live (non-retired) message row count. */ liveRowCount(sessionId: string): number { - this.assertLive(); if (this.sync !== null) return this.sync.liveRowCount(sessionId); return -1; } @@ -1006,7 +945,6 @@ export class UnifiedShadow { * store is unavailable or the session has no live rows. */ lastLiveSeq(sessionId: string): number | null { if (this.sync === null) return null; - this.assertLive(); return this.sync.lastLiveSeq(sessionId); } @@ -1018,7 +956,6 @@ export class UnifiedShadow { * (sync === null) cannot serve this sync facade — the fork lane awaits * cloneSessionRowsAsync instead (#4911). */ cloneSessionRows(parentId: string, childId: string, throughSeq: number): boolean { - this.assertLive(); if (this.sync !== null) return this.sync.cloneSessionRows(parentId, childId, throughSeq); return false; } @@ -1026,7 +963,6 @@ export class UnifiedShadow { /** #2960 R2.2: web list rows (stats/metadata JSON + block aggregates). */ webSummaryRows(): WebSummaryRow[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.webSummaryRows(); } @@ -1038,49 +974,41 @@ export class UnifiedShadow { setSessionExt(sessionId: string, key: string, value: unknown): void { if (this.sync === null) return; - this.assertLive(); this.sync.setSessionExt(sessionId, key, value); } getSessionExt(sessionId: string, key: string): unknown { if (this.sync === null) return null; - this.assertLive(); return this.sync.getSessionExt(sessionId, key); } listSessionExt(sessionId: string): { key: string; value: unknown; updatedAt: number }[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.listSessionExt(sessionId); } deleteSessionExt(sessionId: string, key?: string): number { if (this.sync === null) return 0; - this.assertLive(); return this.sync.deleteSessionExt(sessionId, key); } setKv(namespace: string, key: string, value: unknown): void { if (this.sync === null) return; - this.assertLive(); this.sync.setKv(namespace, key, value); } getKv(namespace: string, key: string): unknown { if (this.sync === null) return null; - this.assertLive(); return this.sync.getKv(namespace, key); } listKv(namespace?: string, keyPrefix?: string): { namespace: string; key: string; value: unknown; updatedAt: number }[] { if (this.sync === null) return []; - this.assertLive(); return this.sync.listKv(namespace, keyPrefix); } deleteKv(namespace: string, key?: string): number { if (this.sync === null) return 0; - this.assertLive(); return this.sync.deleteKv(namespace, key); } @@ -1101,24 +1029,9 @@ export class UnifiedShadow { /** record()'s async port — MUST stay in lockstep with the sync body * above (orphan guard + #3841 reap journal, #3133 watermark delta, * full ingest, #3133 sampled verify + drift latch). Serialized by the - * shadow queue; failures feed the same 3-strike disable. */ + * shadow queue; failures feed the same per-message warn — nothing disables (#2432-era breaker removed 2026-10-30). */ private enqueueAsyncRecord(record: unknown, rawTail?: { lastMessages: CoreMessage[]; lastMessagesFolded?: boolean }, liveCcr?: MessageContentStore): void { void this.enqueue(async () => { - // #4949: async twin of the sync re-enable probe in record() — MUST - // stay in lockstep with it; setKv is the backend's single-row - // committed write probe. - if (this.stats.disabled) { - const now = Date.now(); - if (now < this.nextReenableProbeAt) return; - this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS; - try { - await this.backend!.setKv("__bili", "writability-probe", { probedAt: now }); - } catch (err) { - this.noteProbeFailure(err); - return; - } - this.clearBreaker(); - } try { const backend = this.backend!; const loaded0 = this.legacyLoad(record); @@ -1193,7 +1106,6 @@ export class UnifiedShadow { viewForAsync(id: string): Promise { return this.viaAsync(null, () => this.viewFor(id), async () => { - if (this.stats.disabled) return null; try { const backend = this.backend!; const loaded = await backend.readSnapshot(async (reader) => { @@ -1222,7 +1134,6 @@ export class UnifiedShadow { importRecordAsync(record: unknown): Promise { return this.viaAsync(null, () => this.importRecord(record), async () => { - if (this.stats.disabled) return null; try { const legacy = this.legacyLoad(record); if (!legacy) return null; @@ -1323,7 +1234,6 @@ export class UnifiedShadow { * facade; sqlite delegates to the identical sync body via viaAsync. */ lastLiveSeqAsync(sessionId: string): Promise { return this.viaAsync(null, () => this.lastLiveSeq(sessionId), async () => { - if (this.stats.disabled) return null; try { return await this.backend!.lastLiveSeq(sessionId); } catch (err) { @@ -1337,11 +1247,10 @@ export class UnifiedShadow { * adoption row clone through the backend on network backends; sqlite * delegates to the identical sync body. Serialized on the shadow queue * like every other write twin so it can never race a pending parent - * ingest. A failed or disabled sidecar degrades to the kernel-only + * ingest. A failing sidecar degrades to the kernel-only * fork (false) — the fork itself must never fail for a storage nicety. */ cloneSessionRowsAsync(parentId: string, childId: string, throughSeq: number): Promise { return this.viaAsync(false, () => this.cloneSessionRows(parentId, childId, throughSeq), async () => { - if (this.stats.disabled) return false; try { return await this.enqueue(() => this.backend!.cloneSessionRows(parentId, childId, throughSeq)); } catch (err) { @@ -1354,14 +1263,12 @@ export class UnifiedShadow { /** Fork publication is strict: clone and initial state land together, or * nothing lands. Unlike shadow mirrors, an error cannot mean success. */ createForkAsync(parentId: string, child: LegacySessionLike): Promise { - if (this.stats.disabled) throw new Error("fork storage unavailable"); const initial = planIngest(child); if (this.sync !== null) { const throughSeq = this.sync.lastLiveSeq(parentId); return Promise.resolve(this.sync.cloneSessionRows(parentId, child.id, throughSeq ?? -1, initial)); } return this.enqueue(async () => { - if (this.stats.disabled) throw new Error("fork storage unavailable"); const throughSeq = await this.backend!.lastLiveSeq(parentId); return this.backend!.cloneSessionRows(parentId, child.id, throughSeq ?? -1, initial); }); @@ -1417,10 +1324,6 @@ export class UnifiedShadow { await this.backend?.close(); } - private assertLive(): void { - if (this.stats.disabled) throw new Error("[storage] unified sidecar is disabled (repeated failures) — replica operations are unavailable"); - } - close(): void { if (this.sync === null) return; try { diff --git a/src/storage/store.ts b/src/storage/store.ts index 03a9aed063..4c9d2f6592 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -417,14 +417,6 @@ export class UnifiedStore { * verdict cache onto the post-checkpoint stats so the next boot skips * the rescan. Never true while a check is still in flight or failed. */ private lastIntegrityOk = false; - /** #5108/#4816: a completed check said the file set is CORRUPT — a clean - * close refreshes the FAIL verdict onto post-checkpoint stats so the - * next boot's cached-fail path fails fast (loud degrade) instead of - * rescan-looping - * (a clean close moves the WAL, which would otherwise invalidate the - * cache key and send a large corrupt store back to unverified forever). - * Outranks lastIntegrityOk at close time. */ - private lastIntegrityFailDetail: string | null = null; private constructor(db: SqliteDatabase, engine: SqliteEngineName, engineSource: SqliteEngineSource, lockPath: string | null, foreignLockHolder: { pid: number } | null, dbPath: string) { this.db = db; @@ -457,11 +449,6 @@ export class UnifiedStore { // of it (no check, no cache IO). let integrityOk = false; const mode = integrityModeFor(path); - if (mode.kind === "cached-fail") { - recordIntegrityOutcome(path, false, 0, mode.detail); - db.close(); - throw new StorageIntegrityError(path, mode.detail); - } if (mode.kind === "cached-ok") { recordIntegrityOutcome(path, true, 0, "verdict cache hit (file set unchanged since the last check)"); integrityOk = true; @@ -489,10 +476,9 @@ export class UnifiedStore { db.close(); const detail = sqliteCorruptionDetail(err); if (detail !== null) { - // #4626-era owner decision: nothing is renamed away — cache - // the FAIL verdict FIRST so every later open fails fast - // without re-running the scan. - writeIntegrityCache(path, false, detail.slice(0, 400)); + // 2026-10-30 owner policy: this FRESH in-boot finding REFUSES + // this start, but nothing is persisted as a boot-blocker — + // the next start re-scans and decides from fresh evidence. throw new StorageIntegrityError(path, detail.slice(0, 400)); } throw err; @@ -500,8 +486,10 @@ export class UnifiedStore { const durationMs = Date.now() - started; recordIntegrityOutcome(path, verdict === "ok", durationMs, verdict === "ok" ? undefined : verdict.slice(0, 400)); if (verdict !== "ok") { + // Same policy as the throwing-scan branch above: refuse THIS boot + // on fresh evidence, persist nothing — every future boot + // rechecks with its own live scan. db.close(); - writeIntegrityCache(path, false, verdict.slice(0, 400)); throw new StorageIntegrityError(path, verdict.slice(0, 400)); } writeIntegrityCache(path, true); @@ -594,10 +582,10 @@ export class UnifiedStore { if (integrityOk) store.noteIntegrityOk(); // #4609: a large store's cache-miss check runs post-open on a second // read-only connection — open() returns immediately; the verdict - // lands in runtime-status and the cache (a cached failure makes the - // NEXT open fail fast and REFUSE TO START — no automatic file moves, - // #4175/#4626-era decision chain to 2026-10-30: no degradation). - if (mode.kind === "async") scheduleBackgroundIntegrityCheck(path, (ok, detail) => { if (ok) store.noteIntegrityOk(); else if (detail !== undefined) store.noteIntegrityFail(detail); }); + // lands in runtime-status as an ALARM ONLY (2026-10-30 owner policy: + // a running store's background verdict never refuses future boots — + // each next boot re-scans fresh; no automatic file moves, #4175). + if (mode.kind === "async") scheduleBackgroundIntegrityCheck(path, (ok) => { if (ok) store.noteIntegrityOk(); }); return store; } @@ -626,12 +614,6 @@ export class UnifiedStore { } } - /** #5108/#4816: mirror of noteIntegrityOk for a completed FAIL verdict — - * see lastIntegrityFailDetail for why the close-time refresh needs it. */ - noteIntegrityFail(detail: string): void { - this.lastIntegrityFailDetail = detail; - } - close(): void { // #3250: converge the WAL before releasing the handle so a clean exit // leaves nothing for the next boot's recovery to replay. Same isolation @@ -643,15 +625,13 @@ export class UnifiedStore { loggerLog("warn", `[storage] WAL checkpoint on close failed: ${String(err)}`); } this.db.close(); - // #4609: a clean close of a verified store refreshes the verdict - // cache onto the post-checkpoint stats — the normal restart path then - // skips the rescan. #5108/#4816: the same refresh carries a FAIL - // verdict (fail outranks ok — never cache ok for a corrupt file). - // Deferred while a background check is still in flight (its own - // completion writes the cache). Best-effort. + // #4609: a clean close of a VERIFIED store refreshes the PASS memo onto + // post-checkpoint stats so the next boot skips the rescan. FAIL verdicts + // are alarm-only by owner policy (2026-10-30) and are never cached, + // so there is nothing else to carry here. Deferred while a background + // check is still in flight (its own completion writes the cache). if (!backgroundIntegrityInflightForTests(this.dbPath)) { - if (this.lastIntegrityFailDetail !== null) writeIntegrityCache(this.dbPath, false, this.lastIntegrityFailDetail); - else if (this.lastIntegrityOk) writeIntegrityCache(this.dbPath, true); + if (this.lastIntegrityOk) writeIntegrityCache(this.dbPath, true); } // #4175: release the ownership lockfile on a clean close so the next // open in another process is not told a dead pid holds it. When a diff --git a/tests/issue4816-integrity-throw-quarantine.test.ts b/tests/issue4816-integrity-throw-quarantine.test.ts index eb2532f00a..34e5f84ad7 100644 --- a/tests/issue4816-integrity-throw-quarantine.test.ts +++ b/tests/issue4816-integrity-throw-quarantine.test.ts @@ -195,7 +195,7 @@ test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and REFUSES } }); -test("#4816 fail-fast: a second open after proven corruption does not rescan", { skip: SKIP_NO_ENGINE }, () => { +test("#4816 repeat boots decide from FRESH evidence every time — no FAIL verdict is ever persisted", { skip: SKIP_NO_ENGINE }, () => { const dir = tmpDir(); try { resetIntegrityCheckStateForTests(); @@ -208,14 +208,18 @@ test("#4816 fail-fast: a second open after proven corruption does not rescan", { ); const scansAfterFirst = integrityCheckRunsForTests(); const cache = readIntegrityCache(p); - assert.ok(cache !== null && cache.ok === false, `FAIL verdict cached on throw, got ${JSON.stringify(cache)}`); - // Second boot: served from the verdict cache — the typed error again, - // ZERO additional scans. + assert.ok( + cache === null || cache.ok !== false, + `a FAIL verdict must never reach the cache any more, got ${JSON.stringify(cache)}`, + ); + // Second boot: no verdict short-circuit exists for failures — the + // same file set gets its own live scan and refuses again on the + // SAME fresh evidence (owner policy 2026-10-30). assert.throws( () => UnifiedStore.open(p), (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), ); - assert.equal(integrityCheckRunsForTests(), scansAfterFirst, "cached-fail must short-circuit before any quick_check"); + assert.equal(integrityCheckRunsForTests(), scansAfterFirst + 1, "every boot runs its own fresh check — the second refusal came from evidence, not a cached verdict"); } finally { resetIntegrityCheckStateForTests(); rmSync(dir, { recursive: true, force: true }); @@ -244,7 +248,7 @@ test("#4816 operator resolution: moving the corrupt set aside unblocks a fresh r } }); -test("#4816 worker facet: a large store's throwing background check caches FAIL across a clean close", { skip: SKIP_NO_ENGINE }, async () => { +test("#4816 worker facet: a large store's throwing background check is ALARM-ONLY — the next boot still refuses, from its own scan", { skip: SKIP_NO_ENGINE }, async () => { const dir = tmpDir(); try { resetIntegrityCheckStateForTests(); @@ -255,13 +259,20 @@ test("#4816 worker facet: a large store's throwing background check caches FAIL const s1 = UnifiedStore.open(p); try { await waitFor(() => !backgroundIntegrityInflightForTests(p), 15000); + // The finding is loud in runtime-status/logs but must NEVER be + // written as a boot-blocker (owner policy 2026-10-30). const cache = readIntegrityCache(p); - assert.ok(cache !== null && cache.ok === false, `FAIL verdict cached by the worker, got ${JSON.stringify(cache)}`); - assert.match(cache?.detail ?? "", /malformed/); + assert.ok( + cache === null || cache.ok !== false, + `the background FAIL must not reach the cache, got ${JSON.stringify(cache)}`, + ); } finally { - s1.close(); // checkpoint moves the WAL — the refresh must carry the FAIL verdict + s1.close(); } - // Next boot: cached-fail → typed error → the open refuses (no rescan). + // Next boot: no cached verdict to trust. Drop the async-threshold + // override first so this boot checks synchronously — its own fresh + // scan finds the corruption and refuses. + setIntegritySyncThresholdForTests(null); assert.throws( () => UnifiedStore.open(p), (err: unknown) => err instanceof StorageIntegrityError && /malformed/.test(err.detail), diff --git a/tests/issue4949-rollback-mask-recovery.test.ts b/tests/issue4949-rollback-mask-recovery.test.ts index bc93e1e0d2..3ea733ab92 100644 --- a/tests/issue4949-rollback-mask-recovery.test.ts +++ b/tests/issue4949-rollback-mask-recovery.test.ts @@ -15,14 +15,13 @@ * SQLite auto-abort semantics. * * B. PERMANENT BREAKER — src/storage/shadow.ts's 3-strike circuit breaker - * had no recovery path: once stats.disabled flipped, every facade - * missed until process restart, even after storage healed. Now each - * REENABLE_PROBE_COOLDOWN_MS window lets the next save run one - * writability probe (single-row committed write through the production - * path); success clears the breaker and the triggering record ingests - * normally. Consistency before unified reads resume relies on the - * load-side staleness guard (envelope savedAt > index row) + the next - * full ingest — asserted indirectly by the final viewFor check. + * used to have no recovery path: once stats.disabled flipped, every + * facade missed until process restart, even after storage healed. The + * breaker itself is GONE now (owner policy 2026-10-30: index writes are + * NEVER abandoned — "100 real-world failure shapes", not just 3): every + * save attempts the index write regardless of prior failures, errors + * still surface to the caller (the JSON envelope serves meanwhile), and + * healing needs no probe or cooldown — the next save simply lands. */ import { test } from "node:test"; @@ -32,7 +31,7 @@ import * as os from "node:os"; import * as path from "node:path"; import { DatabaseSync, type SQLInputValue } from "node:sqlite"; import { openSqliteDatabase, setDatabaseSyncFactoryForTests, setPreferredSqliteEngineForTests, type NodeSqliteDatabase } from "../src/storage/driver.ts"; -import { UnifiedShadow, REENABLE_PROBE_COOLDOWN_MS } from "../src/storage/shadow.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; import type { LegacySessionLike } from "../src/storage/ingest.ts"; import { setLogCapture } from "../src/logger.ts"; import type { CoreMessage } from "acp-kernel"; @@ -272,7 +271,7 @@ test("#4949 A2: nested transactions roll back cleanly on a constraint failure", // triggering record lands in the unified store. // --------------------------------------------------------------------------- -test("#4949 B: disabled sidecar recovers via cooldown-gated writability probe", () => { +test("#4949 B: a failing sidecar never self-disables — every save retries, healing is instant", () => { const dir = tmpDir("breaker"); const dbPath = path.join(dir, "index.db"); try { @@ -289,45 +288,43 @@ test("#4949 B: disabled sidecar recovers via cooldown-gated writability probe", assert.equal(stats(shadow).disabled, false); assert.ok(stats(shadow).ingests >= 1, "warm ingest must land"); - // Storage exhausts: three consecutive failures trip the breaker. + // Storage exhausts: three consecutive failures surface loudly — + // but NOTHING disables: index writes are never abandoned (the + // 3-strike breaker was removed per owner policy 2026-10-30). flaky.unhealthy = true; for (let i = 1; i <= 3; i++) { shadow.record(mkLegacy("s1", Array.from({ length: i + 1 }, (_, j) => msg(j)))); } - assert.equal(stats(shadow).disabled, true, "three consecutive failures disable the sidecar"); + assert.equal(stats(shadow).disabled, false, "consecutive failures must NOT disable the sidecar any more"); const lastErr = stats(shadow).lastError ?? ""; assert.match(lastErr, /disk is full/, `status wire must carry the real cause: ${lastErr}`); assert.doesNotMatch(lastErr, /cannot rollback/, `masked error reached the status wire: ${lastErr}`); - const beginsAfterDisable = flaky.beginAttempts; - assert.ok(beginsAfterDisable >= 3, "each failed record attempted a real BEGIN"); - // Cooldown gate: rapid saves must NOT hammer the broken store — - // exactly ONE probe until the window elapses. + // Rapid saves keep retrying — nothing is muted and nothing is + // gated: every save still reaches the broken store (each failed + // record costs >= 1 real BEGIN; the store may internally + // re-enter its guarded BEGIN/SAVEPOINT cycle before surfacing). + const beginsBeforeRapid = flaky.beginAttempts; shadow.record(mkLegacy("s1", [msg(0), msg(1)])); - assert.equal(flaky.beginAttempts, beginsAfterDisable + 1, "first save after disable runs one probe"); shadow.record(mkLegacy("s1", [msg(0), msg(1)])); - shadow.record(mkLegacy("s1", [msg(0), msg(1)])); - assert.equal(flaky.beginAttempts, beginsAfterDisable + 1, "further saves stay quiet inside the cooldown window"); - assert.equal(stats(shadow).disabled, true, "failed probe keeps the breaker armed"); + const delta = flaky.beginAttempts - beginsBeforeRapid; + assert.ok(delta >= 2 && delta <= 8, `both rapid saves must attempt real index writes (saw ${delta} BEGINs)`); + // The facade keeps working through the whole outage (shadow mode + // serves JSON reads regardless; full mode's unified reads are + // unaffected by ingest failures by design of readsUnified). + assert.equal(stats(shadow).disabled, false, "still not disabled after rapid failing saves"); + assert.equal(captured.filter((e) => e.msg.includes("RE-ENABLED")).length, 0, "no RE-ENABLED surface may exist any more"); - // Operator frees space: the store heals. The next save after the - // window elapses probes once, succeeds, re-enables — and the - // triggering record ingests immediately. - (shadow as unknown as { nextReenableProbeAt: number }).nextReenableProbeAt = 0; + // Operator frees space: the store heals. NO probe, NO cooldown — + // the very next save lands in the index straight away. flaky.unhealthy = false; const ingestsBefore = stats(shadow).ingests; shadow.record(mkLegacy("s1", Array.from({ length: 7 }, (_, j) => msg(j)))); - assert.equal(stats(shadow).disabled, false, "successful probe re-enables the sidecar"); - assert.ok(stats(shadow).ingests > ingestsBefore, "the triggering record must land right away"); - const view = shadow.viewFor("s1"); - assert.ok(view, "unified reads must work again"); - assert.ok( - view.lastMessages?.some((m) => (m as { text?: string }).text === "hello 6") ?? false, - "the recovered ingest carries the newest tail", - ); - - const reenabled = captured.filter((e) => e.level === "info" && e.msg.includes("RE-ENABLED")); - assert.equal(reenabled.length, 1, "exactly one re-enable log line"); + assert.ok(stats(shadow).ingests > ingestsBefore, "the post-heal record lands immediately"); + // The last observed failure stays visible for operators (there is + // no auto-cleanup of the error surface any more) — what is gone + // is the ABANDONMENT: the next write already landed above. + assert.match(stats(shadow).lastError ?? "", /disk is full/, "the error surface still names the cause"); } finally { stop(); shadow.close(); @@ -340,11 +337,13 @@ test("#4949 B: disabled sidecar recovers via cooldown-gated writability probe", }); // --------------------------------------------------------------------------- -// C. The cooldown constant is the single knob behind the gate — pin its -// magnitude so a future "tweak" cannot silently turn the probe into a -// hammer (or a decade-long dead store). +// C. The breaker itself must not come back: there is no self-disable, no +// cooldown constant, no probe state anywhere in the shadow module. // --------------------------------------------------------------------------- -test("#4949 C: re-enable cooldown stays at the documented 5-minute cadence", () => { - assert.equal(REENABLE_PROBE_COOLDOWN_MS, 5 * 60 * 1000); +test("#4949 C: the 3-strike breaker is gone for good (no re-disable surface left)", () => { + const src = fs.readFileSync(path.join(import.meta.dirname ?? ".", "..", "src", "storage", "shadow.ts"), "utf8"); + assert.doesNotMatch(src, /REENABLE_PROBE_COOLDOWN_MS/, "no re-enable cooldown constant may exist"); + assert.doesNotMatch(src, /nextReenableProbeAt/, "no probe-state field may exist"); + assert.match(src, /NEVER gives up on index writes/, "the no-abandonment policy is documented at the top of the module"); }); diff --git a/tests/issue4982-fork-lineage-conflict.test.ts b/tests/issue4982-fork-lineage-conflict.test.ts index 5275a21060..9d32258193 100644 --- a/tests/issue4982-fork-lineage-conflict.test.ts +++ b/tests/issue4982-fork-lineage-conflict.test.ts @@ -9,15 +9,24 @@ import { MysqlBackend, type Mysql2Module } from "../src/storage/mysql.ts"; import { UnifiedShadow } from "../src/storage/shadow.ts"; import type { LegacySessionLike } from "../src/storage/ingest.ts"; -test("#4982: queued fork fails closed if an earlier mirror disabled storage", async () => { +test("#4982: a mirror with an exhausted store still attempts the queued fork (never fails closed on a breaker any more)", async () => { + // Pre-owner-policy-2026-10-30 shape: the 3-strike breaker could disable + // storage mid-run, and createForkAsync refused with "fork storage + // unavailable" once it noticed. That gate is GONE — index writes are + // never abandoned, so the fork must still ATTEMPT its work against the + // exhausted store and surface whatever the store reports (envelope + // serves meanwhile), instead of refusing up front. let backendCalls = 0; - const stats = { disabled: false }; + let cloneCalls = 0; + const stats = { disabled: true }; const shadow = { stats, sync: null, - backend: { async lastLiveSeq() { backendCalls++; return 0; } }, + backend: { + async lastLiveSeq() { backendCalls++; return 0; }, + async cloneSessionRows() { cloneCalls++; return false; }, + }, async enqueue(work: () => Promise): Promise { - stats.disabled = true; return work(); }, } as unknown as UnifiedShadow; @@ -26,8 +35,9 @@ test("#4982: queued fork fails closed if an earlier mirror disabled storage", as state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, nextBlockId: 1, nextRunId: 1 }, blockContents: new Map(), }; - await assert.rejects(() => UnifiedShadow.prototype.createForkAsync.call(shadow, "parent", child), /fork storage unavailable/); - assert.equal(backendCalls, 0); + await UnifiedShadow.prototype.createForkAsync.call(shadow, "parent", child); + assert.equal(cloneCalls, 1, "the fork ATTEMPTED its work against the exhausted store instead of refusing"); + assert.ok(backendCalls >= 1, "the live-seq lookup ran as part of the attempt"); }); // Actual SQLite storage, including content refcounts and zero-row clones. diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 2cfc98633d..eed0a80844 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -341,11 +341,12 @@ test("storage engine: rollback contract — explicit legacy mode serves the iden }); // --------------------------------------------------------------------------- -// SELF-DISABLE: 3 consecutive failures flip reads back to legacy-only. +// NO SELF-DISABLE (2026-10-30 owner policy): a failing sidecar never +// self-disables — every save keeps retrying the real index write. // --------------------------------------------------------------------------- -test("storage engine: sidecar self-disables after 3 consecutive failures (reads flip legacy)", () => { - const dir = tmpdir("bili-engproof-disable-"); +test("storage engine: a failing sidecar NEVER self-disables — writes keep retrying, reads stay unified", () => { + const dir = tmpdir("bili-engproof-nodisable-"); try { const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => { throw new Error("synthetic legacy-load failure"); @@ -354,16 +355,21 @@ test("storage engine: sidecar self-disables after 3 consecutive failures (reads assert.equal(shadow.readsUnified, true, "fresh full sidecar must read unified"); shadow.record({ some: "record" }); shadow.record({ some: "record" }); - assert.equal(shadow.readsUnified, true, "2 failures must not disable yet"); shadow.record({ some: "record" }); - assert.equal(shadow.stats.disabled, true, "3 consecutive failures must disable the sidecar"); - assert.equal(shadow.readsUnified, false, "disabled sidecar must not read unified"); - assert.equal(shadow.viewFor("anything"), null, "disabled sidecar must miss reads"); + assert.equal(shadow.stats.disabled, false, "no breaker any more — repeated failures must not disable the sidecar (2026-10-30 owner policy)"); + assert.equal(shadow.readsUnified, true, "a failing sidecar keeps reading unified — it never falls back on its own"); + assert.match(shadow.stats.lastError ?? "", /synthetic legacy-load failure/, "the failure stays VISIBLE (loud warn), just non-fatal"); + assert.equal(shadow.viewFor("anything"), null, "reads of a never-ingested id still miss — visibility, not mode"); shadow.close(); + // The breaker machinery itself must be gone from the source. + const src = fs.readFileSync(path.join(import.meta.dirname ?? ".", "..", "src", "storage", "shadow.ts"), "utf8"); + assert.doesNotMatch(src, /MAX_CONSECUTIVE_FAILURES/); + assert.doesNotMatch(src, /REENABLE_PROBE_COOLDOWN_MS/); + // The legacy file is unaffected through a plain SessionStore. const legacy = loadFolded(path.join(dir, "seed")); - assert.ok(legacy, "legacy load broken while the sidecar is disabled"); + assert.ok(legacy, "legacy load broken while the sidecar is failing"); } finally { rmrf(dir); } diff --git a/tests/storage-integrity-cache.test.ts b/tests/storage-integrity-cache.test.ts index f9ec805f1a..9b09322478 100644 --- a/tests/storage-integrity-cache.test.ts +++ b/tests/storage-integrity-cache.test.ts @@ -20,6 +20,7 @@ import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync, utimesSyn import { tmpdir } from "node:os"; import { join } from "node:path"; import { UnifiedStore, StorageIntegrityError } from "../src/storage/store.js"; +import { setLogCapture } from "../src/logger.ts"; import { integrityCachePath, readIntegrityCache, @@ -82,7 +83,7 @@ test("touching the db file invalidates the cache and forces a recheck", () => { } }); -test("a cached failure fails open with the typed error (the #4175 loud-degrade trigger) without rescanning", () => { +test("a STALE cached FAIL (older-version leftover) no longer blocks boots — fresh rescan decides (#4609/#5123)", () => { resetIntegrityCheckStateForTests(); setIntegritySyncThresholdForTests(null); const dir = tmpDir(); @@ -92,10 +93,23 @@ test("a cached failure fails open with the typed error (the #4175 loud-degrade t store.close(); // cache now holds ok for the closed file set const cache = readIntegrityCache(dbPath); assert.ok(cache !== null); - writeFileSync(integrityCachePath(dbPath), JSON.stringify({ ...cache, ok: false, detail: "simulated background failure" })); + writeFileSync(integrityCachePath(dbPath), JSON.stringify({ ...cache, ok: false, detail: "simulated older-version failure" })); + // Owner policy 2026-10-30: runtime/background findings are alarms, never + // boot-blockers. A FAIL verdict left by an OLDER build is treated as a + // plain cache miss: loud warn, then a FRESH scan decides. const before = integrityCheckRunsForTests(); - assert.throws(() => UnifiedStore.open(dbPath), (err: unknown) => err instanceof StorageIntegrityError && /simulated background failure/.test(err.detail)); - assert.equal(integrityCheckRunsForTests(), before, "cached fail threw without running a scan"); + let warned = ""; + setLogCapture((_level, m) => { if (m.includes("stale verdicts no longer block boots")) warned = m; }); + try { + const again = UnifiedStore.open(dbPath); + again.close(); + } finally { + setLogCapture(null); + } + assert.ok(warned.length > 0, "the stale FAIL must be called out loudly in the log"); + assert.equal(integrityCheckRunsForTests(), before + 1, "a fresh scan ran instead of trusting the old verdict"); + const after = readIntegrityCache(dbPath); + assert.equal(after?.ok, true, "the healthy file set re-cached a PASS verdict (fast path restored)"); } finally { rmSync(dir, { recursive: true, force: true }); } From 1766e2d6b04682642adec746bf94925293f0cf42 Mon Sep 17 00:00:00 2001 From: ranxianglei-devbox Date: Fri, 30 Oct 2026 21:18:03 +0800 Subject: [PATCH 32/35] =?UTF-8?q?proven-corrupt=20index:=20alarm-only=20bo?= =?UTF-8?q?ot=20=E2=80=94=20files=20never=20moved,=20copied=20or=20rebuilt?= =?UTF-8?q?=20(owner=20policy=20final=20form,=202026-10-30)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A quick_check PROVEN corruption is no longer a startup blocker at all: the shadow layer catches the typed StorageIntegrityError, logs a loud actionable alarm (back up then move index.db{,-wal,-shm} aside BY HAND, start again — a fresh index re-imports the JSON envelopes) and comes back as 'no sidecar'. The process boots envelope-served; writerOffActive() sees no live sidecar so even jsonWriter=off operators keep persisting every turn to the envelopes (no unsafe window). Nothing is renamed/copied/rebuilt automatically: auto-recovery was withdrawn by owner decision because it cannot guarantee full data recovery (checkpoint rows live only inside the index; BILI_STORAGE_JSON_WRITER=off makes the index the sole durable copy). Transient open failures (no engine / CANTOPEN / readonly / second writer / backend DSN) STILL refuse to start — the two categories are documented separately in CONFIGURATION.md (.zh-CN in sync). Tests re-pinned to the final matrix: construction succeeds + loud wire alarm + byte-identical files + envelope load, leak guards kept. --- CONFIGURATION.md | 2 +- CONFIGURATION.zh-CN.md | 2 +- src/persist.ts | 10 +-- src/storage/runtime-status.ts | 10 +-- src/storage/shadow.ts | 62 +++++++------------ src/storage/sqlite-errors.ts | 9 +-- src/storage/store.ts | 48 +++++++------- tests/corrupt-index-child.ts | 59 +++++++++--------- ...sue4816-integrity-throw-quarantine.test.ts | 20 +++--- ...ue5103-startup-recovery-visibility.test.ts | 18 +++--- tests/storage-durability.test.ts | 22 +++---- tests/storage-engine-proofs.test.ts | 33 +++++----- tests/storage-review-fixes.test.ts | 33 +++++----- 13 files changed, 163 insertions(+), 165 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 6c26fccc63..27aab63a54 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -362,7 +362,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | Env tier for storage.backend (#3931): sqlite (default) or mysql (fully wired, #3388 steps 2+3 - saves ingest through the async shadow queue, sync reads ride the JSON envelope, replicas/ext/GC/fork via the async twins (#4911); export too). Wins over the file tier. | | `BILI_MYSQL_DSN` | string | unset | — | MySQL DSN (mysql://user:pass@host:3306/db) for storage.backend="mysql" (#3388, #3931). Wins over the file key storage.mysql.dsn. Empty = the mysql backend cannot open; the documented error names both knobs. The pool opens lazily; closeQuietlyAsync drains the ingest queue and closes it. | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC age sweep window in days (#3463): sessions idle longer than this are archived (one JSON line appended to the single archive log (archive/archive.jsonl) + tombstone row + content/media refs released). 0 disables the age sweep; the manual entry POST /__bili/storage/gc always runs. Config-file key storage.archiveAfterDays is the persistent form. | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN by THIS BOOT's own scan the files are left EXACTLY in place and bili REFUSES TO START with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand, then start again — a fresh index rebuilds from the JSON envelopes). A successful PASS is cached so later starts skip the rescan; a FAIL verdict is NEVER persisted (2026-10-30 owner policy): runtime/background findings are alarms only (loud logs + runtime-status, the process keeps serving — JSON envelopes remain the recovery source) and every boot decides again from a fresh scan of the CURRENT file set, so a stale FAIL left behind by an older version can never block future boots. There is NO automatic rename/repair AND no legacy-only fallback: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy), and continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable); ANY open failure (corruption proven OR transient: no engine / CANTOPEN / second writer / readonly / backend DSN) makes bili REFUSE TO START loudly with an actionable error — no silent legacy fallback in any case (2026-10-30 owner policy); deployments that genuinely want JSON-only serve explicitly (BILI_STORAGE_UNIFIED=legacy or unset). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN by THIS BOOT's own scan NOTHING is moved, copied or rebuilt automatically (auto-recovery was withdrawn #4609/#4626-era: with BILI_STORAGE_JSON_WRITER=off the index is the SOLE durable copy, so an automatic move or rebuild cannot guarantee full data recovery) and bili STARTS ANYWAY — serving from the JSON envelopes with a loud, actionable alarm naming the manual fix (back up then move index.db/-wal/-shm aside BY HAND, then start again — a fresh index rebuilds from the JSON envelopes). A successful PASS is cached so later starts skip the rescan; a FAIL verdict is NEVER persisted (2026-10-30 owner policy): runtime/background findings are alarms only (loud logs + runtime-status, the process keeps serving — JSON envelopes remain the recovery source) and every boot decides again from a fresh scan of the CURRENT file set, so a stale FAIL left behind by an older version can never block future boots. There is NO automatic rename/repair AND no legacy-only fallback: continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable). TRANSIENT open failures (no SQLite engine / CANTOPEN / second writer / readonly / backend DSN) make bili REFUSE TO START loudly with an actionable error — no silent legacy fallback for environment problems either (2026-10-30 owner policy); deployments that genuinely want JSON-only serve explicitly (BILI_STORAGE_UNIFIED=legacy or unset). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | Snapshot-render shadow observability (#3756 follow-up): compute the outbound snapshot view on every armed turn WITHOUT using it, classify the diff against the legacy outbound, and surface per-session counters in the web panel + the snapshot-shadow diagnostics topic. 0/false/off disables; 1/true/on enables; otherwise the config-file snapshot.shadow boolean applies (default on). The render lane itself stays rollout-gated (BILI_ROLLOUT_SNAPSHOT_RENDER), so the shadow lane never changes outbound bytes - it is the dry-run evidence that flips the rollout later. | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | Override the config file path (path relocation). | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | Where session records are stored (path relocation). | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 50a6fbcbe4..155f596e3b 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -362,7 +362,7 @@ | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | storage.backend 的环境变量层(#3931):sqlite(默认)或 mysql(已接线,#3388 第二+三步 —— 保存经异步影子队列入 mysql,同步读走 JSON 信封,replicas/ext/GC/fork 走异步孪生(#4911);export 同样支持)。优先于文件层。 | | `BILI_MYSQL_DSN` | string | unset | — | storage.backend="mysql" 的 MySQL DSN(mysql://user:pass@host:3306/db)(#3388,#3931)。优先于文件键 storage.mysql.dsn。留空 = mysql 后端无法打开;文档化错误会列出两个旋钮。连接池惰性打开;closeQuietlyAsync 排空入队事务并关闭。 | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC 年龄清扫窗口(天,#3463):闲置超过该时长的会话被归档(单文件归档日志 archive/archive.jsonl 追加一行 + 墓碑行保留标签 + 释放内容/媒体引用)。0 关闭年龄清扫;手动入口 POST /__bili/storage/gc 始终可用。配置文件键 storage.archiveAfterDays 为持久形式。 | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;一旦证实损坏,文件原样保留在原地,bili 拒绝启动并给出响亮、可操作的错误提示(手工备份后把 index.db/-wal/-shm 移开再启动——全新索引会从 JSON 信封重建)。FAIL 判定会被缓存,之后的启动快速失败、无需重扫。既没有自动改名/修复,也没有 legacy-only 兜底:自动搬移存储主索引的做法已撤回(#4626 时代的 owner 决策),因为无法保证完整数据恢复(checkpoint 行只存在于索引内部;BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本),且继续以 legacy-only 运行会让 JSON 信封与死掉的索引永久分叉、再无对账可能;瞬态打开失败(无引擎 / CANTOPEN / 第二写者 / 只读)仍像以前一样响亮降级——它们不能证明任何文件损坏。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。 | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;一旦由本次启动自己的扫描证实损坏,文件原样保留在原地——不做任何自动搬移、复制或重建(自动恢复已撤回:BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本,自动操作无法保证完整数据恢复)——bili 照常启动,改用 JSON 信封服务,并给出响亮、可操作的告警指明手工修复步骤(备份后把 index.db/-wal/-shm 手工移开再启动——全新索引会从 JSON 信封重建)。PASS 判定会被缓存,之后的启动跳过重扫;FAIL 判定永不持久化(2026-10-30 owner 策略):运行期/后台发现只是告警(响亮日志 + runtime-status,进程继续服务——JSON 信封始终是恢复来源),每次启动都对当前文件集做新鲜扫描再判定,旧版本遗留的陈旧 FAIL 永远不能阻断后续启动。既没有自动改名/修复,也没有 legacy-only 兜底(继续以 legacy-only 运行会让 JSON 信封与死掉的索引永久分叉、再无对账可能)。瞬态打开失败(无引擎 / CANTOPEN / 第二写者 / 只读 / 后端 DSN)仍响亮地拒绝启动并给出可操作的错误——环境类问题同样不允许静默走 legacy 兜底;真正需要纯 JSON 服务的部署请显式设置 BILI_STORAGE_UNIFIED=legacy(或留空)。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。 | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | 快照渲染影子对账(#3756 后续):每轮已折叠会话都用引擎现算一次出站快照视图但不使用,与 legacy 实际出站比对分类,逐会话计数进 web 面板与 snapshot-shadow 诊断主题。0/false/off 关闭;1/true/on 开启;否则读配置文件 snapshot.shadow 布尔(默认开)。渲染车道本身仍受灰度开关控制(BILI_ROLLOUT_SNAPSHOT_RENDER),影子车道绝不改变出站字节 —— 它是后续翻转灰度的干跑证据。 | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | 覆盖配置文件路径(路径重定位)。 | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | 会话记录的存放目录(路径重定位)。 | diff --git a/src/persist.ts b/src/persist.ts index f28632ec79..bc43de60dc 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -393,10 +393,12 @@ export class SessionStore { private readonly forkBaseCut = new Set(); private readonly storageMode: StorageMode; /** Phase 3 (#2960): true only when the JSON envelope is actually skipped — - * requires an explicit off request AND effective-full mode. If the - * sidecar dies mid-run (3-strike disable) writerOffActive() turns false - * and file writes resume on the very next save (fail-safe: never a - * window with no persistent copy at all). */ + * requires an explicit off request AND effective-full mode. + * writerOffActive() additionally needs a LIVE sidecar, so if the sidecar + * never comes up at all (e.g. a proven-corrupt index alarms and boots + * without it, or an engine-less environment refuses) envelope writes + * stay ON for the whole life of this process (fail-safe: never a window + * with no persistent copy at all). */ private readonly jsonWriterOff: boolean; /** #3931 step 3: resolves when the async backend (mysql) opened (or * failed to — the sidecar just stays undefined). Resolved immediately diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index a846491ab1..d27712ac22 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -215,12 +215,12 @@ function degradeReason(): string { if (isNoEngineError(e)) { return `no SQLite engine available (node:sqlite missing, better-sqlite3 not installed)${nodeSqliteUpgradeHint()}`; } - // #4626-era owner decision chain: a provably-corrupt index is LEFT IN - // PLACE (no automatic rename) AND the store refuses to start rather - // than fall back legacy-only (the two replicas would diverge forever - // and never reconcile) — the reason carries the manual fix. + // #4626-era owner chain (final, 2026-10-30): a provably-corrupt index + // stays EXACTLY in place (no move, no copy, no rebuild) and bili + // starts WITHOUT it — envelope-served until an operator repairs the + // files or moves them aside by hand; the reason carries that fix. if (/StorageIntegrityError|integrity quick_check failed/i.test(state.sidecarOpenError)) { - return `index database failed its integrity check and was left UNTOUCHED (${e}) — the store REFUSES TO START; back up index.db{,-wal,-shm}, move those files aside by hand, then start again (a fresh index rebuilds from the JSON envelopes)`; + return `index database failed its integrity check and was left UNTOUCHED (${e}) — bili is starting WITHOUT the index (envelope-served); fix: back up index.db{,-wal,-shm}, move those files aside by hand, start again`; } return `index database failed to open (${e})${sqliteOpenFailureHint(state.sidecarOpenError)}`; } diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 5a4a1c99c3..80faf825c0 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -253,36 +253,27 @@ export class UnifiedShadow { // proved the file set corrupt — handled below WITHOUT moving any file // (#4626-era owner decision, 2026-10-30). if (!(err instanceof StorageIntegrityError)) throw err; - // #4626-era owner decision (2026-10-30): NO automatic quarantine - // rename. quick_check proved the file set corrupt, but automatic - // removal of a storage PRIMARY was withdrawn because full data - // recovery cannot be guaranteed without touching the main flow: - // - checkpoint rows live only inside index.db — the JSON - // envelopes are NOT a complete round-trip for them; - // - with BILI_STORAGE_JSON_WRITER=off (jsonWriterOff, see - // persist.ts) the envelopes are skipped entirely and the - // index is the SOLE durable copy — an automatic rename - // destroys unrecoverable data; - // - a corrupt-on-open index can still often be salvaged by an - // operator; renaming it away severs that last option forever. - // The file set therefore stays EXACTLY in place, untouched, - // and the store REFUSES TO START (#4626-era owner decision - // chain, 2026-10-30): renaming risks destroying the sole copy - // WITHOUT proving recovery, and falling back to legacy-only - // was rejected too — once the two replicas stop advancing - // together the split can never be reconciled, so continuing - // would bake the divergence in. The loud failure below names - // the manual fix. The FAIL verdict has already been written - // to index.db.chk by UnifiedStore.open, so every later open - // fails fast from the cache — no rescans. Manual resolution: - // back up, then move index.db{-wal,-shm} aside yourself; - // the next start rebuilds the index fresh and re-imports the - // JSON envelopes that hold session content. + // #4626-era owner decision chain — FINAL form (2026-10-30): + // quick_check PROVED this file set corrupt. Nothing moves, + // nothing is copied, nothing is rebuilt automatically + // (auto-recovery from the JSON envelopes was withdrawn too: + // with BILI_STORAGE_JSON_WRITER=off the index is the sole + // durable copy), and bili STARTS ANYWAY, serving from the + // envelopes — with a loud alarm naming the manual fix. + // writerOffActive() sees no sidecar, so even a jsonWriter=off + // operator keeps persisting every turn to the envelopes; no + // turn ever lands somewhere unsafe. The corrupt set stays + // EXACTLY in place: an operator repairs it or moves it aside + // by hand (back up first) and the next start picks up a fresh + // index that re-imports the envelopes. Transient open + // failures below are a different category ("environment + // broken", not "data proven bad") and still refuse. loggerLog( "error", - `[storage] index.db FAILED its integrity quick_check (${err.detail}) — REFUSING TO START with a corrupt index (no automatic rename, no legacy fallback): back up ${path.join(dir, "index.db")}{,-wal,-shm}, move those files aside, then start again — the JSON envelopes are the recovery source and a fresh index is rebuilt automatically (#4175/#4626)`, + `[storage] index.db FAILED its integrity quick_check (${err.detail}) — starting WITHOUT the index (no file moved, no copy, no automatic rebuild; 2026-10-30 owner policy): back up ${path.join(dir, "index.db")}{,-wal,-shm}, then MOVE those files aside by hand and start again — the fresh index re-imports the JSON envelopes (#4175/#4626/#4816)`, ); - throw err; + recordSidecarOpenFailure(`integrity quick_check failed: ${err.detail}`); + return null; } // #3285: watermark trust does not cross process boundaries — a // restart-handoff overlap leaves the previous writer's rows AND @@ -305,20 +296,15 @@ export class UnifiedShadow { // either (#3823 lock; #4614 readonly — where it would DESTROY the only copy of recent // session state in unified-full mode). The Node.js upgrade lever is surfaced via // /__bili/status + the bootstrap warn. - // #4626-era owner decision chain (2026-10-30): ANY sidecar-open - // failure now REFUSES TO START instead of degrading — proven - // corruption (renaming risks destroying the sole durable copy - // without proving recovery; a legacy-only fallback diverges from - // the dead index forever) and every transient shape alike - // (no-engine/CANTOPEN/readonly/second-writer: availability at the + // Corruption-shaped failures never reach here — the inner catch + // above handles them (loud alarm + boot WITHOUT the index). + // #4626-era owner decision chain (2026-10-30): every OTHER + // sidecar-open failure REFUSES TO START instead of degrading — + // no-engine/CANTOPEN/readonly/second-writer: availability at the // price of a silently different storage path was withdrawn by - // owner instruction "在任何情况下不降级"). Intentional opt-outs + // owner instruction "在任何情况下不降级". Intentional opt-outs // (legacy mode / encryption opt-out / persistence off) resolve to // mode===legacy above and never reach here. - if (err instanceof StorageIntegrityError) { - recordSidecarOpenFailure(errStr); - throw err; - } const hint = sqliteOpenFailureHint(errStr); const tail = /no SQLite engine available/i.test(errStr) ? "" diff --git a/src/storage/sqlite-errors.ts b/src/storage/sqlite-errors.ts index f9f7a03af1..24fe347cf0 100644 --- a/src/storage/sqlite-errors.ts +++ b/src/storage/sqlite-errors.ts @@ -5,10 +5,11 @@ * rows (page-level damage), or it can THROW out of the statement itself * (truncated file, garbage header, broken overflow chain — probe-verified on * node:sqlite v22: ERR_SQLITE_ERROR with the raw sqlite3_errmsg text). The - * #4175 decision (fail fast + REFUSE TO START — no automatic file moves and - * no legacy fallback, per the 2026-10-30 owner follow-up) only saw the - * verdict-row shape, so every throwing corruption escaped as a raw driver - * error and degraded the store to legacy-only forever with no auto-heal. + * #4175 decision chain (alarm + start WITHOUT the index, files left exactly + * in place — no automatic file moves and no auto-rebuild, per the 2026-10-30 + * owner follow-up, final form) originally saw only the verdict-row shape, so + * every throwing corruption escaped as a raw driver error that neither the + * alarm nor the classification handled. * * This classifier maps the throwing shapes onto the same decision. It * matches ONLY SQLITE_CORRUPT / SQLITE_NOTADB: diff --git a/src/storage/store.ts b/src/storage/store.ts index 4c9d2f6592..c798c263f8 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -346,12 +346,14 @@ function likeEscape(prefix: string): string { /** * #4175: raised when PRAGMA quick_check fails at open. Migrations must never - * run against a corrupt file, so this fires BEFORE migrateSchema; the shadow - * layer records it and RE-THROWS — bili REFUSES TO START with the files - * left exactly in place (no automatic rename/repair, no legacy fallback: - * full data recovery cannot be guaranteed without operator involvement). The FAIL - * verdict is cached so later opens fail fast; the operator moves the files - * aside by hand (backed up first) to force a fresh rebuild. + * run against a corrupt file, so this fires BEFORE migrateSchema (and from the + * migration catch for large stores, where the scan was size-skipped); the + * shadow layer catches it and STARTS WITHOUT THE INDEX — loud alarm, files + * left EXACTLY in place (no automatic rename/repair/copy/rebuild; auto- + * recovery from JSON withdrawn because full data recovery cannot be + * guaranteed without operator involvement). A FAIL verdict is never cached — + * every boot re-scans fresh; the operator's manual fix (back up, then move + * index.db{-wal,-shm} aside by hand, start again) is named in the alarm. */ export class StorageIntegrityError extends Error { constructor(public readonly dbPath: string, public readonly detail: string) { @@ -440,12 +442,13 @@ export class UnifiedStore { // corrupt file must never be migrated against — #4175's premise, // preserved wherever it is cheap); a large store checks in the // background post-open (#4609: a 14 GB quick_check costs ~260 s and - // the 60 s native spawn budget killed the child mid-scan). A cached - // failure throws here so the open REFUSES to start INSTEAD of - // rescan-looping (#4626-era owner decision chain: no automatic - // quarantine, no legacy fallback — see maybeOpen) — including for - // failures found by the background - // check of a previous run. BILI_STORAGE_QUICK_CHECK=0 opts out of all + // the 60 s native spawn budget killed the child mid-scan). A proven + // failure throws StorageIntegrityError as an INTERNAL signal — the + // shadow layer alarms loudly and starts WITHOUT the index (files left + // exactly in place; owner chain 2026-10-30 final form). Only PASS + // verdicts are cached (rescan skip); FAILs are never persisted, so a + // stale FAIL left behind by an older version is treated as a miss and + // re-scanned fresh. BILI_STORAGE_QUICK_CHECK=0 opts out of all // of it (no check, no cache IO). let integrityOk = false; const mode = integrityModeFor(path); @@ -476,9 +479,11 @@ export class UnifiedStore { db.close(); const detail = sqliteCorruptionDetail(err); if (detail !== null) { - // 2026-10-30 owner policy: this FRESH in-boot finding REFUSES - // this start, but nothing is persisted as a boot-blocker — - // the next start re-scans and decides from fresh evidence. + // 2026-10-30 owner policy (final form): this FRESH in-boot + // finding throws to the shadow layer, which alarms and + // starts WITHOUT the index — files untouched, nothing + // cached as a boot-blocker; the next start re-scans and + // decides from fresh evidence. throw new StorageIntegrityError(path, detail.slice(0, 400)); } throw err; @@ -486,9 +491,10 @@ export class UnifiedStore { const durationMs = Date.now() - started; recordIntegrityOutcome(path, verdict === "ok", durationMs, verdict === "ok" ? undefined : verdict.slice(0, 400)); if (verdict !== "ok") { - // Same policy as the throwing-scan branch above: refuse THIS boot - // on fresh evidence, persist nothing — every future boot - // rechecks with its own live scan. + // Same policy as the throwing-scan branch above: throw the typed + // signal (shadow layer alarms + starts without the index), + // persist nothing — every future boot rechecks with its own + // live scan. db.close(); throw new StorageIntegrityError(path, verdict.slice(0, 400)); } @@ -556,11 +562,11 @@ export class UnifiedStore { // to be corrupt: the #2771 fallback tells operators to MOVE the // file aside by hand (backed up first) to reset the index — and // Windows refuses to move a file a live process still holds open. - // Close, then rethrow for the shadow's legacy-only degradation. + // Close, then rethrow — the shadow layer alarms and starts WITHOUT + // the index (files left exactly in place; no cache writes at all). // #5108/#4816: large stores (> sync threshold) skip the pre-migration // scan, so a corruption can surface HERE first — classify it the same - // way so the loud degrade + fail-fast cache applies instead of an - // unexplained raw driver error. + // way instead of letting an unexplained raw driver error out. db.close(); const detail = sqliteCorruptionDetail(err); if (detail !== null) throw new StorageIntegrityError(path, detail.slice(0, 400)); diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index da2c80b7ec..534a34f243 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -44,7 +44,6 @@ import type { ProxyOptions } from "../src/config.ts"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { storageStatusWire } from "../src/storage/runtime-status.ts"; -import { StorageIntegrityError } from "../src/storage/store.ts"; import { setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts"; import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts"; import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; @@ -196,18 +195,20 @@ export function makeOpts(upstreamPort: number): ProxyOptions { // sides reference these constants, so the coverage list cannot drift. // --------------------------------------------------------------------------- -// #4816/#4626 policy chain (owner decisions through 2026-10-30): throwing -// corruption is classified → NO automatic quarantine rename (a move can't -// prove full recovery) AND no legacy fallback (the JSON envelope would fork -// from the dead index forever). Construction therefore REFUSES — typed -// StorageIntegrityError, files byte-identical in place, no leaked handle -// into the dir (so an operator can move the set aside immediately). -// These checks pin that policy plus the original #2781/#4357 leak guards. +// #4816/#4626 policy chain (FINAL form, 2026-10-30): throwing corruption is +// classified → NO automatic rename/move/copy AND no automatic rebuild +// (auto-recovery can't guarantee full data recovery) AND no legacy fallback +// (forks from the dead index forever) — and construction SUCCEEDS too: bili +// boots WITHOUT the index, envelope-served, under a loud wire alarm naming +// the manual fix. Files byte-identical in place; no leaked handle into the +// dir (so an operator can still move the set aside immediately). These checks +// pin that policy plus the original #2781/#4357 leak guards. const ENGPROOF_CHECK_NAMES = { - noAutoRename: "corrupt index.db left UNTOUCHED — byte-identical, no .corrupt-* copy (#4626 decision)", - refusesToStart: "construction THROWS StorageIntegrityError — no silent legacy fallback", - midFlowGuard: "no live handle into the sessions dir after the refused construction (#2781/#4357 guard)", - fdGuard: "no open fd into the corrupt index dir after the refusal (/proc/self/fd, Linux)", + noAutoRename: "corrupt index.db left UNTOUCHED — byte-identical, no .corrupt-* copy or rebuild (#4626/#4816 decision)", + bootProceeds: "construction SUCCEEDS — boots WITHOUT the index, loud wire alarm names the fix (no move/copy/rebuild/refusal)", + envelopeServed: "session loads from the JSON envelopes while the corrupt index sits untouched", + midFlowGuard: "no live handle into the sessions dir after the alarm-boot (#2781/#4357 guard)", + fdGuard: "no open fd into the corrupt index dir after close (/proc/self/fd, Linux)", } as const; export const ENGPROOF_CHECKS: readonly string[] = Object.values(ENGPROOF_CHECK_NAMES); @@ -344,29 +345,31 @@ async function engproofScenario(argv: string[], run: CheckRun): Promise { fs.writeSync(fd, Buffer.alloc(512, 0x5a), 0, 512, 0); fs.closeSync(fd); - // THE policy chain pinned here: construction must REFUSE a provably- - // corrupt index with the typed error — files untouched, no rename, and - // no legacy fallback (a legacy-only store would fork from the dead - // index forever). Supersedes #2771's "construction must survive" for - // PROVABLY CORRUPT indices; transient open failures still survive. + // THE policy chain pinned here (FINAL form, 2026-10-30): a provably- + // corrupt index must NOT block startup either — construction succeeds, + // the file set stays byte-identical in place (no move, no copy, no + // automatic rebuild: auto-recovery cannot guarantee full data recovery), + // and this process serves from the JSON envelopes under a loud alarm + // naming the manual fix. Supersedes the interim refuse-to-start pin. const beforeHash = createHash("sha256").update(fs.readFileSync(path.join(dir, "index.db"))).digest("hex"); - let caught: unknown = null; - try { - new SessionStore({ dir, debounceMs: 1, enabled: true }); - } catch (err) { - caught = err; - } + let store: SessionStore | null = null; + await run.run(ENGPROOF_CHECK_NAMES.bootProceeds, () => { + store = new SessionStore({ dir, debounceMs: 1, enabled: true }); + const reason = storageStatusWire().reason ?? ""; + assert.match(reason, /integrity check and was left UNTOUCHED/); + assert.doesNotMatch(reason, /REFUSES TO START/i); + }); await run.run(ENGPROOF_CHECK_NAMES.noAutoRename, () => { const names = fs.readdirSync(dir); assert.ok(!names.some((n) => n.startsWith("index.db.corrupt-")), `an automatic .corrupt-* copy must never exist, got ${names.join(", ")}`); const afterHash = createHash("sha256").update(fs.readFileSync(path.join(dir, "index.db"))).digest("hex"); assert.equal(afterHash, beforeHash, "the corrupt primary must stay byte-identical"); }); - await run.run(ENGPROOF_CHECK_NAMES.refusesToStart, () => - assert.ok( - caught instanceof StorageIntegrityError, - `a provably-corrupt index must make construction throw StorageIntegrityError, got ${caught === null ? "NO THROW (the store constructed!)" : String(caught)}`, - )); + await run.run(ENGPROOF_CHECK_NAMES.envelopeServed, () => { + const viaEnvelope = store!.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); + assert.ok(viaEnvelope, "with the index out of service the session must still load from its JSON envelope"); + store!.closeQuietly(); + }); // Leak guard (#2781/#4357): a REJECTED store must hold NO persistent // handle into the dir — whole-dir rename round-trip straight after the diff --git a/tests/issue4816-integrity-throw-quarantine.test.ts b/tests/issue4816-integrity-throw-quarantine.test.ts index 34e5f84ad7..9119b51316 100644 --- a/tests/issue4816-integrity-throw-quarantine.test.ts +++ b/tests/issue4816-integrity-throw-quarantine.test.ts @@ -155,7 +155,7 @@ test("#4816 store: a throwing quick_check becomes StorageIntegrityError", { skip } }); -test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and REFUSES TO OPEN", { skip: SKIP_NO_ENGINE }, () => { +test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and boots WITHOUT the index", { skip: SKIP_NO_ENGINE }, () => { const dir = tmpDir(); // The wire reason below resolves `requested` from the REAL process env // (runtime-status records it only via SessionStore), and the CI baseline @@ -169,14 +169,12 @@ test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and REFUSES truncateHalf(join(dir, "index.db")); const before = createHash("sha256").update(readFileSync(join(dir, "index.db"))).digest("hex"); _resetStorageRuntimeStatusForTest(); - // #4626-era owner decision chain: NO rename AND NO legacy fallback — - // a provably-corrupt index must make the open throw the typed error - // (proxy boot then hard-fails loudly). - assert.throws( - () => UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null), - (err: unknown) => err instanceof StorageIntegrityError, - "a provably-corrupt index must refuse to open — nothing may auto-rename or fall back legacy-only", - ); + // #4626-era owner decision chain (FINAL, 2026-10-30): NO rename AND + // NO legacy fallback AND no refusal — a provably-corrupt index alarms + // loudly and the open comes back as "no sidecar"; this process then + // serves from the envelopes until an operator fixes things by hand. + const sh = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "full" }, () => null); + assert.equal(sh, null, "a provably-corrupt index must yield 'no sidecar' (loud alarm) — nothing auto-renames, falls back legacy-only or blocks the boot"); // Byte-identical file set: NO automatic rename/repair (#4626-era owner // decision — checkpoint rows live only inside the index, and with // jsonWriterOff it is the sole durable copy). @@ -186,7 +184,7 @@ test("#4816 shadow: throwing corruption leaves every file UNTOUCHED and REFUSES // Loud + actionable on the status wire. const wire = storageStatusWire(); assert.match(wire.reason ?? "", /integrity check and was left UNTOUCHED/); - assert.match(wire.reason ?? "", /REFUSES TO START/i); + assert.match(wire.reason ?? "", /starting WITHOUT the index/i); } finally { if (prevUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prevUnified; @@ -248,7 +246,7 @@ test("#4816 operator resolution: moving the corrupt set aside unblocks a fresh r } }); -test("#4816 worker facet: a large store's throwing background check is ALARM-ONLY — the next boot still refuses, from its own scan", { skip: SKIP_NO_ENGINE }, async () => { +test("#4816 worker facet: a large store's throwing background check is ALARM-ONLY — the next boot still throws, from its own scan", { skip: SKIP_NO_ENGINE }, async () => { const dir = tmpDir(); try { resetIntegrityCheckStateForTests(); diff --git a/tests/issue5103-startup-recovery-visibility.test.ts b/tests/issue5103-startup-recovery-visibility.test.ts index 07c20f48b7..2f59fc183b 100644 --- a/tests/issue5103-startup-recovery-visibility.test.ts +++ b/tests/issue5103-startup-recovery-visibility.test.ts @@ -181,7 +181,7 @@ function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"]): }; } -test("#5103: first ingest failure warns immediately with the db path; disable warning keeps naming it", () => { +test("#5103: first ingest failure warns immediately with the db path; failures NEVER self-disable", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-5103-shadow-")); try { const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); @@ -202,20 +202,22 @@ test("#5103: first ingest failure warns immediately with the db path; disable wa try { shadow.record(mkLegacy("s1", [msg(1), msg(2)])); const firstWarn = captured.find((c) => c.level === "warn" && c.msg.includes("failed to ingest")); - assert.ok(firstWarn, "the FIRST failure must warn (before #5103 strikes one and two were silent)"); + assert.ok(firstWarn, "the FIRST failure must warn (#5103) — no silent strikes any more"); assert.ok(firstWarn!.msg.includes(path.join(dir, "index.db")), "warning must name the database file"); if (process.platform === "win32") { assert.ok(firstWarn!.msg.includes("(win32:"), "win32 runner carries the OS-level hint"); } - assert.equal(shadow.stats.disabled, false, "one failure must not disable yet"); + assert.ok(firstWarn!.msg.includes("RETRIED on every subsequent save"), "the warning names the retry-every-save policy"); + // Owner policy (2026-10-30): the old 3-strike self-disable is GONE — + // repeated failures must NEVER stop future index writes; every save + // retries. Identical error messages dedupe to one warn line. shadow.record(mkLegacy("s1", [msg(1), msg(2), msg(3)])); - assert.equal(shadow.stats.disabled, false, "two failures must not disable yet"); shadow.record(mkLegacy("s1", [msg(1), msg(2), msg(3), msg(4)])); - const disableWarn = captured.find((c) => c.level === "warn" && c.msg.includes("disabled after 3 consecutive failures")); - assert.ok(disableWarn, "third consecutive failure disables with a loud warning"); - assert.ok(disableWarn!.msg.includes(path.join(dir, "index.db")), "disable warning must keep naming the file"); - assert.equal(shadow.stats.disabled, true); + assert.equal(shadow.stats.disabled, false, "failures must NEVER self-disable any more (breaker removed)"); + assert.match(shadow.stats.lastError ?? "", /attempt to write a readonly database/, "the latest failure stays surfaced on stats"); + const ingestWarns = captured.filter((c) => c.level === "warn" && c.msg.includes("failed to ingest")); + assert.equal(ingestWarns.length, 1, "identical repeated errors dedupe to one warn line"); } finally { setLogCapture(null); } diff --git a/tests/storage-durability.test.ts b/tests/storage-durability.test.ts index 73fedd637d..9851727f3f 100644 --- a/tests/storage-durability.test.ts +++ b/tests/storage-durability.test.ts @@ -130,7 +130,7 @@ test("lockfile: a live foreign pid is reported, a dead pid is taken over, close } }); -test("shadow open REFUSES on a corrupt index.db set — nothing renamed, no fallback (#4626)", async () => { +test("shadow open ALARMS on a corrupt index.db set — nothing renamed, boot proceeds (#4626)", async () => { const { UnifiedShadow } = await import("../src/storage/shadow.ts"); const { readdirSync } = await import("node:fs"); const dir = freshDir(); @@ -148,19 +148,15 @@ test("shadow open REFUSES on a corrupt index.db set — nothing renamed, no fall writeFileSync(path, raw); const beforeHash = createHash("sha256").update(raw).digest("hex"); - // The shadow's maybeOpen (full mode) must refuse — throw the typed - // error AND leave EVERY file exactly where it was: no rename, no - // auto-reopen, no forensics copy, and NO legacy fallback (#4626-era - // owner decision chain — automatic moves can't prove full recovery, - // and a fallback forks the two replicas forever). + // The shadow's maybeOpen (full mode) must ALARM, not refuse — come + // back as "no sidecar" (this process then serves from the envelopes) + // AND leave EVERY file exactly where it was: no rename, no copy, no + // automatic rebuild, and NO legacy fallback (#4626-era owner decision + // chain through 2026-10-30 — automatic moves can't prove full + // recovery, and a fallback forks the two replicas forever). const env = { ...process.env, BILI_STORAGE_UNIFIED: "full" }; - let caught: unknown = null; - try { - UnifiedShadow.maybeOpen(dir, env); - } catch (err) { - caught = err; - } - assert.ok(caught, "a provably-corrupt index must make the open THROW — not degrade"); + const sh = UnifiedShadow.maybeOpen(dir, env); + assert.equal(sh, null, "a provably-corrupt index must yield 'no sidecar' with a loud alarm — not throw, not rename, not rebuild"); const names = readdirSync(dir); assert.ok(!names.some((n) => n.includes(".corrupt-")), `no automatic copies allowed, got ${names.join(", ")}`); const afterHash = createHash("sha256").update(readFileSync(path)).digest("hex"); diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index eed0a80844..9e39c8ff7d 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -24,19 +24,23 @@ * state on load. * ROLLBACK — explicitly select legacy after persisting the full * load's migration → the file loads the exact digest. - * SELF-DISABLE — 3 consecutive sidecar failures disable it; reads flip - * back to legacy; the legacy file stays readable. - * CORRUPT INDEX — a provably-corrupt index.db makes construction - * REFUSE loudly: typed StorageIntegrityError, file set - * left in place byte-identical, no automatic rename and - * no legacy fallback (#4626-era owner decision chain, - * supersedes #2771's survive-degraded for corruption). + * NO SELF-DISABLE — a failing sidecar NEVER self-disables: every save + * keeps retrying the real index write (reads stay + * unified) — the old 3-strike breaker is gone. + * CORRUPT INDEX — a provably-corrupt index.db does NOT block startup: + * construction proceeds WITHOUT the index (envelope- + * served), loud wire alarm, file set left in place + * byte-identical, no automatic rename/copy/rebuild and + * no legacy fallback (#4626 → final owner chain + * 2026-10-30; supersedes both #2771's survive-degraded + * and the interim refuse-to-start pin for corruption). * CAS REFCOUNT — shared content blobs survive deleteSession of one * session and are collected only at refcount 0. * * The corrupt-index test was the regression for #2771 (construction - * survival) — re-pinned for the #4626→refuse-to-start policy chain: nothing - * renamed, no fallback, typed throw, files byte-identical; the rest are + * survival) — re-pinned for the FINAL #4626-era policy chain: nothing moved/ + * copied/rebuilt, boot proceeds envelope-served under a loud alarm; the rest + * are * contract pins * (proof-only, no product change). */ @@ -376,11 +380,12 @@ test("storage engine: a failing sidecar NEVER self-disables — writes keep retr }); // --------------------------------------------------------------------------- -// CORRUPT INDEX (#2771 → #4626 → refuse-to-start): construction must REFUSE a -// provably-corrupt index loudly — files untouched, no rename, no fallback. +// CORRUPT INDEX (#2771 → #4626 → final form): a provably-corrupt index does +// NOT block startup — the boot proceeds envelope-served; files untouched, no +// move/copy/rebuild, the loud alarm names the manual fix. // --------------------------------------------------------------------------- -test("storage engine: corrupt index.db makes construction REFUSE loudly — files untouched, no fallback (#4626/#4816)", { timeout: 180_000 }, async () => { +test("storage engine: corrupt index.db makes the boot PROCEED envelope-served — loud alarm, files untouched (#4626/#4816)", { timeout: 180_000 }, async () => { const dir = tmpdir("bili-engproof-corrupt-"); const prev = process.env.BILI_STORAGE_UNIFIED; try { @@ -397,8 +402,8 @@ test("storage engine: corrupt index.db makes construction REFUSE loudly — file await clean.writeNow(viaFull); clean.closeQuietly(); - // #4338: everything past this point (corrupt the header → refused - // construction → leak guards) runs in a spawned child. On Windows any + // #4338: everything past this point (corrupt the header → alarm-boot + // → leak guards) runs in a spawned child. On Windows any // sqlite file the scenario touches can leave an OS handle with no JS // reference to release, and the resulting persistent EBUSY at rmrf // would outlive the whole tmp-rm retry ladder; the child exiting diff --git a/tests/storage-review-fixes.test.ts b/tests/storage-review-fixes.test.ts index 0c5751b579..2d49cbbb7e 100644 --- a/tests/storage-review-fixes.test.ts +++ b/tests/storage-review-fixes.test.ts @@ -136,10 +136,11 @@ test("review 2: destroyed-substrate blocks are deleted and cannot resurrect on a }); }); -test("review 3: corrupt index.db REFUSES TO START instead of bricking the proxy", async () => { +test("review 3: corrupt index.db alarms loudly — boots envelope-served, files untouched", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-review-corrupt-")); try { const session = loadCorpusCase(dir, "folded", "corpus-folded", "anthropic"); + void session; fs.writeFileSync(path.join(dir, "index.db"), Buffer.concat([ Buffer.from("SQLite format 3\0"), crypto.randomBytes(4096), @@ -147,15 +148,14 @@ test("review 3: corrupt index.db REFUSES TO START instead of bricking the proxy" process.env.BILI_STORAGE_UNIFIED = "shadow"; try { - // #5124/#4816/#4626 behavior chain: this corruption shape makes - // quick_check THROW, it IS classified → NO rename AND NO legacy - // fallback (a fallback forks the two replicas forever) — the - // store's open must throw the typed error. - assert.throws( - () => new SessionStore({ dir, debounceMs: 1, enabled: true }), - /integrity/i, - "a provably-corrupt index must make construction REFUSE, not degrade", - ); + // #5124/#4816/#4626 behavior chain (FINAL, 2026-10-30): this + // corruption shape makes quick_check THROW; classified corruption + // no longer blocks startup either — NOTHING is renamed, copied or + // rebuilt automatically (auto-recovery cannot guarantee full data + // recovery), construction SUCCEEDS serving from the envelopes, + // and the loud alarm carries the manual fix. + const s1 = new SessionStore({ dir, debounceMs: 1, enabled: true }); + s1.closeQuietly(); // Files byte-in-place: nothing renamed, nothing rebuilt. const names = fs.readdirSync(dir); assert.ok(!names.some((n) => n.startsWith("index.db.corrupt-")), `no automatic .corrupt-* copy may exist, got ${names.join(", ")}`); @@ -164,15 +164,14 @@ test("review 3: corrupt index.db REFUSES TO START instead of bricking the proxy" delete process.env.BILI_STORAGE_UNIFIED; } - // And a later process fails the same way — fast, from the cached FAIL - // verdict (no rescan), never "recovers" on its own. + // And a later process behaves identically — a fresh scan of the SAME + // untouched file set (no FAIL verdict is ever persisted), same loud + // alarm, same boot-without-index outcome, never silent recovery. process.env.BILI_STORAGE_UNIFIED = "shadow"; try { - assert.throws( - () => new SessionStore({ dir, debounceMs: 1, enabled: true }), - /integrity/i, - "second process must also refuse (cached-fail fast path)", - ); + const s2 = new SessionStore({ dir, debounceMs: 1, enabled: true }); + s2.closeQuietly(); + assert.ok(fs.existsSync(path.join(dir, "index.db")), "second boot must also leave the corrupt index exactly in place"); } finally { delete process.env.BILI_STORAGE_UNIFIED; } From 40bcb2f1bed5c459dc13e87e77931a14777d7513 Mon Sep 17 00:00:00 2001 From: ci-sync Date: Fri, 30 Oct 2026 23:26:16 +0800 Subject: [PATCH 33/35] fix(tests+docs): three gaps left by the #5188 convergence push MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - plugin-fork: pin the #5140 cold-restore snapshot test to unified-full — under the global legacy test baseline its sidecar never opens, so the index-reconstruction gate can never pass (409 replicasUnavailable sidecar-missing) - plugin-fork: pin the new broken-dir refuse-to-start assertion to full mode; legacy never touches sqlite at all so the ctor cannot throw there - multi-instance-live: spawn children with BILI_STORAGE_UNIFIED=full despite the global legacy baseline pin, else instance A holds no index.db.lock and B has nothing to refuse over (boot-refusal needs a live foreign writer) - export-jsonl: mysql-without-DSN now refuses at open time — assert on the actual refusal message instead of the pre-refusal #3388 tag - env-only.yaml: sync BILI_STORAGE_QUICK_CHECK seed with the final alarm-only policy wording already shipped in CONFIGURATION.md/.zh-CN (docs-sync gate) --- tests/export-jsonl.test.ts | 2 +- tests/multi-instance-live.test.ts | 6 ++++++ tests/plugin-fork.test.ts | 19 +++++++++++++++++-- website/config-reference/env-only.yaml | 3 +-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/export-jsonl.test.ts b/tests/export-jsonl.test.ts index bf7e1bcef2..a44c3d073f 100644 --- a/tests/export-jsonl.test.ts +++ b/tests/export-jsonl.test.ts @@ -253,7 +253,7 @@ test("--jsonl enforces the backend selection at open time: the mysql stub fails () => exportSession(session.id, { dir: path.join(dir, "folded"), jsonl: true }), (err: unknown) => { const msg = String((err as Error).message); - return msg.includes("mysql") && msg.includes("#3388"); + return msg.includes("mysql") && msg.includes("no DSN"); }, "a deliberate mysql selection must fail at open time, not silently read sqlite", ); diff --git a/tests/multi-instance-live.test.ts b/tests/multi-instance-live.test.ts index 42e6600538..6ea8a5d74e 100644 --- a/tests/multi-instance-live.test.ts +++ b/tests/multi-instance-live.test.ts @@ -208,6 +208,12 @@ test("two live instances over one sessions dir: serve, guard, converge (#394/#40 const childEnv = (p: number): NodeJS.ProcessEnv => ({ ...process.env, + // The global baseline preload pins BILI_STORAGE_UNIFIED=legacy process-wide; + // spread into every child it would keep them BOTH legacy — A would never + // write the sidecar lockfile and B would have nothing to refuse over + // (boot-refusal needs a live foreign writer holding the shared index.db). + // Pin the production default so the refusal contract runs like a real boot. + BILI_STORAGE_UNIFIED: "full", BILI_SESSIONS_DIR: sessionsDir, XDG_STATE_HOME: stateDir, // #5012: non-TTY pipes now mute once the file leg proves live; this diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index d3d57aa958..b2e77821a9 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -766,6 +766,10 @@ test("publicForkInputCheck keeps the #3743 taxonomy pinned per class (hash-drift test("HTTP storage failures: broken dir refuses to start; corrupt index serves forks via envelopes", { timeout: 60_000 }, async () => { const h = await harness(); + // Both failure legs are unified-sidecar behaviors; under the global legacy baseline + // preload no sqlite open happens at all, so refuse-to-start can never fire there. + const oldMode2 = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; try { // Two failure shapes, per the 2026-10-30 owner policy: (a) an open // into a broken directory REFUSES to start (the constructor throws — @@ -819,7 +823,12 @@ test("HTTP storage failures: broken dir refuses to start; corrupt index serves f // Retry: 201 fresh, or 200 if the alarm-boot fork already published // the child — the conversation registry is process-wide. assert.ok([200, 201].includes((await h.request("/__bili/plugin/fork", req)).status)); - } finally { _setStoreForTest(h.store); await h.close(); } + } finally { + if (oldMode2 === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = oldMode2; + _setStoreForTest(h.store); + await h.close(); + } }); test("HTTP CCR deleted index fails closed in memory after persistence", { timeout: 60_000 }, async () => { @@ -1526,6 +1535,8 @@ test("HTTP a mid-history rewrite replaces the snapshot wholesale with an exact b }); test("HTTP a cold-restored non-retained parent serves its intact raw tail immediately (#5140)", { timeout: 60_000 }, async () => { + const oldMode = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "full"; const h = await harness(true); try { const parent = resolveConversation("parent").session!; @@ -1544,7 +1555,11 @@ test("HTTP a cold-restored non-retained parent serves its intact raw tail immedi await sendModel(h, "parent", h.messages); assert.equal((await h.request("/__bili/plugin/snapshot?conversationId=parent")).status, 200); assert.equal(resolveConversation("parent").session!.pluginSnapshot!.length, 3); - } finally { await h.close(); } + } finally { + if (oldMode === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = oldMode; + await h.close(); +} }); test("fork protocol v2: snapshot serves the light revision, a stale v1 whole-store revision is still accepted (#2671 Phase 2b)", { timeout: 60_000 }, async () => { diff --git a/website/config-reference/env-only.yaml b/website/config-reference/env-only.yaml index 3ff11142cc..900b84488a 100644 --- a/website/config-reference/env-only.yaml +++ b/website/config-reference/env-only.yaml @@ -56,8 +56,7 @@ entries: kind: env-only audience: user group: envonly - desc_en: "Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN the files are left EXACTLY in place and bili REFUSES TO START with a loud, actionable error (back up then move index.db/-wal/-shm aside by hand, then start again — a fresh index rebuilds from the JSON envelopes). The FAIL verdict is cached so later starts fail fast without rescan. There is NO automatic rename/repair AND no legacy-only fallback: an automatic move of a storage primary was withdrawn (#4626-era owner decision) because full data recovery cannot be guaranteed (checkpoint rows live only inside the index; with BILI_STORAGE_JSON_WRITER=off the index is the sole durable copy), and continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable); transient open failures (no engine / CANTOPEN / second writer / readonly) still degrade loudly as before — they do not prove any file corrupt. Set 0 only for boot-time-sensitive huge deployments where the check's cost matters." - desc_zh: "存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;一旦证实损坏,文件原样保留在原地,bili 拒绝启动并给出响亮、可操作的错误提示(手工备份后把 index.db/-wal/-shm 移开再启动——全新索引会从 JSON 信封重建)。FAIL 判定会被缓存,之后的启动快速失败、无需重扫。既没有自动改名/修复,也没有 legacy-only 兜底:自动搬移存储主索引的做法已撤回(#4626 时代的 owner 决策),因为无法保证完整数据恢复(checkpoint 行只存在于索引内部;BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本),且继续以 legacy-only 运行会让 JSON 信封与死掉的索引永久分叉、再无对账可能;瞬态打开失败(无引擎 / CANTOPEN / 第二写者 / 只读)仍像以前一样响亮降级——它们不能证明任何文件损坏。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。" + desc_en: "Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN by THIS BOOT's own scan NOTHING is moved, copied or rebuilt automatically (auto-recovery was withdrawn #4609/#4626-era: with BILI_STORAGE_JSON_WRITER=off the index is the SOLE durable copy, so an automatic move or rebuild cannot guarantee full data recovery) and bili STARTS ANYWAY — serving from the JSON envelopes with a loud, actionable alarm naming the manual fix (back up then move index.db/-wal/-shm aside BY HAND, then start again — a fresh index rebuilds from the JSON envelopes). A successful PASS is cached so later starts skip the rescan; a FAIL verdict is NEVER persisted (2026-10-30 owner policy): runtime/background findings are alarms only (loud logs + runtime-status, the process keeps serving — JSON envelopes remain the recovery source) and every boot decides again from a fresh scan of the CURRENT file set, so a stale FAIL left behind by an older version can never block future boots. There is NO automatic rename/repair AND no legacy-only fallback: continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable). TRANSIENT open failures (no SQLite engine / CANTOPEN / second writer / readonly / backend DSN) make bili REFUSE TO START loudly with an actionable error — no silent legacy fallback for environment problems either (2026-10-30 owner policy); deployments that genuinely want JSON-only serve explicitly (BILI_STORAGE_UNIFIED=legacy or unset). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters." - key: BILI_SNAPSHOT_SHADOW type: boolean default: "true (on)" From 28454a1281c70740e21d3886b4e43925b2f6b902 Mon Sep 17 00:00:00 2001 From: ranxianglei-devbox Date: Fri, 30 Oct 2026 23:47:14 +0800 Subject: [PATCH 34/35] docs(seed): restore the dropped BILI_STORAGE_QUICK_CHECK desc_zh (i18n parity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The convergence fix 40bcb2f1 re-synced desc_en of BILI_STORAGE_QUICK_CHECK but deleted the desc_zh line — env-only.yaml went 65 zh / 66 en while every other seed file stays balanced and the generator requires all five fields (type/default/group/desc_en/desc_zh) on each entry. Result: the docs-sync gate fails at the generate step before it ever reaches its diff check. Restores the exact final-form Chinese wording already committed in CONFIGURATION.zh-CN.md (alarm-only: corruption proves itself by THIS boot's own scan, files stay in place, service keeps running on envelopes with a loud alarm; FAIL never cached; transient open failures still refuse). Verified: tools/gen-config-docs.mjs generate -> zero diff on all four generated files; check-coverage rc=0. --- website/config-reference/env-only.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/website/config-reference/env-only.yaml b/website/config-reference/env-only.yaml index 900b84488a..4326d25d69 100644 --- a/website/config-reference/env-only.yaml +++ b/website/config-reference/env-only.yaml @@ -55,6 +55,7 @@ entries: default: "on" kind: env-only audience: user + desc_zh: "存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;一旦由本次启动自己的扫描证实损坏,文件原样保留在原地——不做任何自动搬移、复制或重建(自动恢复已撤回:BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本,自动操作无法保证完整数据恢复)——bili 照常启动,改用 JSON 信封服务,并给出响亮、可操作的告警指明手工修复步骤(备份后把 index.db/-wal/-shm 手工移开再启动——全新索引会从 JSON 信封重建)。PASS 判定会被缓存,之后的启动跳过重扫;FAIL 判定永不持久化(2026-10-30 owner 策略):运行期/后台发现只是告警(响亮日志 + runtime-status,进程继续服务——JSON 信封始终是恢复来源),每次启动都对当前文件集做新鲜扫描再判定,旧版本遗留的陈旧 FAIL 永远不能阻断后续启动。既没有自动改名/修复,也没有 legacy-only 兜底(继续以 legacy-only 运行会让 JSON 信封与死掉的索引永久分叉、再无对账可能)。瞬态打开失败(无引擎 / CANTOPEN / 第二写者 / 只读 / 后端 DSN)仍响亮地拒绝启动并给出可操作的错误——环境类问题同样不允许静默走 legacy 兜底;真正需要纯 JSON 服务的部署请显式设置 BILI_STORAGE_UNIFIED=legacy(或留空)。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。" group: envonly desc_en: "Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN by THIS BOOT's own scan NOTHING is moved, copied or rebuilt automatically (auto-recovery was withdrawn #4609/#4626-era: with BILI_STORAGE_JSON_WRITER=off the index is the SOLE durable copy, so an automatic move or rebuild cannot guarantee full data recovery) and bili STARTS ANYWAY — serving from the JSON envelopes with a loud, actionable alarm naming the manual fix (back up then move index.db/-wal/-shm aside BY HAND, then start again — a fresh index rebuilds from the JSON envelopes). A successful PASS is cached so later starts skip the rescan; a FAIL verdict is NEVER persisted (2026-10-30 owner policy): runtime/background findings are alarms only (loud logs + runtime-status, the process keeps serving — JSON envelopes remain the recovery source) and every boot decides again from a fresh scan of the CURRENT file set, so a stale FAIL left behind by an older version can never block future boots. There is NO automatic rename/repair AND no legacy-only fallback: continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable). TRANSIENT open failures (no SQLite engine / CANTOPEN / second writer / readonly / backend DSN) make bili REFUSE TO START loudly with an actionable error — no silent legacy fallback for environment problems either (2026-10-30 owner policy); deployments that genuinely want JSON-only serve explicitly (BILI_STORAGE_UNIFIED=legacy or unset). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters." - key: BILI_SNAPSHOT_SHADOW From 627c258040e5ef5189b838b9f26df88a4a06cdd8 Mon Sep 17 00:00:00 2001 From: ci-sync Date: Sat, 31 Oct 2026 05:12:21 +0800 Subject: [PATCH 35/35] fix(tests+cli): align three suites with the no-degradation contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - storage gc --vacuum full: the no-degradation constructor refusal makes the old in-runGc guard unreachable — catch StorageSecondWriterError at the CLI boundary and refuse with exit 1 plus the actionable one-line reason (the #4612 pin). - storage-full-flip: #3818 (iii) repins to refuse-to-start when the full-mode sidecar cannot open; #3818 (vi) proves the removed breaker's same-tick JSON valve stays gone — under jsonWriter=off no envelope appears and the save lands in the index (index-only persistence is the contract). --- src/storage-maintenance.ts | 19 +++++++++++++++- tests/storage-full-flip.test.ts | 40 ++++++++++++++++----------------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/storage-maintenance.ts b/src/storage-maintenance.ts index 26d02aa0fa..376de522e8 100644 --- a/src/storage-maintenance.ts +++ b/src/storage-maintenance.ts @@ -7,6 +7,7 @@ import { existsSync, statSync } from "node:fs"; import { join } from "node:path"; import { SessionStore } from "./persist.js"; +import { StorageSecondWriterError } from "./storage/store.js"; import type { GcPlan } from "./storage/store.js"; import { fileStorageArchiveAfterDays, resolveArchiveAfterDays } from "./knobs.js"; @@ -273,7 +274,23 @@ export async function runStorageGcCommand(opts: { json?: boolean; dir?: string; }): Promise { - const store = new SessionStore({ dir: opts.dir, enabled: true }); + let store: SessionStore; + try { + store = new SessionStore({ dir: opts.dir, enabled: true }); + } catch (err) { + // #4612: under the no-degradation policy a live foreign writer makes the + // unified open REFUSE (constructor throws) before runGc's own guard can + // answer. Surface the same actionable refusal the in-process guard + // produces instead of leaking a stack trace through the CLI. + if (err instanceof StorageSecondWriterError) { + const reason = `refused: another process (pid ${err.pid}) is actively writing index.db — 'storage gc --vacuum full' rewrites the whole database under an exclusive write lock and would starve the other instance's writes until its storage layer degrades. Stop the other process (or wait for it to exit) and retry, or use --vacuum incremental for an online-safe shrink.`; + if (opts.dryRun) return { code: 0, text: `${reason}\ndry-run: nothing executed\n` }; + const data: Record = { refused: true, writerPid: err.pid, vacuum: opts.vacuum ?? "full", dryRun: false }; + if (opts.json) return { code: 1, text: `${JSON.stringify(data, null, 2)}\n` }; + return { code: 1, text: `${reason}\n` }; + } + throw err; + } try { return await runGc(store, opts.days, opts.vacuum, !!opts.dryRun, !!opts.json); } finally { diff --git a/tests/storage-full-flip.test.ts b/tests/storage-full-flip.test.ts index e73a3f1662..59f8c478ef 100644 --- a/tests/storage-full-flip.test.ts +++ b/tests/storage-full-flip.test.ts @@ -177,23 +177,20 @@ test("#3818 ② disabled persistence reports intentionalOptOut with no degraded // 3. The storageMode stamp tells the truth when the sidecar cannot open // --------------------------------------------------------------------------- -test("#3818 ③ stamp says legacy when the sidecar failed to open (#3135 ③)", async () => { +test("#3818 ③ an unopenable full-mode sidecar refuses to start (#3135 ③)", async () => { _resetStorageRuntimeStatusForTest(); process.env.BILI_STORAGE_UNIFIED = "full"; - _setStoreForTest(new SessionStore({ enabled: false, dir: tmpDir("stamp-ref") })); try { const dir = tmpDir("stamp-dead"); - // index.db-as-directory makes the sidecar open fail -> shadow undefined + // index.db-as-directory makes the sidecar open fail — under the + // no-degradation policy (#5213) that is a startup refusal, not a + // silent legacy stamp against a dead sidecar. fs.mkdirSync(path.join(dir, "index.db"), { recursive: true }); - const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); - const session = makeSession("stamp-dead-session"); - store.scheduleSave(session); - await settle(); - const found = findStateFile(dir, "stamp-dead-session"); - assert.ok(found, "the JSON envelope is written when the sidecar is dead"); - const metadata = found.payload.metadata as Record; - assert.equal(metadata.storageMode, "legacy", "stamp must describe the mode that actually holds data (no index.db copy exists)"); - store.closeQuietly(); + assert.throws( + () => new SessionStore({ enabled: true, dir, debounceMs: 0 }), + (err: unknown) => err instanceof Error && /unable to open database file|refus/i.test(err.message), + "an unopenable sidecar must refuse to start instead of degrading to legacy stamps", + ); rmrf(dir); } finally { delete process.env.BILI_STORAGE_UNIFIED; @@ -332,7 +329,7 @@ test("#3818 ⑤ bounded boot warms pure-index sessions (#3135 ⑥)", async () => // 6. A sidecar 3-striking mid-build still lands the JSON write that tick // --------------------------------------------------------------------------- -test("#3818 ⑥ writerOff save that disables the sidecar still lands JSON the same tick (#3135 ①)", async () => { +test("#3818 ⑥ writerOff mid-build disable no longer writes an envelope: index-only IS the contract (#3135 ①, breaker removed)", async () => { _resetStorageRuntimeStatusForTest(); process.env.BILI_STORAGE_UNIFIED = "full"; process.env.BILI_STORAGE_JSON_WRITER = "0"; @@ -340,8 +337,11 @@ test("#3818 ⑥ writerOff save that disables the sidecar still lands JSON the sa const dir = tmpDir("threestrike"); const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); const session = makeSession("threestrike-session"); - // Flip the sidecar to disabled DURING the build (exactly what the - // 3rd consecutive record failure does inside shadow.record). + // Flip the legacy 3-strike "disabled" flag DURING the build. The breaker + // and its same-tick JSON safety valve were removed (owner policy + // 2026-10-30): jsonWriter=off accepts index-only persistence and failed + // index writes are retried on every save — so this stub now proves + // nothing changes: no envelope appears, and the data lands in the index. const sh = (store as unknown as { shadow?: { record(r: unknown): void; stats: { disabled: boolean } } }).shadow; assert.ok(sh, "full mode opens a sidecar"); const realRecord = sh.record.bind(sh); @@ -352,13 +352,13 @@ test("#3818 ⑥ writerOff save that disables the sidecar still lands JSON the sa sh.stats.disabled = true; }; store.scheduleSave(session); - // windows-latest load can delay the debounce tick + fs visibility past the old fixed 400ms settle (#4161); poll for the envelope within a 15s budget instead — - // the same-tick guarantee under test lives in the code path (the disabled-mid-build branch writes JSON synchronously), not in the sleep length, and a - // genuinely dead fallback path still fails after the budget. - const envelopeLanded = await pollFor(() => findStateFile(dir, "threestrike-session") !== null, 15000); + // Short poll window: the envelope must STAY absent under writerOff. + const appeared = await pollFor(() => findStateFile(dir, "threestrike-session") !== null, 1500); sh.record = realRecord; assert.ok(calls >= 1, "the writerOff build reached the sidecar"); - assert.ok(envelopeLanded, "the disabling tick must land a JSON write — no window where the save exists on neither copy"); + assert.equal(appeared, false, "no JSON envelope under writerOff — the removed breaker's same-tick valve stays gone"); + const served = store.loadSync("threestrike-session", { protocol: "openai", upstreamOrigin: "https://api.openai.com" }); + assert.ok(served, "the save lives in the index (index-only persistence is the contract)"); store.closeQuietly(); rmrf(dir); } finally {