From ef7d05b568dc76ba9b2c5b7eb4a50c5d1c2194d4 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 31 Oct 2026 00:16:56 +0800 Subject: [PATCH] fix(storage): degrade a second long-lived index.db writer to legacy-only instead of co-residing (#5064) When two bili installs share one sessions dir, both RW-open the same index.db. A live foreign long-lived writer now makes this proxy refuse unified storage and degrade to legacy-only at open (via the #2771 shape), instead of logging an error and continuing to double-write (corruption + SQLITE_BUSY turn failures). Ephemeral CLI/export opens keep report-only. - store.open: rejectForeignWriter opt -> StorageForeignWriterError on a live non-benign foreign pid; re-thrown past the lockfile-discipline guard. - shadow.maybeOpen: passes rejectForeignWriter for both long-lived opens; handles the typed error without the destructive delete-file hint. - runtime-status: clean degrade reason naming BILI_SESSIONS_DIR + BILI_CONFIG_FILE isolation (no new config surface). - bidirectional regression test (trigger / no-trigger). --- src/storage/runtime-status.ts | 3 + src/storage/shadow.ts | 14 ++- src/storage/store.ts | 33 +++++- .../issue5064-foreign-writer-degrade.test.ts | 111 ++++++++++++++++++ 4 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 tests/issue5064-foreign-writer-degrade.test.ts diff --git a/src/storage/runtime-status.ts b/src/storage/runtime-status.ts index f2c0aee365..a4f92c5b22 100644 --- a/src/storage/runtime-status.ts +++ b/src/storage/runtime-status.ts @@ -186,6 +186,9 @@ function degradeReason(): string { if (/no SQLite engine available/i.test(e)) { return `no SQLite engine available (node:sqlite missing, better-sqlite3 not installed)${nodeSqliteUpgradeHint()}`; } + if (/refusing second long-lived writer/i.test(state.sidecarOpenError)) { + return "a second bili process owns the sessions index.db — unified storage degraded to legacy-only; isolate installs with BILI_SESSIONS_DIR + BILI_CONFIG_FILE"; + } return `index database failed to open (${e})${sqliteOpenFailureHint(state.sidecarOpenError)}`; } const b = state.bootstrap; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index aec7da44eb..c8448c6547 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -25,7 +25,7 @@ import { log as loggerLog } from "../logger.js"; import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import type { BlockView } from "../session.js"; import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; -import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; +import { UnifiedStore, StorageIntegrityError, StorageForeignWriterError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; import { renameSync } from "node:fs"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; @@ -192,7 +192,7 @@ export class UnifiedShadow { let store: UnifiedStore; try { try { - store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit }); + store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit, rejectForeignWriter: true }); } catch (err) { // #4175: quick_check failed — the file set is provably corrupt. // Quarantine it (db/-wal/-shm → .corrupt-, same suffix @@ -227,7 +227,7 @@ export class UnifiedShadow { "error", `[storage] index.db failed its integrity quick_check (${err.detail}) — quarantined ${quarantined.join(", ")} and reopened fresh; JSON envelopes remain the recovery source (#4175)`, ); - store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit }); + store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit, rejectForeignWriter: true }); } // #3285: watermark trust does not cross process boundaries — a // restart-handoff overlap leaves the previous writer's rows AND @@ -235,6 +235,14 @@ export class UnifiedShadow { // per session re-establishes trust from its own view. store.resetAllIngestWatermarks(); } catch (err) { + // #5064/B: a live second long-lived writer — the DB is NOT corrupt, so skip the + // #2771 "delete the file" hint (it would advise destroying a healthy shared index); + // the actionable isolation recipe was already logged at error level by store.open. + if (err instanceof StorageForeignWriterError) { + loggerLog("warn", `[storage] unified sidecar degraded to legacy-only: another bili process already owns ${dir}/index.db — isolate the installs with BILI_SESSIONS_DIR + BILI_CONFIG_FILE to restore unified storage (#5064/#3923/#3633)`); + recordSidecarOpenFailure(err.message); + return null; + } // #2771: a corrupt/unreadable index.db must degrade to legacy-only // (same shape as the encryption opt-out), never brick SessionStore // construction. No auto-delete: the index may hold pinned checkpoint diff --git a/src/storage/store.ts b/src/storage/store.ts index 0cab41538d..e62e91cdcd 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -324,6 +324,21 @@ export class StorageIntegrityError extends Error { } } +/** #5064/B: raised when a long-lived open finds a LIVE foreign process already + * writing the same index.db. Unlike a corrupt file there is nothing to fix by + * deleting — the remedy is isolating the installs (BILI_SESSIONS_DIR + + * BILI_CONFIG_FILE). The caller (UnifiedShadow.maybeOpen) catches it and degrades + * this proxy to legacy-only via the #2771 shape; the DB is NOT corrupt, so the + * "delete the file" hint must never be attached to this error. Ephemeral + * CLI/export opens leave rejectForeignWriter unset and so never raise this — + * they keep the report-only behavior. */ +export class StorageForeignWriterError extends Error { + constructor(public readonly dbPath: string, public readonly holderPid: number) { + super(`[storage] refusing second long-lived writer on ${dbPath} (live foreign pid ${holderPid}); degrade to legacy-only — isolate installs with BILI_SESSIONS_DIR + BILI_CONFIG_FILE (#5064/#3923/#3633)`); + this.name = "StorageForeignWriterError"; + } +} + /** #4175: true when a pid belongs to a live process (EPERM = alive, foreign user). */ function processAlive(pid: number): boolean { try { @@ -363,7 +378,7 @@ export class UnifiedStore { static open( path: string, engine?: SqliteEngineName, - opts?: { walJournalSizeLimitBytes?: number }, + opts?: { walJournalSizeLimitBytes?: number; rejectForeignWriter?: boolean }, ): UnifiedStore { mkdirSync(dirname(path), { recursive: true }); const { db, engine: used, source } = openSqliteDatabase(path, engine); @@ -437,16 +452,30 @@ export class UnifiedStore { .reduce((best, e) => (best === undefined || (e.startedAt ?? 0) > (best.startedAt ?? 0) ? e : best), undefined); const ownLane = process.env.BILI_LAUNCHER_LANE?.trim() || undefined; const benign = peer !== undefined && ownLane !== undefined && peer.lane !== undefined && peer.lane !== ownLane; + recordSecondProcessOwner(pid, { lane: peer?.lane, instanceId: peer?.instanceId, benign }); if (benign) { loggerLog("info", `[storage] process (pid ${pid}, lane ${peer?.lane}, instance ${peer?.instanceId.slice(0, 8)}) is also using ${path} — different lanes sharing one sessions directory is the supported topology (#3923/#3959); informational only.`); + } else if (opts?.rejectForeignWriter) { + // #5064/B: a long-lived second writer must not co-reside on one index.db — + // two writers corrupt it. Refuse here so the caller's #2771 open-failure + // path degrades THIS proxy to legacy-only (availability preserved, no + // double-write) instead of continuing RW. Ephemeral CLI/export opens leave + // rejectForeignWriter unset and keep the report-only behavior below. + loggerLog("error", `[storage] another bili process (pid ${pid}) already owns ${path} — a second long-lived writer on one index.db would corrupt it (#3923/#3959/#3633), so this proxy refuses unified storage and degrades to legacy-only: sessions keep working from the JSON store, but unified-storage features (fast load / GC / vacuum / checkpoints) are unavailable until isolated. Fix: give each install its own sessions dir + config file — set BILI_SESSIONS_DIR= AND BILI_CONFIG_FILE= per install, then restart both.`); + db.close(); + throw new StorageForeignWriterError(path, pid); } else { loggerLog("error", `[storage] another process (pid ${pid}) is already writing ${path} — two processes on one index.db corrupt it (#3923/#3959). One session dir must have exactly one instance (#3633); this lane keeps running but the condition is reported.`); } - recordSecondProcessOwner(pid, { lane: peer?.lane, instanceId: peer?.instanceId, benign }); } } writeFileSync(lockPath, JSON.stringify({ pid: process.pid, boot: Date.now() })); } catch (err) { + // #5064/B: the deliberate refuse-to-co-reside signal must escape this + // guard (which exists only to make lockfile read/write hiccups + // non-fatal) — otherwise it is swallowed and the open proceeds onto a + // closed handle. + if (err instanceof StorageForeignWriterError) throw err; loggerLog("warn", `[storage] lockfile discipline skipped for ${path}: ${String(err)}`); } try { diff --git a/tests/issue5064-foreign-writer-degrade.test.ts b/tests/issue5064-foreign-writer-degrade.test.ts new file mode 100644 index 0000000000..eb07640a6b --- /dev/null +++ b/tests/issue5064-foreign-writer-degrade.test.ts @@ -0,0 +1,111 @@ +/** + * #5064/B: a live second LONG-LIVED writer on one index.db must degrade this + * proxy to legacy-only instead of co-residing and corrupting the shared store. + * + * Both directions are asserted (gate discipline, #3617): + * - TRIGGER: rejectForeignWriter + a live foreign pid → open throws + * StorageForeignWriterError; maybeOpen degrades to null; status carries the + * isolation reason (and never the destructive "delete the file" hint). + * - NO-TRIGGER: without the flag (ephemeral CLI/export posture) the SAME + * foreign writer keeps the report-only behavior (open succeeds, holder + * surfaced); and a healthy dir (no foreign writer) opens the sidecar cleanly. + */ +import { strict as assert } from "node:assert"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +delete process.env.BILI_STORAGE_UNIFIED; + +import { UnifiedStore, StorageForeignWriterError } from "../src/storage/store.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { storageStatusWire, recordRequestedMode, _resetStorageRuntimeStatusForTest } from "../src/storage/runtime-status.ts"; + +function freshDir(): string { + return mkdtempSync(join(tmpdir(), "issue5064-foreign-writer-")); +} + +/** Spawn a short-lived child owning a genuinely-live pid (same convention as + * tests/storage-durability.test.ts). */ +async function liveChildPid(): Promise<{ pid: number; kill: () => void }> { + const { spawn } = await import("node:child_process"); + const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "ignore" }); + await new Promise((resolve) => setTimeout(resolve, 300)); + return { pid: child.pid!, kill: () => child.kill("SIGKILL") }; +} + +test("#5064 trigger: rejectForeignWriter + live foreign pid → open throws, owner's lock left intact", async () => { + const dir = freshDir(); + const holder = await liveChildPid(); + try { + const path = join(dir, "index.db"); + writeFileSync(`${path}.lock`, JSON.stringify({ pid: holder.pid, boot: Date.now() })); + assert.throws( + () => UnifiedStore.open(path, undefined, { rejectForeignWriter: true }), + (err: unknown) => err instanceof StorageForeignWriterError && err.holderPid === holder.pid, + ); + // We refused BEFORE claiming the lock — the foreign owner's record survives. + const onDisk = JSON.parse(readFileSync(`${path}.lock`, "utf8")) as { pid?: number }; + assert.equal(onDisk.pid, holder.pid, "the foreign owner's lockfile is not clobbered"); + } finally { + holder.kill(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#5064 no-trigger: without the flag the same foreign writer keeps report-only (open succeeds)", async () => { + const dir = freshDir(); + const holder = await liveChildPid(); + try { + const path = join(dir, "index.db"); + writeFileSync(`${path}.lock`, JSON.stringify({ pid: holder.pid, boot: Date.now() })); + const store = UnifiedStore.open(path); // no rejectForeignWriter → ephemeral posture + try { + assert.ok(store.lockHolder() !== null, "ephemeral open still surfaces the foreign holder"); + assert.equal(store.lockHolder()?.pid, holder.pid); + } finally { + store.close(); + } + } finally { + holder.kill(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#5064 trigger (integration): maybeOpen degrades the sidecar to null + status carries the isolation reason", async () => { + const dir = freshDir(); + const holder = await liveChildPid(); + _resetStorageRuntimeStatusForTest(); + try { + recordRequestedMode("shadow", false); + writeFileSync(join(dir, "index.db.lock"), JSON.stringify({ pid: holder.pid, boot: Date.now() })); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, () => null); + assert.equal(shadow, null, "a second long-lived writer degrades the sidecar to legacy-only (null)"); + const wire = storageStatusWire(); + assert.equal(wire.degraded, true, "status reports degraded"); + assert.match(wire.reason ?? "", /BILI_SESSIONS_DIR/, "reason names the isolation lever"); + assert.doesNotMatch(wire.reason ?? "", /delete the file/i, "no destructive delete-file hint for a healthy shared db"); + } finally { + holder.kill(); + _resetStorageRuntimeStatusForTest(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("#5064 no-trigger (integration): a healthy dir (no foreign writer) opens the sidecar cleanly", async () => { + const dir = freshDir(); + _resetStorageRuntimeStatusForTest(); + let shadow: UnifiedShadow | null = null; + try { + recordRequestedMode("shadow", false); + shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, () => null); + assert.ok(shadow !== null, "no foreign writer → sidecar opens (no spurious degrade)"); + const wire = storageStatusWire(); + assert.equal(wire.degraded, false, "healthy sidecar is not reported degraded"); + } finally { + shadow?.close(); + _resetStorageRuntimeStatusForTest(); + rmSync(dir, { recursive: true, force: true }); + } +});