Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/storage/runtime-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ function degradeReason(): string {
if (/no SQLite engine available/i.test(e)) {
return `no SQLite engine available (node:sqlite missing, better-sqlite3 not installed)${nodeSqliteUpgradeHint()}`;
}
if (/refusing second long-lived writer/i.test(state.sidecarOpenError)) {
return "a second bili process owns the sessions index.db — unified storage degraded to legacy-only; isolate installs with BILI_SESSIONS_DIR + BILI_CONFIG_FILE";
}
return `index database failed to open (${e})${sqliteOpenFailureHint(state.sidecarOpenError)}`;
}
const b = state.bootstrap;
Expand Down
14 changes: 11 additions & 3 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import { log as loggerLog } from "../logger.js";
import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts";
import type { BlockView } from "../session.js";
import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts";
import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts";
import { UnifiedStore, StorageIntegrityError, StorageForeignWriterError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts";
import { renameSync } from "node:fs";
import path from "node:path";
import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts";
Expand Down Expand Up @@ -192,7 +192,7 @@ export class UnifiedShadow {
let store: UnifiedStore;
try {
try {
store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit });
store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit, rejectForeignWriter: true });
} catch (err) {
// #4175: quick_check failed — the file set is provably corrupt.
// Quarantine it (db/-wal/-shm → .corrupt-<ts>, same suffix
Expand Down Expand Up @@ -227,14 +227,22 @@ export class UnifiedShadow {
"error",
`[storage] index.db failed its integrity quick_check (${err.detail}) — quarantined ${quarantined.join(", ")} and reopened fresh; JSON envelopes remain the recovery source (#4175)`,
);
store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit });
store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit, rejectForeignWriter: true });
}
// #3285: watermark trust does not cross process boundaries — a
// restart-handoff overlap leaves the previous writer's rows AND
// watermark behind; drop them all so this process's first save
// per session re-establishes trust from its own view.
store.resetAllIngestWatermarks();
} catch (err) {
// #5064/B: a live second long-lived writer — the DB is NOT corrupt, so skip the
// #2771 "delete the file" hint (it would advise destroying a healthy shared index);
// the actionable isolation recipe was already logged at error level by store.open.
if (err instanceof StorageForeignWriterError) {
loggerLog("warn", `[storage] unified sidecar degraded to legacy-only: another bili process already owns ${dir}/index.db — isolate the installs with BILI_SESSIONS_DIR + BILI_CONFIG_FILE to restore unified storage (#5064/#3923/#3633)`);
recordSidecarOpenFailure(err.message);
return null;
}
// #2771: a corrupt/unreadable index.db must degrade to legacy-only
// (same shape as the encryption opt-out), never brick SessionStore
// construction. No auto-delete: the index may hold pinned checkpoint
Expand Down
33 changes: 31 additions & 2 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,21 @@ export class StorageIntegrityError extends Error {
}
}

/** #5064/B: raised when a long-lived open finds a LIVE foreign process already
* writing the same index.db. Unlike a corrupt file there is nothing to fix by
* deleting — the remedy is isolating the installs (BILI_SESSIONS_DIR +
* BILI_CONFIG_FILE). The caller (UnifiedShadow.maybeOpen) catches it and degrades
* this proxy to legacy-only via the #2771 shape; the DB is NOT corrupt, so the
* "delete the file" hint must never be attached to this error. Ephemeral
* CLI/export opens leave rejectForeignWriter unset and so never raise this —
* they keep the report-only behavior. */
export class StorageForeignWriterError extends Error {
constructor(public readonly dbPath: string, public readonly holderPid: number) {
super(`[storage] refusing second long-lived writer on ${dbPath} (live foreign pid ${holderPid}); degrade to legacy-only — isolate installs with BILI_SESSIONS_DIR + BILI_CONFIG_FILE (#5064/#3923/#3633)`);
this.name = "StorageForeignWriterError";
}
}

/** #4175: true when a pid belongs to a live process (EPERM = alive, foreign user). */
function processAlive(pid: number): boolean {
try {
Expand Down Expand Up @@ -363,7 +378,7 @@ export class UnifiedStore {
static open(
path: string,
engine?: SqliteEngineName,
opts?: { walJournalSizeLimitBytes?: number },
opts?: { walJournalSizeLimitBytes?: number; rejectForeignWriter?: boolean },
): UnifiedStore {
mkdirSync(dirname(path), { recursive: true });
const { db, engine: used, source } = openSqliteDatabase(path, engine);
Expand Down Expand Up @@ -437,16 +452,30 @@ export class UnifiedStore {
.reduce<RegistryEntry | undefined>((best, e) => (best === undefined || (e.startedAt ?? 0) > (best.startedAt ?? 0) ? e : best), undefined);
const ownLane = process.env.BILI_LAUNCHER_LANE?.trim() || undefined;
const benign = peer !== undefined && ownLane !== undefined && peer.lane !== undefined && peer.lane !== ownLane;
recordSecondProcessOwner(pid, { lane: peer?.lane, instanceId: peer?.instanceId, benign });
if (benign) {
loggerLog("info", `[storage] process (pid ${pid}, lane ${peer?.lane}, instance ${peer?.instanceId.slice(0, 8)}) is also using ${path} — different lanes sharing one sessions directory is the supported topology (#3923/#3959); informational only.`);
} else if (opts?.rejectForeignWriter) {
// #5064/B: a long-lived second writer must not co-reside on one index.db —
// two writers corrupt it. Refuse here so the caller's #2771 open-failure
// path degrades THIS proxy to legacy-only (availability preserved, no
// double-write) instead of continuing RW. Ephemeral CLI/export opens leave
// rejectForeignWriter unset and keep the report-only behavior below.
loggerLog("error", `[storage] another bili process (pid ${pid}) already owns ${path} — a second long-lived writer on one index.db would corrupt it (#3923/#3959/#3633), so this proxy refuses unified storage and degrades to legacy-only: sessions keep working from the JSON store, but unified-storage features (fast load / GC / vacuum / checkpoints) are unavailable until isolated. Fix: give each install its own sessions dir + config file — set BILI_SESSIONS_DIR=<dir> AND BILI_CONFIG_FILE=<file> per install, then restart both.`);
db.close();
throw new StorageForeignWriterError(path, pid);
} else {
loggerLog("error", `[storage] another process (pid ${pid}) is already writing ${path} — two processes on one index.db corrupt it (#3923/#3959). One session dir must have exactly one instance (#3633); this lane keeps running but the condition is reported.`);
}
recordSecondProcessOwner(pid, { lane: peer?.lane, instanceId: peer?.instanceId, benign });
}
}
writeFileSync(lockPath, JSON.stringify({ pid: process.pid, boot: Date.now() }));
} catch (err) {
// #5064/B: the deliberate refuse-to-co-reside signal must escape this
// guard (which exists only to make lockfile read/write hiccups
// non-fatal) — otherwise it is swallowed and the open proceeds onto a
// closed handle.
if (err instanceof StorageForeignWriterError) throw err;
loggerLog("warn", `[storage] lockfile discipline skipped for ${path}: ${String(err)}`);
}
try {
Expand Down
111 changes: 111 additions & 0 deletions tests/issue5064-foreign-writer-degrade.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/**
* #5064/B: a live second LONG-LIVED writer on one index.db must degrade this
* proxy to legacy-only instead of co-residing and corrupting the shared store.
*
* Both directions are asserted (gate discipline, #3617):
* - TRIGGER: rejectForeignWriter + a live foreign pid → open throws
* StorageForeignWriterError; maybeOpen degrades to null; status carries the
* isolation reason (and never the destructive "delete the file" hint).
* - NO-TRIGGER: without the flag (ephemeral CLI/export posture) the SAME
* foreign writer keeps the report-only behavior (open succeeds, holder
* surfaced); and a healthy dir (no foreign writer) opens the sidecar cleanly.
*/
import { strict as assert } from "node:assert";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";

delete process.env.BILI_STORAGE_UNIFIED;

import { UnifiedStore, StorageForeignWriterError } from "../src/storage/store.ts";
import { UnifiedShadow } from "../src/storage/shadow.ts";
import { storageStatusWire, recordRequestedMode, _resetStorageRuntimeStatusForTest } from "../src/storage/runtime-status.ts";

function freshDir(): string {
return mkdtempSync(join(tmpdir(), "issue5064-foreign-writer-"));
}

/** Spawn a short-lived child owning a genuinely-live pid (same convention as
* tests/storage-durability.test.ts). */
async function liveChildPid(): Promise<{ pid: number; kill: () => void }> {
const { spawn } = await import("node:child_process");
const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "ignore" });
await new Promise<void>((resolve) => setTimeout(resolve, 300));
return { pid: child.pid!, kill: () => child.kill("SIGKILL") };
}

test("#5064 trigger: rejectForeignWriter + live foreign pid → open throws, owner's lock left intact", async () => {
const dir = freshDir();
const holder = await liveChildPid();
try {
const path = join(dir, "index.db");
writeFileSync(`${path}.lock`, JSON.stringify({ pid: holder.pid, boot: Date.now() }));
assert.throws(
() => UnifiedStore.open(path, undefined, { rejectForeignWriter: true }),
(err: unknown) => err instanceof StorageForeignWriterError && err.holderPid === holder.pid,
);
// We refused BEFORE claiming the lock — the foreign owner's record survives.
const onDisk = JSON.parse(readFileSync(`${path}.lock`, "utf8")) as { pid?: number };
assert.equal(onDisk.pid, holder.pid, "the foreign owner's lockfile is not clobbered");
} finally {
holder.kill();
rmSync(dir, { recursive: true, force: true });
}
});

test("#5064 no-trigger: without the flag the same foreign writer keeps report-only (open succeeds)", async () => {
const dir = freshDir();
const holder = await liveChildPid();
try {
const path = join(dir, "index.db");
writeFileSync(`${path}.lock`, JSON.stringify({ pid: holder.pid, boot: Date.now() }));
const store = UnifiedStore.open(path); // no rejectForeignWriter → ephemeral posture
try {
assert.ok(store.lockHolder() !== null, "ephemeral open still surfaces the foreign holder");
assert.equal(store.lockHolder()?.pid, holder.pid);
} finally {
store.close();
}
} finally {
holder.kill();
rmSync(dir, { recursive: true, force: true });
}
});

test("#5064 trigger (integration): maybeOpen degrades the sidecar to null + status carries the isolation reason", async () => {
const dir = freshDir();
const holder = await liveChildPid();
_resetStorageRuntimeStatusForTest();
try {
recordRequestedMode("shadow", false);
writeFileSync(join(dir, "index.db.lock"), JSON.stringify({ pid: holder.pid, boot: Date.now() }));
const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, () => null);
assert.equal(shadow, null, "a second long-lived writer degrades the sidecar to legacy-only (null)");
const wire = storageStatusWire();
assert.equal(wire.degraded, true, "status reports degraded");
assert.match(wire.reason ?? "", /BILI_SESSIONS_DIR/, "reason names the isolation lever");
assert.doesNotMatch(wire.reason ?? "", /delete the file/i, "no destructive delete-file hint for a healthy shared db");
} finally {
holder.kill();
_resetStorageRuntimeStatusForTest();
rmSync(dir, { recursive: true, force: true });
}
});

test("#5064 no-trigger (integration): a healthy dir (no foreign writer) opens the sidecar cleanly", async () => {
const dir = freshDir();
_resetStorageRuntimeStatusForTest();
let shadow: UnifiedShadow | null = null;
try {
recordRequestedMode("shadow", false);
shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, () => null);
assert.ok(shadow !== null, "no foreign writer → sidecar opens (no spurious degrade)");
const wire = storageStatusWire();
assert.equal(wire.degraded, false, "healthy sidecar is not reported degraded");
} finally {
shadow?.close();
_resetStorageRuntimeStatusForTest();
rmSync(dir, { recursive: true, force: true });
}
});