Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/storage/driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,17 @@ function loadHostBetterSqlite3(): BetterSqlite3Ctor {
return require("better-sqlite3") as unknown as BetterSqlite3Ctor;
}

/** #3959: genuine file CORRUPTION (SQLITE_CORRUPT / SQLITE_NOTADB) only —
* the class where quarantining index.db is safe. Deliberately narrow:
* CANTOPEN/BUSY/permission/no-engine failures may face a perfectly healthy
* file behind an OS lock (#3823) and must NOT trip quarantine. */
export function isCorruptIndexError(err: unknown): boolean {
const code = (err as { code?: unknown } | null)?.code;
if (code === "SQLITE_CORRUPT" || code === "SQLITE_NOTADB") return true;
const msg = err instanceof Error ? err.message : String(err);
return /database disk image is malformed/i.test(msg) || /file is not a database/i.test(msg);
}

export function openSqliteDatabase(path: string, engine?: SqliteEngineName): OpenedDatabase {
if (enginesUnavailableForTests) {
throw new SqliteDriverError("no SQLite engine available", "node:sqlite");
Expand Down
79 changes: 69 additions & 10 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,13 @@
* Failure policy: sidecar problems must NEVER break the legacy write path.
* All work is try/catch'd; after MAX_CONSECUTIVE_FAILURES consecutive
* failures the sidecar disables itself for the process (warn once) — in
* full mode that also flips reads back to legacy-only.
* full mode that also flips reads back to legacy-only. Genuine index.db
* CORRUPTION at open time self-heals instead (#3959): the dead file is
* quarantined (renamed, never deleted) and a fresh index opens — the legacy
* JSON files remain the source of truth, so nothing is lost.
*/

import { renameSync } from "node:fs";
import { log as loggerLog } from "../logger.js";
import { planIngest, planIngestDelta, loadLegacyView, completeBlockContents, type LegacySessionLike } from "./ingest.ts";
import type { BlockView } from "../session.js";
Expand All @@ -29,6 +33,7 @@ import { UnifiedStore, type CheckpointRow, type SessionSummaryRow, type WebSumma
import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts";
import { canonicalize, sha256 } from "./canonical.ts";
import { recordSidecarInstance, recordSidecarOpenFailure } from "./runtime-status.ts";
import { isCorruptIndexError } from "./driver.ts";
import type { SqliteEngineName, SqliteEngineSource } from "./driver.ts";
import type { MessageRow } from "./store.js";
import type { CoreMessage, MessageContentStore } from "acp-kernel";
Expand Down Expand Up @@ -169,26 +174,54 @@ export class UnifiedShadow {
const notice = backendSelectionNotice(selected);
if (notice) loggerLog("warn", notice);
}
let store: UnifiedStore;
const indexPath = `${dir}/index.db`;
let store: UnifiedStore | null = null;
try {
store = UnifiedStore.open(`${dir}/index.db`, undefined, { walJournalSizeLimitBytes: walLimit });
store = UnifiedStore.open(indexPath, undefined, { walJournalSizeLimitBytes: walLimit });
// #3285: watermark trust does not cross process boundaries — a
// restart-handoff overlap leaves the previous writer's rows AND
// watermark behind; drop them all so this process's first save
// per session re-establishes trust from its own view.
store.resetAllIngestWatermarks();
} catch (err) {
// #2771: a corrupt/unreadable index.db must degrade to legacy-only
// (same shape as the encryption opt-out), never brick SessionStore
// construction. No auto-delete: the index may hold pinned checkpoint
// blobs (#2671 Phase 2) — destructive recovery is operator policy.
const errStr = String(err);
// #2771 kept corrupt-index recovery manual ("delete the file")
// because DESTRUCTION was operator policy: the index may hold
// pinned checkpoint blobs (#2671 Phase 2). #3959 field report:
// that made one corrupt index.db pin the machine to legacy-only
// forever until someone deleted it by hand. QUARANTINE (rename,
// never delete) removes the manual step without the data risk —
// the legacy JSON session files remain the source of truth in
// BOTH shadow and full mode (full keeps writing them), so a
// fresh index converges as sessions save/load again, and the old
// bytes stay on disk beside the original for forensics.
let quarantinedTo: string | null = null;
if (isCorruptIndexError(err)) {
quarantinedTo = quarantineCorruptIndex(indexPath);
if (quarantinedTo) {
try {
store = UnifiedStore.open(indexPath, undefined, { walJournalSizeLimitBytes: walLimit });
store.resetAllIngestWatermarks();
} catch (retryErr) {
store = null;
loggerLog("error", `[storage] fresh index open after quarantine also failed (${String(retryErr)}) — staying legacy-only`);
}
} else {
loggerLog("error", `[storage] could not quarantine corrupt index ${indexPath} (${errStr}) — staying legacy-only`);
}
}
if (!store) {
loggerLog(
"warn",
`[storage] unified sidecar failed to open ${dir}/index.db (${errStr}) — staying legacy-only${sqliteOpenFailureHint(errStr)}; delete the file to reset the index`,
);
recordSidecarOpenFailure(errStr);
return null;
}
loggerLog(
"warn",
`[storage] unified sidecar failed to open ${dir}/index.db (${errStr}) — staying legacy-only${sqliteOpenFailureHint(errStr)}; delete the file to reset the index`,
`[storage] unified sidecar index was corrupt (${errStr}) — quarantined to ${quarantinedTo} and opened a fresh index; sessions re-ingest from their legacy files as they are saved/loaded`,
);
recordSidecarOpenFailure(errStr);
return null;
}
const shadow = new UnifiedShadow(mode, store, legacyLoad ?? (() => null));
recordSidecarInstance(shadow);
Expand Down Expand Up @@ -1031,6 +1064,32 @@ export class UnifiedShadow {
}
}

/** #3959: rename a corrupt index.db (+ its -wal/-shm/-journal sidecars) out
* of the way under a `.corrupt-<timestamp>` suffix — the same quarantine
* pattern persist.ts applies to undecodable session state (#3157): kept for
* forensics, never deleted, and the suffix keeps directory walkers off it
* (they match *.json). Returns the main file's quarantine path, or null
* when even the main rename failed (caller keeps the legacy-only degrade).
* Sidecar renames are best-effort: a stale WAL left behind cannot poison
* the fresh db — verified empirically (node:sqlite 22.x): a fresh open over
* an orphaned unclean-exit WAL resets the WAL, integrity_check stays ok. */
function quarantineCorruptIndex(file: string): string | null {
const quarantinePath = `${file}.corrupt-${Date.now()}`;
try {
renameSync(file, quarantinePath);
} catch {
return null;
}
for (const suffix of ["-wal", "-shm", "-journal"]) {
try {
renameSync(`${file}${suffix}`, `${quarantinePath}${suffix}`);
} catch {
// absent or locked — harmless (see doc comment)
}
}
return quarantinePath;
}

function shadowDigest(view: unknown): string {
return sha256(JSON.stringify(canonicalize(view)));
}
Expand Down
160 changes: 160 additions & 0 deletions tests/storage-corrupt-index-selfheal.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, readdirSync, readFileSync, writeFileSync, truncateSync, statSync, existsSync } from "node:fs";
import { tmpdir } from "node:os";
import * as path from "node:path";
import { UnifiedShadow } from "../src/storage/shadow.ts";
import { UnifiedStore } from "../src/storage/store.ts";
import { setSqliteEnginesUnavailableForTests } from "../src/storage/driver.ts";
import { _resetStorageRuntimeStatusForTest, storageStatusWire } from "../src/storage/runtime-status.ts";
import { rmrf } from "./tmp-rm.ts";

/**
* #3959: a corrupt index.db used to pin a machine to legacy-only forever —
* open failed with SQLITE_CORRUPT, the log said "delete the file to reset
* the index", and every restart re-failed until someone deleted it by hand.
* Now genuine corruption-class errors quarantine the dead file (rename to
* `.corrupt-<ts>`, never delete) and open a fresh index; the legacy JSON
* session files remain the source of truth, so nothing is lost. Gate
* discipline (#3617): both directions pinned — corruption recovers, and
* non-corruption failures (CANTOPEN, no-engine) do NOT trip quarantine.
*/

const GARBAGE = "this is definitely not a sqlite database, just plain text garbage bytes 0123456789";

const ENV_KEYS = ["BILI_SESSIONS_DIR", "BILI_CONFIG_FILE", "XDG_CACHE_HOME", "XDG_STATE_HOME", "BILI_STORAGE_UNIFIED", "BILI_ENCRYPTION_KEY"] as const;

function isolateEnv(base: string, sessions: string): Record<string, string | undefined> {
const snap: Record<string, string | undefined> = {};
for (const k of ENV_KEYS) snap[k] = process.env[k];
process.env.BILI_SESSIONS_DIR = sessions;
process.env.BILI_STORAGE_UNIFIED = "shadow";
process.env.BILI_CONFIG_FILE = path.join(base, "absent-config.json");
process.env.XDG_CACHE_HOME = path.join(base, "cache");
process.env.XDG_STATE_HOME = path.join(base, "state");
delete process.env.BILI_ENCRYPTION_KEY;
return snap;
}

function restoreEnv(snap: Record<string, string | undefined>): void {
for (const k of ENV_KEYS) {
if (snap[k] === undefined) delete process.env[k];
else process.env[k] = snap[k];
}
}

function quarantineFiles(sessions: string): string[] {
return readdirSync(sessions).filter((f) => f.startsWith("index.db.corrupt-"));
}

test("#3959: non-SQLite bytes in index.db are quarantined and a fresh index opens", () => {
const base = mkdtempSync(path.join(tmpdir(), "corrupt-index-notadb-"));
const sessions = path.join(base, "sessions");
mkdirSync(sessions, { recursive: true });
writeFileSync(path.join(sessions, "index.db"), GARBAGE);
const snap = isolateEnv(base, sessions);
let shadow: UnifiedShadow | null = null;
try {
_resetStorageRuntimeStatusForTest();
shadow = UnifiedShadow.maybeOpen(sessions);
assert.ok(shadow, "sidecar self-heals instead of degrading");
assert.equal(shadow.stats.disabled, false);
assert.equal(shadow.stats.lastError, null);
const wire = storageStatusWire();
assert.equal(wire.degraded, false, "wire reports healthy after self-heal");
assert.equal(wire.effective, "shadow");
const qs = quarantineFiles(sessions);
assert.equal(qs.length, 1, "exactly one quarantine file");
assert.equal(readFileSync(path.join(sessions, qs[0]), "utf8"), GARBAGE, "original bytes preserved for forensics");
assert.ok(existsSync(path.join(sessions, "index.db")), "fresh index.db created in place");
} finally {
shadow?.close();
_resetStorageRuntimeStatusForTest();
restoreEnv(snap);
rmrf(base);
}
});

test("#3959: a truncated real db (database disk image is malformed) is quarantined and a fresh index opens", () => {
const base = mkdtempSync(path.join(tmpdir(), "corrupt-index-truncated-"));
const sessions = path.join(base, "sessions");
mkdirSync(sessions, { recursive: true });
const dbPath = path.join(sessions, "index.db");
// Build a real db with committed data so truncation lands mid-page.
const direct = UnifiedStore.open(dbPath);
direct.setKv("selfheal-probe", "key", "x".repeat(4000));
direct.close();
const size = statSync(dbPath).size;
truncateSync(dbPath, Math.floor(size / 2));
const snap = isolateEnv(base, sessions);
let shadow: UnifiedShadow | null = null;
try {
_resetStorageRuntimeStatusForTest();
shadow = UnifiedShadow.maybeOpen(sessions);
assert.ok(shadow, "sidecar self-heals a malformed db");
const wire = storageStatusWire();
assert.equal(wire.degraded, false, "wire reports healthy after self-heal");
assert.equal(wire.effective, "shadow");
const qs = quarantineFiles(sessions);
assert.equal(qs.length, 1, "exactly one quarantine file");
assert.equal(statSync(path.join(sessions, qs[0])).size, Math.floor(size / 2), "truncated bytes preserved for forensics");
assert.ok(existsSync(dbPath), "fresh index.db created in place");
} finally {
shadow?.close();
_resetStorageRuntimeStatusForTest();
restoreEnv(snap);
rmrf(base);
}
});

test("#3959 negative: CANTOPEN (a directory named index.db) degrades WITHOUT quarantining anything", () => {
const base = mkdtempSync(path.join(tmpdir(), "corrupt-index-cantopen-"));
const sessions = path.join(base, "sessions");
mkdirSync(sessions, { recursive: true });
// A DIRECTORY named index.db: SQLite refuses it (SQLITE_CANTOPEN)
// deterministically on every platform — the same error class as the
// AV-lock / OneDrive / integrity-label field reports (#3823), where the
// "file" situation may be perfectly fine once the OS lock clears.
mkdirSync(path.join(sessions, "index.db"));
const snap = isolateEnv(base, sessions);
try {
_resetStorageRuntimeStatusForTest();
const shadow = UnifiedShadow.maybeOpen(sessions);
assert.equal(shadow, null, "still degrades to legacy-only");
assert.deepEqual(quarantineFiles(sessions), [], "no quarantine side effect on CANTOPEN");
assert.ok(existsSync(path.join(sessions, "index.db")), "the obstructing entry is left untouched");
const wire = storageStatusWire();
assert.equal(wire.degraded, true);
assert.ok(wire.reason?.startsWith("index database failed to open"), `reason names the open failure (got: ${wire.reason})`);
} finally {
_resetStorageRuntimeStatusForTest();
restoreEnv(snap);
rmrf(base);
}
});

test("#3959 negative: a no-engine failure degrades WITHOUT quarantining anything", () => {
const base = mkdtempSync(path.join(tmpdir(), "corrupt-index-noengine-"));
const sessions = path.join(base, "sessions");
mkdirSync(sessions, { recursive: true });
// A valid empty db file sits there — the failure class is "no engine",
// not "bad file": quarantining would move a healthy file aside.
const direct = UnifiedStore.open(path.join(sessions, "index.db"));
direct.close();
const snap = isolateEnv(base, sessions);
try {
_resetStorageRuntimeStatusForTest();
setSqliteEnginesUnavailableForTests(true);
const shadow = UnifiedShadow.maybeOpen(sessions);
assert.equal(shadow, null, "still degrades to legacy-only");
assert.deepEqual(quarantineFiles(sessions), [], "no quarantine side effect without an engine");
const wire = storageStatusWire();
assert.equal(wire.degraded, true);
assert.ok(wire.reason, "reason recorded");
} finally {
setSqliteEnginesUnavailableForTests(false);
_resetStorageRuntimeStatusForTest();
restoreEnv(snap);
rmrf(base);
}
});
Loading
Loading