Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 51 additions & 9 deletions src/storage/driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,26 @@ interface OpenedDatabase {
readonly source: SqliteEngineSource;
}

/** #4949: better-sqlite3's own wrapper runs ROLLBACK after ANY inner failure;
* when BEGIN or COMMIT itself failed there was nothing left to roll back, and
* its "cannot rollback - no transaction is active" REPLACES the real error
* (commonly disk full / SQLITE_FULL) before it reaches us. Name the masking
* at our seam so operators see the likely cause. Both lock modes go through
* it — the wrapper masks identically in deferred mode. */
function surfaceMaskedRollbackError<T>(run: (fn: () => T) => T): (fn: () => T) => T {
return (fn) => {
try {
return run(fn);
} catch (err) {
const msg = String(err);
if (/cannot rollback|no transaction is active/i.test(msg)) {
throw new SqliteDriverError(`${msg} — masked by better-sqlite3's automatic rollback after a failed BEGIN/COMMIT; the original error no longer survives (disk full / SQLITE_FULL is the common cause)`, "better-sqlite3");
}
throw err;
}
};
}

function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase {
return {
exec: (sql) => void db.exec(sql),
Expand All @@ -88,24 +108,37 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase {
// a write lock mid-flight — SQLITE_BUSY_SNAPSHOT, which busy_timeout
// cannot resolve. Taking the write lock up front makes concurrent
// writers queue on busy_timeout instead (#2690 review blocking 5).
transaction: (fn) => db.transaction(fn).immediate(),
readTransaction: (fn) => db.transaction(fn).deferred(),
transaction: surfaceMaskedRollbackError((fn) => db.transaction(fn).immediate()),
readTransaction: surfaceMaskedRollbackError((fn) => db.transaction(fn).deferred()),
close: () => void db.close(),
get open() {
return db.open;
},
};
}

let databaseSyncFactoryForTests: ((path: string) => NodeSqliteDatabase) | null = null;

/** Test seam: substitute how the node:sqlite emulation obtains its database
* handle (e.g. an adapter injecting engine-level I/O failures, #4949).
* Null = the real builtin node:sqlite module. */
export function setDatabaseSyncFactoryForTests(factory: ((path: string) => NodeSqliteDatabase) | null): void {
databaseSyncFactoryForTests = factory;
}

function openNodeSqlite(path: string): SqliteDatabase {
// node:sqlite is synchronous like better-sqlite3, but has no
// .transaction() helper — emulate with BEGIN/COMMIT + savepoints for
// nesting, which is the only behavioral difference the store relies on.
// eslint-disable-next-line @typescript-eslint/no-require-imports
const { DatabaseSync } = require("node:sqlite") as {
DatabaseSync: new (path: string) => NodeSqliteDatabase;
};
const db = new DatabaseSync(path);
const db = databaseSyncFactoryForTests
? databaseSyncFactoryForTests(path)
: (() => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const { DatabaseSync } = require("node:sqlite") as {
DatabaseSync: new (path: string) => NodeSqliteDatabase;
};
return new DatabaseSync(path);
})();
let depth = 0;
const begin = (readOnly: boolean) => db.exec(depth === 0 ? (readOnly ? "BEGIN DEFERRED" : "BEGIN IMMEDIATE") : `SAVEPOINT sp${depth}`);
const commit = () => db.exec(depth === 1 ? "COMMIT" : `RELEASE sp${depth - 1}`);
Expand All @@ -123,7 +156,16 @@ function openNodeSqlite(path: string): SqliteDatabase {
commit();
return out;
} catch (err) {
rollback();
// #4949: a failed BEGIN/COMMIT can leave NO transaction active
// (SQLite auto-aborts on I/O-class failures such as disk full);
// the follow-up ROLLBACK then throws "cannot rollback - no
// transaction is active" and would MASK the original failure.
// The original error below is always authoritative.
try {
rollback();
} catch {
/* secondary rollback failure — original error wins (#4949) */
}
throw err;
} finally {
depth--;
Expand All @@ -146,7 +188,7 @@ interface NodeSqliteStatement {
get(...args: unknown[]): unknown;
all(...args: unknown[]): unknown[];
}
interface NodeSqliteDatabase {
export interface NodeSqliteDatabase {
exec(sql: string): unknown;
prepare(sql: string): NodeSqliteStatement;
getAutoCommit(): boolean;
Expand Down
59 changes: 57 additions & 2 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@ interface ResidentFields {
blockContents?: Map<string, { one: BlockView | null; full: BlockView }>;
}

/** #4949: re-enable cadence for the disabled sidecar. The 3-strike breaker used
* to be permanent until process restart even after the store healed. Now each
* cooldown window lets the NEXT save run one writability probe (a single-row
* committed write through the production path); success clears the breaker and
* the triggering record ingests normally. Missed saves converge via the
* load-side staleness guard (envelope savedAt > index row) + next full ingest,
* so no global pre-check is needed before unified reads resume. */
export const REENABLE_PROBE_COOLDOWN_MS = 5 * 60 * 1000;

/** #5188: ingest errors meaning "the resource was briefly unavailable",
* not "the storage is broken" — a concurrent lock held past busy_timeout,
* a momentary disk-full or fd spike, an EBUSY from the OS. These ride
Expand Down Expand Up @@ -191,6 +200,9 @@ export class UnifiedShadow {
* path left users hunting %TEMP% logs for which file said what. */
private readonly dbPath: string | null;
private consecutiveFailures = 0;
/** #4949: timestamp until which the disabled sidecar stays quiet; the first
* save after it runs the writability probe (REENABLE_PROBE_COOLDOWN_MS). */
private nextReenableProbeAt = 0;
/** #3133: per-session ingest counters (verify sampling). Per-process
* only — nothing persisted, no schema change. */
private readonly ingestCounts = new Map<string, number>();
Expand Down Expand Up @@ -409,7 +421,21 @@ export class UnifiedShadow {
this.enqueueAsyncRecord(record, rawTail, liveCcr);
return;
}
if (this.stats.disabled) return;
// #4949: a disabled sidecar is no longer dead for the process lifetime —
// one cooldown-gated writability probe per window; on success the
// breaker clears and THIS record falls through to the normal ingest.
if (this.stats.disabled) {
const now = Date.now();
if (now < this.nextReenableProbeAt) return;
this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS;
try {
this.sync.probeWritable();
} catch (err) {
this.noteProbeFailure(err);
return;
}
this.clearBreaker();
}
try {
const loaded = this.legacyLoad(record);
if (!loaded) return;
Expand Down Expand Up @@ -547,6 +573,21 @@ export class UnifiedShadow {
}
}

/** #4949: re-enable-probe failure tail — keeps the breaker armed while the
* latest error stays visible on the status wire via stats.lastError. */
private noteProbeFailure(err: unknown): void {
this.stats.lastError = String(err);
loggerLog("debug", `[storage] unified store re-enable probe failed (${String(err)}) — staying disabled, next probe after ${REENABLE_PROBE_COOLDOWN_MS / 60_000}min (#4949)`);
}

/** #4949: clears the 3-strike breaker after a successful writability probe. */
private clearBreaker(): void {
this.stats.disabled = false;
this.stats.lastError = null;
this.consecutiveFailures = 0;
loggerLog("info", "[storage] unified store RE-ENABLED: writability probe succeeded after disable (#4949); sessions missed during the outage converge via the envelope staleness guard + next full ingest");
}

/** #3133 delta ingest attempt for one record's legacy view — returns
* true when the O(new-messages) fast path committed. Every doubt
* routes to `false` (→ the caller's full ingest); failures here must
Expand Down Expand Up @@ -1066,7 +1107,21 @@ export class UnifiedShadow {
* shadow queue; failures feed the same 3-strike disable. */
private enqueueAsyncRecord(record: unknown, rawTail?: { lastMessages: CoreMessage[]; lastMessagesFolded?: boolean }, liveCcr?: MessageContentStore): void {
void this.enqueue(async () => {
if (this.stats.disabled) return;
// #4949: async twin of the sync re-enable probe in record() — MUST
// stay in lockstep with it; setKv is the backend's single-row
// committed write probe.
if (this.stats.disabled) {
const now = Date.now();
if (now < this.nextReenableProbeAt) return;
this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS;
try {
await this.backend!.setKv("__bili", "writability-probe", { probedAt: now });
} catch (err) {
this.noteProbeFailure(err);
return;
}
this.clearBreaker();
}
try {
const backend = this.backend!;
const loaded0 = this.legacyLoad(record);
Expand Down
10 changes: 10 additions & 0 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2153,6 +2153,16 @@ export class UnifiedStore {
return Number(this.db.prepare("DELETE FROM session_ext WHERE session_id = ? AND key = ?").run(sessionId, key).changes);
}

/** #4949: circuit-breaker re-enable probe — one REAL committed write
* through the production transaction path (proves lock acquisition +
* disk space, not just connectivity). A single fixed kv key keeps the
* trace constant; the row doubles as a liveness marker. Throws on failure. */
probeWritable(): void {
this.db.transaction(() => {
this.setKv("__bili", "writability-probe", { probedAt: Date.now() });
});
}

setKv(namespace: string, key: string, value: unknown): void {
validateExtKey(namespace, "namespace");
validateExtKey(key, "key");
Expand Down
Loading
Loading