diff --git a/src/storage/driver.ts b/src/storage/driver.ts index 63799382d4..596e7a79f4 100644 --- a/src/storage/driver.ts +++ b/src/storage/driver.ts @@ -78,6 +78,26 @@ interface OpenedDatabase { readonly source: SqliteEngineSource; } +/** #4949: better-sqlite3's own wrapper runs ROLLBACK after ANY inner failure; + * when BEGIN or COMMIT itself failed there was nothing left to roll back, and + * its "cannot rollback - no transaction is active" REPLACES the real error + * (commonly disk full / SQLITE_FULL) before it reaches us. Name the masking + * at our seam so operators see the likely cause. Both lock modes go through + * it — the wrapper masks identically in deferred mode. */ +function surfaceMaskedRollbackError(run: (fn: () => T) => T): (fn: () => T) => T { + return (fn) => { + try { + return run(fn); + } catch (err) { + const msg = String(err); + if (/cannot rollback|no transaction is active/i.test(msg)) { + throw new SqliteDriverError(`${msg} — masked by better-sqlite3's automatic rollback after a failed BEGIN/COMMIT; the original error no longer survives (disk full / SQLITE_FULL is the common cause)`, "better-sqlite3"); + } + throw err; + } + }; +} + function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { return { exec: (sql) => void db.exec(sql), @@ -88,8 +108,8 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { // a write lock mid-flight — SQLITE_BUSY_SNAPSHOT, which busy_timeout // cannot resolve. Taking the write lock up front makes concurrent // writers queue on busy_timeout instead (#2690 review blocking 5). - transaction: (fn) => db.transaction(fn).immediate(), - readTransaction: (fn) => db.transaction(fn).deferred(), + transaction: surfaceMaskedRollbackError((fn) => db.transaction(fn).immediate()), + readTransaction: surfaceMaskedRollbackError((fn) => db.transaction(fn).deferred()), close: () => void db.close(), get open() { return db.open; @@ -97,15 +117,28 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { }; } +let databaseSyncFactoryForTests: ((path: string) => NodeSqliteDatabase) | null = null; + +/** Test seam: substitute how the node:sqlite emulation obtains its database + * handle (e.g. an adapter injecting engine-level I/O failures, #4949). + * Null = the real builtin node:sqlite module. */ +export function setDatabaseSyncFactoryForTests(factory: ((path: string) => NodeSqliteDatabase) | null): void { + databaseSyncFactoryForTests = factory; +} + function openNodeSqlite(path: string): SqliteDatabase { // node:sqlite is synchronous like better-sqlite3, but has no // .transaction() helper — emulate with BEGIN/COMMIT + savepoints for // nesting, which is the only behavioral difference the store relies on. - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { DatabaseSync } = require("node:sqlite") as { - DatabaseSync: new (path: string) => NodeSqliteDatabase; - }; - const db = new DatabaseSync(path); + const db = databaseSyncFactoryForTests + ? databaseSyncFactoryForTests(path) + : (() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { DatabaseSync } = require("node:sqlite") as { + DatabaseSync: new (path: string) => NodeSqliteDatabase; + }; + return new DatabaseSync(path); + })(); let depth = 0; const begin = (readOnly: boolean) => db.exec(depth === 0 ? (readOnly ? "BEGIN DEFERRED" : "BEGIN IMMEDIATE") : `SAVEPOINT sp${depth}`); const commit = () => db.exec(depth === 1 ? "COMMIT" : `RELEASE sp${depth - 1}`); @@ -123,7 +156,16 @@ function openNodeSqlite(path: string): SqliteDatabase { commit(); return out; } catch (err) { - rollback(); + // #4949: a failed BEGIN/COMMIT can leave NO transaction active + // (SQLite auto-aborts on I/O-class failures such as disk full); + // the follow-up ROLLBACK then throws "cannot rollback - no + // transaction is active" and would MASK the original failure. + // The original error below is always authoritative. + try { + rollback(); + } catch { + /* secondary rollback failure — original error wins (#4949) */ + } throw err; } finally { depth--; @@ -146,7 +188,7 @@ interface NodeSqliteStatement { get(...args: unknown[]): unknown; all(...args: unknown[]): unknown[]; } -interface NodeSqliteDatabase { +export interface NodeSqliteDatabase { exec(sql: string): unknown; prepare(sql: string): NodeSqliteStatement; getAutoCommit(): boolean; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index db9efa522b..e9095f9f05 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -47,6 +47,15 @@ interface ResidentFields { blockContents?: Map; } +/** #4949: re-enable cadence for the disabled sidecar. The 3-strike breaker used + * to be permanent until process restart even after the store healed. Now each + * cooldown window lets the NEXT save run one writability probe (a single-row + * committed write through the production path); success clears the breaker and + * the triggering record ingests normally. Missed saves converge via the + * load-side staleness guard (envelope savedAt > index row) + next full ingest, + * so no global pre-check is needed before unified reads resume. */ +export const REENABLE_PROBE_COOLDOWN_MS = 5 * 60 * 1000; + /** #5188: ingest errors meaning "the resource was briefly unavailable", * not "the storage is broken" — a concurrent lock held past busy_timeout, * a momentary disk-full or fd spike, an EBUSY from the OS. These ride @@ -191,6 +200,9 @@ export class UnifiedShadow { * path left users hunting %TEMP% logs for which file said what. */ private readonly dbPath: string | null; private consecutiveFailures = 0; + /** #4949: timestamp until which the disabled sidecar stays quiet; the first + * save after it runs the writability probe (REENABLE_PROBE_COOLDOWN_MS). */ + private nextReenableProbeAt = 0; /** #3133: per-session ingest counters (verify sampling). Per-process * only — nothing persisted, no schema change. */ private readonly ingestCounts = new Map(); @@ -409,7 +421,21 @@ export class UnifiedShadow { this.enqueueAsyncRecord(record, rawTail, liveCcr); return; } - if (this.stats.disabled) return; + // #4949: a disabled sidecar is no longer dead for the process lifetime — + // one cooldown-gated writability probe per window; on success the + // breaker clears and THIS record falls through to the normal ingest. + if (this.stats.disabled) { + const now = Date.now(); + if (now < this.nextReenableProbeAt) return; + this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS; + try { + this.sync.probeWritable(); + } catch (err) { + this.noteProbeFailure(err); + return; + } + this.clearBreaker(); + } try { const loaded = this.legacyLoad(record); if (!loaded) return; @@ -547,6 +573,21 @@ export class UnifiedShadow { } } + /** #4949: re-enable-probe failure tail — keeps the breaker armed while the + * latest error stays visible on the status wire via stats.lastError. */ + private noteProbeFailure(err: unknown): void { + this.stats.lastError = String(err); + loggerLog("debug", `[storage] unified store re-enable probe failed (${String(err)}) — staying disabled, next probe after ${REENABLE_PROBE_COOLDOWN_MS / 60_000}min (#4949)`); + } + + /** #4949: clears the 3-strike breaker after a successful writability probe. */ + private clearBreaker(): void { + this.stats.disabled = false; + this.stats.lastError = null; + this.consecutiveFailures = 0; + loggerLog("info", "[storage] unified store RE-ENABLED: writability probe succeeded after disable (#4949); sessions missed during the outage converge via the envelope staleness guard + next full ingest"); + } + /** #3133 delta ingest attempt for one record's legacy view — returns * true when the O(new-messages) fast path committed. Every doubt * routes to `false` (→ the caller's full ingest); failures here must @@ -1066,7 +1107,21 @@ export class UnifiedShadow { * shadow queue; failures feed the same 3-strike disable. */ private enqueueAsyncRecord(record: unknown, rawTail?: { lastMessages: CoreMessage[]; lastMessagesFolded?: boolean }, liveCcr?: MessageContentStore): void { void this.enqueue(async () => { - if (this.stats.disabled) return; + // #4949: async twin of the sync re-enable probe in record() — MUST + // stay in lockstep with it; setKv is the backend's single-row + // committed write probe. + if (this.stats.disabled) { + const now = Date.now(); + if (now < this.nextReenableProbeAt) return; + this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS; + try { + await this.backend!.setKv("__bili", "writability-probe", { probedAt: now }); + } catch (err) { + this.noteProbeFailure(err); + return; + } + this.clearBreaker(); + } try { const backend = this.backend!; const loaded0 = this.legacyLoad(record); diff --git a/src/storage/store.ts b/src/storage/store.ts index f57b897c83..14a6f0f27b 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2153,6 +2153,16 @@ export class UnifiedStore { return Number(this.db.prepare("DELETE FROM session_ext WHERE session_id = ? AND key = ?").run(sessionId, key).changes); } + /** #4949: circuit-breaker re-enable probe — one REAL committed write + * through the production transaction path (proves lock acquisition + + * disk space, not just connectivity). A single fixed kv key keeps the + * trace constant; the row doubles as a liveness marker. Throws on failure. */ + probeWritable(): void { + this.db.transaction(() => { + this.setKv("__bili", "writability-probe", { probedAt: Date.now() }); + }); + } + setKv(namespace: string, key: string, value: unknown): void { validateExtKey(namespace, "namespace"); validateExtKey(key, "key"); diff --git a/tests/issue4949-rollback-mask-recovery.test.ts b/tests/issue4949-rollback-mask-recovery.test.ts new file mode 100644 index 0000000000..c2f71f12f1 --- /dev/null +++ b/tests/issue4949-rollback-mask-recovery.test.ts @@ -0,0 +1,354 @@ +/** + * #4949: two defects in the unified-store resilience path, pinned together + * because they share one failure domain (transient storage failure): + * + * A. ERROR MASKING — src/storage/driver.ts's node:sqlite transaction + * emulation ran ROLLBACK unconditionally in its catch block. When an + * I/O-class failure (disk full / SQLITE_FULL) aborts BEGIN or COMMIT, + * SQLite has already rolled the transaction back by itself, so the + * follow-up ROLLBACK throws "cannot rollback - no transaction is + * active" and REPLACES the original error — operators saw the + * confusing secondary error instead of the real disk-full cause (the + * exact signature reported in #4949). Constraint-class failures (FK + * etc.) do NOT trigger the mask — verified during triage — which is + * why the repro below injects a COMMIT-phase I/O abort with real + * SQLite auto-abort semantics. + * + * B. PERMANENT BREAKER — src/storage/shadow.ts's 3-strike circuit breaker + * had no recovery path: once stats.disabled flipped, every facade + * missed until process restart, even after storage healed. Now each + * REENABLE_PROBE_COOLDOWN_MS window lets the next save run one + * writability probe (single-row committed write through the production + * path); success clears the breaker and the triggering record ingests + * normally. Consistency before unified reads resume relies on the + * load-side staleness guard (envelope savedAt > index row) + the next + * full ingest — asserted indirectly by the final viewFor check. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { DatabaseSync, type SQLInputValue } from "node:sqlite"; +import { openSqliteDatabase, setDatabaseSyncFactoryForTests, setPreferredSqliteEngineForTests, type NodeSqliteDatabase } from "../src/storage/driver.ts"; +import { UnifiedShadow, REENABLE_PROBE_COOLDOWN_MS } from "../src/storage/shadow.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import { setLogCapture } from "../src/logger.ts"; +import type { CoreMessage } from "acp-kernel"; + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites: a developer machine with +// storage.unified in its real config must not leak a tier into these tests. +const HERMETIC = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4949-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(HERMETIC, "no-config.json"); +delete process.env.BILI_STORAGE_UNIFIED; + +type Captured = { level: string; msg: string }; + +function captureLogs(): { captured: Captured[]; stop: () => void } { + const captured: Captured[] = []; + setLogCapture((level, msg) => captured.push({ level, msg })); + return { captured, stop: () => setLogCapture(null) }; +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +function tmpDir(tag: string): string { + return fs.mkdtempSync(path.join(HERMETIC, `${tag}-`)); +} + +function msg(i: number, text = `hello ${i}`): CoreMessage { + return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text } as CoreMessage; +} + +function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"]): LegacySessionLike { + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "anthropic" }, + stats: { requests: 1 }, + metadata: {}, + state: { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages, + lastMessagesFolded: false, + }; +} + +interface ShadowStatsView { + ingests: number; + disabled: boolean; + lastError: string | null; +} + +function stats(shadow: UnifiedShadow): ShadowStatsView { + return (shadow as unknown as { stats: ShadowStatsView }).stats; +} + +/** Adapter over a REAL DatabaseSync: while unhealthy, write-control + * statements fail with a BREAKAGE-class error ('file is not a database') + * exactly where the engine would (BEGIN/COMMIT/savepoint/row writes). The + * class matters since #5188: contention-class errors (disk full, locks, + * fd exhaustion) refresh lastError but never strike the disable latch, so + * only breakage-class evidence can trip the breaker this test drives. + * Reads stay alive — a corrupt database file is still readable as bytes. + * ROLLBACK is deliberately NOT gated: the real engine answers it, which is + * what makes the pre-fix mask ("cannot rollback - no transaction is active") + * observable. */ +class FlakyHandle implements NodeSqliteDatabase { + unhealthy = false; + beginAttempts = 0; + constructor(private readonly real: InstanceType) {} + exec(sql: string): unknown { + const t = sql.trim(); + if (this.unhealthy && /^(BEGIN|SAVEPOINT|INSERT|UPDATE|DELETE|COMMIT)\b/i.test(t)) { + if (/^BEGIN\b/i.test(t)) this.beginAttempts++; + throw new Error("SQLITE_NOTADB: file is not a database"); + } + return this.real.exec(sql); + } + prepare(sql: string) { + const s = this.real.prepare(sql); + return { + run: (...args: unknown[]) => s.run(...(args as SQLInputValue[])), + get: (...args: unknown[]) => s.get(...(args as SQLInputValue[])), + all: (...args: unknown[]) => s.all(...(args as SQLInputValue[])), + }; + } + getAutoCommit(): boolean { + return true; + } + get open(): boolean { + return this.real.isOpen; + } + close(): void { + this.real.close(); + } +} + +/** Adapter whose COMMIT behaves like a real I/O-class commit failure: + * SQLite aborts the transaction by itself before surfacing the error. */ +class CommitAbortHandle implements NodeSqliteDatabase { + private abortMessage: string | null = null; + constructor(private readonly real: InstanceType) {} + armCommitAbort(message: string): void { + this.abortMessage = message; + } + exec(sql: string): unknown { + const t = sql.trim(); + if (this.abortMessage !== null && /^COMMIT\b/i.test(t)) { + const m = this.abortMessage; + this.abortMessage = null; + this.real.exec("ROLLBACK"); + throw new Error(m); + } + return this.real.exec(sql); + } + prepare(sql: string) { + const s = this.real.prepare(sql); + return { + run: (...args: unknown[]) => s.run(...(args as SQLInputValue[])), + get: (...args: unknown[]) => s.get(...(args as SQLInputValue[])), + all: (...args: unknown[]) => s.all(...(args as SQLInputValue[])), + }; + } + getAutoCommit(): boolean { + return true; + } + get open(): boolean { + return this.real.isOpen; + } + close(): void { + this.real.close(); + } +} + +// --------------------------------------------------------------------------- +// A. Commit-phase I/O failure must surface the ORIGINAL error, never the +// secondary "cannot rollback" mask. +// --------------------------------------------------------------------------- + +test("#4949 A: a failed COMMIT surfaces the original error, not 'cannot rollback'", () => { + const dir = tmpDir("mask"); + const dbPath = path.join(dir, "index.db"); + try { + const real = new DatabaseSync(dbPath); + real.exec("CREATE TABLE t(x INTEGER)"); + real.exec("INSERT INTO t(x) VALUES (0)"); + const handle = new CommitAbortHandle(real); + setDatabaseSyncFactoryForTests(() => handle); + const { db } = openSqliteDatabase(dbPath, "node:sqlite"); + try { + assert.equal(db.transaction(() => 42), 42, "sanity: plain transactions work"); + + handle.armCommitAbort("SQLITE_FULL: database or disk is full"); + let thrown: unknown; + try { + db.transaction(() => { + db.exec("INSERT INTO t(x) VALUES (2)"); + return 1; + }); + } catch (e) { + thrown = e; + } + assert.ok(thrown instanceof Error, "the failed transaction must throw"); + const err = thrown as Error; + // The reported #4949 signature: pre-fix this threw + // "cannot rollback - no transaction is active" instead. + assert.match(err.message, /disk is full/, `original error must survive: ${err.message}`); + assert.doesNotMatch(err.message, /cannot rollback/, `secondary mask leaked: ${err.message}`); + + // State bookkeeping survived the aborted transaction: the next + // real transaction commits cleanly and the aborted row is gone. + assert.equal(db.transaction(() => { db.exec("INSERT INTO t(x) VALUES (3)"); return 7; }), 7); + const rows = db.prepare("SELECT x FROM t ORDER BY x").all() as { x: number | bigint }[]; + assert.deepEqual(rows.map((r) => Number(r.x)), [0, 3], "aborted insert must not persist"); + } finally { + setDatabaseSyncFactoryForTests(null); + handle.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// A2. The nested-savepoint path keeps working through the guarded rollback +// (constraint-class commit failure: the transaction survives, so the +// rollback MUST still run — this pins that the guard swallows ONLY the +// secondary error, never the primary cleanup). +// --------------------------------------------------------------------------- + +test("#4949 A2: nested transactions roll back cleanly on a constraint failure", () => { + const dir = tmpDir("nest"); + const dbPath = path.join(dir, "index.db"); + try { + setPreferredSqliteEngineForTests("node:sqlite"); + const { db } = openSqliteDatabase(dbPath, "node:sqlite"); + try { + db.exec("CREATE TABLE p(id INTEGER PRIMARY KEY)"); + db.exec("CREATE TABLE c(id INTEGER PRIMARY KEY, pid INTEGER REFERENCES p(id) DEFERRABLE INITIALLY DEFERRED)"); + + let thrown: unknown; + try { + db.transaction(() => { + db.transaction(() => { + db.exec("INSERT INTO c(id, pid) VALUES (1, 999)"); + }); + return "outer"; + }); + } catch (e) { + thrown = e; + } + assert.ok(thrown instanceof Error, "the failed transaction must throw"); + const err = thrown as Error; + assert.match(err.message, /FOREIGN KEY/, `original constraint error must survive: ${err.message}`); + assert.doesNotMatch(err.message, /cannot rollback/, `unexpected secondary error: ${err.message}`); + + const rows = db.prepare("SELECT COUNT(*) AS n FROM c").get() as { n: number | bigint }; + assert.equal(Number(rows.n), 0, "nothing may persist from a failed nested tree"); + assert.equal(db.transaction(() => { db.exec("INSERT INTO p(id) VALUES (1)"); return "ok"; }), "ok"); + } finally { + db.close(); + setPreferredSqliteEngineForTests(null); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// B. Full breaker loop: 3 strikes disable with the TRUE error visible, the +// cooldown gates probe frequency, healing + one probe re-enables, and the +// triggering record lands in the unified store. +// --------------------------------------------------------------------------- + +test("#4949 B: disabled sidecar recovers via cooldown-gated writability probe", () => { + const dir = tmpDir("breaker"); + const dbPath = path.join(dir, "index.db"); + try { + const real = new DatabaseSync(dbPath); + const flaky = new FlakyHandle(real); + setDatabaseSyncFactoryForTests(() => flaky); + setPreferredSqliteEngineForTests("node:sqlite"); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "shadow sidecar must open"); + const { captured, stop } = captureLogs(); + try { + // Warm: the store is healthy, one real ingest lands. + shadow.record(mkLegacy("s1", [msg(0)])); + assert.equal(stats(shadow).disabled, false); + assert.ok(stats(shadow).ingests >= 1, "warm ingest must land"); + + // Storage exhausts: three consecutive failures trip the breaker. + flaky.unhealthy = true; + for (let i = 1; i <= 3; i++) { + shadow.record(mkLegacy("s1", Array.from({ length: i + 1 }, (_, j) => msg(j)))); + } + assert.equal(stats(shadow).disabled, true, "three consecutive failures disable the sidecar"); + const lastErr = stats(shadow).lastError ?? ""; + assert.match(lastErr, /file is not a database/, `status wire must carry the real cause: ${lastErr}`); + assert.doesNotMatch(lastErr, /cannot rollback/, `masked error reached the status wire: ${lastErr}`); + const beginsAfterDisable = flaky.beginAttempts; + assert.ok(beginsAfterDisable >= 3, "each failed record attempted a real BEGIN"); + + // Cooldown gate: rapid saves must NOT hammer the broken store — + // exactly ONE probe until the window elapses. + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + assert.equal(flaky.beginAttempts, beginsAfterDisable + 1, "first save after disable runs one probe"); + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + assert.equal(flaky.beginAttempts, beginsAfterDisable + 1, "further saves stay quiet inside the cooldown window"); + assert.equal(stats(shadow).disabled, true, "failed probe keeps the breaker armed"); + + // Operator restores a valid database file: the store heals. The + // next save after the window elapses probes once, succeeds, + // re-enables — and the triggering record ingests immediately. + (shadow as unknown as { nextReenableProbeAt: number }).nextReenableProbeAt = 0; + flaky.unhealthy = false; + const ingestsBefore = stats(shadow).ingests; + shadow.record(mkLegacy("s1", Array.from({ length: 7 }, (_, j) => msg(j)))); + assert.equal(stats(shadow).disabled, false, "successful probe re-enables the sidecar"); + assert.ok(stats(shadow).ingests > ingestsBefore, "the triggering record must land right away"); + const view = shadow.viewFor("s1"); + assert.ok(view, "unified reads must work again"); + assert.ok( + view.lastMessages?.some((m) => (m as { text?: string }).text === "hello 6") ?? false, + "the recovered ingest carries the newest tail", + ); + + const reenabled = captured.filter((e) => e.level === "info" && e.msg.includes("RE-ENABLED")); + assert.equal(reenabled.length, 1, "exactly one re-enable log line"); + } finally { + stop(); + shadow.close(); + setDatabaseSyncFactoryForTests(null); + setPreferredSqliteEngineForTests(null); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// C. The cooldown constant is the single knob behind the gate — pin its +// magnitude so a future "tweak" cannot silently turn the probe into a +// hammer (or a decade-long dead store). +// --------------------------------------------------------------------------- + +test("#4949 C: re-enable cooldown stays at the documented 5-minute cadence", () => { + assert.equal(REENABLE_PROBE_COOLDOWN_MS, 5 * 60 * 1000); +}); diff --git a/tests/issue5140-contention-not-disabling.test.ts b/tests/issue5140-contention-not-disabling.test.ts index 3a79b3eb28..2e7e091498 100644 --- a/tests/issue5140-contention-not-disabling.test.ts +++ b/tests/issue5140-contention-not-disabling.test.ts @@ -17,6 +17,13 @@ * evidence never latches, structural evidence latches once and warns once, * one healthy save resets the streak. * + * #4949 composition note: a latch no longer survives a following save when the + * write path is healthy — the cooldown-gated writability probe (its first run + * is ungated by design) re-enables the sidecar. The structural-latch test + * below therefore asserts the latch at the 3rd strike and pins the probe's + * re-enable as the expected outcome of the 4th save (a store whose WRITES are + * broken still fails its probe and stays disabled — the pre-#4949 behavior). + * * The open-side half of #5188 (a first synchronous open that dies transiently * is retried from the save path) has its own suite: * tests/issue5188-sidecar-recoverable-open.test.ts @@ -134,15 +141,20 @@ test("structural failures latch exactly once with one warn line", () => { setLogCapture((level, msg) => { if (level === "warn") lines.push(msg); }); try { const off = installFault(shadow, NOTADB); - for (let i = 0; i < 4; i++) shadow.record(fix()); - off(); + for (let i = 0; i < 3; i++) shadow.record(fix()); assert.equal(shadow.stats.disabled, true, "three structural strikes latch the sidecar"); assert.match(String(shadow.stats.lastError), /NOTADB|file is not a database/); const disables = lines.filter((l) => /disabled after 3 consecutive failures/.test(l)); assert.equal(disables.length, 1, "latch warn fires exactly once (no per-save spam after)"); assert.equal(lines.filter((l) => /failed to ingest/.test(l)).length, 1, "first-failure warn also fires exactly once"); + // The fault breaks the VERIFY leg (read-back), not writability — so + // under #4949 the next save's writability probe succeeds and lifts the + // fresh latch. Pin that union behavior here; a write-broken store + // fails the probe and stays disabled (tests/issue4949-… covers it). shadow.record(fix()); - assert.equal(lines.filter((l) => /disabled after/.test(l)).length, 1, "still one after a later save"); + assert.equal(shadow.stats.disabled, false, "#4949: successful writability probe re-enables after a fresh structural latch"); + assert.equal(lines.filter((l) => /disabled after/.test(l)).length, 1, "still exactly one latch warn after the re-enabling save"); + off(); } finally { setLogCapture(null); shadow.close();