From fc6a5cfd6a677cfa3f04ca1ad0cc461fe92771b3 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 27 Oct 2026 18:37:52 +0800 Subject: [PATCH 1/2] fix: surface real storage errors after failed BEGIN/COMMIT and auto-recover the unified-store breaker (#4949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects in one failure domain, both observed in production on v0.1.206 after host storage exhaustion: 1. Error masking: when an I/O-class failure (disk full / SQLITE_FULL) aborts BEGIN or COMMIT, SQLite auto-aborts the transaction, so the follow-up ROLLBACK in the node:sqlite emulation throws 'cannot rollback - no transaction is active' and replaces the real error before it reaches the sidecar breaker's lastError / status wire. The rollback attempt is now guarded so the original error always wins. The better-sqlite3 seam gets an annotated message at our boundary because that library loses the original inside its own wrapper. 2. Permanent circuit breaker: stats.disabled was set on 3 consecutive failures but never cleared — the unified store stayed disabled for the whole process lifetime even after the disk was freed (only a restart recovered it). Both sync and async sidecar paths now run a cooldown-gated writability probe (REENABLE_PROBE_COOLDOWN_MS = 5min, one probe per window on the next save) through the same transaction path production writes use; success re-enables the store and the triggering record ingests immediately. Missed sessions converge via the existing envelope staleness guard at load plus the next full ingest. --- src/storage/driver.ts | 54 +++++++++++++++++++++++++++++++++------ src/storage/shadow.ts | 59 +++++++++++++++++++++++++++++++++++++++++-- src/storage/store.ts | 10 ++++++++ 3 files changed, 113 insertions(+), 10 deletions(-) diff --git a/src/storage/driver.ts b/src/storage/driver.ts index 6bc44be0c7..e15cfc9c32 100644 --- a/src/storage/driver.ts +++ b/src/storage/driver.ts @@ -86,7 +86,23 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { // a write lock mid-flight — SQLITE_BUSY_SNAPSHOT, which busy_timeout // cannot resolve. Taking the write lock up front makes concurrent // writers queue on busy_timeout instead (#2690 review blocking 5). - transaction: (fn) => db.transaction(fn).immediate(), + transaction: (fn) => { + try { + return db.transaction(fn).immediate(); + } catch (err) { + // #4949: better-sqlite3's own wrapper runs ROLLBACK after ANY + // inner failure; when BEGIN or COMMIT itself failed there was + // nothing left to roll back, and its "cannot rollback - no + // transaction is active" REPLACES the real error (commonly + // disk full / SQLITE_FULL) before it reaches us. Name the + // masking at our seam so operators see the likely cause. + const msg = String(err); + if (/cannot rollback|no transaction is active/i.test(msg)) { + throw new SqliteDriverError(`${msg} — masked by better-sqlite3's automatic rollback after a failed BEGIN/COMMIT; the original error no longer survives (disk full / SQLITE_FULL is the common cause)`, "better-sqlite3"); + } + throw err; + } + }, close: () => void db.close(), get open() { return db.open; @@ -94,15 +110,28 @@ function wrapBetterSqlite3(db: BetterSqlite3Database): SqliteDatabase { }; } +let databaseSyncFactoryForTests: ((path: string) => NodeSqliteDatabase) | null = null; + +/** Test seam: substitute how the node:sqlite emulation obtains its database + * handle (e.g. an adapter injecting engine-level I/O failures, #4949). + * Null = the real builtin node:sqlite module. */ +export function setDatabaseSyncFactoryForTests(factory: ((path: string) => NodeSqliteDatabase) | null): void { + databaseSyncFactoryForTests = factory; +} + function openNodeSqlite(path: string): SqliteDatabase { // node:sqlite is synchronous like better-sqlite3, but has no // .transaction() helper — emulate with BEGIN/COMMIT + savepoints for // nesting, which is the only behavioral difference the store relies on. - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { DatabaseSync } = require("node:sqlite") as { - DatabaseSync: new (path: string) => NodeSqliteDatabase; - }; - const db = new DatabaseSync(path); + const db = databaseSyncFactoryForTests + ? databaseSyncFactoryForTests(path) + : (() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { DatabaseSync } = require("node:sqlite") as { + DatabaseSync: new (path: string) => NodeSqliteDatabase; + }; + return new DatabaseSync(path); + })(); let depth = 0; const begin = () => db.exec(depth === 0 ? "BEGIN IMMEDIATE" : `SAVEPOINT sp${depth}`); // IMMEDIATE at top level: multi-process SQLITE_BUSY_SNAPSHOT guard (see openBetterSqlite3) const commit = () => db.exec(depth === 1 ? "COMMIT" : `RELEASE sp${depth - 1}`); @@ -123,7 +152,16 @@ function openNodeSqlite(path: string): SqliteDatabase { commit(); return out; } catch (err) { - rollback(); + // #4949: a failed BEGIN/COMMIT can leave NO transaction active + // (SQLite auto-aborts on I/O-class failures such as disk full); + // the follow-up ROLLBACK then throws "cannot rollback - no + // transaction is active" and would MASK the original failure. + // The original error below is always authoritative. + try { + rollback(); + } catch { + /* secondary rollback failure — original error wins (#4949) */ + } throw err; } finally { depth--; @@ -141,7 +179,7 @@ interface NodeSqliteStatement { get(...args: unknown[]): unknown; all(...args: unknown[]): unknown[]; } -interface NodeSqliteDatabase { +export interface NodeSqliteDatabase { exec(sql: string): unknown; prepare(sql: string): NodeSqliteStatement; getAutoCommit(): boolean; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 3a8c9c4ee5..2775526471 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -37,6 +37,15 @@ import type { CoreMessage, MessageContentStore } from "acp-kernel"; export const MAX_CONSECUTIVE_FAILURES = 3; +/** #4949: re-enable cadence for the disabled sidecar. The 3-strike breaker used + * to be permanent until process restart even after the store healed. Now each + * cooldown window lets the NEXT save run one writability probe (a single-row + * committed write through the production path); success clears the breaker and + * the triggering record ingests normally. Missed saves converge via the + * load-side staleness guard (envelope savedAt > index row) + next full ingest, + * so no global pre-check is needed before unified reads resume. */ +export const REENABLE_PROBE_COOLDOWN_MS = 5 * 60 * 1000; + /** #3133: read-back verification cadence — the verify leg (full read-back + * double shadowDigest) is O(log) per save, while the drift it guards is a * migration-parity property, not a per-commit one. The FIRST ingest of a @@ -140,6 +149,9 @@ export class UnifiedShadow { private queueTail: Promise = Promise.resolve(); private readonly legacyLoad: (record: unknown) => LegacySessionLike | null; private consecutiveFailures = 0; + /** #4949: timestamp until which the disabled sidecar stays quiet; the first + * save after it runs the writability probe (REENABLE_PROBE_COOLDOWN_MS). */ + private nextReenableProbeAt = 0; /** #3133: per-session ingest counters (verify sampling). Per-process * only — nothing persisted, no schema change. */ private readonly ingestCounts = new Map(); @@ -328,7 +340,21 @@ export class UnifiedShadow { this.enqueueAsyncRecord(record, rawTail, liveCcr); return; } - if (this.stats.disabled) return; + // #4949: a disabled sidecar is no longer dead for the process lifetime — + // one cooldown-gated writability probe per window; on success the + // breaker clears and THIS record falls through to the normal ingest. + if (this.stats.disabled) { + const now = Date.now(); + if (now < this.nextReenableProbeAt) return; + this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS; + try { + this.sync.probeWritable(); + } catch (err) { + this.noteProbeFailure(err); + return; + } + this.clearBreaker(); + } try { const loaded = this.legacyLoad(record); if (!loaded) return; @@ -471,6 +497,21 @@ export class UnifiedShadow { } } + /** #4949: re-enable-probe failure tail — keeps the breaker armed while the + * latest error stays visible on the status wire via stats.lastError. */ + private noteProbeFailure(err: unknown): void { + this.stats.lastError = String(err); + loggerLog("debug", `[storage] unified store re-enable probe failed (${String(err)}) — staying disabled, next probe after ${REENABLE_PROBE_COOLDOWN_MS / 60_000}min (#4949)`); + } + + /** #4949: clears the 3-strike breaker after a successful writability probe. */ + private clearBreaker(): void { + this.stats.disabled = false; + this.stats.lastError = null; + this.consecutiveFailures = 0; + loggerLog("info", "[storage] unified store RE-ENABLED: writability probe succeeded after disable (#4949); sessions missed during the outage converge via the envelope staleness guard + next full ingest"); + } + /** #3133 delta ingest attempt for one record's legacy view — returns * true when the O(new-messages) fast path committed. Every doubt * routes to `false` (→ the caller's full ingest); failures here must @@ -917,7 +958,21 @@ export class UnifiedShadow { * shadow queue; failures feed the same 3-strike disable. */ private enqueueAsyncRecord(record: unknown, rawTail?: { lastMessages: CoreMessage[]; lastMessagesFolded?: boolean }, liveCcr?: MessageContentStore): void { void this.enqueue(async () => { - if (this.stats.disabled) return; + // #4949: async twin of the sync re-enable probe in record() — MUST + // stay in lockstep with it; setKv is the backend's single-row + // committed write probe. + if (this.stats.disabled) { + const now = Date.now(); + if (now < this.nextReenableProbeAt) return; + this.nextReenableProbeAt = now + REENABLE_PROBE_COOLDOWN_MS; + try { + await this.backend!.setKv("__bili", "writability-probe", { probedAt: now }); + } catch (err) { + this.noteProbeFailure(err); + return; + } + this.clearBreaker(); + } try { const backend = this.backend!; const loaded0 = this.legacyLoad(record); diff --git a/src/storage/store.ts b/src/storage/store.ts index 930a55b8f4..59c7f8643b 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1966,6 +1966,16 @@ export class UnifiedStore { return Number(this.db.prepare("DELETE FROM session_ext WHERE session_id = ? AND key = ?").run(sessionId, key).changes); } + /** #4949: circuit-breaker re-enable probe — one REAL committed write + * through the production transaction path (proves lock acquisition + + * disk space, not just connectivity). A single fixed kv key keeps the + * trace constant; the row doubles as a liveness marker. Throws on failure. */ + probeWritable(): void { + this.db.transaction(() => { + this.setKv("__bili", "writability-probe", { probedAt: Date.now() }); + }); + } + setKv(namespace: string, key: string, value: unknown): void { validateExtKey(namespace, "namespace"); validateExtKey(key, "key"); From 4508a28f4763ef19b16320f7e91a2c40ac665d68 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 27 Oct 2026 18:37:52 +0800 Subject: [PATCH 2/2] test: pin #4949 rollback-masking and breaker recovery regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A: injected engine-level COMMIT abort over a real DatabaseSync proves the original error surfaces (pre-fix this failed with exactly the reported 'cannot rollback - no transaction is active' mask); aborted rows do not persist. - A2: negative control — nested transactions on a constraint failure still roll back cleanly and propagate the original (gate both directions, #3617). - B: full breaker loop on a real store — 3 strikes disable with the true cause visible, cooldown gates probe frequency (exactly one probe per window), healing + one probe re-enables, triggering record lands, unified reads work again, exactly one RE-ENABLED log line. - C: pins REENABLE_PROBE_COOLDOWN_MS at the documented 5-minute cadence. --- .../issue4949-rollback-mask-recovery.test.ts | 350 ++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 tests/issue4949-rollback-mask-recovery.test.ts diff --git a/tests/issue4949-rollback-mask-recovery.test.ts b/tests/issue4949-rollback-mask-recovery.test.ts new file mode 100644 index 0000000000..bc93e1e0d2 --- /dev/null +++ b/tests/issue4949-rollback-mask-recovery.test.ts @@ -0,0 +1,350 @@ +/** + * #4949: two defects in the unified-store resilience path, pinned together + * because they share one failure domain (transient storage failure): + * + * A. ERROR MASKING — src/storage/driver.ts's node:sqlite transaction + * emulation ran ROLLBACK unconditionally in its catch block. When an + * I/O-class failure (disk full / SQLITE_FULL) aborts BEGIN or COMMIT, + * SQLite has already rolled the transaction back by itself, so the + * follow-up ROLLBACK throws "cannot rollback - no transaction is + * active" and REPLACES the original error — operators saw the + * confusing secondary error instead of the real disk-full cause (the + * exact signature reported in #4949). Constraint-class failures (FK + * etc.) do NOT trigger the mask — verified during triage — which is + * why the repro below injects a COMMIT-phase I/O abort with real + * SQLite auto-abort semantics. + * + * B. PERMANENT BREAKER — src/storage/shadow.ts's 3-strike circuit breaker + * had no recovery path: once stats.disabled flipped, every facade + * missed until process restart, even after storage healed. Now each + * REENABLE_PROBE_COOLDOWN_MS window lets the next save run one + * writability probe (single-row committed write through the production + * path); success clears the breaker and the triggering record ingests + * normally. Consistency before unified reads resume relies on the + * load-side staleness guard (envelope savedAt > index row) + the next + * full ingest — asserted indirectly by the final viewFor check. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { DatabaseSync, type SQLInputValue } from "node:sqlite"; +import { openSqliteDatabase, setDatabaseSyncFactoryForTests, setPreferredSqliteEngineForTests, type NodeSqliteDatabase } from "../src/storage/driver.ts"; +import { UnifiedShadow, REENABLE_PROBE_COOLDOWN_MS } from "../src/storage/shadow.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import { setLogCapture } from "../src/logger.ts"; +import type { CoreMessage } from "acp-kernel"; + +process.env.BILI_PERSIST_ZSTD = "0"; +// Same hermeticity pin as the other storage suites: a developer machine with +// storage.unified in its real config must not leak a tier into these tests. +const HERMETIC = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4949-hermetic-")); +process.env.BILI_CONFIG_FILE = path.join(HERMETIC, "no-config.json"); +delete process.env.BILI_STORAGE_UNIFIED; + +type Captured = { level: string; msg: string }; + +function captureLogs(): { captured: Captured[]; stop: () => void } { + const captured: Captured[] = []; + setLogCapture((level, msg) => captured.push({ level, msg })); + return { captured, stop: () => setLogCapture(null) }; +} + +function rmBestEffort(p: string): void { + try { + fs.rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch { + /* leaked temp dir on a Windows runner is harmless */ + } +} + +function tmpDir(tag: string): string { + return fs.mkdtempSync(path.join(HERMETIC, `${tag}-`)); +} + +function msg(i: number, text = `hello ${i}`): CoreMessage { + return { id: `h_msg${i}`, role: i % 2 === 0 ? "user" : "assistant", contentType: "text", text } as CoreMessage; +} + +function mkLegacy(id: string, lastMessages: LegacySessionLike["lastMessages"]): LegacySessionLike { + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "anthropic" }, + stats: { requests: 1 }, + metadata: {}, + state: { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages, + lastMessagesFolded: false, + }; +} + +interface ShadowStatsView { + ingests: number; + disabled: boolean; + lastError: string | null; +} + +function stats(shadow: UnifiedShadow): ShadowStatsView { + return (shadow as unknown as { stats: ShadowStatsView }).stats; +} + +/** Adapter over a REAL DatabaseSync: while unhealthy, write-control + * statements fail with a disk-full-shaped error exactly where the engine + * would (BEGIN/COMMIT/savepoint/row writes). Reads stay alive — a full disk + * does not make the database unreadable. ROLLBACK is deliberately NOT + * gated: the real engine answers it, which is what makes the pre-fix mask + * ("cannot rollback - no transaction is active") observable. */ +class FlakyHandle implements NodeSqliteDatabase { + unhealthy = false; + beginAttempts = 0; + constructor(private readonly real: InstanceType) {} + exec(sql: string): unknown { + const t = sql.trim(); + if (this.unhealthy && /^(BEGIN|SAVEPOINT|INSERT|UPDATE|DELETE|COMMIT)\b/i.test(t)) { + if (/^BEGIN\b/i.test(t)) this.beginAttempts++; + throw new Error("SQLITE_FULL: database or disk is full"); + } + return this.real.exec(sql); + } + prepare(sql: string) { + const s = this.real.prepare(sql); + return { + run: (...args: unknown[]) => s.run(...(args as SQLInputValue[])), + get: (...args: unknown[]) => s.get(...(args as SQLInputValue[])), + all: (...args: unknown[]) => s.all(...(args as SQLInputValue[])), + }; + } + getAutoCommit(): boolean { + return true; + } + get open(): boolean { + return this.real.isOpen; + } + close(): void { + this.real.close(); + } +} + +/** Adapter whose COMMIT behaves like a real I/O-class commit failure: + * SQLite aborts the transaction by itself before surfacing the error. */ +class CommitAbortHandle implements NodeSqliteDatabase { + private abortMessage: string | null = null; + constructor(private readonly real: InstanceType) {} + armCommitAbort(message: string): void { + this.abortMessage = message; + } + exec(sql: string): unknown { + const t = sql.trim(); + if (this.abortMessage !== null && /^COMMIT\b/i.test(t)) { + const m = this.abortMessage; + this.abortMessage = null; + this.real.exec("ROLLBACK"); + throw new Error(m); + } + return this.real.exec(sql); + } + prepare(sql: string) { + const s = this.real.prepare(sql); + return { + run: (...args: unknown[]) => s.run(...(args as SQLInputValue[])), + get: (...args: unknown[]) => s.get(...(args as SQLInputValue[])), + all: (...args: unknown[]) => s.all(...(args as SQLInputValue[])), + }; + } + getAutoCommit(): boolean { + return true; + } + get open(): boolean { + return this.real.isOpen; + } + close(): void { + this.real.close(); + } +} + +// --------------------------------------------------------------------------- +// A. Commit-phase I/O failure must surface the ORIGINAL error, never the +// secondary "cannot rollback" mask. +// --------------------------------------------------------------------------- + +test("#4949 A: a failed COMMIT surfaces the original error, not 'cannot rollback'", () => { + const dir = tmpDir("mask"); + const dbPath = path.join(dir, "index.db"); + try { + const real = new DatabaseSync(dbPath); + real.exec("CREATE TABLE t(x INTEGER)"); + real.exec("INSERT INTO t(x) VALUES (0)"); + const handle = new CommitAbortHandle(real); + setDatabaseSyncFactoryForTests(() => handle); + const { db } = openSqliteDatabase(dbPath, "node:sqlite"); + try { + assert.equal(db.transaction(() => 42), 42, "sanity: plain transactions work"); + + handle.armCommitAbort("SQLITE_FULL: database or disk is full"); + let thrown: unknown; + try { + db.transaction(() => { + db.exec("INSERT INTO t(x) VALUES (2)"); + return 1; + }); + } catch (e) { + thrown = e; + } + assert.ok(thrown instanceof Error, "the failed transaction must throw"); + const err = thrown as Error; + // The reported #4949 signature: pre-fix this threw + // "cannot rollback - no transaction is active" instead. + assert.match(err.message, /disk is full/, `original error must survive: ${err.message}`); + assert.doesNotMatch(err.message, /cannot rollback/, `secondary mask leaked: ${err.message}`); + + // State bookkeeping survived the aborted transaction: the next + // real transaction commits cleanly and the aborted row is gone. + assert.equal(db.transaction(() => { db.exec("INSERT INTO t(x) VALUES (3)"); return 7; }), 7); + const rows = db.prepare("SELECT x FROM t ORDER BY x").all() as { x: number | bigint }[]; + assert.deepEqual(rows.map((r) => Number(r.x)), [0, 3], "aborted insert must not persist"); + } finally { + setDatabaseSyncFactoryForTests(null); + handle.close(); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// A2. The nested-savepoint path keeps working through the guarded rollback +// (constraint-class commit failure: the transaction survives, so the +// rollback MUST still run — this pins that the guard swallows ONLY the +// secondary error, never the primary cleanup). +// --------------------------------------------------------------------------- + +test("#4949 A2: nested transactions roll back cleanly on a constraint failure", () => { + const dir = tmpDir("nest"); + const dbPath = path.join(dir, "index.db"); + try { + setPreferredSqliteEngineForTests("node:sqlite"); + const { db } = openSqliteDatabase(dbPath, "node:sqlite"); + try { + db.exec("CREATE TABLE p(id INTEGER PRIMARY KEY)"); + db.exec("CREATE TABLE c(id INTEGER PRIMARY KEY, pid INTEGER REFERENCES p(id) DEFERRABLE INITIALLY DEFERRED)"); + + let thrown: unknown; + try { + db.transaction(() => { + db.transaction(() => { + db.exec("INSERT INTO c(id, pid) VALUES (1, 999)"); + }); + return "outer"; + }); + } catch (e) { + thrown = e; + } + assert.ok(thrown instanceof Error, "the failed transaction must throw"); + const err = thrown as Error; + assert.match(err.message, /FOREIGN KEY/, `original constraint error must survive: ${err.message}`); + assert.doesNotMatch(err.message, /cannot rollback/, `unexpected secondary error: ${err.message}`); + + const rows = db.prepare("SELECT COUNT(*) AS n FROM c").get() as { n: number | bigint }; + assert.equal(Number(rows.n), 0, "nothing may persist from a failed nested tree"); + assert.equal(db.transaction(() => { db.exec("INSERT INTO p(id) VALUES (1)"); return "ok"; }), "ok"); + } finally { + db.close(); + setPreferredSqliteEngineForTests(null); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// B. Full breaker loop: 3 strikes disable with the TRUE error visible, the +// cooldown gates probe frequency, healing + one probe re-enables, and the +// triggering record lands in the unified store. +// --------------------------------------------------------------------------- + +test("#4949 B: disabled sidecar recovers via cooldown-gated writability probe", () => { + const dir = tmpDir("breaker"); + const dbPath = path.join(dir, "index.db"); + try { + const real = new DatabaseSync(dbPath); + const flaky = new FlakyHandle(real); + setDatabaseSyncFactoryForTests(() => flaky); + setPreferredSqliteEngineForTests("node:sqlite"); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as LegacySessionLike); + assert.ok(shadow, "shadow sidecar must open"); + const { captured, stop } = captureLogs(); + try { + // Warm: the store is healthy, one real ingest lands. + shadow.record(mkLegacy("s1", [msg(0)])); + assert.equal(stats(shadow).disabled, false); + assert.ok(stats(shadow).ingests >= 1, "warm ingest must land"); + + // Storage exhausts: three consecutive failures trip the breaker. + flaky.unhealthy = true; + for (let i = 1; i <= 3; i++) { + shadow.record(mkLegacy("s1", Array.from({ length: i + 1 }, (_, j) => msg(j)))); + } + assert.equal(stats(shadow).disabled, true, "three consecutive failures disable the sidecar"); + const lastErr = stats(shadow).lastError ?? ""; + assert.match(lastErr, /disk is full/, `status wire must carry the real cause: ${lastErr}`); + assert.doesNotMatch(lastErr, /cannot rollback/, `masked error reached the status wire: ${lastErr}`); + const beginsAfterDisable = flaky.beginAttempts; + assert.ok(beginsAfterDisable >= 3, "each failed record attempted a real BEGIN"); + + // Cooldown gate: rapid saves must NOT hammer the broken store — + // exactly ONE probe until the window elapses. + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + assert.equal(flaky.beginAttempts, beginsAfterDisable + 1, "first save after disable runs one probe"); + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + shadow.record(mkLegacy("s1", [msg(0), msg(1)])); + assert.equal(flaky.beginAttempts, beginsAfterDisable + 1, "further saves stay quiet inside the cooldown window"); + assert.equal(stats(shadow).disabled, true, "failed probe keeps the breaker armed"); + + // Operator frees space: the store heals. The next save after the + // window elapses probes once, succeeds, re-enables — and the + // triggering record ingests immediately. + (shadow as unknown as { nextReenableProbeAt: number }).nextReenableProbeAt = 0; + flaky.unhealthy = false; + const ingestsBefore = stats(shadow).ingests; + shadow.record(mkLegacy("s1", Array.from({ length: 7 }, (_, j) => msg(j)))); + assert.equal(stats(shadow).disabled, false, "successful probe re-enables the sidecar"); + assert.ok(stats(shadow).ingests > ingestsBefore, "the triggering record must land right away"); + const view = shadow.viewFor("s1"); + assert.ok(view, "unified reads must work again"); + assert.ok( + view.lastMessages?.some((m) => (m as { text?: string }).text === "hello 6") ?? false, + "the recovered ingest carries the newest tail", + ); + + const reenabled = captured.filter((e) => e.level === "info" && e.msg.includes("RE-ENABLED")); + assert.equal(reenabled.length, 1, "exactly one re-enable log line"); + } finally { + stop(); + shadow.close(); + setDatabaseSyncFactoryForTests(null); + setPreferredSqliteEngineForTests(null); + } + } finally { + rmBestEffort(dir); + } +}); + +// --------------------------------------------------------------------------- +// C. The cooldown constant is the single knob behind the gate — pin its +// magnitude so a future "tweak" cannot silently turn the probe into a +// hammer (or a decade-long dead store). +// --------------------------------------------------------------------------- + +test("#4949 C: re-enable cooldown stays at the documented 5-minute cadence", () => { + assert.equal(REENABLE_PROBE_COOLDOWN_MS, 5 * 60 * 1000); +});