From 3e82f42a5c39a8645bff04f57b70a7cdb5908928 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 20 Nov 2026 21:13:16 +0800 Subject: [PATCH] fix(storage): the FULL-mode boot line states the actual selection and writer state (#7408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-#6480 wording ("storage.unified=full or BILI_STORAGE_UNIFIED=full: ... legacy files keep being written — unset the knob to roll back") survived the pure-sqlite default flip. Under the default posture — no knob set, JSON envelopes NOT written — all three clauses were false, contradicting the accurate jsonWriter-off line persist.ts prints in the same boot and steering field diagnosis into chasing a non-existent dual-write (#7408 §3b). The line now branches on the raw selection (unifiedSelectionIsSqlite) and the effective writer state (effectiveJsonWriterOff, which at that point equals persist.ts's jsonWriterOff: mode is full, backend is sqlite, and an encryption-blocked open never reaches the line): - writer off → "index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on" - writer on + pure-sqlite selection → names the explicit storage.jsonWriter=on as the reason envelopes are written - writer on + explicit full → the historical dual-write wording, unchanged maybeOpen/openAsync take the file-tier jsonWriter value as a new optional parameter (persist.ts passes fileStorageJsonWriter() at both call sites); the module doc header is updated to the four current postures. Regression test: tests/issue7408-storage-boot-log.test.ts pins the exact boot line for all five reachable postures. Log wording only — no wire, config, or storage-semantics change. --- src/persist.ts | 4 +- src/storage/shadow.ts | 44 ++++++-- tests/issue7408-storage-boot-log.test.ts | 127 +++++++++++++++++++++++ 3 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 tests/issue7408-storage-boot-log.test.ts diff --git a/src/persist.ts b/src/persist.ts index ecdc7d59d4..838d62a3ee 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -730,7 +730,7 @@ export class SessionStore { // triggered a paced retry — stamping here kept that read failing // closed for a full cooldown window. this.syncSidecarOpen = { legacyLoad }; - this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend()) ?? undefined; + this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend(), fileStorageJsonWriter()) ?? undefined; } else { this.shadow = undefined; } @@ -3887,7 +3887,7 @@ export class SessionStore { const cooldown = shadowReopenCooldownMsOverride ?? SHADOW_REOPEN_COOLDOWN_MS; this.sidecarReopenNextAttemptAt = now + cooldown; try { - this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, args.legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend()) ?? undefined; + this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, args.legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend(), fileStorageJsonWriter()) ?? undefined; } catch (err) { // Transient refusal class re-occurred mid-process (unreachable via // boot — that path exits first — kept defensive per #5188): diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index c70be67ba9..183f7d2df8 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1,19 +1,25 @@ /** * Unified-storage sidecar (#2671). * - * Two opt-in levels via BILI_STORAGE_UNIFIED: + * Selections via BILI_STORAGE_UNIFIED / storage.unified: + * - absent or `sqlite` (the DEFAULT since #6480, "pure-sqlite"): the + * drop-in replacement. Reads go unified-first with organic import (a + * session the index has not seen is imported from its legacy file on + * first load) and legacy fallback on any miss, and the JSON envelope + * WRITER defaults off — index.db is the only persistent copy (restore + * dual-write with storage.jsonWriter=on). + * - `full`: the historical #3818 posture — the same unified-first + * machinery, but the JSON envelope writer defaults ON (dual-write; + * rollback = unset the knob). * - `1`/`true`/`shadow` (Phase 1): double-write dry-run. Every session * record the legacy SessionStore commits is ALSO ingested into * /index.db, and the reconstructed view is digest-checked * against what the legacy loader (buildSession) would serve after a * restart. Any drift is logged as an error — that is the entire point: * the unified layer must replay legacy bytes exactly, and production - * traffic becomes the migration test corpus. - * - `full` (Phase 1.5): the drop-in replacement. Reads flip to - * unified-first with organic import (a session the index has not seen - * is imported from its legacy file on first load) and legacy fallback - * on any miss. Legacy files keep being written (rollback = unset the - * env), but the unified index is the load path. + * traffic becomes the migration test corpus. The JSON envelope stays + * the sole source of truth at this level. + * - `legacy`: the one-line escape hatch — no sidecar at all. * * Failure policy: sidecar problems must NEVER break the legacy write path. * All work is try/catch'd. The sidecar NEVER gives up on index writes @@ -28,7 +34,7 @@ import { log as loggerLog } from "../logger.js"; import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts"; import { splitMediaContent } from "./identity.js"; import type { BlockView } from "../session.js"; -import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts"; +import { effectiveJsonWriterOff, resolveStorageMode, resolveStorageWalJournalSizeLimit, unifiedSelectionIsSqlite, type StorageMode } from "../knobs.ts"; import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts"; import path from "node:path"; import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts"; @@ -289,6 +295,7 @@ export class UnifiedShadow { fileUnified?: string, fileWalLimit?: number, fileBackend?: string, + fileJsonWriter?: string, ): UnifiedShadow | null { const mode = resolveStorageMode(env, fileUnified); if (mode === "legacy") return null; @@ -394,10 +401,26 @@ export class UnifiedShadow { recordSidecarInstance(shadow); const engineTag = ` [engine: ${store.engine}/${store.engineSource}]`; const walTag = ` [wal-cap: ${walLimit}]`; + // #7408: the FULL-mode line must state the ACTUAL writer state and + // selection — the pre-#6480 wording ("legacy files keep being + // written — unset the knob to roll back") is false under the + // pure-sqlite default (no knob set, envelopes NOT written) and + // contradicted the accurate jsonWriter-off line printed by persist. + // At this point mode is full, the backend is sqlite (mysql logs its + // own line in openAsync), and encryption-blocking never reaches here + // (persist gates the open), so effectiveJsonWriterOff equals the + // jsonWriterOff persist.ts computes for save decisions. + const sqliteSelection = unifiedSelectionIsSqlite(env, fileUnified); + const selection = sqliteSelection + ? "pure-sqlite selection (storage.unified absent or =sqlite)" + : "storage.unified=full or BILI_STORAGE_UNIFIED=full"; + const writerOff = effectiveJsonWriterOff(env, fileJsonWriter, fileUnified); loggerLog( "info", mode === "full" - ? `[storage] unified FULL mode ENABLED (storage.unified=full or BILI_STORAGE_UNIFIED=full): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — unset the knob to roll back${engineTag}${walTag}` + ? writerOff + ? `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on${engineTag}${walTag}` + : `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — ${sqliteSelection ? "remove the explicit storage.jsonWriter=on to return to the pure-sqlite default" : "unset the knob to roll back"}${engineTag}${walTag}` : `[storage] unified shadow mode ENABLED (storage.unified=shadow or BILI_STORAGE_UNIFIED): every session write is double-ingested into index.db and digest-reconciled${engineTag}${walTag}`, ); return shadow; @@ -416,9 +439,10 @@ export class UnifiedShadow { fileWalLimit?: number, fileBackend?: string, fileMysqlDsn?: string, + fileJsonWriter?: string, ): Promise { const backendName = resolveBackendName(env, fileBackend); - if (backendName !== "mysql") return UnifiedShadow.maybeOpen(dir, env, legacyLoad, fileUnified, fileWalLimit, fileBackend); + if (backendName !== "mysql") return UnifiedShadow.maybeOpen(dir, env, legacyLoad, fileUnified, fileWalLimit, fileBackend, fileJsonWriter); const mode = resolveStorageMode(env, fileUnified); if (mode === "legacy") return null; const dsn = (env.BILI_MYSQL_DSN ?? fileMysqlDsn ?? "").trim(); diff --git a/tests/issue7408-storage-boot-log.test.ts b/tests/issue7408-storage-boot-log.test.ts new file mode 100644 index 0000000000..33d944778d --- /dev/null +++ b/tests/issue7408-storage-boot-log.test.ts @@ -0,0 +1,127 @@ +/** + * #7408 §3b — the full-mode storage boot line must state the ACTUAL + * selection and writer state. The pre-#6480 wording ("legacy files keep + * being written — unset the knob to roll back") survived the pure-sqlite + * default flip: under the default (no knob set, envelopes NOT written) it + * claimed a knob that was never set, a dual-write that does not happen, and + * a rollback that does not exist — contradicting the accurate jsonWriter-off + * line persist.ts prints in the same boot and steering field diagnosis into + * chasing a non-existent dual-write. Pinned for every reachable posture: + * default pure-sqlite, explicit full, and the two explicit jsonWriter + * overrides (env tier + file tier through the new fileJsonWriter parameter). + */ +import assert from "node:assert"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { after, before, test } from "node:test"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import type { LegacySessionLike } from "../src/storage/ingest.ts"; +import { setLogCapture } from "../src/logger.ts"; +import { _resetStorageRuntimeStatusForTest } from "../src/storage/runtime-status.ts"; + +let root: string; +let engineAvailable = true; + +function freshDir(prefix: string): string { + return mkdtempSync(path.join(root, prefix)); +} + +function withEnv(vars: Record, fn: () => void): void { + const prev: Record = {}; + for (const [k, v] of Object.entries(vars)) { + prev[k] = process.env[k]; + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + try { + fn(); + } finally { + for (const [k, v] of Object.entries(prev)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +before(() => { + root = mkdtempSync(path.join(tmpdir(), "bili-7408-")); + const probe = freshDir("probe-"); + try { + const s = UnifiedStore.open(path.join(probe, "index.db")); + s.close(); + } catch { + engineAvailable = false; + } +}); + +after(() => { + rmSync(root, { recursive: true, force: true }); + setLogCapture(null); +}); + +/** Open the sidecar under the given knob posture and return the exact + * full-mode boot line (both state homes kept out of the way so no test + * instance touches a real state dir). */ +function bootLine(env: NodeJS.ProcessEnv, fileUnified?: string, fileJsonWriter?: string): string { + const state = freshDir("state-"); + const sessions = freshDir("sessions-"); + const logs: string[] = []; + setLogCapture((_level, msg) => logs.push(msg)); + try { + withEnv({ XDG_STATE_HOME: state }, () => { + const shadow = UnifiedShadow.maybeOpen(sessions, env, (r) => r as LegacySessionLike, fileUnified, undefined, undefined, fileJsonWriter); + assert.ok(shadow, "sidecar must open"); + shadow.close(); + }); + const line = logs.find((m) => m.includes("unified FULL mode ENABLED")); + assert.ok(line, `FULL-mode boot line expected, got: ${logs.join(" | ")}`); + return line; + } finally { + setLogCapture(null); + } +} + +test("#7408 §3b: default (pure-sqlite) — the boot line says index.db is the only persistent copy and envelopes are NOT written", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({}); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on"), line); + assert.ok(!line.includes("legacy files keep being written"), "the pre-#6480 dual-write claim must be gone under the default"); + assert.ok(!line.includes("unset the knob"), "no knob is set under the default — the rollback hint must not claim otherwise"); +}); + +test("#7408 §3b: explicit full — the historical dual-write wording stays (knob set, envelopes written)", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_UNIFIED: "full" }); + assert.ok(line.includes("storage.unified=full or BILI_STORAGE_UNIFIED=full"), line); + assert.ok(line.includes("legacy files keep being written — unset the knob to roll back"), line); +}); + +test("#7408 §3b: pure-sqlite + explicit jsonWriter=on (env) — envelopes ARE written; the line names the explicit on, not a unified knob", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_JSON_WRITER: "on" }); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("legacy files keep being written — remove the explicit storage.jsonWriter=on to return to the pure-sqlite default"), line); +}); + +test("#7408 §3b: explicit full + explicit jsonWriter=off — the writer-off wording wins over the selection wording", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({ BILI_STORAGE_UNIFIED: "full", BILI_STORAGE_JSON_WRITER: "off" }); + assert.ok(line.includes("storage.unified=full or BILI_STORAGE_UNIFIED=full"), line); + assert.ok(line.includes("JSON envelopes are not written"), line); + assert.ok(!line.includes("legacy files keep being written"), line); +}); + +test("#7408 §3b: file tier — storage.unified=sqlite + storage.jsonWriter=on reaches the line through the fileJsonWriter parameter (the persist.ts plumbing)", () => { + if (!engineAvailable) return; + _resetStorageRuntimeStatusForTest(); + const line = bootLine({}, "sqlite", "on"); + assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line); + assert.ok(line.includes("legacy files keep being written — remove the explicit storage.jsonWriter=on to return to the pure-sqlite default"), line); +});