Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -730,7 +730,7 @@ export class SessionStore {
// triggered a paced retry — stamping here kept that read failing
// closed for a full cooldown window.
this.syncSidecarOpen = { legacyLoad };
this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend()) ?? undefined;
this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend(), fileStorageJsonWriter()) ?? undefined;
} else {
this.shadow = undefined;
}
Expand Down Expand Up @@ -3887,7 +3887,7 @@ export class SessionStore {
const cooldown = shadowReopenCooldownMsOverride ?? SHADOW_REOPEN_COOLDOWN_MS;
this.sidecarReopenNextAttemptAt = now + cooldown;
try {
this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, args.legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend()) ?? undefined;
this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, args.legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend(), fileStorageJsonWriter()) ?? undefined;
} catch (err) {
// Transient refusal class re-occurred mid-process (unreachable via
// boot — that path exits first — kept defensive per #5188):
Expand Down
44 changes: 34 additions & 10 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
/**
* Unified-storage sidecar (#2671).
*
* Two opt-in levels via BILI_STORAGE_UNIFIED:
* Selections via BILI_STORAGE_UNIFIED / storage.unified:
* - absent or `sqlite` (the DEFAULT since #6480, "pure-sqlite"): the
* drop-in replacement. Reads go unified-first with organic import (a
* session the index has not seen is imported from its legacy file on
* first load) and legacy fallback on any miss, and the JSON envelope
* WRITER defaults off — index.db is the only persistent copy (restore
* dual-write with storage.jsonWriter=on).
* - `full`: the historical #3818 posture — the same unified-first
* machinery, but the JSON envelope writer defaults ON (dual-write;
* rollback = unset the knob).
* - `1`/`true`/`shadow` (Phase 1): double-write dry-run. Every session
* record the legacy SessionStore commits is ALSO ingested into
* <sessionsDir>/index.db, and the reconstructed view is digest-checked
* against what the legacy loader (buildSession) would serve after a
* restart. Any drift is logged as an error — that is the entire point:
* the unified layer must replay legacy bytes exactly, and production
* traffic becomes the migration test corpus.
* - `full` (Phase 1.5): the drop-in replacement. Reads flip to
* unified-first with organic import (a session the index has not seen
* is imported from its legacy file on first load) and legacy fallback
* on any miss. Legacy files keep being written (rollback = unset the
* env), but the unified index is the load path.
* traffic becomes the migration test corpus. The JSON envelope stays
* the sole source of truth at this level.
* - `legacy`: the one-line escape hatch — no sidecar at all.
*
* Failure policy: sidecar problems must NEVER break the legacy write path.
* All work is try/catch'd. The sidecar NEVER gives up on index writes
Expand All @@ -28,7 +34,7 @@ import { log as loggerLog } from "../logger.js";
import { planIngest, planIngestDelta, loadLegacyView, LegacyViewBuilder, appendBlockContents, mergeOrphanBlockContents, completeBlockContents, type LegacySessionLike } from "./ingest.ts";
import { splitMediaContent } from "./identity.js";
import type { BlockView } from "../session.js";
import { resolveStorageMode, resolveStorageWalJournalSizeLimit, type StorageMode } from "../knobs.ts";
import { effectiveJsonWriterOff, resolveStorageMode, resolveStorageWalJournalSizeLimit, unifiedSelectionIsSqlite, type StorageMode } from "../knobs.ts";
import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSummaryRow, type WebSummaryRow } from "./store.ts";
import path from "node:path";
import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts";
Expand Down Expand Up @@ -289,6 +295,7 @@ export class UnifiedShadow {
fileUnified?: string,
fileWalLimit?: number,
fileBackend?: string,
fileJsonWriter?: string,
): UnifiedShadow | null {
const mode = resolveStorageMode(env, fileUnified);
if (mode === "legacy") return null;
Expand Down Expand Up @@ -394,10 +401,26 @@ export class UnifiedShadow {
recordSidecarInstance(shadow);
const engineTag = ` [engine: ${store.engine}/${store.engineSource}]`;
const walTag = ` [wal-cap: ${walLimit}]`;
// #7408: the FULL-mode line must state the ACTUAL writer state and
// selection — the pre-#6480 wording ("legacy files keep being
// written — unset the knob to roll back") is false under the
// pure-sqlite default (no knob set, envelopes NOT written) and
// contradicted the accurate jsonWriter-off line printed by persist.
// At this point mode is full, the backend is sqlite (mysql logs its
// own line in openAsync), and encryption-blocking never reaches here
// (persist gates the open), so effectiveJsonWriterOff equals the
// jsonWriterOff persist.ts computes for save decisions.
const sqliteSelection = unifiedSelectionIsSqlite(env, fileUnified);
const selection = sqliteSelection
? "pure-sqlite selection (storage.unified absent or =sqlite)"
: "storage.unified=full or BILI_STORAGE_UNIFIED=full";
const writerOff = effectiveJsonWriterOff(env, fileJsonWriter, fileUnified);
loggerLog(
"info",
mode === "full"
? `[storage] unified FULL mode ENABLED (storage.unified=full or BILI_STORAGE_UNIFIED=full): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — unset the knob to roll back${engineTag}${walTag}`
? writerOff
? `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on${engineTag}${walTag}`
: `[storage] unified FULL mode ENABLED (${selection}): index.db is the load path (unified-first, legacy fallback); legacy files keep being written — ${sqliteSelection ? "remove the explicit storage.jsonWriter=on to return to the pure-sqlite default" : "unset the knob to roll back"}${engineTag}${walTag}`
: `[storage] unified shadow mode ENABLED (storage.unified=shadow or BILI_STORAGE_UNIFIED): every session write is double-ingested into index.db and digest-reconciled${engineTag}${walTag}`,
);
return shadow;
Expand All @@ -416,9 +439,10 @@ export class UnifiedShadow {
fileWalLimit?: number,
fileBackend?: string,
fileMysqlDsn?: string,
fileJsonWriter?: string,
): Promise<UnifiedShadow | null> {
const backendName = resolveBackendName(env, fileBackend);
if (backendName !== "mysql") return UnifiedShadow.maybeOpen(dir, env, legacyLoad, fileUnified, fileWalLimit, fileBackend);
if (backendName !== "mysql") return UnifiedShadow.maybeOpen(dir, env, legacyLoad, fileUnified, fileWalLimit, fileBackend, fileJsonWriter);
const mode = resolveStorageMode(env, fileUnified);
if (mode === "legacy") return null;
const dsn = (env.BILI_MYSQL_DSN ?? fileMysqlDsn ?? "").trim();
Expand Down
127 changes: 127 additions & 0 deletions tests/issue7408-storage-boot-log.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/**
* #7408 §3b — the full-mode storage boot line must state the ACTUAL
* selection and writer state. The pre-#6480 wording ("legacy files keep
* being written — unset the knob to roll back") survived the pure-sqlite
* default flip: under the default (no knob set, envelopes NOT written) it
* claimed a knob that was never set, a dual-write that does not happen, and
* a rollback that does not exist — contradicting the accurate jsonWriter-off
* line persist.ts prints in the same boot and steering field diagnosis into
* chasing a non-existent dual-write. Pinned for every reachable posture:
* default pure-sqlite, explicit full, and the two explicit jsonWriter
* overrides (env tier + file tier through the new fileJsonWriter parameter).
*/
import assert from "node:assert";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { after, before, test } from "node:test";
import { UnifiedShadow } from "../src/storage/shadow.ts";
import { UnifiedStore } from "../src/storage/store.ts";
import type { LegacySessionLike } from "../src/storage/ingest.ts";
import { setLogCapture } from "../src/logger.ts";
import { _resetStorageRuntimeStatusForTest } from "../src/storage/runtime-status.ts";

let root: string;
let engineAvailable = true;

function freshDir(prefix: string): string {
return mkdtempSync(path.join(root, prefix));
}

function withEnv(vars: Record<string, string | undefined>, fn: () => void): void {
const prev: Record<string, string | undefined> = {};
for (const [k, v] of Object.entries(vars)) {
prev[k] = process.env[k];
if (v === undefined) delete process.env[k];
else process.env[k] = v;
}
try {
fn();
} finally {
for (const [k, v] of Object.entries(prev)) {
if (v === undefined) delete process.env[k];
else process.env[k] = v;
}
}
}

before(() => {
root = mkdtempSync(path.join(tmpdir(), "bili-7408-"));
const probe = freshDir("probe-");
try {
const s = UnifiedStore.open(path.join(probe, "index.db"));
s.close();
} catch {
engineAvailable = false;
}
});

after(() => {
rmSync(root, { recursive: true, force: true });
setLogCapture(null);
});

/** Open the sidecar under the given knob posture and return the exact
* full-mode boot line (both state homes kept out of the way so no test
* instance touches a real state dir). */
function bootLine(env: NodeJS.ProcessEnv, fileUnified?: string, fileJsonWriter?: string): string {
const state = freshDir("state-");
const sessions = freshDir("sessions-");
const logs: string[] = [];
setLogCapture((_level, msg) => logs.push(msg));
try {
withEnv({ XDG_STATE_HOME: state }, () => {
const shadow = UnifiedShadow.maybeOpen(sessions, env, (r) => r as LegacySessionLike, fileUnified, undefined, undefined, fileJsonWriter);
assert.ok(shadow, "sidecar must open");
shadow.close();
});
const line = logs.find((m) => m.includes("unified FULL mode ENABLED"));
assert.ok(line, `FULL-mode boot line expected, got: ${logs.join(" | ")}`);
return line;
} finally {
setLogCapture(null);
}
}

test("#7408 §3b: default (pure-sqlite) — the boot line says index.db is the only persistent copy and envelopes are NOT written", () => {
if (!engineAvailable) return;
_resetStorageRuntimeStatusForTest();
const line = bootLine({});
assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line);
assert.ok(line.includes("index.db is the load path AND the only persistent copy — JSON envelopes are not written; restore dual-write with storage.jsonWriter=on"), line);
assert.ok(!line.includes("legacy files keep being written"), "the pre-#6480 dual-write claim must be gone under the default");
assert.ok(!line.includes("unset the knob"), "no knob is set under the default — the rollback hint must not claim otherwise");
});

test("#7408 §3b: explicit full — the historical dual-write wording stays (knob set, envelopes written)", () => {
if (!engineAvailable) return;
_resetStorageRuntimeStatusForTest();
const line = bootLine({ BILI_STORAGE_UNIFIED: "full" });
assert.ok(line.includes("storage.unified=full or BILI_STORAGE_UNIFIED=full"), line);
assert.ok(line.includes("legacy files keep being written — unset the knob to roll back"), line);
});

test("#7408 §3b: pure-sqlite + explicit jsonWriter=on (env) — envelopes ARE written; the line names the explicit on, not a unified knob", () => {
if (!engineAvailable) return;
_resetStorageRuntimeStatusForTest();
const line = bootLine({ BILI_STORAGE_JSON_WRITER: "on" });
assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line);
assert.ok(line.includes("legacy files keep being written — remove the explicit storage.jsonWriter=on to return to the pure-sqlite default"), line);
});

test("#7408 §3b: explicit full + explicit jsonWriter=off — the writer-off wording wins over the selection wording", () => {
if (!engineAvailable) return;
_resetStorageRuntimeStatusForTest();
const line = bootLine({ BILI_STORAGE_UNIFIED: "full", BILI_STORAGE_JSON_WRITER: "off" });
assert.ok(line.includes("storage.unified=full or BILI_STORAGE_UNIFIED=full"), line);
assert.ok(line.includes("JSON envelopes are not written"), line);
assert.ok(!line.includes("legacy files keep being written"), line);
});

test("#7408 §3b: file tier — storage.unified=sqlite + storage.jsonWriter=on reaches the line through the fileJsonWriter parameter (the persist.ts plumbing)", () => {
if (!engineAvailable) return;
_resetStorageRuntimeStatusForTest();
const line = bootLine({}, "sqlite", "on");
assert.ok(line.includes("pure-sqlite selection (storage.unified absent or =sqlite)"), line);
assert.ok(line.includes("legacy files keep being written — remove the explicit storage.jsonWriter=on to return to the pure-sqlite default"), line);
});
Loading