Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import { loadOptions, ensureConfigTemplate } from "./config.js";
import { startServer, restartForceCloseOf } from "./server.js";
import { ensureSqliteEngine } from "./storage/engine-bootstrap.js";
import { recordEngineBootstrap } from "./storage/runtime-status.js";
import { configFile as defaultConfigFile } from "./paths.js";
import { log as loggerLog } from "./logger.js";
import { createAutoRestartHandler } from "./restart.js";
Expand Down Expand Up @@ -597,9 +598,10 @@ export async function main(): Promise<void> {
// fetches when unified storage is requested AND no builtin/host engine
// exists, so the common case costs nothing. Must complete before the first
// session store opens (the store resolves its engine at construction).
await ensureSqliteEngine({
const bootstrapOutcome = await ensureSqliteEngine({
resolveProxy: (url) => resolveProxy(opts.routes, opts.proxy, url, opts.proxyFallback),
});
recordEngineBootstrap(bootstrapOutcome);
const server = await startServer(opts);

// Start background auto-update after the server is listening so a slow
Expand Down
18 changes: 18 additions & 0 deletions src/knobs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,24 @@ export function fakeBufCapBytes(): number {
return Number.isFinite(v) && v > 0 ? v : 16 * 1024 * 1024;
}

/** Config-file tiers of the storage knobs (#3040): single readers so the
* runtime-status surface and the store constructor report exactly what the
* file says (moved out of persist.ts, which used to be the only home). */
export function fileStorageUnified(): string | undefined {
const v = loadConfigFile().storage?.unified;
return typeof v === "string" ? v : undefined;
}

export function fileStorageJsonWriter(): string | undefined {
const v = loadConfigFile().storage?.jsonWriter;
return typeof v === "string" ? v : undefined;
}

export function fileStorageAutoDownload(): boolean | undefined {
const v = loadConfigFile().storage?.sqliteAutoDownload;
return typeof v === "boolean" ? v : undefined;
}

/** Tiered storage-migration switch (#2671): legacy | shadow | full. Lives here
* per the env-var discipline — env tier BILI_STORAGE_UNIFIED (1/true = shadow,
* full/unified = full, 0/false/legacy/garbage = legacy) wins over the
Expand Down
21 changes: 4 additions & 17 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ import {
persistEpermAlertThreshold as knobPersistEpermAlertThreshold,
persistEpermAlertRepeatMs as knobPersistEpermAlertRepeatMs,
resolveJsonWriterOff as knobResolveJsonWriterOff,
fileStorageUnified,
fileStorageJsonWriter,
} from "./knobs.js";
import { log as loggerLog } from "./logger.js";
import { loadConfigFile } from "./config.js";
import { recordRequestedMode } from "./storage/runtime-status.js";
import { VERSION } from "./version.js";
import { createStorageCodec, parseEncryptionKey } from "./encrypt.js";
import { PersistEpermAlert } from "./persist-eperm.js";
Expand Down Expand Up @@ -367,6 +369,7 @@ export class SessionStore {
baseLog("warn", "[storage] unified sidecar DISABLED: BILI_ENCRYPTION_KEY is active and index.db would hold cleartext ingest payloads — staying legacy-only");
}
this.storageMode = storageMode;
recordRequestedMode(storageMode, unifiedBlockedByEncryption);
// Phase 3 (#2960): stop writing the legacy JSON envelope while the
// unified store runs in effective-full mode. Ignored (warned once)
// under shadow/legacy/encryption — there the JSON file is the sole
Expand Down Expand Up @@ -1630,22 +1633,6 @@ function viewToRecord(view: LegacySessionLike): PersistedSession {
return record;
}

/** File-config `storage.unified` value for this lane, when present and
* string-shaped (#2671). The config reader caches on file content, so this
* is a page-cache-hot read at store construction; mode itself is
* boot-decided and does not follow hot reloads. */
function fileStorageUnified(): string | undefined {
const section = (loadConfigFile() as { storage?: { unified?: unknown; jsonWriter?: unknown } }).storage;
return typeof section?.unified === "string" ? section.unified : undefined;
}

/** Config-file tier for the Phase 3 JSON-envelope switch (#2960):
* storage.jsonWriter = "on" | "off" (env tier wins; see knobs.ts). */
function fileStorageJsonWriter(): string | undefined {
const section = (loadConfigFile() as { storage?: { jsonWriter?: unknown } }).storage;
return typeof section?.jsonWriter === "string" ? section.jsonWriter : undefined;
}

/** resolveStorageMode guarded for the constructor: env parsing cannot throw
* (resolveStorageMode is total), so this only exists to keep the call site
* honest about the default when the knob is absent. */
Expand Down
19 changes: 18 additions & 1 deletion src/server/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ import { detectStaleInstall } from "../update.js";
import { PACKAGE_NAME, VERSION, BUILD_COMMIT } from "../version.js";
import { buildOverview, buildSessionDetail, buildSessionList, buildSessionPage, handleConfigGet, handleConfigPut, hiddenEmptyCount, renderUI } from "../web/index.js";
import { queryLogLines } from "../web/logs-query.js";
import { buildDiagnostics } from "../web/diagnostics.js";
import { storageStatusWire } from "../storage/runtime-status.js";

interface AdminCtx {
opts: ProxyOptions;
Expand Down Expand Up @@ -155,6 +157,7 @@ export async function handleAdminRoute(req: http.IncomingMessage, res: http.Serv
if (req.method === "GET" && req.url === "/__bili/overview") return sendOverview(res, opts);
if (req.method === "GET" && (req.url === "/__bili/sessions" || req.url?.startsWith("/__bili/sessions?"))) return sendWebSessions(res, req);
if (req.method === "GET" && req.url?.startsWith("/__bili/logs")) return sendWebLogs(res, req);
if (req.method === "GET" && req.url?.startsWith("/__bili/diagnostics")) return sendDiagnostics(res, req.url);
if (req.method === "GET" && req.url?.startsWith("/__bili/sessions/") && req.url.endsWith("/detail")) return sendWebSessionDetail(res, req.url);
if (req.method === "GET" && req.url === "/") {
// Browser visits root → redirect to the web UI. curl / health probes
Expand Down Expand Up @@ -783,7 +786,7 @@ async function sendStatus(res: http.ServerResponse, opts: ProxyOptions): Promise
loggerLog("warn", `split-session canary (#2170): conversation ${w.base} has live traffic under multiple session keys (design persona forks are excluded): ${w.sessions.map((s) => `${s.id} (requests=${s.requests})`).join("; ")}. For a non-persona host this is the #2165 failure shape (stolen anchor / never-compressing split) — investigate if unexpected.`);
}
}
res.end(JSON.stringify({ version: VERSION, commit: BUILD_COMMIT, diskVersion, stale, updateReady: updateReadyPayload(), autoRestartOnUpdate: opts.autoRestartOnUpdate, advisory: currentAdvisoryPayload(), pluginAdvisories: detectCostAdvisories(process.env), inFlight: totalInFlight(), splitSessions: splitWarnings, conflicts: summarizeConflicts(listSessions()) }, null, 2));
res.end(JSON.stringify({ version: VERSION, commit: BUILD_COMMIT, diskVersion, stale, updateReady: updateReadyPayload(), autoRestartOnUpdate: opts.autoRestartOnUpdate, advisory: currentAdvisoryPayload(), pluginAdvisories: detectCostAdvisories(process.env), inFlight: totalInFlight(), splitSessions: splitWarnings, conflicts: summarizeConflicts(listSessions()), storage: storageStatusWire() }, null, 2));
}

// #2090 plan A — read-only view backing the web UI's "Signed upstreams" card:
Expand Down Expand Up @@ -841,9 +844,23 @@ async function sendOverview(res: http.ServerResponse, opts: ProxyOptions): Promi
conflicts: summarizeConflicts(listSessions()),
passthrough: { enabled: !!opts.passthrough, source: opts.passthroughSource },
alerts: getUpstreamAlerts(),
storage: storageStatusWire(),
}, null, 2));
}

/** #3040: generic diagnostics report — copyable blob + prefilled GitHub issue
* link. Loopback-gated like every other /__bili/* route; never throws. */
function sendDiagnostics(res: http.ServerResponse, url: string | null): void {
let topic: string | undefined;
try {
topic = new URL(url ?? "/__bili/diagnostics", "http://localhost").searchParams.get("topic") ?? undefined;
} catch {
topic = undefined;
}
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify(buildDiagnostics(topic), null, 2));
}

/** #1937: optional ?q= / ?page= / ?pageSize= (≤200) switch the endpoint to
* server-side filtered paging. Without params the response shape is unchanged
* ({sessions, hiddenEmpty} + additive total) so older UIs keep working. */
Expand Down
46 changes: 21 additions & 25 deletions src/storage/engine-bootstrap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,16 @@
*/

import { createHash, randomBytes } from "node:crypto";
import { existsSync } from "node:fs";
import { existsSync, readFileSync } from "node:fs";
import { promises as fsp } from "node:fs";
import { createRequire } from "node:module";
import path from "node:path";
import { gunzipSync } from "node:zlib";
import { log as loggerLog } from "../logger.js";
import { cacheDir } from "../paths.js";
import { loadConfigFile } from "../config.js";

import { probeBuiltinSqlite, probeHostBetterSqlite3, registerSelfDownloadedBetterSqlite3, type BetterSqlite3Ctor } from "./driver.js";
import { resolveStorageMode, resolveSqliteAutoDownload, updateRegistryBase } from "../knobs.js";
import { fileStorageAutoDownload, fileStorageUnified, resolveStorageMode, resolveSqliteAutoDownload, updateRegistryBase } from "../knobs.js";
import { normalizeRegistryBase } from "../update.js";
import { proxyDispatcher } from "../upstream-proxy.js";

Expand All @@ -59,7 +59,7 @@ const SUPPORTED_TARGETS = [
];

const BOOTSTRAP_TIMEOUT_MS = 10_000;
const COOLDOWN_MS = 6 * 60 * 60 * 1000;
export const COOLDOWN_MS = 6 * 60 * 60 * 1000;

type BootstrapStatus =
| "not-needed"
Expand Down Expand Up @@ -128,11 +128,25 @@ function cooldownFile(): string {
return path.join(cacheRoot(), ".bootstrap-failed.json");
}

interface CooldownMarker {
export interface CooldownMarker {
atMs: number;
reason: string;
}

/** Sync read of the failure marker for status surfaces (#3040): same file and
* shape validation as readCooldown, but synchronous so the web endpoints can
* report remaining cooldown without awaiting. */
export function readBootstrapFailureMarker(): CooldownMarker | null {
try {
const raw = readFileSync(cooldownFile(), "utf8");
const j = JSON.parse(raw) as Partial<CooldownMarker>;
if (typeof j.atMs === "number" && typeof j.reason === "string") return { atMs: j.atMs, reason: j.reason };
return null;
} catch {
return null; // corrupt/absent marker: fail open toward retrying
}
}

async function readCooldown(): Promise<CooldownMarker | null> {
try {
const raw = await fsp.readFile(cooldownFile(), "utf8");
Expand Down Expand Up @@ -165,24 +179,6 @@ function cacheUsable(dir: string, target: string): boolean {
&& existsSync(path.join(dir, "prebuilds", `${target}.node`));
}

function fileStorageUnifiedValue(): string | undefined {
try {
const section = (loadConfigFile() as { storage?: { unified?: unknown } }).storage;
return typeof section?.unified === "string" ? section.unified : undefined;
} catch {
return undefined;
}
}

function fileSqliteAutoDownloadValue(): boolean | undefined {
try {
const v = (loadConfigFile() as { storage?: { sqliteAutoDownload?: unknown } }).storage?.sqliteAutoDownload;
return typeof v === "boolean" ? v : undefined;
} catch {
return undefined;
}
}

interface EnsureSqliteEngineOptions {
/** Egress proxy resolver (same chain as updater/model traffic, #609). Absent = direct. */
resolveProxy?: (url: string) => string | undefined;
Expand All @@ -199,7 +195,7 @@ export function ensureSqliteEngine(opts: EnsureSqliteEngineOptions = {}): Promis

async function runBootstrap(opts: EnsureSqliteEngineOptions): Promise<BootstrapOutcome> {
const env = opts.env ?? process.env;
if (resolveStorageMode(env, fileStorageUnifiedValue()) === "legacy") {
if (resolveStorageMode(env, fileStorageUnified()) === "legacy") {
return { status: "not-needed", detail: "unified storage disabled" };
}
if (probeBuiltinSqlite()) {
Expand All @@ -222,7 +218,7 @@ async function runBootstrap(opts: EnsureSqliteEngineOptions): Promise<BootstrapO
if (!SUPPORTED_TARGETS.includes(target)) {
return { status: "unsupported-platform", detail: target };
}
if (!resolveSqliteAutoDownload(env, fileSqliteAutoDownloadValue())) {
if (!resolveSqliteAutoDownload(env, fileStorageAutoDownload())) {
return { status: "disabled", detail: "storage.sqliteAutoDownload=false" };
}
const cd = await readCooldown();
Expand Down
Loading
Loading