From 514e8a56e0334f78aa885d18985680e9a52975e7 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 27 Oct 2026 16:31:43 +0800 Subject: [PATCH 1/3] fix: announce the synchronous pre-migration integrity scan before startup blocks on it (#4940) --- src/storage/store.ts | 11 +++++++++++ tests/storage-integrity-cache.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/storage/store.ts b/src/storage/store.ts index 930a55b8f4..85f98739b5 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -393,6 +393,17 @@ export class UnifiedStore { const started = Date.now(); let verdict: string; try { + // #4940: announce the phase BEFORE the scan starts — this is + // the only integrity path that blocks startup, and while it + // runs the child's log stays empty, so a slow scan read as a + // hung child to the launcher (a 5 GB cold scan cost ~60 s and + // outran the 60 s spawn budget on pre-#4609 releases). + const pageCountRow = db.prepare("PRAGMA page_count").get() as { page_count?: unknown } | null | undefined; + const pageSizeRow = db.prepare("PRAGMA page_size").get() as { page_size?: unknown } | null | undefined; + const pages = typeof pageCountRow?.page_count === "number" ? pageCountRow.page_count : null; + const pageSize = typeof pageSizeRow?.page_size === "number" ? pageSizeRow.page_size : null; + const sizeSummary = pages !== null && pageSize !== null ? ` (${(pages * pageSize / (1024 * 1024)).toFixed(1)} MiB, ${pages} pages)` : ""; + loggerLog("info", `[storage] integrity quick_check started for ${path}${sizeSummary} — synchronous pre-migration scan; startup stays blocked until it finishes (#4940)`); const rows = db.prepare("PRAGMA quick_check").all() as Array<{ quick_check?: unknown }>; verdict = rows.map((row) => String(row.quick_check ?? "")).join(" ").trim(); } catch (err) { diff --git a/tests/storage-integrity-cache.test.ts b/tests/storage-integrity-cache.test.ts index 12f5f0be47..afa962393d 100644 --- a/tests/storage-integrity-cache.test.ts +++ b/tests/storage-integrity-cache.test.ts @@ -12,6 +12,8 @@ * the cache, and the counter reaches 1. * 5. BILI_STORAGE_QUICK_CHECK=0: no check, no cache IO. * 6. corrupt small store still throws synchronously (cache or no cache). + * 7. sync path announces its phase before scanning (#4940); the opt-out + * stays silent. */ import test from "node:test"; import assert from "node:assert/strict"; @@ -27,6 +29,7 @@ import { setIntegritySyncThresholdForTests, backgroundIntegrityInflightForTests, } from "../src/storage/integrity-check.js"; +import { setLogCapture } from "../src/logger.js"; function tmpDir(): string { return mkdtempSync(join(tmpdir(), "bili-integrity-")); @@ -153,3 +156,27 @@ test("a corrupt small store still fails open synchronously and writes no ok-cach rmSync(dir, { recursive: true, force: true }); } }); + +test("the sync check announces its phase before scanning, and the opt-out stays silent (#4940)", () => { + resetIntegrityCheckStateForTests(); + setIntegritySyncThresholdForTests(null); + const dir = tmpDir(); + const lines: string[] = []; + setLogCapture((_level, msg) => lines.push(msg)); + const prev = process.env.BILI_STORAGE_QUICK_CHECK; + try { + const dbPath = join(dir, "index.db"); + UnifiedStore.open(dbPath).close(); + const announces = lines.filter((m) => m.includes("integrity quick_check started")); + assert.equal(announces.length, 1, `one pre-scan announcement; got ${JSON.stringify(lines)}`); + assert.match(announces[0], /\(\d+(\.\d+)? MiB, \d+ pages\) — synchronous pre-migration scan/); + process.env.BILI_STORAGE_QUICK_CHECK = "0"; + UnifiedStore.open(dbPath).close(); + assert.equal(lines.filter((m) => m.includes("integrity quick_check started")).length, 1, "the opt-out added no announcement"); + } finally { + setLogCapture(null); + if (prev === undefined) delete process.env.BILI_STORAGE_QUICK_CHECK; + else process.env.BILI_STORAGE_QUICK_CHECK = prev; + rmSync(dir, { recursive: true, force: true }); + } +}); From 7d0016054a74c12cc47def12f539e2785cc2dd70 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 27 Oct 2026 16:31:43 +0800 Subject: [PATCH 2/3] docs: correct BILI_STORAGE_QUICK_CHECK behavior and cost docs post-#4609 (#4940) --- CONFIGURATION.md | 2 +- CONFIGURATION.zh-CN.md | 2 +- LIFECYCLE.md | 2 +- website/config-reference/env-only.yaml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index cd922567c8..d65baec284 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -360,7 +360,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | Env tier for storage.backend (#3931): sqlite (default) or mysql (fully wired, #3388 steps 2+3 - saves ingest through the async shadow queue, sync reads ride the JSON envelope, replicas/ext/GC via the async twins; export too). Wins over the file tier. | | `BILI_MYSQL_DSN` | string | unset | — | MySQL DSN (mysql://user:pass@host:3306/db) for storage.backend="mysql" (#3388, #3931). Wins over the file key storage.mysql.dsn. Empty = the mysql backend cannot open; the documented error names both knobs. The pool opens lazily; closeQuietlyAsync drains the ingest queue and closes it. | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC age sweep window in days (#3463): sessions idle longer than this are archived (one JSON line appended to the single archive log (archive/archive.jsonl) + tombstone row + content/media refs released). 0 disables the age sweep; the manual entry POST /__bili/storage/gc always runs. Config-file key storage.archiveAfterDays is the persistent form. | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; on failure the shadow layer quarantines the file set (index.db/-wal/-shm renamed to .corrupt-) and reopens fresh (JSON envelopes remain the recovery source). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175/#4609): index.db opens of 512 MiB or less run PRAGMA quick_check synchronously BEFORE migrations (that window blocks startup); larger stores check in the background post-open on a second read-only connection so boot never blocks. Verdicts are cached in an index.db.chk sidecar keyed by schema version + file-set stats — an unchanged file set skips the rescan. Measured cost (#4940, 4.96 GB store, 4 KB pages): ~54–73 s from a cold page cache under memory pressure, ~10–15 s warm — the scan is bound by per-page read overhead (~45 µs/page), not bandwidth; on releases predating #4609 expect a first start after boot on a multi-GB store to exceed the 60 s native spawn budget. On failure the shadow layer quarantines the file set (index.db/-wal/-shm renamed to .corrupt-) and reopens fresh (JSON envelopes remain the recovery source). Set 0 to skip the check entirely (no check, no cache IO) — also the escape hatch on those older releases. | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | Snapshot-render shadow observability (#3756 follow-up): compute the outbound snapshot view on every armed turn WITHOUT using it, classify the diff against the legacy outbound, and surface per-session counters in the web panel + the snapshot-shadow diagnostics topic. 0/false/off disables; 1/true/on enables; otherwise the config-file snapshot.shadow boolean applies (default on). The render lane itself stays rollout-gated (BILI_ROLLOUT_SNAPSHOT_RENDER), so the shadow lane never changes outbound bytes - it is the dry-run evidence that flips the rollout later. | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | Override the config file path (path relocation). | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | Where session records are stored (path relocation). | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 0bb8498f59..6314738b5e 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -360,7 +360,7 @@ | `BILI_STORAGE_BACKEND` | sqlite\|mysql | sqlite | — | storage.backend 的环境变量层(#3931):sqlite(默认)或 mysql(已接线,#3388 第二+三步 —— 保存经异步影子队列入 mysql,同步读走 JSON 信封,replicas/ext/GC 走异步孪生;export 同样支持)。优先于文件层。 | | `BILI_MYSQL_DSN` | string | unset | — | storage.backend="mysql" 的 MySQL DSN(mysql://user:pass@host:3306/db)(#3388,#3931)。优先于文件键 storage.mysql.dsn。留空 = mysql 后端无法打开;文档化错误会列出两个旋钮。连接池惰性打开;closeQuietlyAsync 排空入队事务并关闭。 | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC 年龄清扫窗口(天,#3463):闲置超过该时长的会话被归档(单文件归档日志 archive/archive.jsonl 追加一行 + 墓碑行保留标签 + 释放内容/媒体引用)。0 关闭年龄清扫;手动入口 POST /__bili/storage/gc 始终可用。配置文件键 storage.archiveAfterDays 为持久形式。 | -| `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;失败时影子层把文件组(index.db/-wal/-shm 改名为 .corrupt-)隔离并重建新库(JSON 信封仍是恢复源)。仅当超大库的启动时间敏感时才设 0 跳过。 | +| `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175/#4609):≤512 MiB 的 index.db 每次打开在迁移前同步跑 PRAGMA quick_check(该窗口阻塞启动);更大的库在 open 之后用第二条只读连接后台检查,启动不阻塞。判决缓存在 index.db.chk sidecar(按 schema 版本 + 文件组 size/mtime 判定),文件集不变直接跳过重扫。实测代价(#4940,4.96 GB 库、4 KB 页):冷页缓存且内存紧张时 ~54–73 s,暖缓存扫描本身 ~10–15 s——瓶颈是每页读开销(~45 µs/页)而非带宽;#4609 之前的版本上,多 GB 库重启后首次启动会超过 60 s 原生 spawn 预算。失败时影子层把文件组(index.db/-wal/-shm 改名为 .corrupt-)隔离并重建新库(JSON 信封仍是恢复源)。设 0 = 完全跳过(不检查、不读写缓存),也是那些旧版本上的逃生口。 | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | 快照渲染影子对账(#3756 后续):每轮已折叠会话都用引擎现算一次出站快照视图但不使用,与 legacy 实际出站比对分类,逐会话计数进 web 面板与 snapshot-shadow 诊断主题。0/false/off 关闭;1/true/on 开启;否则读配置文件 snapshot.shadow 布尔(默认开)。渲染车道本身仍受灰度开关控制(BILI_ROLLOUT_SNAPSHOT_RENDER),影子车道绝不改变出站字节 —— 它是后续翻转灰度的干跑证据。 | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | 覆盖配置文件路径(路径重定位)。 | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | 会话记录的存放目录(路径重定位)。 | diff --git a/LIFECYCLE.md b/LIFECYCLE.md index fdb7f323a4..e3e1650197 100644 --- a/LIFECYCLE.md +++ b/LIFECYCLE.md @@ -129,7 +129,7 @@ | `BILI_PERSIST_ZSTD` | `=1/true` 启用 zstd 压缩持久化(owner 决定 #1080,默认 off) | | `BILI_SESSIONS_DIR` | 状态目录重定位 | | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | storage.archiveAfterDays,默认 365 天——超龄会话由 gc 归档(export + tombstone + CAS 释放;0 关闭年龄清扫,#3901/#3463) | -| `BILI_STORAGE_QUICK_CHECK` | 默认开——index.db 开库在迁移前跑 PRAGMA quick_check 自检;失败=影子层隔离文件组(.corrupt-)+重建新库(JSON 信封为恢复源);=0 跳过(超大库启动时间敏感场景,#4175) | +| `BILI_STORAGE_QUICK_CHECK` | 默认开——开库自检(#4175/#4609):≤512 MiB 迁移前同步 quick_check(阻塞启动),更大库 open 后后台检查不阻塞;判决缓存 index.db.chk(文件集不变跳过重扫);实测 4.96 GB 冷缓存 ~54–73 s / 暖 ~10–15 s(#4940,~45 µs/页);失败=影子层隔离文件组(.corrupt-)+重建新库(JSON 信封为恢复源);=0 完全跳过(旧版本多 GB 库的逃生口) | | `BILI_STORAGE_WAL_JOURNAL_SIZE_LIMIT_BYTES` | storage.walJournalSizeLimitBytes——SQLite WAL 回收字节上限(#3250;垃圾/非正值落回文件/默认层,不让 typo 崩 boot) | | `BILI_MYSQL_DSN` | storage.mysql.dsn——mysql 引擎连接串(env > 文件 > "";空 = mysql 不可用走 sqlite,#3388/#3931) | diff --git a/website/config-reference/env-only.yaml b/website/config-reference/env-only.yaml index f615a194c9..56bbbea80f 100644 --- a/website/config-reference/env-only.yaml +++ b/website/config-reference/env-only.yaml @@ -56,8 +56,8 @@ entries: kind: env-only audience: user group: envonly - desc_en: "Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; on failure the shadow layer quarantines the file set (index.db/-wal/-shm renamed to .corrupt-) and reopens fresh (JSON envelopes remain the recovery source). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters." - desc_zh: "存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;失败时影子层把文件组(index.db/-wal/-shm 改名为 .corrupt-)隔离并重建新库(JSON 信封仍是恢复源)。仅当超大库的启动时间敏感时才设 0 跳过。" + desc_en: "Storage integrity self-check at open (#4175/#4609): index.db opens of 512 MiB or less run PRAGMA quick_check synchronously BEFORE migrations (that window blocks startup); larger stores check in the background post-open on a second read-only connection so boot never blocks. Verdicts are cached in an index.db.chk sidecar keyed by schema version + file-set stats — an unchanged file set skips the rescan. Measured cost (#4940, 4.96 GB store, 4 KB pages): ~54–73 s from a cold page cache under memory pressure, ~10–15 s warm — the scan is bound by per-page read overhead (~45 µs/page), not bandwidth; on releases predating #4609 expect a first start after boot on a multi-GB store to exceed the 60 s native spawn budget. On failure the shadow layer quarantines the file set (index.db/-wal/-shm renamed to .corrupt-) and reopens fresh (JSON envelopes remain the recovery source). Set 0 to skip the check entirely (no check, no cache IO) — also the escape hatch on those older releases." + desc_zh: "存储开库自检(#4175/#4609):≤512 MiB 的 index.db 每次打开在迁移前同步跑 PRAGMA quick_check(该窗口阻塞启动);更大的库在 open 之后用第二条只读连接后台检查,启动不阻塞。判决缓存在 index.db.chk sidecar(按 schema 版本 + 文件组 size/mtime 判定),文件集不变直接跳过重扫。实测代价(#4940,4.96 GB 库、4 KB 页):冷页缓存且内存紧张时 ~54–73 s,暖缓存扫描本身 ~10–15 s——瓶颈是每页读开销(~45 µs/页)而非带宽;#4609 之前的版本上,多 GB 库重启后首次启动会超过 60 s 原生 spawn 预算。失败时影子层把文件组(index.db/-wal/-shm 改名为 .corrupt-)隔离并重建新库(JSON 信封仍是恢复源)。设 0 = 完全跳过(不检查、不读写缓存),也是那些旧版本上的逃生口。" - key: BILI_SNAPSHOT_SHADOW type: boolean default: "true (on)" From 8f332eef4cb73e407d2123933c351dd87e5c8048 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 28 Oct 2026 00:33:06 +0800 Subject: [PATCH 3/3] test: pin the real 512 MiB sync/async integrity dispatch boundary with a sparse-padded store (#4940) --- tests/storage-integrity-cache.test.ts | 28 ++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/storage-integrity-cache.test.ts b/tests/storage-integrity-cache.test.ts index afa962393d..92f9f45d82 100644 --- a/tests/storage-integrity-cache.test.ts +++ b/tests/storage-integrity-cache.test.ts @@ -14,10 +14,12 @@ * 6. corrupt small store still throws synchronously (cache or no cache). * 7. sync path announces its phase before scanning (#4940); the opt-out * stays silent. + * 8. a store padded past the REAL 512 MiB boundary (sparse tail, tiny page + * count) takes the background path — no synchronous rescan (#4940). */ import test from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync, utimesSync } from "node:fs"; +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync, utimesSync, truncateSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { UnifiedStore, StorageIntegrityError } from "../src/storage/store.js"; @@ -157,6 +159,30 @@ test("a corrupt small store still fails open synchronously and writes no ok-cach } }); +test("a store padded past the real 512 MiB boundary takes the background path without a synchronous rescan (#4940)", () => { + resetIntegrityCheckStateForTests(); + setIntegritySyncThresholdForTests(null); // exercise the REAL threshold constant + const dir = tmpDir(); + const lines: string[] = []; + setLogCapture((_level, msg) => lines.push(msg)); + try { + const dbPath = join(dir, "index.db"); + UnifiedStore.open(dbPath).close(); // small store: sync check + verdict cache written + lines.length = 0; + truncateSync(dbPath, 513 * 1024 * 1024); // sparse tail: stat sees >512 MiB while the page count stays tiny + const store = UnifiedStore.open(dbPath); + assert.ok(lines.some((m) => m.includes("background quick_check started for") && m.includes("over the 512 MiB sync threshold")), `padded open dispatched to the background check; got ${JSON.stringify(lines)}`); + assert.ok(!lines.some((m) => m.includes("synchronous pre-migration scan")), "no synchronous rescan on the padded open"); + const done = waitFor(() => !backgroundIntegrityInflightForTests(dbPath) && integrityCheckRunsForTests() === 2 && existsSync(integrityCachePath(dbPath)), 15000); + assert.ok(done, "background check completed within 15s"); + assert.ok(readIntegrityCache(dbPath)?.ok === true, "background verdict cached as ok"); + store.close(); + } finally { + setLogCapture(null); + rmSync(dir, { recursive: true, force: true }); + } +}); + test("the sync check announces its phase before scanning, and the opt-out stays silent (#4940)", () => { resetIntegrityCheckStateForTests(); setIntegritySyncThresholdForTests(null);