From 21480babf9e15798c852c6240ca8f5801de44f53 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 07:06:15 +0800 Subject: [PATCH 1/5] fix(storage): refuse gc full-VACUUM while another live process holds the db (#4612) Field report (bili 0.1.204, 14 GB index.db): an online 'storage gc' (vacuum=full) held the exclusive write lock across the whole O(db-size) VACUUM rewrite for 14 minutes with zero progress output, starving the running proxy until its unified shadow layer degraded (3 consecutive 'database is locked' failures -> legacy-only for the process lifetime). - UnifiedStore.activeForeignWriter(): union of the open-time captured #4175 lock holder (revalidated for liveness) and a fresh .lock read (catches an opener that started after us); fail-open on unreadable file. - UnifiedStore.gc(): vacuum:'full' refuses up front when a live foreign writer is present (actionable error naming the pid); incremental / no-vacuum sweeps are short transactions and stay allowed. - Zero-ref sweep now commits in bounded batches (rowid subselect; SQLite DELETE takes no LIMIT) with per-batch progress lines; batch grows with the table so log volume stays bounded. Crash-safe: rows are garbage by definition, leftovers are collected by the next sweep. - Full-vacuum sizing logs: main/WAL size before, elapsed + size delta after. - CLI lane surfaces the refusal as exit 1 + actionable text; panel endpoint returns 409. Dry-run stays read-only. --- src/persist.ts | 12 +++++ src/server/admin.ts | 11 ++++ src/storage-maintenance.ts | 15 ++++++ src/storage/shadow.ts | 8 +++ src/storage/store.ts | 105 +++++++++++++++++++++++++++++++++++-- 5 files changed, 146 insertions(+), 5 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index bdf9abc0d1..7c5264370e 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1764,6 +1764,18 @@ export class SessionStore { } } + /** #4612: live foreign writer on the unified db (#4175 lock discipline) — + * the gc full-vacuum preflight signal for the CLI/HTTP surfaces. Null when + * the unified store is unavailable or nobody else is provably writing. */ + unifiedActiveWriter(): number | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + try { + return this.shadow.activeForeignWriter(); + } catch { + return null; // sidecar degraded mid-call: no evidence of a writer + } + } + /** #3901/#3463: restore an archived session from its export file. */ unifiedRestoreArchived(sessionId: string, archiveDir?: string): { ok: boolean; reason?: string } | null { if (!this.shadow || this.shadow.stats.disabled) return null; diff --git a/src/server/admin.ts b/src/server/admin.ts index 6f57a24bf2..5b02a0d91a 100644 --- a/src/server/admin.ts +++ b/src/server/admin.ts @@ -540,6 +540,17 @@ export async function handleAdminRoute(req: http.IncomingMessage, res: http.Serv } const qVacuum = (() => { try { return new URL(req.url!, "http://localhost").searchParams.get("vacuum"); } catch { return null; } })(); const vacuum = qVacuum === "full" || qVacuum === "incremental" ? qVacuum : parsed.vacuum === "full" ? "full" : parsed.vacuum === "incremental" ? "incremental" : undefined; + // #4612: full VACUUM refuses to start while another live process + // writes the same db (exclusive write lock across the whole + // O(db-size) rewrite) — same preflight as the CLI lane. + if (vacuum === "full") { + const writer = store.unifiedActiveWriter(); + if (writer !== null) { + res.writeHead(409, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: `another process (pid ${writer}) is actively writing index.db — full VACUUM holds the exclusive write lock across the whole rewrite; stop the other process first or use ?vacuum=incremental`, writerPid: writer })); + return; + } + } const result = await store.unifiedGc({ archiveAfterDays: typeof parsed.archiveAfterDays === "number" && Number.isFinite(parsed.archiveAfterDays) && parsed.archiveAfterDays >= 0 ? parsed.archiveAfterDays : undefined, vacuum, diff --git a/src/storage-maintenance.ts b/src/storage-maintenance.ts index a78f76cbbd..26d02aa0fa 100644 --- a/src/storage-maintenance.ts +++ b/src/storage-maintenance.ts @@ -212,6 +212,21 @@ async function runGc(store: SessionStore, days: number | undefined, vacuum: "ful if (json) return { code: 0, text: `${JSON.stringify(data, null, 2)}\n` }; return { code: 0, text: `${text}\ndry-run: nothing executed\n` }; } + // #4612: full VACUUM is an O(db-size) atomic rewrite under the exclusive + // write lock with no intermediate progress — running it while another live + // process (e.g. a running proxy) writes the same db starves that process + // until its storage layer degrades. Refuse with an actionable message; + // incremental / no-vacuum sweeps are short transactions and stay allowed. + if (vacuumMode === "full") { + const writer = store.unifiedActiveWriter(); + if (writer !== null) { + data.refused = true; + data.writerPid = writer; + const reason = `refused: another process (pid ${writer}) is actively writing index.db — 'storage gc --vacuum full' rewrites the whole database under an exclusive write lock (minutes on large dbs, no intermediate progress) and would starve the other instance's writes until its storage layer degrades. Stop the other process (or wait for it to exit) and retry, or use --vacuum incremental for an online-safe shrink.`; + if (json) return { code: 1, text: `${JSON.stringify(data, null, 2)}\n` }; + return fail(`${text}\n${reason}`, json, data); + } + } const dbBefore = indexDbSize(store.dir); // Same facade call the HTTP endpoint makes (admin.ts POST /__bili/storage/gc) // — results are therefore comparable lane-for-lane. diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index d527be96c4..9cdaf38022 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -725,6 +725,14 @@ export class UnifiedShadow { return this.sync.gcPlan(archiveAfterDays, now); } + /** #4612: live foreign writer on the unified db (#4175 lock discipline) — + * the gc full-vacuum preflight signal. Null when unavailable or nobody + * else is provably writing. */ + activeForeignWriter(): number | null { + if (this.sync === null) return null; + return this.sync.activeForeignWriter(); + } + rawMessages(sessionId: string): { seq: number; content: string }[] { if (this.sync === null) return []; this.assertLive(); diff --git a/src/storage/store.ts b/src/storage/store.ts index 8903060935..9b76f7ad00 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -315,6 +315,24 @@ export class StorageIntegrityError extends Error { } } +/** #4612: zero-ref sweep batching — minimum rows per commit transaction, and a + * cap on how many progress lines one sweep may emit (the batch grows with the + * table, so a 1M-row sweep logs ~500 lines instead of 1000). */ +const GC_SWEEP_BATCH_MIN_ROWS = 1000; +const GC_SWEEP_MAX_PROGRESS_LINES = 500; + +function formatDbSize(bytes: number): string { + if (bytes < 0) return "?"; + const units = ["B", "KiB", "MiB", "GiB"]; + let v = bytes; + let i = 0; + while (v >= 1024 && i < units.length - 1) { + v /= 1024; + i++; + } + return i === 0 ? `${v} B` : `${v.toFixed(1)} ${units[i]}`; +} + /** #4175: true when a pid belongs to a live process (EPERM = alive, foreign user). */ function processAlive(pid: number): boolean { try { @@ -330,13 +348,16 @@ export class UnifiedStore { /** Where the engine came from (#2833): builtin / host / self-downloaded — logged at enable so field diagnosis can tell which acquisition path a lane ran on. */ readonly engineSource: SqliteEngineSource; private readonly db: SqliteDatabase; + /** #4612: main database file path (VACUUM sizing logs). */ + private readonly path: string; /** #3250: successful-ingest counter driving the WAL_CHECKPOINT_EVERY_INGESTS cadence. */ private ingestCount = 0; /** #4175: the live foreign pid found holding .lock at open, if any. */ private readonly foreignLockHolder: { pid: number } | null; private readonly lockPath: string | null; - private constructor(db: SqliteDatabase, engine: SqliteEngineName, engineSource: SqliteEngineSource, lockPath: string | null, foreignLockHolder: { pid: number } | null) { + private constructor(path: string, db: SqliteDatabase, engine: SqliteEngineName, engineSource: SqliteEngineSource, lockPath: string | null, foreignLockHolder: { pid: number } | null) { + this.path = path; this.db = db; this.engine = engine; this.engineSource = engineSource; @@ -438,7 +459,7 @@ export class UnifiedStore { } catch (err) { loggerLog("warn", `[storage] WAL checkpoint on open failed for ${path}: ${String(err)}`); } - return new UnifiedStore(db, used, source, lockPath, foreignLockHolder); + return new UnifiedStore(path, db, used, source, lockPath, foreignLockHolder); } close(): void { @@ -469,6 +490,40 @@ export class UnifiedStore { return this.foreignLockHolder; } + /** #4612: a live pid besides ours currently writing this db — the union of + * two evidence sources: (1) the foreign holder captured at open, re-validated + * for liveness (it may have exited since); (2) a fresh read of .lock + * pointing at another live pid — every opener overwrites the lockfile with + * its own pid on open, so this catches an instance that started AFTER us. + * Null when no live foreign writer is provably present; an unreadable + * lockfile means "no evidence" (fail-open) — callers need proof of + * liveness to refuse, not certainty of absence. */ + activeForeignWriter(): number | null { + if (this.foreignLockHolder !== null && processAlive(this.foreignLockHolder.pid)) { + return this.foreignLockHolder.pid; + } + if (this.lockPath !== null) { + try { + if (existsSync(this.lockPath)) { + const parsed = JSON.parse(readFileSync(this.lockPath, "utf8")) as { pid?: unknown }; + const pid = typeof parsed.pid === "number" ? parsed.pid : null; + if (pid !== null && pid !== process.pid && processAlive(pid)) return pid; + } + } catch { + /* unreadable lockfile: no evidence of a writer */ + } + } + return null; + } + + private dbFileSize(): number { + try { + return statSync(this.path).size; + } catch { + return -1; + } + } + /** * #4175: online consistent snapshot via SQLite's native VACUUM INTO — * safe while WAL writers are active (it reads a consistent snapshot and @@ -1685,6 +1740,19 @@ export class UnifiedStore { * pages incrementally; a one-time VACUUM reclaims the initial bloat of * databases created before that pragma). */ gc(opts: { archiveAfterDays: number; archiveDir: string; now?: number; vacuum?: "full" | "incremental" }): { archived: number; contentRowsDeleted: number } { + // #4612: full VACUUM is an O(db-size) atomic rewrite under the exclusive + // write lock with no intermediate progress. Running it against a db that + // another live process is writing starves that process until its storage + // layer degrades (field report: shadow 3-strike during a 14-min run on a + // 14 GB db). Refuse up front with an actionable error instead of silently + // contending. Incremental / no-vacuum sweeps are short transactions — the + // existing busy_timeout handling covers them. + if (opts.vacuum === "full") { + const writer = this.activeForeignWriter(); + if (writer !== null) { + throw new Error(`[#4612] gc full VACUUM refused: another process (pid ${writer}) is actively writing this database — the rewrite holds the exclusive write lock across the whole pass and would starve the peer's writes until its storage layer degrades; stop the other process (or wait for it to exit) and retry, or use vacuum:"incremental"`); + } + } const now = opts.now ?? Date.now(); let archived = 0; let contentRowsDeleted = 0; @@ -1699,14 +1767,41 @@ export class UnifiedStore { } const zero = this.db.prepare("SELECT COUNT(*) AS n FROM content WHERE refcount <= 0").get() as { n: number }; if (zero.n > 0) { - this.db.prepare("DELETE FROM content WHERE refcount <= 0").run(); - contentRowsDeleted += zero.n; + // #4612: batched sweep. One unbounded DELETE held the exclusive write + // lock across the entire scan (field: 82k rows / 11.3 GiB, 14 minutes + // of silence) with no progress visible. Per-batch commits release the + // lock between batches and give the operator something to watch. Rows + // are garbage by definition (refcount <= 0), so partial progress is + // safe: whatever survives a crash is collected by the next sweep. The + // batch grows with the table so total log lines stay bounded. + const batch = Math.max(GC_SWEEP_BATCH_MIN_ROWS, Math.ceil(zero.n / GC_SWEEP_MAX_PROGRESS_LINES)); + // SQLite DELETE takes no LIMIT clause: bound the batch through a + // rowid subselect instead. + const del = this.db.prepare("DELETE FROM content WHERE rowid IN (SELECT rowid FROM content WHERE refcount <= 0 ORDER BY rowid LIMIT ?)"); + let deleted = 0; + for (;;) { + const r = del.run(batch); + const n = Number(r.changes); + if (n === 0) break; + deleted += n; + loggerLog("info", `[#4612] gc zero-ref sweep: ${deleted}/${zero.n} rows (${Math.round((deleted / zero.n) * 100)}%)`); + } + contentRowsDeleted += deleted; } if (opts.vacuum === "full") { - loggerLog("info", "[#3901] gc: running full VACUUM (this can take a while on large databases)"); + const sizeBefore = this.dbFileSize(); + let walPart = ""; + try { + walPart = `, ${formatDbSize(statSync(`${this.path}-wal`).size)} WAL`; + } catch { + /* no WAL file present */ + } + loggerLog("info", `[#4612] gc: full VACUUM starting — ${formatDbSize(sizeBefore)} main file${walPart}; atomic rewrite holding the exclusive write lock until done (no intermediate progress)`); + const startedAt = Date.now(); // #4390: delegate to vacuum() — a bare VACUUM leaves the compacted // image in the WAL; only the TRUNCATE checkpoint shrinks the files. this.vacuum("full"); + loggerLog("info", `[#4612] gc: full VACUUM finished in ${((Date.now() - startedAt) / 1000).toFixed(1)} s — ${formatDbSize(sizeBefore)} -> ${formatDbSize(this.dbFileSize())}`); } else if (opts.vacuum === "incremental") { this.vacuum("incremental"); } From 5203110aaec21d10c5b7f989c32912356863e602 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 07:06:57 +0800 Subject: [PATCH 2/5] test: pin the #4612 gc online guard and sweep batching Dummy live process (spawned node child) stands in for the running proxy: open-time holder capture, later-opener lockfile detection, liveness re-validation on exit, incremental/dry-run exemptions, batched zero-ref sweep outcome + progress lines, and the CLI refusal surface (code 1, actionable text, json fields). --- tests/issue4612-gc-online-guard.test.ts | 237 ++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 tests/issue4612-gc-online-guard.test.ts diff --git a/tests/issue4612-gc-online-guard.test.ts b/tests/issue4612-gc-online-guard.test.ts new file mode 100644 index 0000000000..ab845881d0 --- /dev/null +++ b/tests/issue4612-gc-online-guard.test.ts @@ -0,0 +1,237 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { spawn, type ChildProcess } from "node:child_process"; +import { DatabaseSync } from "node:sqlite"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { setLogCapture } from "../src/logger.ts"; +import { runStorageGcCommand } from "../src/storage-maintenance.ts"; + +/** + * #4612 — storage gc full-VACUUM online guard + progress. + * + * Field report: an online `bili storage gc` (vacuum=full) against a 14 GB + * index.db held the exclusive write lock across the whole O(db-size) VACUUM + * rewrite for 14 minutes with zero progress output, starving the running + * proxy until its unified shadow layer degraded (3 consecutive "database is + * locked" failures → legacy-only for the process lifetime). + * + * Fix under test: (1) gc full-vacuum refuses up front when another live + * process holds the db (#4175 .lock pid discipline — both the holder + * captured at open and a later opener that overwrote our lockfile); (2) the + * zero-ref sweep commits in bounded batches with per-batch progress; (3) the + * CLI lane surfaces the refusal as code 1 + actionable text instead of a + * generic mid-run failure. Dry-run stays read-only; incremental/no-vacuum + * sweeps stay online-safe. + */ + +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_STORAGE_UNIFIED = "full"; +process.env.BILI_CONFIG_FILE = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), "bili-4612-gc-guard-")), + "no-config.json", +); + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function waitUntil(fn: () => boolean, ms: number, what: string): Promise { + const deadline = Date.now() + ms; + while (!fn()) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((r) => setTimeout(r, 25)); + } +} + +/** A live stand-in for "the other instance" (e.g. a running proxy). */ +function spawnDummyWriter(): { pid: number; done(): Promise; kill(): void } { + const child: ChildProcess = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" }); + const pid = child.pid!; + return { + pid, + done: () => waitUntil(() => !alive(pid), 10_000, `dummy writer ${pid} to exit`), + kill: () => child.kill("SIGTERM"), + }; +} + +interface CoreMsg { id: string; role: string; contentType: string; text: string } +const mkMsg = (id: string, text: string): CoreMsg => ({ id, role: "user", contentType: "text", text }); + +const mkLegacy = (id: string, msgs: CoreMsg[], lastSeen: number): LegacySessionLike => ({ + id, + createdAt: 1_000, + lastSeen, + meta: { protocol: "openai" }, + stats: {}, + metadata: {}, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw: {}, byRef: {} } }, + blockContents: new Map(), + lastMessages: msgs as unknown as LegacySessionLike["lastMessages"], + lastMessagesFolded: false, + pluginSnapshot: undefined, + contentStore: undefined, +} as unknown as LegacySessionLike); + +/** DB-direct seeding (no JSON files). */ +function seedDb(dir: string, entries: Array<[string, CoreMsg[], number]>): void { + const store = UnifiedStore.open(path.join(dir, "index.db")); + try { + for (const [id, msgs, lastSeen] of entries) store.ingestSession(planIngest(mkLegacy(id, msgs, lastSeen))); + } finally { + store.close(); + } +} + +test("#4612: full gc refuses while an open-time foreign holder is live; incremental proceeds; refusal lifts on exit", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4612-guard-open-")); + const dbPath = path.join(dir, "index.db"); + const dummy = spawnDummyWriter(); + try { + await waitUntil(() => alive(dummy.pid), 10_000, "dummy writer to start"); + // The peer opened first: its lockfile entry is what our open() captures. + fs.writeFileSync(`${dbPath}.lock`, JSON.stringify({ pid: dummy.pid, boot: Date.now() })); + const store = UnifiedStore.open(dbPath); + try { + assert.equal(store.lockHolder()?.pid, dummy.pid, "open captures the foreign holder (#4175)"); + assert.equal(store.activeForeignWriter(), dummy.pid, "fresh liveness re-validation sees the live peer"); + + assert.throws( + () => store.gc({ archiveAfterDays: 0, archiveDir: path.join(dir, "archive"), vacuum: "full" }), + (err: Error) => /another process \(pid \d+\)/.test(err.message) && /vacuum:"incremental"/.test(err.message), + "full gc refuses up front with an actionable error", + ); + // Incremental stays online-safe: short transactions, no exclusive rewrite. + const inc = store.gc({ archiveAfterDays: 0, archiveDir: path.join(dir, "archive"), vacuum: "incremental" }); + assert.equal(typeof inc.archived, "number", "incremental gc proceeds while the peer writes"); + + dummy.kill(); + await dummy.done(); + assert.equal(store.activeForeignWriter(), null, "dead holder no longer blocks"); + const full = store.gc({ archiveAfterDays: 0, archiveDir: path.join(dir, "archive"), vacuum: "full" }); + assert.equal(full.archived, 0, "full gc runs once the peer has exited"); + } finally { + store.close(); + } + } finally { + dummy.kill(); + await dummy.done().catch(() => {}); + } +}); + +test("#4612: detects a writer that started AFTER us via the lockfile fresh-read path", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4612-guard-late-")); + const dbPath = path.join(dir, "index.db"); + const store = UnifiedStore.open(dbPath); + const dummy = spawnDummyWriter(); + try { + await waitUntil(() => alive(dummy.pid), 10_000, "dummy writer to start"); + assert.equal(store.activeForeignWriter(), null, "clean open: no foreign writer"); + // The peer started after us and overwrote the lockfile with its own pid + // (every opener does, #4175) — the open-time snapshot cannot see it. + fs.writeFileSync(`${dbPath}.lock`, JSON.stringify({ pid: dummy.pid, boot: Date.now() })); + assert.equal(store.activeForeignWriter(), dummy.pid, "fresh lockfile read catches the later opener"); + assert.throws( + () => store.gc({ archiveAfterDays: 0, archiveDir: path.join(dir, "archive"), vacuum: "full" }), + /another process \(pid \d+\)/, + "full gc refuses against the later opener too", + ); + dummy.kill(); + await dummy.done(); + assert.equal(store.activeForeignWriter(), null, "stale lockfile entry (dead pid) is not a writer"); + store.gc({ archiveAfterDays: 0, archiveDir: path.join(dir, "archive"), vacuum: "full" }); + } finally { + dummy.kill(); + await dummy.done().catch(() => {}); + store.close(); + } +}); + +test("#4612: zero-ref sweep deletes every garbage row in batches with progress lines", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4612-sweep-")); + const dbPath = path.join(dir, "index.db"); + const store = UnifiedStore.open(dbPath); + try { + const N = 2500; // batch floor 1000 → three commits (1000/1000/500) + const raw = new DatabaseSync(dbPath); + try { + const ins = raw.prepare("INSERT INTO content (hash, byte_len, refcount, algo, bytes) VALUES (?, ?, 0, 'sha256', ?)"); + for (let i = 0; i < N; i++) { + const body = Buffer.from(`zero-ref body ${i}`); + ins.run(`4612-zero-ref-${i}`, body.length, body); + } + } finally { + raw.close(); + } + const lines: string[] = []; + setLogCapture((_level, msg) => { lines.push(msg); }); + let result!: { archived: number; contentRowsDeleted: number }; + try { + result = store.gc({ archiveAfterDays: 0, archiveDir: path.join(dir, "archive") }); + } finally { + setLogCapture(null); + } + assert.equal(result.contentRowsDeleted, N, "every zero-ref row collected"); + const progress = lines.filter((m) => /gc zero-ref sweep: \d+\/\d+ rows \(\d+%\)/.test(m)); + assert.ok(progress.length >= 2, `sweep committed in multiple batches (got ${progress.length} progress lines)`); + assert.match(progress[progress.length - 1], /rows \(100%\)$/, "final progress line reports completion"); + const raw2 = new DatabaseSync(dbPath, { readOnly: true }); + try { + const left = raw2.prepare("SELECT COUNT(*) AS n FROM content WHERE refcount <= 0").get() as { n: number }; + assert.equal(left.n, 0, "no garbage rows survive"); + } finally { + raw2.close(); + } + } finally { + store.close(); + } +}); + +test("#4612: CLI 'storage gc --vacuum full' exits 1 with actionable refusal while a writer is live", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4612-cli-")); + seedDb(dir, [["sess-1", [mkMsg("m1", "cli guard body ".repeat(20))], Date.now()]]); + const dummy = spawnDummyWriter(); + // Each CLI invocation is its own process that reads whatever the peer last + // wrote to .lock; an in-process invocation overwrites the file with its + // own pid on open, so re-arm the fixture before every guarded call. + const armPeerLock = (): void => fs.writeFileSync(path.join(dir, "index.db.lock"), JSON.stringify({ pid: dummy.pid, boot: Date.now() })); + try { + await waitUntil(() => alive(dummy.pid), 10_000, "dummy writer to start"); + armPeerLock(); + + const refused = await runStorageGcCommand({ dir, vacuum: "full" }); + assert.equal(refused.code, 1, "refusal is a hard failure"); + assert.match(refused.text, /refused: another process \(pid \d+\)/); + assert.match(refused.text, new RegExp(String(dummy.pid))); + assert.match(refused.text, /--vacuum incremental/); + + armPeerLock(); + const dry = await runStorageGcCommand({ dir, vacuum: "full", dryRun: true }); + assert.equal(dry.code, 0, "dry-run stays read-only even with a writer present"); + assert.match(dry.text, /dry-run: nothing executed/); + + armPeerLock(); + const jsonOut = await runStorageGcCommand({ dir, vacuum: "full", json: true }); + assert.equal(jsonOut.code, 1); + const parsed = JSON.parse(jsonOut.text) as { refused?: unknown; writerPid?: unknown }; + assert.equal(parsed.refused, true); + assert.equal(parsed.writerPid, dummy.pid); + + dummy.kill(); + await dummy.done(); + const ok = await runStorageGcCommand({ dir, vacuum: "full" }); + assert.equal(ok.code, 0, "succeeds once the writer has exited"); + assert.match(ok.text, /content rows deleted:/); + } finally { + dummy.kill(); + await dummy.done().catch(() => {}); + } +}); From 822df1fc85fcbf639480dd2ea9ada0bf086563cb Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 07:06:57 +0800 Subject: [PATCH 3/5] docs: large-db full vacuum is an offline operation (#4612) README en+zh: 'storage gc --vacuum full' refuses to start while another live process holds the database (CLI exit 1 / panel HTTP 409); use --vacuum incremental for online-safe shrink. --- README.md | 2 ++ README.zh-CN.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 01cef318a1..558ca8b702 100644 --- a/README.md +++ b/README.md @@ -665,6 +665,8 @@ bili storage gc [--days N] [--vacuum full|incremental] [--dry-run] `archive`/`restore`/`gc` require the unified store (`BILI_STORAGE_UNIFIED=shadow|full` or `storage.unified` in the config file) — legacy-only installs refuse with a hint. `--days` defaults to `storage.archiveAfterDays` (365). No scheduler: these run on demand. All commands accept `--json`. +**Full vacuum is an offline operation on large databases (#4612).** `--vacuum full` (the default) rewrites the entire index.db under an exclusive write lock — O(db size), minutes to tens of minutes on multi-GB databases, with no intermediate progress (SQLite's VACUUM is atomic; it cannot be split). Running it while a proxy is serving from the same sessions dir starves that proxy's writes until its storage layer degrades, so `storage gc --vacuum full` **refuses to start when another live process holds the database** (`index.db.lock` pid discipline, #4175): the CLI exits 1 with an actionable message, the panel endpoint returns HTTP 409. Stop the other instance first (or wait for it to exit) and retry; for online-safe shrink use `--vacuum incremental`, which reclaims freed pages in short transactions without taking the exclusive lock. The zero-ref delete sweep itself is batched with per-batch progress output either way. + ## Status Early. Protocol handling and compression work against mock tests (500+ passing). Real-model integration testing is the next milestone. Expect rough edges. diff --git a/README.zh-CN.md b/README.zh-CN.md index 59aa0cf060..25a0814f10 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -483,6 +483,8 @@ bili storage gc [--days N] [--vacuum full|incremental] [--dry-run] `archive`/`restore`/`gc` 需要统一存储(`BILI_STORAGE_UNIFIED=shadow|full` 或配置文件里的 `storage.unified`)—— legacy-only 安装会拒绝并给出提示。`--days` 默认取 `storage.archiveAfterDays`(365)。没有调度器:全部按需运行。所有命令都支持 `--json`。 +**大库上 full vacuum 是离线操作(#4612)。** `--vacuum full`(默认)会在独占写锁下重写整个 index.db —— O(库大小),多 GB 的库要几分钟到几十分钟,且没有中间进度(SQLite 的 VACUUM 是原子操作,拆不开)。在有代理正从同一会话目录服务时跑它,会把那个代理的写入饿死到其存储层降级为止,所以 `storage gc --vacuum full` **在另一个存活进程持有该数据库时会拒绝启动**(`index.db.lock` pid 纪律,#4175):CLI 以退出码 1 给出可操作提示,面板端点返回 HTTP 409。先停掉另一个实例(或等它退出)再重试;要在线安全地缩库用 `--vacuum incremental`,它以短事务回收空闲页、不拿独占锁。零引用删除清扫无论哪种模式都分批执行并逐批打印进度。 + ## 状态 早期。协议处理和压缩已通过 mock 测试(500+ 项通过)。真实模型集成测试是下一里程碑。预期会有粗糙的地方。 From d08d4e6359b5f8ac645c21f03fdb5407360b7c6d Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 07:20:30 +0800 Subject: [PATCH 4/5] fix(ci): register issue4612 spawn-based suite in LIVE_FILES (#4612) The new gc online-guard test spawns an idle node child as its writer fixture; the run-ci-tests SPAWN_RE tripwire requires every such suite in LIVE_FILES or it aborts all shards before any test runs (#4604 class). --- scripts/live-files.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/live-files.mjs b/scripts/live-files.mjs index 41cb876080..151c50e152 100644 --- a/scripts/live-files.mjs +++ b/scripts/live-files.mjs @@ -61,6 +61,9 @@ export const LIVE_FILES = [ // #4434 (split-brain surface): spawns bili CLI children against a throwaway // origin; own probe server is listen(0) so only the zone band matters "tests/issue-split-brain-surface.test.ts", + // #4612: spawns an idle node child as the "other live process" fixture; + // binds no ports, mkdtemp-isolated per test + "tests/issue4612-gc-online-guard.test.ts", ]; /** Band width per live file: the zone ladder drifts +1 per EADDRINUSE retry; From f5fd932edfd6800cd880d6594441c2b81b29062d Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 25 Oct 2026 07:55:37 +0800 Subject: [PATCH 5/5] fix(ci): de-export internal-only ClientBuildCheck type (master-inherited red) Unused-exports gate fails on every PR head rebased onto master tip 146b34f27 with the same line master itself fails on: INTERNAL_ONLY src/agent/client-build-note.ts: ClientBuildCheck (#4545 train left the type exported though nothing outside the file imports it; missed by the 4d34abfaf de-export sweep). One-word fix, identical to #4606 commit 1; drop this commit when rebasing after #4606 merges. --- src/agent/client-build-note.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/client-build-note.ts b/src/agent/client-build-note.ts index 8ca7e9ddaf..6c08149309 100644 --- a/src/agent/client-build-note.ts +++ b/src/agent/client-build-note.ts @@ -29,7 +29,7 @@ import path from "node:path"; import { isVersionNewer } from "../update.js"; import { logHostEvent } from "./host-log.js"; -export type ClientBuildCheck = { +type ClientBuildCheck = { /** Short human label used in the note text ("dsh", "pi", ...). */ label: string; /** The client's npm id, as shown in the note (@deepseek-ai/dsh, ...). */