From c0d89e27ccad421f2f9d07387d8bcb542466a154 Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 15 Oct 2026 04:28:53 +0800 Subject: [PATCH 1/3] fix(web): serve /__bili lists from the unified index and bound boot parsing in unified storage modes (#3199) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /__bili/overview (and /__bili/sessions) froze the event loop for minutes on cold instances with large session corpora in unified storage: shadow mode deliberately bypassed the index fast path (rows may lag unrecorded files), so the first list hit decoded the entire corpus on the main thread, and boot parsed the whole corpus before the listener started — feeding the host-side respawn churn (#3044 family). Two mechanisms, one failure domain (main-thread cold-corpus cost): 1. Shadow list surfaces now read the index as a BASELINE and reconcile against the file tree inside refreshIndex: fresh rows are adopted without decoding (file mtime within 5 min of row updated_at — normal writes always land after their row), files newer than their row or never recorded are decoded individually, ghost rows drop because only files produce entries. Full mode stays authoritative (vanished-file sessions remain listed); legacy keeps the plain walk. Cold-start cost drops from "decode the whole corpus" to one SQL + one stat per file + O(delta) decodes. 2. Boot in unified modes (shadow or full) parses only a bounded window of the newest files (mtime desc) before listen; label aliases are seeded synchronously from index rows and the remaining alias/canonical bookkeeping finishes in the background (backgroundScanComplete()). Survivor arbitration still uses the original savedAt-desc trim, so the #404 freshness semantics hold. Legacy mode keeps the synchronous full parse verbatim. WAL reclamation (issue suggestion 3a) landed independently on master via #3250/#3255 (open/close TRUNCATE checkpoints + periodic PASSIVE + configurable cap) — this PR no longer carries its own checkpoint code. Behavior changes (disclosed per owner policy): - shadow-mode web lists may lag bypass-written files by up to 5 minutes, display-only (detail views always read the file; any activity re-ingests); - unified-mode boot readiness = warm-window parse only; label aliases for UNRECORDED files fill post-boot in the background (recorded labels are seeded synchronously from the index). No new config surface (reuses sessions.warmMax / sessions.warmMaxBytes). Kernel untouched. Plugin/proxy wire modes unaffected (admin/web surface). Tests: new tests/issue3199-overview-cold-scan.test.ts pins all paths; targeted suites green locally (web-sessions, storage-unified, store, persist-zstd, persist-encryption, bounded-storage, issue404, identity-registry, search-context-cross-session); full verdict via remote CI. --- src/persist.ts | 222 +++++++++++++++++++++++++++++++++++---- src/storage/store.ts | 10 +- src/web/sessions-data.ts | 70 ++++++++++-- 3 files changed, 269 insertions(+), 33 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index d13508bd6a..dd35595820 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; import { readdir, readFile, rm } from "node:fs/promises"; import * as path from "node:path"; import { StateStore, flatFileNameFor, type PersistedEnvelope, type StateStoreCodec } from "acp-kernel/persist"; @@ -330,6 +330,9 @@ export class SessionStore { private diskFiles: string[] = []; private readonly legacyAliases = new Map(); private readonly sourceFiles = new Map(); + /** #3199: post-boot background alias scan (unified modes only) — null in + * legacy mode and until bootBounded defers work. */ + private remainderScan: Promise | null = null; /** #2674: monotonic per-process counter for unique content-store temp * names (pid + seq makes collisions across processes/rapid saves * unreachable). */ @@ -622,7 +625,7 @@ export class SessionStore { } /** Keep only file metadata while indexing. Legacy aliases are adopted on - * resume without deleting their historical state or original companions. */ + * resume without deleting their historical state or original companions. */ private async bootBounded(limits: { maxSessions: number; maxBytes: number }): Promise> { if (!Number.isFinite(limits.maxSessions) || limits.maxSessions < 0 || !Number.isFinite(limits.maxBytes) || limits.maxBytes < 0) { throw new Error("Invalid session boot limits"); @@ -633,30 +636,86 @@ export class SessionStore { this.legacyAliases.clear(); const warm: Array<{ file: string; id: string; meta: PersistedSession["meta"]; savedAt: number; bytes: number }> = []; const canonical = new Set(); + const filesByRoot = new Map(); for (const root of [this.sessionsDir, `${this.sessionsDir}-cold`]) { const files = existsSync(root) ? await walkJsonFiles(root) : []; for (const file of files) { if (path.basename(file).endsWith(".content-store.json")) continue; this.diskFiles.push(file); - const parsed = await this.readRawFileWithBytes(file); + filesByRoot.set(root, [...(filesByRoot.get(root) ?? []), file]); + } + } + // #3199: unified mode (shadow or full) — the listener must not wait + // for the whole corpus to parse. Two stages: (1) parse a bounded + // WINDOW of the newest files (mtime desc, stopped once the window + // holds a full budget's worth of candidates), (2) arbitrate the final + // warm set by savedAt desc with the original while-pop trim (#404 — + // mtime only orders the parse window, never the survivor choice). + // Seed label aliases from the index rows first (no decode); the rest + // of the bookkeeping finishes in the background. Gate is + // replicasAvailable() — a live non-degraded sidecar EXISTS (shadow OR + // full) — NOT readsUnified, which is the full-mode-only "rows are + // authoritative" predicate. + if (this.replicasAvailable()) { + this.seedIndexedAliases(canonical); + const hot = (filesByRoot.get(this.sessionsDir) ?? []) + .map((file) => { + try { + const st = statSync(file); + return { file, mtimeMs: st.mtimeMs, size: st.size }; + } catch { + return null; // raced deletion between walk and stat + } + }) + .filter((c): c is { file: string; mtimeMs: number; size: number } => c !== null) + .sort((a, b) => b.mtimeMs - a.mtimeMs); + const parsedFiles = new Set(); + const parsedCands: Array<{ file: string; id: string; meta: PersistedSession["meta"]; savedAt: number; bytes: number }> = []; + let windowBytes = 0; + for (const cand of hot) { + if (parsedCands.length >= limits.maxSessions && windowBytes >= limits.maxBytes) break; + // on-disk size lower-bounds decoded size (#2803) — cheap pre-filter + if (cand.size > limits.maxBytes) continue; + const parsed = await this.readRawFileWithBytes(cand.file); + parsedFiles.add(cand.file); if (!parsed) continue; - const raw = parsed.value; - const payload = raw && typeof raw === "object" && "payload" in raw ? raw.payload : raw; - if (!isValidRecord(payload)) continue; - const meta = payload.meta ?? { protocol: payload.protocol, upstreamOrigin: payload.upstreamOrigin, label: payload.label }; - if (!fileNameMatchesId(path.basename(file), payload.id, meta.protocol, meta.upstreamOrigin)) continue; - const savedAt = payload.savedAt ?? 0; - if (payload.id === meta.label || payload.id.startsWith("pfa-")) canonical.add(payload.id); - else if (meta.label) { - const previous = this.legacyAliases.get(meta.label); - if (!previous || savedAt >= previous.savedAt) this.legacyAliases.set(meta.label, { file, id: payload.id, savedAt }); - } - if (root !== this.sessionsDir) continue; + const rec = this.parseBootPayload(parsed.value); + if (!rec || !fileNameMatchesId(path.basename(cand.file), rec.payload.id, rec.meta.protocol, rec.meta.upstreamOrigin)) continue; + const savedAt = rec.payload.savedAt ?? 0; + if (rec.payload.id === rec.meta.label || rec.payload.id.startsWith("pfa-")) this.noteCanonical(rec.payload.id, canonical); + else if (rec.meta.label) this.upsertLegacyAlias(rec.meta.label, { file: cand.file, id: rec.payload.id, savedAt }, canonical); const bytes = parsed.bytes; if (bytes > limits.maxBytes) continue; - warm.push({ file, id: payload.id, meta, savedAt, bytes }); - warm.sort((a, b) => b.savedAt - a.savedAt); - while (warm.length > limits.maxSessions || warm.reduce((sum, item) => sum + item.bytes, 0) > limits.maxBytes) warm.pop(); + windowBytes += bytes; + parsedCands.push({ file: cand.file, id: rec.payload.id, meta: rec.meta, savedAt, bytes }); + } + parsedCands.sort((a, b) => b.savedAt - a.savedAt); + while (parsedCands.length > limits.maxSessions || parsedCands.reduce((sum, item) => sum + item.bytes, 0) > limits.maxBytes) parsedCands.pop(); + warm.push(...parsedCands); + this.remainderScan = this.scanRemainderInBackground(parsedFiles, canonical); + void this.remainderScan; + } else { + // legacy mode: keep the synchronous full parse exactly as before — + // there is no index to lean on, so nothing can be deferred. + for (const root of [this.sessionsDir, `${this.sessionsDir}-cold`]) { + for (const file of filesByRoot.get(root) ?? []) { + const parsed = await this.readRawFileWithBytes(file); + if (!parsed) continue; + const rec = this.parseBootPayload(parsed.value); + if (!rec || !fileNameMatchesId(path.basename(file), rec.payload.id, rec.meta.protocol, rec.meta.upstreamOrigin)) continue; + const savedAt = rec.payload.savedAt ?? 0; + if (rec.payload.id === rec.meta.label || rec.payload.id.startsWith("pfa-")) canonical.add(rec.payload.id); + else if (rec.meta.label) { + const previous = this.legacyAliases.get(rec.meta.label); + if (!previous || savedAt >= previous.savedAt) this.legacyAliases.set(rec.meta.label, { file, id: rec.payload.id, savedAt }); + } + if (root !== this.sessionsDir) continue; + const bytes = parsed.bytes; + if (bytes > limits.maxBytes) continue; + warm.push({ file, id: rec.payload.id, meta: rec.meta, savedAt, bytes }); + warm.sort((a, b) => b.savedAt - a.savedAt); + while (warm.length > limits.maxSessions || warm.reduce((sum, item) => sum + item.bytes, 0) > limits.maxBytes) warm.pop(); + } } } for (const id of canonical) this.legacyAliases.delete(id); @@ -672,6 +731,105 @@ export class SessionStore { return out; } + /** Shared envelope→(record, meta) extraction for the boot passes; null on + * any parse/validation failure (same gates as before the split). Meta is + * resolved to the flat fallbacks so callers never see undefined. */ + private parseBootPayload(raw: unknown): { payload: PersistedSession; meta: NonNullable } | null { + const payload = raw && typeof raw === "object" && "payload" in raw ? raw.payload : raw; + if (!isValidRecord(payload)) return null; + const meta = payload.meta ?? { protocol: payload.protocol, upstreamOrigin: payload.upstreamOrigin, label: payload.label }; + return { payload, meta }; + } + + /** Incremental canonical registration: adding an id to the canonical set + * voids any alias under the same name NOW instead of at end-of-scan — + * the incremental result equals the old final "delete all canonical + * labels from the alias map" pass. */ + private noteCanonical(id: string, canonical: Set): void { + canonical.add(id); + this.legacyAliases.delete(id); + } + + /** Alias upsert with the newest-savedAt-wins rule; refuses names that are + * (or became) canonical ids. Returns whether the map changed. */ + private upsertLegacyAlias(label: string, entry: { file: string; id: string; savedAt: number }, canonical: Set): boolean { + if (canonical.has(label)) return false; + const previous = this.legacyAliases.get(label); + if (previous && entry.savedAt < previous.savedAt) return false; + this.legacyAliases.set(label, entry); + return true; + } + + /** #3199: pre-seed label aliases from the unified index rows so label-based + * resume of recorded sessions works WITHOUT parsing their files during + * boot. Unrecorded files get their aliases from the background scan. */ + private seedIndexedAliases(canonical: Set): void { + const shadow = this.shadow; + if (!shadow) return; + let rows; + try { + rows = shadow.sessionSummaries(); + } catch { + return; // sidecar degraded mid-boot: background scan still covers it + } + for (const r of rows) { + if (!r.label || r.label === r.sessionId || r.sessionId.startsWith("pfa-")) continue; + const file = this.resolveIndexedFile(r.sessionId, r.protocol, r.origin); + if (!file) continue; + this.upsertLegacyAlias(r.label, { file, id: r.sessionId, savedAt: r.updatedAt }, canonical); + } + } + + /** #3199: locate a recorded session's JSON file by its id alone — the same + * three rel-path candidates the live probes use, across both roots; first + * existing regular file wins (stat-only, no decode). */ + private resolveIndexedFile(id: string, protocol?: string, origin?: string): string | null { + const candidates = [relPathFor(id, protocol, origin), relPathFor(id), flatFileNameFor(id)]; + for (const root of [this.sessionsDir, `${this.sessionsDir}-cold`]) { + for (const rel of candidates) { + const file = path.join(root, rel); + try { + if (statSync(file).isFile()) return file; + } catch { + // missing candidate: next + } + } + } + return null; + } + + /** #3199: finish alias/canonical bookkeeping over the corpus AFTER boot + * returned, so the listener is not held by files outside the warm window. + * Idempotent with the synchronous part (same upsert rules); yields the + * event loop between files like the web disk index does. */ + private async scanRemainderInBackground(parsedFiles: Set, canonical: Set): Promise { + let scanned = 0; + let aliased = 0; + for (const file of this.diskFiles) { + if (parsedFiles.has(file)) continue; + scanned++; + const parsed = await this.readRawFileWithBytes(file); + if (parsed) { + const rec = this.parseBootPayload(parsed.value); + if (rec && fileNameMatchesId(path.basename(file), rec.payload.id, rec.meta.protocol, rec.meta.upstreamOrigin)) { + const savedAt = rec.payload.savedAt ?? 0; + if (rec.payload.id === rec.meta.label || rec.payload.id.startsWith("pfa-")) this.noteCanonical(rec.payload.id, canonical); + else if (rec.meta.label) aliased += this.upsertLegacyAlias(rec.meta.label, { file, id: rec.payload.id, savedAt }, canonical) ? 1 : 0; + } + } + await new Promise((resolve) => setImmediate(resolve)); + } + if (scanned > 0) { + this.log("info", `[persist] background index scan finished after boot: ${scanned} file(s) scanned, ${aliased} alias(es) recorded`); + } + } + + /** #3199: resolves when the post-boot background alias scan has finished + * (immediate in legacy mode or when nothing was deferred). */ + backgroundScanComplete(): Promise { + return this.remainderScan ?? Promise.resolve(); + } + private readEnvelopeFile(file: string, id: string): PersistedEnvelope | null { let bytes: Buffer; try { @@ -1066,13 +1224,31 @@ export class SessionStore { } } - /** #2960 R2.2: web list rows — stats/metadata JSON + block aggregates - * from one SQL. Full mode only (same gate as above); null → the web - * falls back to the bounded disk index walk. */ + /** #3199: which authority backs the web list surfaces for THIS store. + * "authoritative" (full mode): the rows alone are complete — organic + * import on every load/write keeps them whole, so a session whose JSON + * file vanished stays listed (R2.1 self-sufficient index). "reconcile" + * (shadow mode): the rows are a BASELINE only — the JSON tree is still + * the truth base, so callers must reconcile against it (adopt fresh + * rows, decode files newer than their row or never recorded, drop ghost + * rows whose file vanished). null = legacy/degraded — plain file walk. */ + webSummaryBaseline(): "authoritative" | "reconcile" | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + return this.shadow.mode === "full" ? "authoritative" : "reconcile"; + } + + /** #2960 R2.2 / #3199: web list rows — stats/metadata JSON + block + * aggregates from one SQL. Full mode serves them as the authoritative + * answer; shadow mode (#3199) serves them as a BASELINE the caller must + * reconcile against the JSON tree (see webSummaryBaseline) — shadow rows + * may lag files never yet recorded, and ghost rows must be dropped there. + * Null → the web falls back to the bounded disk index walk. */ unifiedWebSummaries(): import("./storage/store.js").WebSummaryRow[] | null { - if (!this.shadow?.readsUnified) return null; + if (this.webSummaryBaseline() === null) return null; + const shadow = this.shadow; + if (!shadow) return null; try { - return this.shadow.webSummaryRows(); + return shadow.webSummaryRows(); } catch { return null; // sidecar degraded mid-call: fall back to the walk } diff --git a/src/storage/store.ts b/src/storage/store.ts index 4db22f1a4c..1612c6674e 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -141,7 +141,9 @@ export interface CheckpointRow { } /** Summary row for the export/web list fast path (#2960 R2.1). Parity - * contract with the legacy walk in export.ts listSessions. */ + * contract with the legacy walk in export.ts listSessions. updatedAt (#3199) + * = sessions.updated_at (ms epoch, last ingest time) — boot uses it as the + * alias savedAt when seeding label aliases from rows without decoding files. */ export interface SessionSummaryRow { sessionId: string; title: string | undefined; @@ -152,6 +154,7 @@ export interface SessionSummaryRow { blocks: number; latestBlockAt: number | undefined; contextTokens: number | undefined; + updatedAt: number; } /** #2960 R2.2: row backing the WEB session list in full mode. Carries the @@ -1041,9 +1044,10 @@ export class UnifiedStore { .prepare( `SELECT s.session_id AS sessionId, s.host_title AS hostTitle, s.title AS title, s.label AS label, s.protocol AS protocol, s.origin AS origin, s.namespace AS namespace, s.stats_json AS statsJson, + s.updated_at AS updatedAt, (SELECT COUNT(*) FROM blocks b WHERE b.session_id = s.session_id) AS blocks, (SELECT MAX(b.created_at) FROM blocks b WHERE b.session_id = s.session_id) AS latestBlockAt - FROM sessions s`, + FROM sessions s`, ) .all() as { sessionId: string; @@ -1054,6 +1058,7 @@ export class UnifiedStore { origin: string | null; namespace: string; statsJson: string; + updatedAt: number; blocks: number; latestBlockAt: number | null; }[]; @@ -1075,6 +1080,7 @@ export class UnifiedStore { blocks: r.blocks, latestBlockAt: r.latestBlockAt ?? undefined, contextTokens, + updatedAt: r.updatedAt, }; }); } diff --git a/src/web/sessions-data.ts b/src/web/sessions-data.ts index b37fbb6ab0..c8f1636817 100644 --- a/src/web/sessions-data.ts +++ b/src/web/sessions-data.ts @@ -4,6 +4,7 @@ import { conflictClientOf, conflictEventsOf } from "../conflict-watch.js"; import { isDisplayOnlyConflictDetail, isSiblingConflictDetail } from "../thirdparty-scan.js"; import { SessionStore, fileNameMatchesId, isValidRecord, relPathFor } from "../persist.js"; import { flatFileNameFor } from "acp-kernel/persist"; +import type { WebSummaryRow } from "../storage/store.js"; import { renderHandoff } from "../export.js"; import { buildSessionCacheReport } from "../cache-ledger.js"; import { METADATA_FOLD_COVERAGE } from "../fold-reconcile.js"; @@ -302,6 +303,37 @@ function liveCoveredPaths(dir: string): Set { return out; } +/** #3199: how much newer a session file's mtime may be than its unified row's + * updated_at before the row counts as stale and the file must be decoded. + * Normal writes always land AFTER their row (guardedBuild ingests before the + * JSON flush; updated_at is stamped at build time), so a bare mtime >= + * updatedAt test would decode everything — the window absorbs that lag while + * still catching bypass writes (#408 clamp, #286 rekey) and manual edits once + * they age out. Exposure is display-only and self-healing: shadow detail + * views always read the file, and any activity re-ingests the row. */ +const SHADOW_ROW_STALE_WINDOW_MS = 5 * 60 * 1000; + +/** #3199: basename → freshest unified row claiming that filename. Session + * filenames derive from the session id (sha256 prefix), so row→file needs no + * stat and no decode — the walk's own basenames are the join key (the same + * three candidates liveCoveredPaths builds per live id). */ +function rowClaimsByBasename(rows: WebSummaryRow[]): Map { + const claims = new Map(); + for (const r of rows) { + const candidates = [ + relPathFor(r.sessionId, r.protocol, r.origin), + relPathFor(r.sessionId), + flatFileNameFor(r.sessionId), + ]; + for (const rel of candidates) { + const base = path.basename(rel); + const cur = claims.get(base); + if (!cur || r.updatedAt > cur.updatedAt) claims.set(base, r); + } + } + return claims; +} + // #2260(F)/#2180 follow-up: bili's own infra dirs created under the data root // BEFORE any session exists (ensureRootCA() makes /ca at proxy start) — // their presence alone must not read as "misconfigured". The set is closed by @@ -356,6 +388,16 @@ async function refreshIndex(): Promise { const prev = byFile ?? new Map(); const next = new Map(); const covered = liveCoveredPaths(dir); + // #3199: shadow mode adopts fresh unified rows instead of decoding the + // files they describe — cold-start cost drops from "decode the whole + // corpus" to one SQL + one stat per file. Full mode never walks here + // (its rows are authoritative); legacy has no rows; both keep the + // plain decode path below. + let claims: Map | null = null; + if (store.webSummaryBaseline() === "reconcile") { + const rows = store.unifiedWebSummaries(); + if (rows !== null) claims = rowClaimsByBasename(rows); + } for (const f of files) { if (covered.has(f.abs)) continue; const p = prev.get(f.abs); @@ -363,6 +405,11 @@ async function refreshIndex(): Promise { next.set(f.abs, p); continue; } + const claim = claims?.get(path.basename(f.abs)); + if (claim && f.mtimeMs - claim.updatedAt <= SHADOW_ROW_STALE_WINDOW_MS) { + next.set(f.abs, { abs: f.abs, mtimeMs: f.mtimeMs, size: f.size, savedAt: claim.updatedAt, summary: summaryFromRow(claim) }); + continue; + } decodeCount++; const entry: DiskEntry = { abs: f.abs, mtimeMs: f.mtimeMs, size: f.size, savedAt: 0, summary: null }; const relName = path.relative(dir, f.abs); @@ -695,14 +742,21 @@ export function hiddenEmptyCount(): number { return lastHiddenEmpty; } async function mergedSummaries(): Promise { const out = new Map(); - // #2960 R2.2: full mode serves the disk side from the unified index (one - // SQL — no directory walk, no envelope decode, and sessions whose JSON - // file vanished stay listed). Legacy/shadow keep the bounded file walk: - // there the JSON tree is the truth base and rows may lag unrecorded files. - const rows = getDiskStore().unifiedWebSummaries(); - if (rows !== null) { - for (const r of rows) out.set(r.sessionId, summaryFromRow(r)); - } else { + // #2960 R2.2 / #3199: full mode serves the disk side from the unified index + // alone (one SQL — no walk, no decode; sessions whose JSON file vanished + // stay listed). Shadow RECONCILES the rows against the tree inside + // refreshIndex (fresh rows adopted without decode; stale/unrecorded files + // decoded; ghost rows dropped because only files produce entries). Legacy + // keeps the plain bounded file walk. + let fromRows = false; + if (getDiskStore().webSummaryBaseline() === "authoritative") { + const rows = getDiskStore().unifiedWebSummaries(); + if (rows !== null) { + for (const r of rows) out.set(r.sessionId, summaryFromRow(r)); + fromRows = true; + } + } + if (!fromRows) { const files = await ensureIndex(); for (const e of files.values()) if (e.summary) out.set(e.summary.id, e.summary); } From 3a76715b2e964c5521782c029fd6d92187a8b7bc Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 15 Oct 2026 04:29:16 +0800 Subject: [PATCH 2/3] fix(tests): make #841 fixture dir scan sidecar-aware (pre-existing ENOTDIR under unified storage) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Small in-passing fix found while testing #3199, its own labeled commit. Pre-existing failure verified on pristine master with the node:sqlite builtin engine: the #841 cold-load case assumed every top-level entry under the sessions dir is a protocol directory and readdir'd each one — unified storage puts index.db (+WAL files) at top level, so the snapshot scan died with ENOTDIR. Guard the descent with isDirectory(); unrelated to the #3199 change itself. --- tests/search-context-cross-session.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/search-context-cross-session.test.ts b/tests/search-context-cross-session.test.ts index 499fe5d2ec..2e8f71de93 100644 --- a/tests/search-context-cross-session.test.ts +++ b/tests/search-context-cross-session.test.ts @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, readFileSync, readdirSync } from "node:fs"; +import { mkdtempSync, readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; import { tmpdir } from "node:os"; import { createHash } from "node:crypto"; @@ -199,7 +199,9 @@ test("#841 cold-loaded historical session: read-only, file untouched, no save sc const allFiles: string[] = []; for (const d of readdirSync(dir)) { - if (!d.startsWith(".")) for (const f of readdirSync(path.join(dir, d))) allFiles.push(path.join(dir, d, f)); + // Unified-storage sidecars (index.db, WAL files) sit at the top + // level next to the protocol dirs — only descend into directories. + if (!d.startsWith(".") && statSync(path.join(dir, d)).isDirectory()) for (const f of readdirSync(path.join(dir, d))) allFiles.push(path.join(dir, d, f)); } assert.ok(allFiles.length > 0, "session file must exist"); const before = allFiles.map((f) => readFileSync(f, "utf8")); From 22dfa122e3c8d2c88e94c2eb725e55c8136ebd63 Mon Sep 17 00:00:00 2001 From: awork Date: Thu, 15 Oct 2026 04:29:16 +0800 Subject: [PATCH 3/3] test: pin #3199 cold-scan behavior (shadow reconcile, bounded boot, legacy control) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - T1: shadow list = index baseline + tree delta — fresh row adopted with zero decodes; externally rewritten file decoded and wins; unrecorded file listed; ghost row dropped; live session overrides. Structural assert that decode count equals the delta count only (no full-corpus decode). - T2: unified boot parses only the newest window before listen; survivors come from the savedAt-desc trim within the window (#404 contract kept); label aliases seeded synchronously from index rows; remainder scan completes post-boot and covers unrecorded labels. - T3: legacy control — synchronous full parse, aliases complete immediately after boot (zero regression for non-unified installs). --- tests/issue3199-overview-cold-scan.test.ts | 341 +++++++++++++++++++++ 1 file changed, 341 insertions(+) create mode 100644 tests/issue3199-overview-cold-scan.test.ts diff --git a/tests/issue3199-overview-cold-scan.test.ts b/tests/issue3199-overview-cold-scan.test.ts new file mode 100644 index 0000000000..e411908b66 --- /dev/null +++ b/tests/issue3199-overview-cold-scan.test.ts @@ -0,0 +1,341 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createInitialState } from "acp-kernel"; +import type { CompressionBlock } from "acp-kernel"; +import { flatFileNameFor } from "acp-kernel/persist"; +import { SessionStore, _setStoreForTest, fileNameMatchesId } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; +import { getSession, _resetSessionsForTest } from "../src/session.ts"; +import { buildSessionList, _resetDiskCacheForTest, _diskScanStatsForTest } from "../src/web/sessions-data.ts"; +import { rmrf } from "./tmp-rm.ts"; + +// #3199: Web UI first paint stalled 78-180s+ on a cold instance — in shadow +// mode the web lists ignored the unified index (full-only gate) and decoded +// the ENTIRE corpus on the event loop, and boot parsed the whole corpus +// before the listener started. These tests pin both fixes: +// T1 — shadow mode adopts fresh index rows WITHOUT decoding their files; +// only files newer than their row (or never recorded) are decoded; +// ghost rows (file vanished) drop from the shadow list. +// T2 — unified-mode boot parses only the warm window (newest files within +// the budget); label aliases for recorded sessions are SQL-seeded +// synchronously; the rest fills in the background. +// T3 — legacy-mode boot keeps the synchronous full parse (control). + +// Plain JSON session files so seeds and round-trips stay deterministic (#1080) +process.env.BILI_PERSIST_ZSTD = "0"; + +// Hermetic config: a developer machine's real billion-context.json must not +// leak storage settings into this suite's tmp-dir stores (same discipline as +// tests/web-sessions.test.ts). +process.env.BILI_CONFIG_FILE = path.join( + mkdtempSync(path.join(tmpdir(), "bili-3199-hermetic-")), + "no-config.json", +); +// Absent everywhere resolves to the shadow DEFAULT (#2815 rollout flip); pin +// the explicit legacy baseline — legs flip per test. +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +interface Stats { + requests: number; + tokensSaved: number; + inputTokens: number; + cachedTokens: number; + outputTokens: number; + cacheSamples: number; + lastInputTokens: number; + contextTokens: number; + retrieveCalls: number; + retrieveHits: number; + retrieveMisses: number; + storedBytes: number; + storeBytesSaved: number; + compressCreditTokens: number; + rangeRestores: number; +} + +function zeroStats(): Stats { + return { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, compressCreditTokens: 0, rangeRestores: 0 }; +} + +function makeSession(id: string, meta: Session["meta"] = {}, patch: Partial = {}, lastSeen = Date.now()): Session { + return { + id, + meta, + stats: { ...zeroStats(), ...patch }, + metadata: {}, + state: createInitialState(), + createdAt: lastSeen, + lastSeen, + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +/** Full persisted-state shape (kernel initial state + our blocks) so hand- + * written records survive both summary extraction AND buildSession's compat + * pass in the loadSync legs. */ +function stateWith(blocks: CompressionBlock[]): unknown { + const s = JSON.parse(JSON.stringify(createInitialState())) as { blocks: unknown[] }; + s.blocks = blocks; + return s; +} + +const mkBlock = (blockId: string, compressedTokens: number, active: boolean, topic = "T"): CompressionBlock => ({ + blockId, + runId: "r1", + tier: 1, + topic, + summary: "S", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + compressedTokens, + createdAt: Date.now(), + survivedCount: 1, + generation: "young", + active, +}); + +function withSessionsDir(name: string, fn: (dir: string) => Promise): void { + test(name, async () => { + const dir = mkdtempSync(path.join(tmpdir(), "bili-3199-")); + const prev = process.env.BILI_SESSIONS_DIR; + process.env.BILI_SESSIONS_DIR = dir; + try { + await fn(dir); + } finally { + if (prev === undefined) delete process.env.BILI_SESSIONS_DIR; else process.env.BILI_SESSIONS_DIR = prev; + _resetSessionsForTest(); + _resetDiskCacheForTest(); + rmrf(dir); + } + }); +} + +/** Flip the suite's legacy baseline to shadow for one body and restore after + * — _resetDiskCacheForTest() re-creates the disk-store singleton (mode is + * resolved at construction), so each leg sees its own engine honestly. */ +function withShadowMode(name: string, fn: (dir: string) => Promise): void { + withSessionsDir(name, async (dir) => { + const prev = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "shadow"; + try { + await fn(dir); + } finally { + if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prev; + } + }); +} + +/** id → absolute envelope path, by scanning the corpus (layout-independent). */ +function fileById(root: string, id: string): string | null { + const walk = (r: string): string[] => { + const out: string[] = []; + for (const e of readdirSync(r, { withFileTypes: true })) { + const p = path.join(r, e.name); + if (e.isDirectory()) out.push(...walk(p)); + else if (e.name.endsWith(".json")) out.push(p); + } + return out; + }; + for (const p of walk(root)) { + try { + const env = JSON.parse(readFileSync(p, "utf8")) as { id?: unknown; payload?: { id?: unknown } }; + if (env.id === id || env.payload?.id === id) return p; + } catch { + // non-JSON / unreadable: not our file + } + } + return null; +} + +withShadowMode("#3199 T1: shadow list adopts fresh rows without decoding; stale/unrecorded files win; ghosts drop", async (dir) => { + const store = new SessionStore({ dir, debounceMs: 0, enabled: true }); + assert.ok(store.replicasAvailable(), "sidecar must be live for this leg (engine available)"); + + const sFresh = makeSession("s3199-fresh", { protocol: "openai", title: "Fresh", label: "s3199-fresh" }, { requests: 3, inputTokens: 800, cachedTokens: 200, outputTokens: 40, contextTokens: 400 }); + sFresh.state.blocks.push(mkBlock("bf1", 1500, true, "Fresh topic")); + await store.writeNow(sFresh); + + const sStale = makeSession("s3199-stale", { protocol: "openai", title: "Stale Title", label: "s3199-stale" }, { requests: 2, inputTokens: 500, cachedTokens: 100, contextTokens: 300, tokensSaved: 111 }); + sStale.state.blocks.push(mkBlock("bs1", 700, true, "Old topic")); + await store.writeNow(sStale); + + const sGhost = makeSession("s3199-ghost", { protocol: "openai", title: "Ghost" }, { requests: 2, inputTokens: 500, cachedTokens: 100, contextTokens: 300 }); + await store.writeNow(sGhost); + store.closeQuietly(); + + // External rewrite of the stale session AFTER its row was ingested: bump + // the values, then stamp the mtime 6 minutes into the future so + // mtime - row.updated_at deterministically exceeds the 5-minute freshness + // window (normal writes always land slightly AFTER their row). + const staleFile = fileById(dir, "s3199-stale"); + assert.ok(staleFile, "stale session file present"); + const env = JSON.parse(readFileSync(staleFile, "utf8")) as Record; + const payload = (env.payload ?? env) as Record; + payload.meta = { protocol: "openai", title: "Stale Rewritten", label: "s3199-stale" }; + payload.stats = { ...zeroStats(), requests: 9, inputTokens: 500, cachedTokens: 100, contextTokens: 300, tokensSaved: 4242 }; + payload.state = stateWith([mkBlock("bs2", 3333, true, "New topic")]); + payload.savedAt = Date.now(); + env.savedAt = payload.savedAt; + writeFileSync(staleFile, JSON.stringify(env), "utf8"); + const future = Date.now() / 1000 + 6 * 60; + utimesSync(staleFile, future, future); + + // Ghost: the JSON twin vanishes (hand-deleted) — the shadow list must DROP + // the orphan row (in shadow the tree is the truth base; full mode keeps + // such rows, pinned by the R2.2 file-less test). + const ghostFile = fileById(dir, "s3199-ghost"); + assert.ok(ghostFile, "ghost session file present before deletion"); + rmSync(ghostFile); + + // Unrecorded: a valid session file that was never ingested (pre-shadow + // era file) — no row exists, so the walk must decode and list it. + const unrecId = "s3199-unrecorded"; + const unrecName = flatFileNameFor(unrecId); + assert.ok(fileNameMatchesId(unrecName, unrecId), "flat name is addressable for its id"); + writeFileSync(path.join(dir, unrecName), JSON.stringify({ + version: 2, + savedAt: Date.now(), + id: unrecId, + meta: { protocol: "openai", title: "Unrecorded", label: unrecId }, + stats: { ...zeroStats(), requests: 5, inputTokens: 900, cachedTokens: 100, contextTokens: 500, tokensSaved: 2222 }, + state: stateWith([mkBlock("bu1", 2000, true, "Unrecorded topic")]), + }), "utf8"); + + _setStoreForTest(new SessionStore({ enabled: false })); + _resetDiskCacheForTest(); + + const list = await buildSessionList(); + assert.deepEqual(list.map((x) => x.id).sort(), ["s3199-fresh", "s3199-stale", "s3199-unrecorded"], "ghost row dropped, unrecorded file listed"); + + // Structural proof of the fix: exactly the two files whose rows were + // missing/stale got decoded — the fresh row was adopted from SQL alone. + assert.equal(_diskScanStatsForTest().decodedTotal, 2, "only stale + unrecorded files may be decoded"); + + const fresh = list.find((x) => x.id === "s3199-fresh"); + assert.ok(fresh, "fresh row listed"); + assert.equal(fresh!.title, "Fresh"); + assert.equal(fresh!.requests, 3); + assert.equal(fresh!.tokensSaved, 1500, "active-block sum floor from the row"); + assert.equal(fresh!.blocks, 1); + assert.equal(fresh!.firstBlockHint, "Fresh topic"); + assert.equal(fresh!.live, false); + + const stale = list.find((x) => x.id === "s3199-stale"); + assert.ok(stale, "stale file listed"); + assert.equal(stale!.title, "Stale Rewritten", "file values beat the lagging row"); + assert.equal(stale!.requests, 9); + assert.equal(stale!.tokensSaved, 4242); + + const unrec = list.find((x) => x.id === unrecId); + assert.ok(unrec, "unrecorded file listed"); + assert.equal(unrec!.title, "Unrecorded"); + assert.equal(unrec!.requests, 5); + + // Live pool still overrides disk entries in the merge. + const live = getSession("s3199-live", { protocol: "openai", label: "s3199-live" }); + live.stats.requests = 2; + live.stats.contextTokens = 100; + const list2 = await buildSessionList(); + const liveRow = list2.find((x) => x.id === "s3199-live"); + assert.ok(liveRow, "live session listed"); + assert.equal(liveRow!.live, true); + assert.equal(_diskScanStatsForTest().decodedTotal, 2, "second list served from the (mtime,size) cache"); +}); + +withShadowMode("#3199 T2: unified boot parses only the warm window; aliases SQL-seeded; rest fills in background", async (dir) => { + const N = 12; + const seeder = new SessionStore({ dir, debounceMs: 0, enabled: true }); + assert.ok(seeder.replicasAvailable(), "sidecar must be live for this leg (engine available)"); + for (let i = 0; i < N; i++) { + const id = `s3199-boot-${i}`; + // Older half carries a distinct label (alias candidate); the newest + // four use label === id (canonical — never aliased). + const label = i >= N - 4 ? id : `s3199-label-${i}`; + const s = makeSession(id, { protocol: "openai", title: `Boot ${i}`, label }, { requests: i + 1, contextTokens: 100 * (i + 1) }); + await seeder.writeNow(s); + } + seeder.closeQuietly(); + + // Stagger mtimes by one minute: i = N-1 is newest. Track the largest + // on-disk size so the byte budget below can never trim an intended winner + // (file sizes grow a few bytes as token counts cross digit boundaries). + const base = Math.floor(Date.now() / 1000) - N * 60; + let maxFileSize = 0; + for (let i = 0; i < N; i++) { + const f = fileById(dir, `s3199-boot-${i}`); + assert.ok(f, `session file for i=${i}`); + maxFileSize = Math.max(maxFileSize, statSync(f).size); + const t = base + i * 60; + utimesSync(f, t, t); + } + + // An UNRECORDED labeled file: no index row exists, so its alias can only + // come from the post-boot background scan. Stamped OLDER than every boot + // file on BOTH signals (mtime and savedAt) so it cannot win a warm slot + // in the savedAt arbitration. + const strayId = "s3199-stray"; + const strayLabel = "s3199-stray-label"; + const strayFile = path.join(dir, flatFileNameFor(strayId)); + writeFileSync(strayFile, JSON.stringify({ + version: 2, + savedAt: (base - 60) * 1000, + id: strayId, + meta: { protocol: "openai", title: "Stray", label: strayLabel }, + stats: { ...zeroStats(), requests: 7, contextTokens: 700 }, + state: stateWith([]), + }), "utf8"); + utimesSync(strayFile, base - 60, base - 60); + + // Byte budget sized to ~4-5 files: the parse window stops after a full + // budget's worth of candidates instead of draining the whole corpus, yet + // 4×maxFileSize always fits so the trim can never drop an intended winner. + const store = new SessionStore({ dir, debounceMs: 0, enabled: true }); + const booted = await store.boot({ maxSessions: 4, maxBytes: maxFileSize * 5 }); + // Lexicographic order ("boot-10" < "boot-8") — same comparator on both sides. + assert.deepEqual([...booted.keys()].sort(), ["s3199-boot-10", "s3199-boot-11", "s3199-boot-8", "s3199-boot-9"], "warm set = 4 newest by mtime, arbitrated by savedAt"); + + // Label-based resume of a NON-warm recorded session works SYNCHRONOUSLY — + // the alias came from the index rows, not from parsing the file. Alias + // resume re-keys the session to the requested label by design, so assert + // the data (contextTokens 800 = i 7) proves the right file was read. + const resumed = store.loadSync("s3199-label-7"); + assert.ok(resumed, "SQL-seeded alias resolves before any background work"); + assert.equal(resumed!.id, "s3199-label-7"); + assert.equal(resumed!.stats.contextTokens, 800); + + // The unrecorded label is only resolvable once the background scan reaches + // its file (it was outside the warm window by construction). + await store.backgroundScanComplete(); + const stray = store.loadSync(strayLabel); + assert.ok(stray, "background scan recorded the unrecorded label alias"); + assert.equal(stray!.id, strayLabel); + assert.equal(stray!.stats.contextTokens, 700); + store.closeQuietly(); +}); + +withSessionsDir("#3199 T3: legacy boot keeps the synchronous full parse (control)", async (dir) => { + const seeder = new SessionStore({ dir, debounceMs: 0, enabled: true }); + for (let i = 0; i < 4; i++) { + const s = makeSession(`s3199-leg-${i}`, { protocol: "openai", title: `Leg ${i}`, label: `s3199-leg-label-${i}` }, { requests: i + 1, contextTokens: 100 * (i + 1) }); + await seeder.writeNow(s); + } + seeder.closeQuietly(); + + const store = new SessionStore({ dir, debounceMs: 0, enabled: true }); + await store.boot({ maxSessions: 2, maxBytes: 10 * 1024 * 1024 }); + // No background phase in legacy mode: even the OLDEST alias resolves + // immediately after boot returns. Alias resume re-keys to the label by + // design; the data proves the right file was read. + const loaded = store.loadSync("s3199-leg-label-0"); + assert.ok(loaded, "legacy boot completes alias bookkeeping synchronously"); + assert.equal(loaded!.id, "s3199-leg-label-0"); + assert.equal(loaded!.stats.contextTokens, 100); + store.closeQuietly(); +});