diff --git a/src/persist.ts b/src/persist.ts index d13508bd6a..dd35595820 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; import { readdir, readFile, rm } from "node:fs/promises"; import * as path from "node:path"; import { StateStore, flatFileNameFor, type PersistedEnvelope, type StateStoreCodec } from "acp-kernel/persist"; @@ -330,6 +330,9 @@ export class SessionStore { private diskFiles: string[] = []; private readonly legacyAliases = new Map(); private readonly sourceFiles = new Map(); + /** #3199: post-boot background alias scan (unified modes only) — null in + * legacy mode and until bootBounded defers work. */ + private remainderScan: Promise | null = null; /** #2674: monotonic per-process counter for unique content-store temp * names (pid + seq makes collisions across processes/rapid saves * unreachable). */ @@ -622,7 +625,7 @@ export class SessionStore { } /** Keep only file metadata while indexing. Legacy aliases are adopted on - * resume without deleting their historical state or original companions. */ + * resume without deleting their historical state or original companions. */ private async bootBounded(limits: { maxSessions: number; maxBytes: number }): Promise> { if (!Number.isFinite(limits.maxSessions) || limits.maxSessions < 0 || !Number.isFinite(limits.maxBytes) || limits.maxBytes < 0) { throw new Error("Invalid session boot limits"); @@ -633,30 +636,86 @@ export class SessionStore { this.legacyAliases.clear(); const warm: Array<{ file: string; id: string; meta: PersistedSession["meta"]; savedAt: number; bytes: number }> = []; const canonical = new Set(); + const filesByRoot = new Map(); for (const root of [this.sessionsDir, `${this.sessionsDir}-cold`]) { const files = existsSync(root) ? await walkJsonFiles(root) : []; for (const file of files) { if (path.basename(file).endsWith(".content-store.json")) continue; this.diskFiles.push(file); - const parsed = await this.readRawFileWithBytes(file); + filesByRoot.set(root, [...(filesByRoot.get(root) ?? []), file]); + } + } + // #3199: unified mode (shadow or full) — the listener must not wait + // for the whole corpus to parse. Two stages: (1) parse a bounded + // WINDOW of the newest files (mtime desc, stopped once the window + // holds a full budget's worth of candidates), (2) arbitrate the final + // warm set by savedAt desc with the original while-pop trim (#404 — + // mtime only orders the parse window, never the survivor choice). + // Seed label aliases from the index rows first (no decode); the rest + // of the bookkeeping finishes in the background. Gate is + // replicasAvailable() — a live non-degraded sidecar EXISTS (shadow OR + // full) — NOT readsUnified, which is the full-mode-only "rows are + // authoritative" predicate. + if (this.replicasAvailable()) { + this.seedIndexedAliases(canonical); + const hot = (filesByRoot.get(this.sessionsDir) ?? []) + .map((file) => { + try { + const st = statSync(file); + return { file, mtimeMs: st.mtimeMs, size: st.size }; + } catch { + return null; // raced deletion between walk and stat + } + }) + .filter((c): c is { file: string; mtimeMs: number; size: number } => c !== null) + .sort((a, b) => b.mtimeMs - a.mtimeMs); + const parsedFiles = new Set(); + const parsedCands: Array<{ file: string; id: string; meta: PersistedSession["meta"]; savedAt: number; bytes: number }> = []; + let windowBytes = 0; + for (const cand of hot) { + if (parsedCands.length >= limits.maxSessions && windowBytes >= limits.maxBytes) break; + // on-disk size lower-bounds decoded size (#2803) — cheap pre-filter + if (cand.size > limits.maxBytes) continue; + const parsed = await this.readRawFileWithBytes(cand.file); + parsedFiles.add(cand.file); if (!parsed) continue; - const raw = parsed.value; - const payload = raw && typeof raw === "object" && "payload" in raw ? raw.payload : raw; - if (!isValidRecord(payload)) continue; - const meta = payload.meta ?? { protocol: payload.protocol, upstreamOrigin: payload.upstreamOrigin, label: payload.label }; - if (!fileNameMatchesId(path.basename(file), payload.id, meta.protocol, meta.upstreamOrigin)) continue; - const savedAt = payload.savedAt ?? 0; - if (payload.id === meta.label || payload.id.startsWith("pfa-")) canonical.add(payload.id); - else if (meta.label) { - const previous = this.legacyAliases.get(meta.label); - if (!previous || savedAt >= previous.savedAt) this.legacyAliases.set(meta.label, { file, id: payload.id, savedAt }); - } - if (root !== this.sessionsDir) continue; + const rec = this.parseBootPayload(parsed.value); + if (!rec || !fileNameMatchesId(path.basename(cand.file), rec.payload.id, rec.meta.protocol, rec.meta.upstreamOrigin)) continue; + const savedAt = rec.payload.savedAt ?? 0; + if (rec.payload.id === rec.meta.label || rec.payload.id.startsWith("pfa-")) this.noteCanonical(rec.payload.id, canonical); + else if (rec.meta.label) this.upsertLegacyAlias(rec.meta.label, { file: cand.file, id: rec.payload.id, savedAt }, canonical); const bytes = parsed.bytes; if (bytes > limits.maxBytes) continue; - warm.push({ file, id: payload.id, meta, savedAt, bytes }); - warm.sort((a, b) => b.savedAt - a.savedAt); - while (warm.length > limits.maxSessions || warm.reduce((sum, item) => sum + item.bytes, 0) > limits.maxBytes) warm.pop(); + windowBytes += bytes; + parsedCands.push({ file: cand.file, id: rec.payload.id, meta: rec.meta, savedAt, bytes }); + } + parsedCands.sort((a, b) => b.savedAt - a.savedAt); + while (parsedCands.length > limits.maxSessions || parsedCands.reduce((sum, item) => sum + item.bytes, 0) > limits.maxBytes) parsedCands.pop(); + warm.push(...parsedCands); + this.remainderScan = this.scanRemainderInBackground(parsedFiles, canonical); + void this.remainderScan; + } else { + // legacy mode: keep the synchronous full parse exactly as before — + // there is no index to lean on, so nothing can be deferred. + for (const root of [this.sessionsDir, `${this.sessionsDir}-cold`]) { + for (const file of filesByRoot.get(root) ?? []) { + const parsed = await this.readRawFileWithBytes(file); + if (!parsed) continue; + const rec = this.parseBootPayload(parsed.value); + if (!rec || !fileNameMatchesId(path.basename(file), rec.payload.id, rec.meta.protocol, rec.meta.upstreamOrigin)) continue; + const savedAt = rec.payload.savedAt ?? 0; + if (rec.payload.id === rec.meta.label || rec.payload.id.startsWith("pfa-")) canonical.add(rec.payload.id); + else if (rec.meta.label) { + const previous = this.legacyAliases.get(rec.meta.label); + if (!previous || savedAt >= previous.savedAt) this.legacyAliases.set(rec.meta.label, { file, id: rec.payload.id, savedAt }); + } + if (root !== this.sessionsDir) continue; + const bytes = parsed.bytes; + if (bytes > limits.maxBytes) continue; + warm.push({ file, id: rec.payload.id, meta: rec.meta, savedAt, bytes }); + warm.sort((a, b) => b.savedAt - a.savedAt); + while (warm.length > limits.maxSessions || warm.reduce((sum, item) => sum + item.bytes, 0) > limits.maxBytes) warm.pop(); + } } } for (const id of canonical) this.legacyAliases.delete(id); @@ -672,6 +731,105 @@ export class SessionStore { return out; } + /** Shared envelope→(record, meta) extraction for the boot passes; null on + * any parse/validation failure (same gates as before the split). Meta is + * resolved to the flat fallbacks so callers never see undefined. */ + private parseBootPayload(raw: unknown): { payload: PersistedSession; meta: NonNullable } | null { + const payload = raw && typeof raw === "object" && "payload" in raw ? raw.payload : raw; + if (!isValidRecord(payload)) return null; + const meta = payload.meta ?? { protocol: payload.protocol, upstreamOrigin: payload.upstreamOrigin, label: payload.label }; + return { payload, meta }; + } + + /** Incremental canonical registration: adding an id to the canonical set + * voids any alias under the same name NOW instead of at end-of-scan — + * the incremental result equals the old final "delete all canonical + * labels from the alias map" pass. */ + private noteCanonical(id: string, canonical: Set): void { + canonical.add(id); + this.legacyAliases.delete(id); + } + + /** Alias upsert with the newest-savedAt-wins rule; refuses names that are + * (or became) canonical ids. Returns whether the map changed. */ + private upsertLegacyAlias(label: string, entry: { file: string; id: string; savedAt: number }, canonical: Set): boolean { + if (canonical.has(label)) return false; + const previous = this.legacyAliases.get(label); + if (previous && entry.savedAt < previous.savedAt) return false; + this.legacyAliases.set(label, entry); + return true; + } + + /** #3199: pre-seed label aliases from the unified index rows so label-based + * resume of recorded sessions works WITHOUT parsing their files during + * boot. Unrecorded files get their aliases from the background scan. */ + private seedIndexedAliases(canonical: Set): void { + const shadow = this.shadow; + if (!shadow) return; + let rows; + try { + rows = shadow.sessionSummaries(); + } catch { + return; // sidecar degraded mid-boot: background scan still covers it + } + for (const r of rows) { + if (!r.label || r.label === r.sessionId || r.sessionId.startsWith("pfa-")) continue; + const file = this.resolveIndexedFile(r.sessionId, r.protocol, r.origin); + if (!file) continue; + this.upsertLegacyAlias(r.label, { file, id: r.sessionId, savedAt: r.updatedAt }, canonical); + } + } + + /** #3199: locate a recorded session's JSON file by its id alone — the same + * three rel-path candidates the live probes use, across both roots; first + * existing regular file wins (stat-only, no decode). */ + private resolveIndexedFile(id: string, protocol?: string, origin?: string): string | null { + const candidates = [relPathFor(id, protocol, origin), relPathFor(id), flatFileNameFor(id)]; + for (const root of [this.sessionsDir, `${this.sessionsDir}-cold`]) { + for (const rel of candidates) { + const file = path.join(root, rel); + try { + if (statSync(file).isFile()) return file; + } catch { + // missing candidate: next + } + } + } + return null; + } + + /** #3199: finish alias/canonical bookkeeping over the corpus AFTER boot + * returned, so the listener is not held by files outside the warm window. + * Idempotent with the synchronous part (same upsert rules); yields the + * event loop between files like the web disk index does. */ + private async scanRemainderInBackground(parsedFiles: Set, canonical: Set): Promise { + let scanned = 0; + let aliased = 0; + for (const file of this.diskFiles) { + if (parsedFiles.has(file)) continue; + scanned++; + const parsed = await this.readRawFileWithBytes(file); + if (parsed) { + const rec = this.parseBootPayload(parsed.value); + if (rec && fileNameMatchesId(path.basename(file), rec.payload.id, rec.meta.protocol, rec.meta.upstreamOrigin)) { + const savedAt = rec.payload.savedAt ?? 0; + if (rec.payload.id === rec.meta.label || rec.payload.id.startsWith("pfa-")) this.noteCanonical(rec.payload.id, canonical); + else if (rec.meta.label) aliased += this.upsertLegacyAlias(rec.meta.label, { file, id: rec.payload.id, savedAt }, canonical) ? 1 : 0; + } + } + await new Promise((resolve) => setImmediate(resolve)); + } + if (scanned > 0) { + this.log("info", `[persist] background index scan finished after boot: ${scanned} file(s) scanned, ${aliased} alias(es) recorded`); + } + } + + /** #3199: resolves when the post-boot background alias scan has finished + * (immediate in legacy mode or when nothing was deferred). */ + backgroundScanComplete(): Promise { + return this.remainderScan ?? Promise.resolve(); + } + private readEnvelopeFile(file: string, id: string): PersistedEnvelope | null { let bytes: Buffer; try { @@ -1066,13 +1224,31 @@ export class SessionStore { } } - /** #2960 R2.2: web list rows — stats/metadata JSON + block aggregates - * from one SQL. Full mode only (same gate as above); null → the web - * falls back to the bounded disk index walk. */ + /** #3199: which authority backs the web list surfaces for THIS store. + * "authoritative" (full mode): the rows alone are complete — organic + * import on every load/write keeps them whole, so a session whose JSON + * file vanished stays listed (R2.1 self-sufficient index). "reconcile" + * (shadow mode): the rows are a BASELINE only — the JSON tree is still + * the truth base, so callers must reconcile against it (adopt fresh + * rows, decode files newer than their row or never recorded, drop ghost + * rows whose file vanished). null = legacy/degraded — plain file walk. */ + webSummaryBaseline(): "authoritative" | "reconcile" | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + return this.shadow.mode === "full" ? "authoritative" : "reconcile"; + } + + /** #2960 R2.2 / #3199: web list rows — stats/metadata JSON + block + * aggregates from one SQL. Full mode serves them as the authoritative + * answer; shadow mode (#3199) serves them as a BASELINE the caller must + * reconcile against the JSON tree (see webSummaryBaseline) — shadow rows + * may lag files never yet recorded, and ghost rows must be dropped there. + * Null → the web falls back to the bounded disk index walk. */ unifiedWebSummaries(): import("./storage/store.js").WebSummaryRow[] | null { - if (!this.shadow?.readsUnified) return null; + if (this.webSummaryBaseline() === null) return null; + const shadow = this.shadow; + if (!shadow) return null; try { - return this.shadow.webSummaryRows(); + return shadow.webSummaryRows(); } catch { return null; // sidecar degraded mid-call: fall back to the walk } diff --git a/src/storage/store.ts b/src/storage/store.ts index 4db22f1a4c..1612c6674e 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -141,7 +141,9 @@ export interface CheckpointRow { } /** Summary row for the export/web list fast path (#2960 R2.1). Parity - * contract with the legacy walk in export.ts listSessions. */ + * contract with the legacy walk in export.ts listSessions. updatedAt (#3199) + * = sessions.updated_at (ms epoch, last ingest time) — boot uses it as the + * alias savedAt when seeding label aliases from rows without decoding files. */ export interface SessionSummaryRow { sessionId: string; title: string | undefined; @@ -152,6 +154,7 @@ export interface SessionSummaryRow { blocks: number; latestBlockAt: number | undefined; contextTokens: number | undefined; + updatedAt: number; } /** #2960 R2.2: row backing the WEB session list in full mode. Carries the @@ -1041,9 +1044,10 @@ export class UnifiedStore { .prepare( `SELECT s.session_id AS sessionId, s.host_title AS hostTitle, s.title AS title, s.label AS label, s.protocol AS protocol, s.origin AS origin, s.namespace AS namespace, s.stats_json AS statsJson, + s.updated_at AS updatedAt, (SELECT COUNT(*) FROM blocks b WHERE b.session_id = s.session_id) AS blocks, (SELECT MAX(b.created_at) FROM blocks b WHERE b.session_id = s.session_id) AS latestBlockAt - FROM sessions s`, + FROM sessions s`, ) .all() as { sessionId: string; @@ -1054,6 +1058,7 @@ export class UnifiedStore { origin: string | null; namespace: string; statsJson: string; + updatedAt: number; blocks: number; latestBlockAt: number | null; }[]; @@ -1075,6 +1080,7 @@ export class UnifiedStore { blocks: r.blocks, latestBlockAt: r.latestBlockAt ?? undefined, contextTokens, + updatedAt: r.updatedAt, }; }); } diff --git a/src/web/sessions-data.ts b/src/web/sessions-data.ts index b37fbb6ab0..c8f1636817 100644 --- a/src/web/sessions-data.ts +++ b/src/web/sessions-data.ts @@ -4,6 +4,7 @@ import { conflictClientOf, conflictEventsOf } from "../conflict-watch.js"; import { isDisplayOnlyConflictDetail, isSiblingConflictDetail } from "../thirdparty-scan.js"; import { SessionStore, fileNameMatchesId, isValidRecord, relPathFor } from "../persist.js"; import { flatFileNameFor } from "acp-kernel/persist"; +import type { WebSummaryRow } from "../storage/store.js"; import { renderHandoff } from "../export.js"; import { buildSessionCacheReport } from "../cache-ledger.js"; import { METADATA_FOLD_COVERAGE } from "../fold-reconcile.js"; @@ -302,6 +303,37 @@ function liveCoveredPaths(dir: string): Set { return out; } +/** #3199: how much newer a session file's mtime may be than its unified row's + * updated_at before the row counts as stale and the file must be decoded. + * Normal writes always land AFTER their row (guardedBuild ingests before the + * JSON flush; updated_at is stamped at build time), so a bare mtime >= + * updatedAt test would decode everything — the window absorbs that lag while + * still catching bypass writes (#408 clamp, #286 rekey) and manual edits once + * they age out. Exposure is display-only and self-healing: shadow detail + * views always read the file, and any activity re-ingests the row. */ +const SHADOW_ROW_STALE_WINDOW_MS = 5 * 60 * 1000; + +/** #3199: basename → freshest unified row claiming that filename. Session + * filenames derive from the session id (sha256 prefix), so row→file needs no + * stat and no decode — the walk's own basenames are the join key (the same + * three candidates liveCoveredPaths builds per live id). */ +function rowClaimsByBasename(rows: WebSummaryRow[]): Map { + const claims = new Map(); + for (const r of rows) { + const candidates = [ + relPathFor(r.sessionId, r.protocol, r.origin), + relPathFor(r.sessionId), + flatFileNameFor(r.sessionId), + ]; + for (const rel of candidates) { + const base = path.basename(rel); + const cur = claims.get(base); + if (!cur || r.updatedAt > cur.updatedAt) claims.set(base, r); + } + } + return claims; +} + // #2260(F)/#2180 follow-up: bili's own infra dirs created under the data root // BEFORE any session exists (ensureRootCA() makes /ca at proxy start) — // their presence alone must not read as "misconfigured". The set is closed by @@ -356,6 +388,16 @@ async function refreshIndex(): Promise { const prev = byFile ?? new Map(); const next = new Map(); const covered = liveCoveredPaths(dir); + // #3199: shadow mode adopts fresh unified rows instead of decoding the + // files they describe — cold-start cost drops from "decode the whole + // corpus" to one SQL + one stat per file. Full mode never walks here + // (its rows are authoritative); legacy has no rows; both keep the + // plain decode path below. + let claims: Map | null = null; + if (store.webSummaryBaseline() === "reconcile") { + const rows = store.unifiedWebSummaries(); + if (rows !== null) claims = rowClaimsByBasename(rows); + } for (const f of files) { if (covered.has(f.abs)) continue; const p = prev.get(f.abs); @@ -363,6 +405,11 @@ async function refreshIndex(): Promise { next.set(f.abs, p); continue; } + const claim = claims?.get(path.basename(f.abs)); + if (claim && f.mtimeMs - claim.updatedAt <= SHADOW_ROW_STALE_WINDOW_MS) { + next.set(f.abs, { abs: f.abs, mtimeMs: f.mtimeMs, size: f.size, savedAt: claim.updatedAt, summary: summaryFromRow(claim) }); + continue; + } decodeCount++; const entry: DiskEntry = { abs: f.abs, mtimeMs: f.mtimeMs, size: f.size, savedAt: 0, summary: null }; const relName = path.relative(dir, f.abs); @@ -695,14 +742,21 @@ export function hiddenEmptyCount(): number { return lastHiddenEmpty; } async function mergedSummaries(): Promise { const out = new Map(); - // #2960 R2.2: full mode serves the disk side from the unified index (one - // SQL — no directory walk, no envelope decode, and sessions whose JSON - // file vanished stay listed). Legacy/shadow keep the bounded file walk: - // there the JSON tree is the truth base and rows may lag unrecorded files. - const rows = getDiskStore().unifiedWebSummaries(); - if (rows !== null) { - for (const r of rows) out.set(r.sessionId, summaryFromRow(r)); - } else { + // #2960 R2.2 / #3199: full mode serves the disk side from the unified index + // alone (one SQL — no walk, no decode; sessions whose JSON file vanished + // stay listed). Shadow RECONCILES the rows against the tree inside + // refreshIndex (fresh rows adopted without decode; stale/unrecorded files + // decoded; ghost rows dropped because only files produce entries). Legacy + // keeps the plain bounded file walk. + let fromRows = false; + if (getDiskStore().webSummaryBaseline() === "authoritative") { + const rows = getDiskStore().unifiedWebSummaries(); + if (rows !== null) { + for (const r of rows) out.set(r.sessionId, summaryFromRow(r)); + fromRows = true; + } + } + if (!fromRows) { const files = await ensureIndex(); for (const e of files.values()) if (e.summary) out.set(e.summary.id, e.summary); } diff --git a/tests/issue3199-overview-cold-scan.test.ts b/tests/issue3199-overview-cold-scan.test.ts new file mode 100644 index 0000000000..e411908b66 --- /dev/null +++ b/tests/issue3199-overview-cold-scan.test.ts @@ -0,0 +1,341 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createInitialState } from "acp-kernel"; +import type { CompressionBlock } from "acp-kernel"; +import { flatFileNameFor } from "acp-kernel/persist"; +import { SessionStore, _setStoreForTest, fileNameMatchesId } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; +import { getSession, _resetSessionsForTest } from "../src/session.ts"; +import { buildSessionList, _resetDiskCacheForTest, _diskScanStatsForTest } from "../src/web/sessions-data.ts"; +import { rmrf } from "./tmp-rm.ts"; + +// #3199: Web UI first paint stalled 78-180s+ on a cold instance — in shadow +// mode the web lists ignored the unified index (full-only gate) and decoded +// the ENTIRE corpus on the event loop, and boot parsed the whole corpus +// before the listener started. These tests pin both fixes: +// T1 — shadow mode adopts fresh index rows WITHOUT decoding their files; +// only files newer than their row (or never recorded) are decoded; +// ghost rows (file vanished) drop from the shadow list. +// T2 — unified-mode boot parses only the warm window (newest files within +// the budget); label aliases for recorded sessions are SQL-seeded +// synchronously; the rest fills in the background. +// T3 — legacy-mode boot keeps the synchronous full parse (control). + +// Plain JSON session files so seeds and round-trips stay deterministic (#1080) +process.env.BILI_PERSIST_ZSTD = "0"; + +// Hermetic config: a developer machine's real billion-context.json must not +// leak storage settings into this suite's tmp-dir stores (same discipline as +// tests/web-sessions.test.ts). +process.env.BILI_CONFIG_FILE = path.join( + mkdtempSync(path.join(tmpdir(), "bili-3199-hermetic-")), + "no-config.json", +); +// Absent everywhere resolves to the shadow DEFAULT (#2815 rollout flip); pin +// the explicit legacy baseline — legs flip per test. +process.env.BILI_STORAGE_UNIFIED = "legacy"; + +interface Stats { + requests: number; + tokensSaved: number; + inputTokens: number; + cachedTokens: number; + outputTokens: number; + cacheSamples: number; + lastInputTokens: number; + contextTokens: number; + retrieveCalls: number; + retrieveHits: number; + retrieveMisses: number; + storedBytes: number; + storeBytesSaved: number; + compressCreditTokens: number; + rangeRestores: number; +} + +function zeroStats(): Stats { + return { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, compressCreditTokens: 0, rangeRestores: 0 }; +} + +function makeSession(id: string, meta: Session["meta"] = {}, patch: Partial = {}, lastSeen = Date.now()): Session { + return { + id, + meta, + stats: { ...zeroStats(), ...patch }, + metadata: {}, + state: createInitialState(), + createdAt: lastSeen, + lastSeen, + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + }; +} + +/** Full persisted-state shape (kernel initial state + our blocks) so hand- + * written records survive both summary extraction AND buildSession's compat + * pass in the loadSync legs. */ +function stateWith(blocks: CompressionBlock[]): unknown { + const s = JSON.parse(JSON.stringify(createInitialState())) as { blocks: unknown[] }; + s.blocks = blocks; + return s; +} + +const mkBlock = (blockId: string, compressedTokens: number, active: boolean, topic = "T"): CompressionBlock => ({ + blockId, + runId: "r1", + tier: 1, + topic, + summary: "S", + directMessageIds: [], + effectiveMessageIds: [], + directBlockIds: [], + compressedTokens, + createdAt: Date.now(), + survivedCount: 1, + generation: "young", + active, +}); + +function withSessionsDir(name: string, fn: (dir: string) => Promise): void { + test(name, async () => { + const dir = mkdtempSync(path.join(tmpdir(), "bili-3199-")); + const prev = process.env.BILI_SESSIONS_DIR; + process.env.BILI_SESSIONS_DIR = dir; + try { + await fn(dir); + } finally { + if (prev === undefined) delete process.env.BILI_SESSIONS_DIR; else process.env.BILI_SESSIONS_DIR = prev; + _resetSessionsForTest(); + _resetDiskCacheForTest(); + rmrf(dir); + } + }); +} + +/** Flip the suite's legacy baseline to shadow for one body and restore after + * — _resetDiskCacheForTest() re-creates the disk-store singleton (mode is + * resolved at construction), so each leg sees its own engine honestly. */ +function withShadowMode(name: string, fn: (dir: string) => Promise): void { + withSessionsDir(name, async (dir) => { + const prev = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "shadow"; + try { + await fn(dir); + } finally { + if (prev === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = prev; + } + }); +} + +/** id → absolute envelope path, by scanning the corpus (layout-independent). */ +function fileById(root: string, id: string): string | null { + const walk = (r: string): string[] => { + const out: string[] = []; + for (const e of readdirSync(r, { withFileTypes: true })) { + const p = path.join(r, e.name); + if (e.isDirectory()) out.push(...walk(p)); + else if (e.name.endsWith(".json")) out.push(p); + } + return out; + }; + for (const p of walk(root)) { + try { + const env = JSON.parse(readFileSync(p, "utf8")) as { id?: unknown; payload?: { id?: unknown } }; + if (env.id === id || env.payload?.id === id) return p; + } catch { + // non-JSON / unreadable: not our file + } + } + return null; +} + +withShadowMode("#3199 T1: shadow list adopts fresh rows without decoding; stale/unrecorded files win; ghosts drop", async (dir) => { + const store = new SessionStore({ dir, debounceMs: 0, enabled: true }); + assert.ok(store.replicasAvailable(), "sidecar must be live for this leg (engine available)"); + + const sFresh = makeSession("s3199-fresh", { protocol: "openai", title: "Fresh", label: "s3199-fresh" }, { requests: 3, inputTokens: 800, cachedTokens: 200, outputTokens: 40, contextTokens: 400 }); + sFresh.state.blocks.push(mkBlock("bf1", 1500, true, "Fresh topic")); + await store.writeNow(sFresh); + + const sStale = makeSession("s3199-stale", { protocol: "openai", title: "Stale Title", label: "s3199-stale" }, { requests: 2, inputTokens: 500, cachedTokens: 100, contextTokens: 300, tokensSaved: 111 }); + sStale.state.blocks.push(mkBlock("bs1", 700, true, "Old topic")); + await store.writeNow(sStale); + + const sGhost = makeSession("s3199-ghost", { protocol: "openai", title: "Ghost" }, { requests: 2, inputTokens: 500, cachedTokens: 100, contextTokens: 300 }); + await store.writeNow(sGhost); + store.closeQuietly(); + + // External rewrite of the stale session AFTER its row was ingested: bump + // the values, then stamp the mtime 6 minutes into the future so + // mtime - row.updated_at deterministically exceeds the 5-minute freshness + // window (normal writes always land slightly AFTER their row). + const staleFile = fileById(dir, "s3199-stale"); + assert.ok(staleFile, "stale session file present"); + const env = JSON.parse(readFileSync(staleFile, "utf8")) as Record; + const payload = (env.payload ?? env) as Record; + payload.meta = { protocol: "openai", title: "Stale Rewritten", label: "s3199-stale" }; + payload.stats = { ...zeroStats(), requests: 9, inputTokens: 500, cachedTokens: 100, contextTokens: 300, tokensSaved: 4242 }; + payload.state = stateWith([mkBlock("bs2", 3333, true, "New topic")]); + payload.savedAt = Date.now(); + env.savedAt = payload.savedAt; + writeFileSync(staleFile, JSON.stringify(env), "utf8"); + const future = Date.now() / 1000 + 6 * 60; + utimesSync(staleFile, future, future); + + // Ghost: the JSON twin vanishes (hand-deleted) — the shadow list must DROP + // the orphan row (in shadow the tree is the truth base; full mode keeps + // such rows, pinned by the R2.2 file-less test). + const ghostFile = fileById(dir, "s3199-ghost"); + assert.ok(ghostFile, "ghost session file present before deletion"); + rmSync(ghostFile); + + // Unrecorded: a valid session file that was never ingested (pre-shadow + // era file) — no row exists, so the walk must decode and list it. + const unrecId = "s3199-unrecorded"; + const unrecName = flatFileNameFor(unrecId); + assert.ok(fileNameMatchesId(unrecName, unrecId), "flat name is addressable for its id"); + writeFileSync(path.join(dir, unrecName), JSON.stringify({ + version: 2, + savedAt: Date.now(), + id: unrecId, + meta: { protocol: "openai", title: "Unrecorded", label: unrecId }, + stats: { ...zeroStats(), requests: 5, inputTokens: 900, cachedTokens: 100, contextTokens: 500, tokensSaved: 2222 }, + state: stateWith([mkBlock("bu1", 2000, true, "Unrecorded topic")]), + }), "utf8"); + + _setStoreForTest(new SessionStore({ enabled: false })); + _resetDiskCacheForTest(); + + const list = await buildSessionList(); + assert.deepEqual(list.map((x) => x.id).sort(), ["s3199-fresh", "s3199-stale", "s3199-unrecorded"], "ghost row dropped, unrecorded file listed"); + + // Structural proof of the fix: exactly the two files whose rows were + // missing/stale got decoded — the fresh row was adopted from SQL alone. + assert.equal(_diskScanStatsForTest().decodedTotal, 2, "only stale + unrecorded files may be decoded"); + + const fresh = list.find((x) => x.id === "s3199-fresh"); + assert.ok(fresh, "fresh row listed"); + assert.equal(fresh!.title, "Fresh"); + assert.equal(fresh!.requests, 3); + assert.equal(fresh!.tokensSaved, 1500, "active-block sum floor from the row"); + assert.equal(fresh!.blocks, 1); + assert.equal(fresh!.firstBlockHint, "Fresh topic"); + assert.equal(fresh!.live, false); + + const stale = list.find((x) => x.id === "s3199-stale"); + assert.ok(stale, "stale file listed"); + assert.equal(stale!.title, "Stale Rewritten", "file values beat the lagging row"); + assert.equal(stale!.requests, 9); + assert.equal(stale!.tokensSaved, 4242); + + const unrec = list.find((x) => x.id === unrecId); + assert.ok(unrec, "unrecorded file listed"); + assert.equal(unrec!.title, "Unrecorded"); + assert.equal(unrec!.requests, 5); + + // Live pool still overrides disk entries in the merge. + const live = getSession("s3199-live", { protocol: "openai", label: "s3199-live" }); + live.stats.requests = 2; + live.stats.contextTokens = 100; + const list2 = await buildSessionList(); + const liveRow = list2.find((x) => x.id === "s3199-live"); + assert.ok(liveRow, "live session listed"); + assert.equal(liveRow!.live, true); + assert.equal(_diskScanStatsForTest().decodedTotal, 2, "second list served from the (mtime,size) cache"); +}); + +withShadowMode("#3199 T2: unified boot parses only the warm window; aliases SQL-seeded; rest fills in background", async (dir) => { + const N = 12; + const seeder = new SessionStore({ dir, debounceMs: 0, enabled: true }); + assert.ok(seeder.replicasAvailable(), "sidecar must be live for this leg (engine available)"); + for (let i = 0; i < N; i++) { + const id = `s3199-boot-${i}`; + // Older half carries a distinct label (alias candidate); the newest + // four use label === id (canonical — never aliased). + const label = i >= N - 4 ? id : `s3199-label-${i}`; + const s = makeSession(id, { protocol: "openai", title: `Boot ${i}`, label }, { requests: i + 1, contextTokens: 100 * (i + 1) }); + await seeder.writeNow(s); + } + seeder.closeQuietly(); + + // Stagger mtimes by one minute: i = N-1 is newest. Track the largest + // on-disk size so the byte budget below can never trim an intended winner + // (file sizes grow a few bytes as token counts cross digit boundaries). + const base = Math.floor(Date.now() / 1000) - N * 60; + let maxFileSize = 0; + for (let i = 0; i < N; i++) { + const f = fileById(dir, `s3199-boot-${i}`); + assert.ok(f, `session file for i=${i}`); + maxFileSize = Math.max(maxFileSize, statSync(f).size); + const t = base + i * 60; + utimesSync(f, t, t); + } + + // An UNRECORDED labeled file: no index row exists, so its alias can only + // come from the post-boot background scan. Stamped OLDER than every boot + // file on BOTH signals (mtime and savedAt) so it cannot win a warm slot + // in the savedAt arbitration. + const strayId = "s3199-stray"; + const strayLabel = "s3199-stray-label"; + const strayFile = path.join(dir, flatFileNameFor(strayId)); + writeFileSync(strayFile, JSON.stringify({ + version: 2, + savedAt: (base - 60) * 1000, + id: strayId, + meta: { protocol: "openai", title: "Stray", label: strayLabel }, + stats: { ...zeroStats(), requests: 7, contextTokens: 700 }, + state: stateWith([]), + }), "utf8"); + utimesSync(strayFile, base - 60, base - 60); + + // Byte budget sized to ~4-5 files: the parse window stops after a full + // budget's worth of candidates instead of draining the whole corpus, yet + // 4×maxFileSize always fits so the trim can never drop an intended winner. + const store = new SessionStore({ dir, debounceMs: 0, enabled: true }); + const booted = await store.boot({ maxSessions: 4, maxBytes: maxFileSize * 5 }); + // Lexicographic order ("boot-10" < "boot-8") — same comparator on both sides. + assert.deepEqual([...booted.keys()].sort(), ["s3199-boot-10", "s3199-boot-11", "s3199-boot-8", "s3199-boot-9"], "warm set = 4 newest by mtime, arbitrated by savedAt"); + + // Label-based resume of a NON-warm recorded session works SYNCHRONOUSLY — + // the alias came from the index rows, not from parsing the file. Alias + // resume re-keys the session to the requested label by design, so assert + // the data (contextTokens 800 = i 7) proves the right file was read. + const resumed = store.loadSync("s3199-label-7"); + assert.ok(resumed, "SQL-seeded alias resolves before any background work"); + assert.equal(resumed!.id, "s3199-label-7"); + assert.equal(resumed!.stats.contextTokens, 800); + + // The unrecorded label is only resolvable once the background scan reaches + // its file (it was outside the warm window by construction). + await store.backgroundScanComplete(); + const stray = store.loadSync(strayLabel); + assert.ok(stray, "background scan recorded the unrecorded label alias"); + assert.equal(stray!.id, strayLabel); + assert.equal(stray!.stats.contextTokens, 700); + store.closeQuietly(); +}); + +withSessionsDir("#3199 T3: legacy boot keeps the synchronous full parse (control)", async (dir) => { + const seeder = new SessionStore({ dir, debounceMs: 0, enabled: true }); + for (let i = 0; i < 4; i++) { + const s = makeSession(`s3199-leg-${i}`, { protocol: "openai", title: `Leg ${i}`, label: `s3199-leg-label-${i}` }, { requests: i + 1, contextTokens: 100 * (i + 1) }); + await seeder.writeNow(s); + } + seeder.closeQuietly(); + + const store = new SessionStore({ dir, debounceMs: 0, enabled: true }); + await store.boot({ maxSessions: 2, maxBytes: 10 * 1024 * 1024 }); + // No background phase in legacy mode: even the OLDEST alias resolves + // immediately after boot returns. Alias resume re-keys to the label by + // design; the data proves the right file was read. + const loaded = store.loadSync("s3199-leg-label-0"); + assert.ok(loaded, "legacy boot completes alias bookkeeping synchronously"); + assert.equal(loaded!.id, "s3199-leg-label-0"); + assert.equal(loaded!.stats.contextTokens, 100); + store.closeQuietly(); +}); diff --git a/tests/search-context-cross-session.test.ts b/tests/search-context-cross-session.test.ts index 499fe5d2ec..2e8f71de93 100644 --- a/tests/search-context-cross-session.test.ts +++ b/tests/search-context-cross-session.test.ts @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtempSync, readFileSync, readdirSync } from "node:fs"; +import { mkdtempSync, readFileSync, readdirSync, statSync } from "node:fs"; import path from "node:path"; import { tmpdir } from "node:os"; import { createHash } from "node:crypto"; @@ -199,7 +199,9 @@ test("#841 cold-loaded historical session: read-only, file untouched, no save sc const allFiles: string[] = []; for (const d of readdirSync(dir)) { - if (!d.startsWith(".")) for (const f of readdirSync(path.join(dir, d))) allFiles.push(path.join(dir, d, f)); + // Unified-storage sidecars (index.db, WAL files) sit at the top + // level next to the protocol dirs — only descend into directories. + if (!d.startsWith(".") && statSync(path.join(dir, d)).isDirectory()) for (const f of readdirSync(path.join(dir, d))) allFiles.push(path.join(dir, d, f)); } assert.ok(allFiles.length > 0, "session file must exist"); const before = allFiles.map((f) => readFileSync(f, "utf8"));