From 0371089db6cd381eb0292a25f862274a6f1a2b25 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 31 Oct 2026 16:16:10 +0800 Subject: [PATCH 1/3] docs(lifecycle): register BILI_SESSION_CONTENT_STORE_PIN and BILI_SPAWN_FINGERPRINT in LIFECYCLE.md (#4599) The lifecycle-ledger-guard job fails on every tree containing master since c8d7a802 (#4788 phase 2) and fc53fd46 (#4125): two BILI_* env vars read from src/ were never registered in LIFECYCLE.md, so every PR based on current master goes red within seconds (first observed on #5037). Backfill only, no code change: BILI_SESSION_CONTENT_STORE_PIN -> section 1 transition ledger (escape hatch for #4788 phase 2 idle-fold, default fold); BILI_SPAWN_FINGERPRINT -> section 2.2 internal spawn channel (#4125 spawner-side entry-script hash, launcher to child). --- LIFECYCLE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/LIFECYCLE.md b/LIFECYCLE.md index 9f6fdde0a8..97fa591cdb 100644 --- a/LIFECYCLE.md +++ b/LIFECYCLE.md @@ -52,6 +52,7 @@ | `BILI_OPENCODE_INLINE` | #3278 opencode 移植(#4687) | opencode inline 模式(需手动开启,默认保持旧行为 spawn,owner 决定不改默认):与 pi 同构——不起 proxy 进程,模型 URL 被 V2 route/intercept 改写成伪 origin 后由进程内引擎接住(bun 宿主已验:引擎 boot/dispatch 全绿);`=1/on/yes/TRUE` 或 file 家 `"opencode": {"inline": true}` 显式开启(显式开启压过 attach 目标);`bili opencode-inline` 一键车道 | 同 pi inline 车道节奏 | opt-in(不设置时 attach/spawn 行为与旧版完全一致) | | `BILI_INLINE_DEBUG` | #4669 | 仅 inline 车道:引擎 stderr 日志腿默认静默(stderr 是 pi 的交互终端,日志会挤掉输入框;默认只写 bili.log),设 1 恢复 stderr 输出用于排查 | 随 inline 车道(#3278)走 | 排查逃生口 | | 测试钩子:`BILI_DSH_RECOVERY_INTERVAL_MS` / `BILI_DSH_RETRY_INTERVAL_MS` / `BILI_MODEL_INFO_RETRY_MS` | #2082 等 | 缩短测试/运维节奏(dsh-native.ts;NODE_TEST_CONTEXT 感知,避免生产误缩) | 永久测试设施,随宿主代码走 | 测试钩子 | +| `BILI_SESSION_CONTENT_STORE_PIN` | #4788 phase 2(c8d7a802) | idle-fold 逃生口:`=1/true` 让每会话 CCR content store({byRef}/{byHash} 原文)常驻内存、save 落盘后不折叠(默认折叠;读点 src/knobs.ts:451,store 可从 unified ccr mirror 或 .content-store.json sidecar 重新物化) | #4788 phase 2 idle-fold 生产站稳后评估退役(owner 拍板) | 在用(默认 fold) | ## 2. 常设面全量登记表(src/ 实际读取的其余 BILI_* 环境变量) @@ -77,6 +78,7 @@ | `BILI_NON_HTTP_PROVIDERS` | #1392 非 http(s) baseUrl provider 白名单传递 | | `BILI_PROVIDER_REWRITES` | #535 provider 改写 manifest JSON(存在即关 zcode native) | | `BILI_UPGRADE_REPLACEMENT` | #3044 self-restart 替换进程交接标记(UPGRADE_REPLACEMENT_ENV) | +| `BILI_SPAWN_FINGERPRINT` | #4125 spawner 侧入口脚本 sha256 指纹(SPAWN_FINGERPRINT_ENV,src/instance.ts:343),spawn 时注入供子进程记录自身指纹,避免 auto-update 窗口内 re-hash 漂移 | ### 2.3 网络调优(#2030 迁移家族:env > 文件 network.* > 默认) From 29fde221fe5b25c0ac773d47e98ec3dc1ec53a16 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 31 Oct 2026 15:59:34 +0800 Subject: [PATCH 2/3] fix: sweep provably-orphan collapsed blocks out of upgraded sessions (D2, #5308) A collapsed block committed under a shared conversation id during the <=v0.1.208 era reappears forever in the parent session after an upgrade: the #2695 e-gate exemption shelters it (never-present substrate never accrues a death streak) while the render side keeps it active on any covered-id collision. No wire-side path reaps it. Add a host-side maintenance sweep (bili sessions orphan-sweep [id] [--fix]) that flags blocks whose covered ids have ZERO overlap with their own session's unified-store transcript (active + retired rows) - provably foreign. Dry-run by default; --fix archives via a metadata-backed exclusion list that buildSession honors on load (reversible, no kstate rewrite). Legit folds always retain their covered rows, so they are never swept. --- src/cli.ts | 27 ++++-- src/orphan-sweep.ts | 185 +++++++++++++++++++++++++++++++++++++ src/persist.ts | 36 +++++++- src/storage-maintenance.ts | 23 ++++- src/storage/shadow.ts | 15 +++ src/storage/store.ts | 23 +++++ tests/orphan-sweep.test.ts | 131 ++++++++++++++++++++++++++ 7 files changed, 430 insertions(+), 10 deletions(-) create mode 100644 src/orphan-sweep.ts create mode 100644 tests/orphan-sweep.test.ts diff --git a/src/cli.ts b/src/cli.ts index 9b76bbfb1d..e57d1b274f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -14,7 +14,7 @@ * bili export [id] [--full] export a persisted session as a handoff doc * (--jsonl: deterministic JSONL dump #2671) * bili acp-cache diff attribute cache breaks from ACP_DUMP_BODY dumps - * bili sessions ls|rm|archive|restore [id] manage persisted sessions + * bili sessions ls|rm|archive|restore|orphan-sweep [id] manage persisted sessions * bili storage gc [--days N] run the storage sweep now (--dry-run plans only) * bili test pi non-polluting pi smoke test * bili --version @@ -89,6 +89,11 @@ Usage: bili sessions archive move one session out of index.db into the single append-only archive log (rows swept, restorable) bili sessions restore bring an archived session back into index.db + bili sessions orphan-sweep [id] #5308: find collapsed blocks whose covered ids + have zero overlap with their own session's stored + transcript (provably foreign, e.g. shared-id leftovers + from <=v0.1.208). Dry-run by default; --fix archives + them (reversible). Needs full unified storage. bili storage gc run the storage sweep now (#3911): archive sessions older than storage.archiveAfterDays (default 365), release unreferenced content, shrink index.db @@ -200,9 +205,10 @@ type Parsed = { acpCacheLog?: string; acpCacheNoLog?: boolean; acpCacheSession?: string; - sessionsSub?: "ls" | "rm" | "archive" | "restore"; + sessionsSub?: "ls" | "rm" | "archive" | "restore" | "orphan-sweep"; sessionsTarget?: string; sessionsNoVacuum?: boolean; + orphanSweepFix?: boolean; storageDays?: number; storageVacuum?: "full" | "incremental"; storageDryRun?: boolean; @@ -239,6 +245,7 @@ export function parseArgs(argv: string[]): Parsed { let jsonOutput = false; let doctorJson = false; let doctorFix = false; + let orphanSweepFix = false; for (let i = 0; i < argv.length; i++) { const a = argv[i]!; @@ -307,7 +314,11 @@ export function parseArgs(argv: string[]): Parsed { doctorJson = true; break; case "--fix": + // Shared "perform" token: doctor uses it for dsh-ledger repair, + // `sessions orphan-sweep` for archiving orphans. Commands are + // mutually exclusive, so each branch reads only its own flag. doctorFix = true; + orphanSweepFix = true; break; case "--no-vacuum": sessionsNoVacuum = true; @@ -465,15 +476,15 @@ export function parseArgs(argv: string[]): Parsed { } else if (cmd === "sessions") { command = "sessions"; const sub = positional[1]; - if (sub === "ls" || sub === "rm" || sub === "archive" || sub === "restore") { + if (sub === "ls" || sub === "rm" || sub === "archive" || sub === "restore" || sub === "orphan-sweep") { sessionsSub = sub; sessionsTarget = positional[2]; - if (sub !== "ls" && sessionsTarget === undefined) { + if (sub !== "ls" && sub !== "orphan-sweep" && sessionsTarget === undefined) { console.error(`bili sessions ${sub}: is required (try "bili sessions ${sub} ")`); process.exit(2); } } else { - console.error(`bili sessions: unknown action "${sub ?? ""}" (try "bili sessions ls|rm|archive|restore")`); + console.error(`bili sessions: unknown action "${sub ?? ""}" (try "bili sessions ls|rm|archive|restore|orphan-sweep")`); process.exit(2); } } else if (cmd === "storage") { @@ -488,11 +499,11 @@ export function parseArgs(argv: string[]): Parsed { } } - return { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix }; + return { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, orphanSweepFix }; } export async function main(): Promise { - const { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix } = parseArgs(process.argv.slice(2)); + const { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, orphanSweepFix } = parseArgs(process.argv.slice(2)); if (command === "help") { process.stdout.write(HELP); return; @@ -621,7 +632,7 @@ export async function main(): Promise { // carry their own exit codes (1 = operation failed, 2 = usage). try { const r = command === "sessions" - ? await runSessionsCommand({ sub: sessionsSub!, target: sessionsTarget, json: jsonOutput, noVacuum: sessionsNoVacuum }) + ? await runSessionsCommand({ sub: sessionsSub!, target: sessionsTarget, json: jsonOutput, noVacuum: sessionsNoVacuum, orphanSweepFix }) : await runStorageGcCommand({ days: storageDays, vacuum: storageVacuum, dryRun: storageDryRun, json: jsonOutput }); process.stdout.write(r.text); if (r.code !== 0) process.exit(r.code); diff --git a/src/orphan-sweep.ts b/src/orphan-sweep.ts new file mode 100644 index 0000000000..6a35fd67f1 --- /dev/null +++ b/src/orphan-sweep.ts @@ -0,0 +1,185 @@ +// #5308 (D2) — sweep provably-foreign collapsed blocks out of upgraded legacy +// sessions. +// +// A collapsed block committed under a SHARED conversation id during the +// <=v0.1.208 era can reappear forever in the parent session after an upgrade: +// its covered substrate was never part of the parent's durable record, so the +// #2695 e-gate exemption ("never-present substrate never accrues a death +// streak") gives it permanent shelter, while the render side (kernel sync.ts +// stillPresent) keeps it active whenever ANY covered id happens to collide with +// a message the parent actually read. No existing wire-side path reaps it +// (orphan-gc only looks at the CURRENT wire, where those collisions land). +// +// The discriminator here is STORE presence, which is independent of wire +// presence: a block whose covered ids have ZERO overlap with the session's +// unified-store transcript rows could never have been folded out of THIS +// session, so it is provably foreign. Legit folds always retain their covered +// rows (active OR retired — rewind/prune retire, never delete), so they are +// never swept. This is the safe half of option A from #5308; option B (blocks +// carry creation-time conversation-key lineage) is a separate kernel-side item. +// +// Dry-run by default; --fix archives via a metadata-backed exclusion list that +// buildSession honors on load (reversible, no kstate rewrite). + +import { baseIdOf } from "acp-kernel"; +import type { SessionStore } from "./persist.js"; +import type { Session } from "./session.js"; +import { log as loggerLog } from "./logger.js"; + +export interface OrphanBlockFinding { + sessionId: string; + blockId: string; + coveredCount: number; + /** First few covered base ids, for reporting. */ + coveredSample: string[]; +} + +export interface OrphanSweepPlan { + storageMode: "full" | "shadow" | "legacy"; + /** True only when --fix can durably apply (full unified storage). */ + performable: boolean; + /** Sessions that had >=1 unified transcript row and were scanned. */ + scannedSessions: number; + /** Sessions skipped because they have no unified transcript (criterion vacuous). */ + skippedNoTranscript: number; + findings: OrphanBlockFinding[]; +} + +export interface OrphanSweepResult { + plan: OrphanSweepPlan; + dryRun: boolean; + sweptBlocks: number; + touchedSessions: number; + errors: string[]; + lines: string[]; +} + +async function scan(store: SessionStore, loaded: Map): Promise { + const full = store.unifiedReads(); + const replicas = store.replicasAvailable(); + const storageMode: OrphanSweepPlan["storageMode"] = full ? "full" : replicas ? "shadow" : "legacy"; + if (!replicas) { + return { storageMode, performable: false, scannedSessions: 0, skippedNoTranscript: 0, findings: [] }; + } + const findings: OrphanBlockFinding[] = []; + let scanned = 0; + let skipped = 0; + for (const [id, session] of loaded) { + const msgIds = store.unifiedSessionTranscriptMsgIds(id); + // No unified transcript -> the zero-overlap criterion is vacuous (every + // block would look orphan). Skip rather than false-positive. + if (!msgIds || msgIds.length === 0) { + skipped++; + continue; + } + scanned++; + const transcriptBases = new Set(); + for (const mid of msgIds) transcriptBases.add(baseIdOf(mid)); + const blocks = session.state?.blocks; + if (!blocks || blocks.length === 0) continue; + for (const block of blocks) { + const covered = block.effectiveMessageIds; + if (!covered || covered.length === 0) continue; + if (!block.active) continue; + let intersect = 0; + for (const cid of covered) { + if (transcriptBases.has(baseIdOf(cid))) intersect++; + } + if (intersect > 0) continue; + findings.push({ + sessionId: id, + blockId: block.blockId, + coveredCount: covered.length, + coveredSample: covered.slice(0, 6).map((x) => baseIdOf(x)), + }); + } + } + return { storageMode, performable: full, scannedSessions: scanned, skippedNoTranscript: skipped, findings }; +} + +/** Read-only detection: every active block whose covered ids have zero overlap + * with its own session's unified-store transcript. Never mutates. */ +export async function detectOrphanBlocks(store: SessionStore): Promise { + const loaded = await store.loadAll(); + return scan(store, loaded); +} + +/** Detect, then (unless dryRun) durably archive the orphans via the metadata + * exclusion list. dryRun defaults to true — mutation requires an explicit + * `dryRun: false` AND full-mode storage. Optionally scoped to one session. */ +export async function applyOrphanSweep( + store: SessionStore, + opts: { dryRun?: boolean; sessionId?: string }, +): Promise { + const dryRun = opts.dryRun !== false; + const loaded = await store.loadAll(); + let plan = await scan(store, loaded); + if (opts.sessionId) { + plan = { ...plan, findings: plan.findings.filter((f) => f.sessionId === opts.sessionId) }; + } + const lines: string[] = []; + const errors: string[] = []; + + if (plan.findings.length === 0) { + lines.push( + `scanned ${plan.scannedSessions} session(s) with a unified transcript (${plan.storageMode}); ` + + `${plan.skippedNoTranscript} skipped (no unified transcript); no provably-orphan blocks found`, + ); + return { plan, dryRun, sweptBlocks: 0, touchedSessions: 0, errors, lines }; + } + + if (dryRun) { + for (const f of plan.findings) { + lines.push(`[dry-run] ${f.sessionId} :: ${f.blockId} covers ${f.coveredCount} id(s), zero overlap with this session's transcript [${f.coveredSample.join(", ")}]`); + } + lines.push(`dry run only - nothing changed. Re-run with --fix to archive ${plan.findings.length} orphan block(s).`); + return { plan, dryRun: true, sweptBlocks: 0, touchedSessions: 0, errors, lines }; + } + + if (!plan.performable) { + for (const f of plan.findings) lines.push(`[blocked] ${f.sessionId} :: ${f.blockId} (covers ${f.coveredCount})`); + lines.push('storage is not in "full" mode - cannot durably archive here. Enable full unified storage (storage.unified=full / BILI_STORAGE_UNIFIED=full) and re-run --fix.'); + return { plan, dryRun: false, sweptBlocks: 0, touchedSessions: 0, errors: ["not performable: storage not in full mode"], lines }; + } + + const bySession = new Map(); + for (const f of plan.findings) { + const arr = bySession.get(f.sessionId) ?? []; + arr.push(f); + bySession.set(f.sessionId, arr); + } + let swept = 0; + const touched = new Set(); + const now = Date.now(); + for (const [sid, fs] of bySession) { + const session = loaded.get(sid); + if (!session) { + errors.push(`${sid}: session vanished between scan and apply`); + continue; + } + try { + const curMeta: Record = + session.metadata && typeof session.metadata === "object" ? { ...(session.metadata as Record) } : {}; + const prevRaw = curMeta.orphanExcludedBlocks; + const prev: Record = + prevRaw && typeof prevRaw === "object" && !Array.isArray(prevRaw) ? { ...(prevRaw as Record) } : {}; + for (const f of fs) { + prev[f.blockId] = { at: now, reason: `#5308 orphan sweep: ${f.coveredCount} covered id(s), zero transcript overlap` }; + } + curMeta.orphanExcludedBlocks = prev; + store.setUnifiedSessionMetadataJson(sid, JSON.stringify(curMeta)); + touched.add(sid); + swept += fs.length; + for (const f of fs) lines.push(`[archived] ${sid} :: ${f.blockId} (covers ${f.coveredCount} id(s), zero transcript overlap)`); + loggerLog("info", `[orphan-sweep] ${sid}: archived ${fs.length} orphan block(s): ${fs.map((f) => f.blockId).join(", ")}`); + } catch (err) { + const detail = /SQLITE_BUSY|database is locked/i.test(String(err)) + ? "database busy (another bili instance holds index.db)" + : String(err); + errors.push(`${sid}: ${detail}`); + lines.push(`[error] ${sid}: could not persist exclusion - ${detail}`); + } + } + lines.push(`applied: archived ${swept} orphan block(s) across ${touched.size} session(s); takes effect on next load/restart of each affected client.`); + return { plan, dryRun: false, sweptBlocks: swept, touchedSessions: touched.size, errors, lines }; +} diff --git a/src/persist.ts b/src/persist.ts index 3537509881..20545c6412 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2201,6 +2201,26 @@ export class SessionStore { } } + /** #5308 orphan sweep (read-only): every transcript msg_id for a session + * (active + retired rows). Null when the unified store is unavailable or + * degraded mid-call — the sweep degrades to "cannot prove", never guesses. */ + unifiedSessionTranscriptMsgIds(sessionId: string): string[] | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + try { + return this.shadow.listSessionTranscriptMsgIds(sessionId); + } catch { + return null; + } + } + + /** #5308 orphan sweep (perform): persist an out-of-process metadata mutation + * (the orphan-block exclusion list) on the session's index.db row. Full mode + * only — callers gate on unifiedReads() first so the write is authoritative + * and picked up on reload. Throws on failure (e.g. SQLITE_BUSY). */ + setUnifiedSessionMetadataJson(sessionId: string, metadataJson: string): void { + this.liveShadow().updateSessionMetadataJson(sessionId, metadataJson); + } + /** #D5-5 render windowing: content rows outside the covered intervals. * Null when the unified store is unavailable. */ unifiedUncoveredRows(sessionId: string, intervals: readonly [number, number][]): import("./storage/store.js").MessageRow[] | null { @@ -3603,6 +3623,20 @@ function restoreBlockContents(entries: Iterable): Ma return blockContents; } +/** #5308 orphan sweep: drop blocks doctor has durably marked foreign. The + * exclusion lives in metadata.orphanExcludedBlocks (blockId -> {at, reason}), + * written out-of-process by `bili sessions orphan-sweep --fix` and honored here + * so a provably-orphan block can never render into THIS session again. Absent / + * malformed list = no-op fast path. Data stays in kstate -> fully reversible by + * removing the entry. */ +function filterExcludedOrphanBlocks(state: CompressionState, excluded: unknown): CompressionState { + if (!excluded || typeof excluded !== "object" || Array.isArray(excluded)) return state; + const ids = Object.keys(excluded as Record); + if (ids.length === 0) return state; + const exSet = new Set(ids); + return { ...state, blocks: state.blocks.filter((b) => !exSet.has(b.blockId)) }; +} + /** #4942: the record→legacy-view adapter the sidecar ingest path runs on * every committed record — exported so the hermetic seam suite can build a * sidecar against a stub backend with the SAME adapter the store uses. */ @@ -3690,7 +3724,7 @@ function buildSession(parsed: PersistedSession): Session { wholeBlockRestoresPreciseAvailable: stats.wholeBlockRestoresPreciseAvailable ?? 0, }, metadata: parsed.metadata ?? {}, - state: mergeState(parsed.state), + state: filterExcludedOrphanBlocks(mergeState(parsed.state), (parsed.metadata as { orphanExcludedBlocks?: unknown } | undefined)?.orphanExcludedBlocks), createdAt: parsed.createdAt ?? Date.now(), // #404: lastSeen reflects the on-disk savedAt (the last real // activity), NOT the restore moment — a restart must not fabricate diff --git a/src/storage-maintenance.ts b/src/storage-maintenance.ts index 6b64d06a21..3f539de8d4 100644 --- a/src/storage-maintenance.ts +++ b/src/storage-maintenance.ts @@ -7,6 +7,7 @@ import { existsSync, statSync } from "node:fs"; import { join } from "node:path"; import { SessionStore } from "./persist.js"; +import { applyOrphanSweep, type OrphanSweepResult } from "./orphan-sweep.js"; import type { GcPlan } from "./storage/store.js"; import { fileStorageArchiveAfterDays, resolveArchiveAfterDays } from "./knobs.js"; @@ -242,11 +243,29 @@ async function runGc(store: SessionStore, days: number | undefined, vacuum: "ful return { code: 0, text: `${text}\n` }; } +/** #5308 (D2): exit code for the orphan sweep. Non-zero when writes failed or + * when --fix found orphans but could not durably archive them (non-full mode), + * so automation knows the problem persists; a dry-run that merely reports is 0. */ +function orphanSweepExitCode(result: OrphanSweepResult): number { + if (result.errors.length > 0) return 1; + if (!result.dryRun && result.plan.findings.length > 0 && result.sweptBlocks === 0) return 1; + return 0; +} + +async function runOrphanSweep(store: SessionStore, target: string | undefined, json: boolean, fix: boolean): Promise { + const result = await applyOrphanSweep(store, { dryRun: !fix, sessionId: target }); + if (json) return { code: orphanSweepExitCode(result), text: `${JSON.stringify(result, null, 2)}\n` }; + const mode = `${result.plan.storageMode}${result.dryRun ? ", dry-run" : result.sweptBlocks > 0 ? ", applied" : ", no-op"}`; + const text = [`orphan-block sweep (${mode})`, ...result.lines].join("\n"); + return { code: orphanSweepExitCode(result), text: `${text}\n` }; +} + export async function runSessionsCommand(opts: { - sub: "ls" | "rm" | "archive" | "restore"; + sub: "ls" | "rm" | "archive" | "restore" | "orphan-sweep"; target?: string; json?: boolean; noVacuum?: boolean; + orphanSweepFix?: boolean; dir?: string; }): Promise { const store = new SessionStore({ dir: opts.dir, enabled: true }); @@ -260,6 +279,8 @@ export async function runSessionsCommand(opts: { return await runArchive(store, opts.target, !!opts.json); case "restore": return await runRestore(store, opts.target, !!opts.json); + case "orphan-sweep": + return await runOrphanSweep(store, opts.target, !!opts.json, !!opts.orphanSweepFix); } } finally { store.closeQuietly(); diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 8a6ed079d1..bc2c4dc8b6 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -855,6 +855,21 @@ export class UnifiedShadow { return this.sync.messageIdRows(sessionId); } + /** #5308 orphan sweep (read-only): all transcript msg_ids (active + retired), + * empty when there is no local unified store. */ + listSessionTranscriptMsgIds(sessionId: string): string[] { + if (this.sync === null) return []; + return this.sync.listSessionTranscriptMsgIds(sessionId); + } + + /** #5308 orphan sweep (perform): persist an out-of-process metadata mutation + * on the local unified store. Fails fast on an async backend (the sweep only + * runs against full-mode local storage). */ + updateSessionMetadataJson(sessionId: string, metadataJson: string): void { + if (this.sync === null) throw new Error("[orphan-sweep] unified store unavailable (async backend)"); + this.sync.updateSessionMetadataJson(sessionId, metadataJson); + } + /** #D5-5: content rows outside the covered intervals only. */ uncoveredRows(sessionId: string, intervals: readonly [number, number][]): (MessageRow & { contentHash: string })[] { if (this.sync === null) return []; diff --git a/src/storage/store.ts b/src/storage/store.ts index b73b85607c..58f8438602 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2845,6 +2845,29 @@ export class UnifiedStore { return rows.map((r) => ({ seq: r.seq as number, msgId: r.msg_id as string, ts: (r.ts as number | null) ?? null })); } + /** #5308 orphan sweep (read-only): every transcript msg_id for a session, + * active AND retired rows. A folded message's row is RETIRED (tombstoned), + * never deleted, so retired rows still prove the substrate belonged to this + * session's durable record. Selects only msg_id — no content hydration. */ + listSessionTranscriptMsgIds(sessionId: string): string[] { + const rows = this.db + .prepare("SELECT msg_id FROM messages WHERE session_id = ?") + .all(sessionId) as Record[]; + return rows.map((r) => r.msg_id as string); + } + + /** #5308 orphan sweep (perform): durably record an out-of-process metadata + * mutation on one session's authoritative index.db row. Updates metadata_json + * and advances updated_at so this write wins the full-mode load staleness + * guard (buildForLoadAll re-imports a JSON file newer than the row, which + * would otherwise clobber the exclusion). Atomic; no watermark/refcount/ + * message/kstate/tail touched. buildSession honors the list on next load. */ + updateSessionMetadataJson(sessionId: string, metadataJson: string): void { + this.db + .prepare("UPDATE sessions SET metadata_json = ?, updated_at = ? WHERE session_id = ?") + .run(metadataJson, Date.now(), sessionId); + } + /** #D5-5 render windowing: full rows (content + media) ONLY for seqs * outside the covered intervals. Inclusive [start, end] pairs; overlaps * are harmless (each pair contributes an outside-of-pair conjunct). diff --git a/tests/orphan-sweep.test.ts b/tests/orphan-sweep.test.ts new file mode 100644 index 0000000000..68372bd1f4 --- /dev/null +++ b/tests/orphan-sweep.test.ts @@ -0,0 +1,131 @@ +// #5308 (D2) orphan-block sweep. A collapsed block whose covered substrate was +// never part of the session's durable record (shared-id leftovers from the +// <=v0.1.208 era) reappears forever in the parent session: the #2695 e-gate +// exemption shelters it and the render side keeps it active on any covered-id +// collision. This pins BOTH directions of the discriminator (#3617): +// negative — a block whose covered ids ARE in the session's unified-store +// transcript is NEVER flagged (the dangerous false-positive dir); +// positive — a block whose covered ids have ZERO transcript overlap IS +// flagged, and under --fix is archived (excluded on reload) without +// touching the legit block; the archive is reversible and idempotent. +process.env.BILI_STORAGE_UNIFIED = "full"; +process.env.BILI_PERSIST_ZSTD = "0"; +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { CompressionBlock, CoreMessage } from "acp-kernel"; +import { createInitialState } from "acp-kernel"; +import { SessionStore } from "../src/persist.ts"; +import { applyOrphanSweep, detectOrphanBlocks } from "../src/orphan-sweep.ts"; + +const MESSAGES = Array.from({ length: 80 }, (_, i) => ({ + id: `raw${i}`, + role: (i % 2 ? "assistant" : "user") as "assistant" | "user", + text: "lorem ipsum dolor sit amet conseq ".repeat(8) + i, +})) as unknown as CoreMessage[]; + +function makeBlock(blockId: string, effectiveMessageIds: string[]): CompressionBlock { + return { + blockId, + runId: `run-${blockId}`, + tier: 1, + summary: "S".repeat(60), + directMessageIds: [...effectiveMessageIds], + effectiveMessageIds, + directBlockIds: [], + compressedTokens: 1000, + createdAt: Date.now(), + survivedCount: 0, + generation: "young", + active: true, + }; +} + +function makeStoredSession(id: string, lastMessages: CoreMessage[], blocks: CompressionBlock[] = []) { + return { + id, + meta: { protocol: "anthropic" }, + stats: { + requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, + cacheSamples: 0, lastInputTokens: 0, compressCreditTokens: 0, contextTokens: 0, + retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0, + }, + metadata: {}, + state: { ...createInitialState(), blocks }, + createdAt: Date.now(), + lastSeen: Date.now(), + blockContents: new Map(), + inFlight: 0, + persisted: false, + pendingRetrievals: [], + lastMessages, + lastMessagesFolded: false, + pluginSnapshot: undefined, + contentStore: undefined, + } as never; +} + +test("#5308 orphan-sweep: zero-overlap blocks are swept, overlapping folds never are", async () => { + const cfgDir = mkdtempSync(join(tmpdir(), "bili-orphan-cfg-")); + process.env.BILI_CONFIG_FILE = join(cfgDir, "no-config.json"); + const dir = mkdtempSync(join(tmpdir(), "bili-orphan-")); + const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + try { + // Seed the unified store with a real transcript (80 message rows). + await store.writeNow(makeStoredSession("sess-orphan", MESSAGES)); + const tids = store.unifiedSessionTranscriptMsgIds("sess-orphan"); + assert.ok(tids && tids.length > 0, "seeded session must have unified transcript rows"); + + // Legit block: covered ids ARE in the transcript. Foreign block: covered + // ids were never ingested into this session (the shared-id leftover). + const legit = makeBlock("b_legit", tids!.slice(0, 5)); + const foreign = makeBlock("b_foreign", ["orphan-fx-1", "orphan-fx-2"]); + const seeded = (await store.loadAll()).get("sess-orphan")!; + seeded.state.blocks = [legit, foreign]; + await store.writeNow(seeded); + + // NEGATIVE + POSITIVE gate: only the zero-overlap block is flagged. + const plan = await detectOrphanBlocks(store); + const flagged = new Set(plan.findings.map((f) => `${f.sessionId}:${f.blockId}`)); + assert.ok(flagged.has("sess-orphan:b_foreign"), "foreign block (zero transcript overlap) must be flagged"); + assert.ok(!flagged.has("sess-orphan:b_legit"), "legit fold (overlapping transcript) must NEVER be flagged"); + assert.equal(plan.performable, true, "full mode must be performable"); + + // DRY-RUN default: reports but mutates nothing. + const dry = await applyOrphanSweep(store, {}); + assert.equal(dry.dryRun, true); + assert.equal(dry.sweptBlocks, 0, "dry-run must not archive"); + assert.ok((await store.loadAll()).get("sess-orphan")!.state.blocks.some((b) => b.blockId === "b_foreign"), + "dry-run leaves the foreign block in place"); + + // PERFORM: archives only the foreign block. + const done = await applyOrphanSweep(store, { dryRun: false }); + assert.equal(done.dryRun, false); + assert.equal(done.sweptBlocks, 1, "exactly the one foreign block archived"); + assert.equal(done.touchedSessions, 1); + assert.ok(done.lines.some((l) => l.includes("[archived]") && l.includes("b_foreign")), "audit line names the archived block"); + + // Reload: foreign block gone, legit block untouched. + const reloaded = (await store.loadAll()).get("sess-orphan")!; + const ids = reloaded.state.blocks.map((b) => b.blockId); + assert.ok(!ids.includes("b_foreign"), "archived foreign block must not render into the session anymore"); + assert.ok(ids.includes("b_legit"), "legit fold must survive the sweep"); + + // IDEMPOTENT: nothing left to sweep after the archive. + const plan2 = await detectOrphanBlocks(store); + assert.ok(!plan2.findings.some((f) => f.blockId === "b_foreign"), "swept block must not be re-flagged"); + + // REVERSIBLE: clearing the exclusion restores the block (data was kept in kstate). + const meta = { ...(reloaded.metadata as Record) }; + delete meta.orphanExcludedBlocks; + store.setUnifiedSessionMetadataJson("sess-orphan", JSON.stringify(meta)); + const restored = (await store.loadAll()).get("sess-orphan")!; + assert.ok(restored.state.blocks.some((b) => b.blockId === "b_foreign"), "clearing the exclusion must restore the block"); + } finally { + store.closeQuietly(); + rmSync(dir, { recursive: true, force: true }); + rmSync(cfgDir, { recursive: true, force: true }); + } +}); From 978ab77886498a4c512ece0240262d2cc345f3fd Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 31 Oct 2026 18:09:57 +0800 Subject: [PATCH 3/3] fix: drop unused exports flagged by unused-exports gate (OrphanBlockFinding, OrphanSweepPlan) The two helper interfaces are only used inside src/orphan-sweep.ts; the module-boundary surface is detectOrphanBlocks/applyOrphanSweep/OrphanSweepResult. Self-introduced violation of the #2643-era gate on this PR branch (the other finding, CodexWindowSource, is inherited master red awaiting ci-fixes). --- src/orphan-sweep.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/orphan-sweep.ts b/src/orphan-sweep.ts index 6a35fd67f1..7c73a70fce 100644 --- a/src/orphan-sweep.ts +++ b/src/orphan-sweep.ts @@ -26,7 +26,7 @@ import type { SessionStore } from "./persist.js"; import type { Session } from "./session.js"; import { log as loggerLog } from "./logger.js"; -export interface OrphanBlockFinding { +interface OrphanBlockFinding { sessionId: string; blockId: string; coveredCount: number; @@ -34,7 +34,7 @@ export interface OrphanBlockFinding { coveredSample: string[]; } -export interface OrphanSweepPlan { +interface OrphanSweepPlan { storageMode: "full" | "shadow" | "legacy"; /** True only when --fix can durably apply (full unified storage). */ performable: boolean;