Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions LIFECYCLE.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
| `BILI_OPENCODE_INLINE` | #3278 opencode 移植(#4687) | opencode inline 模式(需手动开启,默认保持旧行为 spawn,owner 决定不改默认):与 pi 同构——不起 proxy 进程,模型 URL 被 V2 route/intercept 改写成伪 origin 后由进程内引擎接住(bun 宿主已验:引擎 boot/dispatch 全绿);`=1/on/yes/TRUE` 或 file 家 `"opencode": {"inline": true}` 显式开启(显式开启压过 attach 目标);`bili opencode-inline` 一键车道 | 同 pi inline 车道节奏 | opt-in(不设置时 attach/spawn 行为与旧版完全一致) |
| `BILI_INLINE_DEBUG` | #4669 | 仅 inline 车道:引擎 stderr 日志腿默认静默(stderr 是 pi 的交互终端,日志会挤掉输入框;默认只写 bili.log),设 1 恢复 stderr 输出用于排查 | 随 inline 车道(#3278)走 | 排查逃生口 |
| 测试钩子:`BILI_DSH_RECOVERY_INTERVAL_MS` / `BILI_DSH_RETRY_INTERVAL_MS` / `BILI_MODEL_INFO_RETRY_MS` | #2082 等 | 缩短测试/运维节奏(dsh-native.ts;NODE_TEST_CONTEXT 感知,避免生产误缩) | 永久测试设施,随宿主代码走 | 测试钩子 |
| `BILI_SESSION_CONTENT_STORE_PIN` | #4788 phase 2(c8d7a802) | idle-fold 逃生口:`=1/true` 让每会话 CCR content store({byRef}/{byHash} 原文)常驻内存、save 落盘后不折叠(默认折叠;读点 src/knobs.ts:451,store 可从 unified ccr mirror 或 .content-store.json sidecar 重新物化) | #4788 phase 2 idle-fold 生产站稳后评估退役(owner 拍板) | 在用(默认 fold) |

## 2. 常设面全量登记表(src/ 实际读取的其余 BILI_* 环境变量)

Expand All @@ -78,6 +79,7 @@
| `BILI_NON_HTTP_PROVIDERS` | #1392 非 http(s) baseUrl provider 白名单传递 |
| `BILI_PROVIDER_REWRITES` | #535 provider 改写 manifest JSON(存在即关 zcode native) |
| `BILI_UPGRADE_REPLACEMENT` | #3044 self-restart 替换进程交接标记(UPGRADE_REPLACEMENT_ENV) |
| `BILI_SPAWN_FINGERPRINT` | #4125 spawner 侧入口脚本 sha256 指纹(SPAWN_FINGERPRINT_ENV,src/instance.ts:343),spawn 时注入供子进程记录自身指纹,避免 auto-update 窗口内 re-hash 漂移 |

### 2.3 网络调优(#2030 迁移家族:env > 文件 network.* > 默认)

Expand Down
27 changes: 19 additions & 8 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* bili export [id] [--full] export a persisted session as a handoff doc
* (--jsonl: deterministic JSONL dump #2671)
* bili acp-cache diff <dir> attribute cache breaks from ACP_DUMP_BODY dumps
* bili sessions ls|rm|archive|restore [id] manage persisted sessions
* bili sessions ls|rm|archive|restore|orphan-sweep [id] manage persisted sessions
* bili storage gc [--days N] run the storage sweep now (--dry-run plans only)
* bili test pi non-polluting pi smoke test
* bili --version
Expand Down Expand Up @@ -89,6 +89,11 @@ Usage:
bili sessions archive <id> move one session out of index.db into the single
append-only archive log (rows swept, restorable)
bili sessions restore <id> bring an archived session back into index.db
bili sessions orphan-sweep [id] #5308: find collapsed blocks whose covered ids
have zero overlap with their own session's stored
transcript (provably foreign, e.g. shared-id leftovers
from <=v0.1.208). Dry-run by default; --fix archives
them (reversible). Needs full unified storage.
bili storage gc run the storage sweep now (#3911): archive sessions
older than storage.archiveAfterDays (default 365),
release unreferenced content, shrink index.db
Expand Down Expand Up @@ -200,9 +205,10 @@ type Parsed = {
acpCacheLog?: string;
acpCacheNoLog?: boolean;
acpCacheSession?: string;
sessionsSub?: "ls" | "rm" | "archive" | "restore";
sessionsSub?: "ls" | "rm" | "archive" | "restore" | "orphan-sweep";
sessionsTarget?: string;
sessionsNoVacuum?: boolean;
orphanSweepFix?: boolean;
storageDays?: number;
storageVacuum?: "full" | "incremental";
storageDryRun?: boolean;
Expand Down Expand Up @@ -239,6 +245,7 @@ export function parseArgs(argv: string[]): Parsed {
let jsonOutput = false;
let doctorJson = false;
let doctorFix = false;
let orphanSweepFix = false;

for (let i = 0; i < argv.length; i++) {
const a = argv[i]!;
Expand Down Expand Up @@ -307,7 +314,11 @@ export function parseArgs(argv: string[]): Parsed {
doctorJson = true;
break;
case "--fix":
// Shared "perform" token: doctor uses it for dsh-ledger repair,
// `sessions orphan-sweep` for archiving orphans. Commands are
// mutually exclusive, so each branch reads only its own flag.
doctorFix = true;
orphanSweepFix = true;
break;
case "--no-vacuum":
sessionsNoVacuum = true;
Expand Down Expand Up @@ -465,15 +476,15 @@ export function parseArgs(argv: string[]): Parsed {
} else if (cmd === "sessions") {
command = "sessions";
const sub = positional[1];
if (sub === "ls" || sub === "rm" || sub === "archive" || sub === "restore") {
if (sub === "ls" || sub === "rm" || sub === "archive" || sub === "restore" || sub === "orphan-sweep") {
sessionsSub = sub;
sessionsTarget = positional[2];
if (sub !== "ls" && sessionsTarget === undefined) {
if (sub !== "ls" && sub !== "orphan-sweep" && sessionsTarget === undefined) {
console.error(`bili sessions ${sub}: <id> is required (try "bili sessions ${sub} <session-id>")`);
process.exit(2);
}
} else {
console.error(`bili sessions: unknown action "${sub ?? ""}" (try "bili sessions ls|rm|archive|restore")`);
console.error(`bili sessions: unknown action "${sub ?? ""}" (try "bili sessions ls|rm|archive|restore|orphan-sweep")`);
process.exit(2);
}
} else if (cmd === "storage") {
Expand All @@ -488,11 +499,11 @@ export function parseArgs(argv: string[]): Parsed {
}
}

return { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix };
return { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, orphanSweepFix };
}

export async function main(): Promise<void> {
const { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix } = parseArgs(process.argv.slice(2));
const { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, orphanSweepFix } = parseArgs(process.argv.slice(2));
if (command === "help") {
process.stdout.write(HELP);
return;
Expand Down Expand Up @@ -621,7 +632,7 @@ export async function main(): Promise<void> {
// carry their own exit codes (1 = operation failed, 2 = usage).
try {
const r = command === "sessions"
? await runSessionsCommand({ sub: sessionsSub!, target: sessionsTarget, json: jsonOutput, noVacuum: sessionsNoVacuum })
? await runSessionsCommand({ sub: sessionsSub!, target: sessionsTarget, json: jsonOutput, noVacuum: sessionsNoVacuum, orphanSweepFix })
: await runStorageGcCommand({ days: storageDays, vacuum: storageVacuum, dryRun: storageDryRun, json: jsonOutput });
process.stdout.write(r.text);
if (r.code !== 0) process.exit(r.code);
Expand Down
185 changes: 185 additions & 0 deletions src/orphan-sweep.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
// #5308 (D2) — sweep provably-foreign collapsed blocks out of upgraded legacy
// sessions.
//
// A collapsed block committed under a SHARED conversation id during the
// <=v0.1.208 era can reappear forever in the parent session after an upgrade:
// its covered substrate was never part of the parent's durable record, so the
// #2695 e-gate exemption ("never-present substrate never accrues a death
// streak") gives it permanent shelter, while the render side (kernel sync.ts
// stillPresent) keeps it active whenever ANY covered id happens to collide with
// a message the parent actually read. No existing wire-side path reaps it
// (orphan-gc only looks at the CURRENT wire, where those collisions land).
//
// The discriminator here is STORE presence, which is independent of wire
// presence: a block whose covered ids have ZERO overlap with the session's
// unified-store transcript rows could never have been folded out of THIS
// session, so it is provably foreign. Legit folds always retain their covered
// rows (active OR retired — rewind/prune retire, never delete), so they are
// never swept. This is the safe half of option A from #5308; option B (blocks
// carry creation-time conversation-key lineage) is a separate kernel-side item.
//
// Dry-run by default; --fix archives via a metadata-backed exclusion list that
// buildSession honors on load (reversible, no kstate rewrite).

import { baseIdOf } from "acp-kernel";
import type { SessionStore } from "./persist.js";
import type { Session } from "./session.js";
import { log as loggerLog } from "./logger.js";

interface OrphanBlockFinding {
sessionId: string;
blockId: string;
coveredCount: number;
/** First few covered base ids, for reporting. */
coveredSample: string[];
}

interface OrphanSweepPlan {
storageMode: "full" | "shadow" | "legacy";
/** True only when --fix can durably apply (full unified storage). */
performable: boolean;
/** Sessions that had >=1 unified transcript row and were scanned. */
scannedSessions: number;
/** Sessions skipped because they have no unified transcript (criterion vacuous). */
skippedNoTranscript: number;
findings: OrphanBlockFinding[];
}

export interface OrphanSweepResult {
plan: OrphanSweepPlan;
dryRun: boolean;
sweptBlocks: number;
touchedSessions: number;
errors: string[];
lines: string[];
}

async function scan(store: SessionStore, loaded: Map<string, Session>): Promise<OrphanSweepPlan> {
const full = store.unifiedReads();
const replicas = store.replicasAvailable();
const storageMode: OrphanSweepPlan["storageMode"] = full ? "full" : replicas ? "shadow" : "legacy";
if (!replicas) {
return { storageMode, performable: false, scannedSessions: 0, skippedNoTranscript: 0, findings: [] };
}
const findings: OrphanBlockFinding[] = [];
let scanned = 0;
let skipped = 0;
for (const [id, session] of loaded) {
const msgIds = store.unifiedSessionTranscriptMsgIds(id);
// No unified transcript -> the zero-overlap criterion is vacuous (every
// block would look orphan). Skip rather than false-positive.
if (!msgIds || msgIds.length === 0) {
skipped++;
continue;
}
scanned++;
const transcriptBases = new Set<string>();
for (const mid of msgIds) transcriptBases.add(baseIdOf(mid));
const blocks = session.state?.blocks;
if (!blocks || blocks.length === 0) continue;
for (const block of blocks) {
const covered = block.effectiveMessageIds;
if (!covered || covered.length === 0) continue;
if (!block.active) continue;
let intersect = 0;
for (const cid of covered) {
if (transcriptBases.has(baseIdOf(cid))) intersect++;
}
if (intersect > 0) continue;
findings.push({
sessionId: id,
blockId: block.blockId,
coveredCount: covered.length,
coveredSample: covered.slice(0, 6).map((x) => baseIdOf(x)),
});
}
}
return { storageMode, performable: full, scannedSessions: scanned, skippedNoTranscript: skipped, findings };
}

/** Read-only detection: every active block whose covered ids have zero overlap
* with its own session's unified-store transcript. Never mutates. */
export async function detectOrphanBlocks(store: SessionStore): Promise<OrphanSweepPlan> {
const loaded = await store.loadAll();
return scan(store, loaded);
}

/** Detect, then (unless dryRun) durably archive the orphans via the metadata
* exclusion list. dryRun defaults to true — mutation requires an explicit
* `dryRun: false` AND full-mode storage. Optionally scoped to one session. */
export async function applyOrphanSweep(
store: SessionStore,
opts: { dryRun?: boolean; sessionId?: string },
): Promise<OrphanSweepResult> {
const dryRun = opts.dryRun !== false;
const loaded = await store.loadAll();
let plan = await scan(store, loaded);
if (opts.sessionId) {
plan = { ...plan, findings: plan.findings.filter((f) => f.sessionId === opts.sessionId) };
}
const lines: string[] = [];
const errors: string[] = [];

if (plan.findings.length === 0) {
lines.push(
`scanned ${plan.scannedSessions} session(s) with a unified transcript (${plan.storageMode}); ` +
`${plan.skippedNoTranscript} skipped (no unified transcript); no provably-orphan blocks found`,
);
return { plan, dryRun, sweptBlocks: 0, touchedSessions: 0, errors, lines };
}

if (dryRun) {
for (const f of plan.findings) {
lines.push(`[dry-run] ${f.sessionId} :: ${f.blockId} covers ${f.coveredCount} id(s), zero overlap with this session's transcript [${f.coveredSample.join(", ")}]`);
}
lines.push(`dry run only - nothing changed. Re-run with --fix to archive ${plan.findings.length} orphan block(s).`);
return { plan, dryRun: true, sweptBlocks: 0, touchedSessions: 0, errors, lines };
}

if (!plan.performable) {
for (const f of plan.findings) lines.push(`[blocked] ${f.sessionId} :: ${f.blockId} (covers ${f.coveredCount})`);
lines.push('storage is not in "full" mode - cannot durably archive here. Enable full unified storage (storage.unified=full / BILI_STORAGE_UNIFIED=full) and re-run --fix.');
return { plan, dryRun: false, sweptBlocks: 0, touchedSessions: 0, errors: ["not performable: storage not in full mode"], lines };
}

const bySession = new Map<string, OrphanBlockFinding[]>();
for (const f of plan.findings) {
const arr = bySession.get(f.sessionId) ?? [];
arr.push(f);
bySession.set(f.sessionId, arr);
}
let swept = 0;
const touched = new Set<string>();
const now = Date.now();
for (const [sid, fs] of bySession) {
const session = loaded.get(sid);
if (!session) {
errors.push(`${sid}: session vanished between scan and apply`);
continue;
}
try {
const curMeta: Record<string, unknown> =
session.metadata && typeof session.metadata === "object" ? { ...(session.metadata as Record<string, unknown>) } : {};
const prevRaw = curMeta.orphanExcludedBlocks;
const prev: Record<string, unknown> =
prevRaw && typeof prevRaw === "object" && !Array.isArray(prevRaw) ? { ...(prevRaw as Record<string, unknown>) } : {};
for (const f of fs) {
prev[f.blockId] = { at: now, reason: `#5308 orphan sweep: ${f.coveredCount} covered id(s), zero transcript overlap` };
}
curMeta.orphanExcludedBlocks = prev;
store.setUnifiedSessionMetadataJson(sid, JSON.stringify(curMeta));
touched.add(sid);
swept += fs.length;
for (const f of fs) lines.push(`[archived] ${sid} :: ${f.blockId} (covers ${f.coveredCount} id(s), zero transcript overlap)`);
loggerLog("info", `[orphan-sweep] ${sid}: archived ${fs.length} orphan block(s): ${fs.map((f) => f.blockId).join(", ")}`);
} catch (err) {
const detail = /SQLITE_BUSY|database is locked/i.test(String(err))
? "database busy (another bili instance holds index.db)"
: String(err);
errors.push(`${sid}: ${detail}`);
lines.push(`[error] ${sid}: could not persist exclusion - ${detail}`);
}
}
lines.push(`applied: archived ${swept} orphan block(s) across ${touched.size} session(s); takes effect on next load/restart of each affected client.`);
return { plan, dryRun: false, sweptBlocks: swept, touchedSessions: touched.size, errors, lines };
}
Loading