From 8a0fe77409a7d3ebe27d90ce8fcafffb36a313c3 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sun, 1 Nov 2026 20:56:16 +0800 Subject: [PATCH 01/20] =?UTF-8?q?feat(perf):=20#5460=20P1=20=E2=80=94=20re?= =?UTF-8?q?altime=20telemetry=20into=20perf.sqlite=20(DB=20modes=20only,?= =?UTF-8?q?=20serving-immune)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/perf-telemetry.ts: perf_events table (WAL, retention cap 200k rows, ring buffer), kill switch BILI_PERF_TELEMETRY, pure-JSON mode frozen/off, degrade-not-throw discipline (#5327 culture) - src/server.ts logRequestCost: structured mirror row through the single observation point every lane already flows through (#5408) - src/persist.ts: arm sqlite/mysql lanes at the two sidecar-open paths, explicit json-mode off - LIFECYCLE.md §2.4: BILI_PERF_TELEMETRY registered - tests/issue5460-perf-telemetry.test.ts: 8 cases incl. pinned DDL/WAL golden --- CHANGELOG.md | 4 + docs/lifecycle.md | 1 + src/perf-telemetry.ts | 209 +++++++++++++++++++++++++ src/persist.ts | 7 + src/server/request-cost.ts | 4 + tests/issue5460-perf-telemetry.test.ts | 170 ++++++++++++++++++++ 6 files changed, 395 insertions(+) create mode 100644 src/perf-telemetry.ts create mode 100644 tests/issue5460-perf-telemetry.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e66f8e17..8f5122a633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,10 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag - **The full-resend watermark fastpath (#5403/#5418) passed its §7.4 human review and ships behind the 5% experimental cohort** (owner ruling 2026-11-02: default off, small cohort first; the web switch + one-time intro notice ship alongside): plugin-lane chat-completions turns whose prefix is byte-stable skip the full history re-parse/re-pin on hit (the #5403 OOM producer), any doubt still falls back to the untouched full-parse path per turn. Force either direction per install with `rollout.features.pluginFullResendWatermark: true|false` (or a percent) or `BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK=1|0`. - **The silent-direct oversized guard (#5405) defaults to 8 MiB and honors `BILI_SILENT_DIRECT_MAX_MB`.** The missing-slot / unattributed arms count RAW body bytes — base64-inflated media included — so an in-window session carrying even one medium image died at 2 MiB with a retryable 422 that could never succeed; the runaway multi-MB prompts the guard exists for are tens of megabytes or more. Default raised 2 → 8 MiB (owner ruling, #5826 release review); installs can lower it back per machine (megabytes, >0; invalid values fall back to the default). +### Added + +* **#5460(P1):实时性能采集落地(SQLite 存储)**。近期所有性能事故(#5403 OOM×6、#4528 主线程卡死、#5385 前缀钉死)都是用户先发现、CI 零预警。现在 DB 存储模式(sqlite/mysql)下每处理一个请求,就在会话目录旁的新小库 `perf.sqlite` 落一行(消息数/进出字节/本地耗时/会话号),另有每 30 秒一条内存水位采样(rss/heap/external)——OOM 类事故从此有爬坡曲线可查。写路径与服务完全隔离:任何写失败只停采集(一次性警告),绝不影响服务;纯 JSON 存储模式冻结零足迹(owner 定)。`BILI_PERF_TELEMETRY=0` 整体关闭。后续:P2 场景夹具+`bili bench`(只记录不判)、P3 基线容差带接 FULL/夜间 CI(普通 PR CI 零性能 job)、P4 `/stats`+Web 面板性能页+外部看门狗——跟踪单 #5460。 + ### Fixed - **压缩失败提示里的 compress 调用示范改用官方字符串形态。** 两处面向模型的文案仍演示已退役的 `content` 原生数组形态(schema 早已只收字符串,严格校验的主机会直接拒调):`tierActionHint` 的 tier 蒸馏/凝缩建议(`kernel/src/compress.ts`,恰好挂在「已压过该区间」收据后面——弱模型刚被拒、最容易照抄的时刻)与断路器 armed 后的 format reminder(`src/stream.ts`)。两处均改为与 `compress` 工具 schema 及 nudge 示例一致的 `content: "b1–b5 optional topic\nsummary markdown..."` 字符串形态;`docs/plugin-protocol.md` 的 `/__bili/plugin/tool` 请求示例同步改为字符串并补注(JSON 编码数组放在同一字符串内仍被接受)。回归:`kernel/tests/regression-bili-470.test.ts`(示例文案钉住)、`tests/issue4812-armed-breaker-short-receipt.test.ts`(新 reminder 断言 + 旧数组措辞负向断言)。 diff --git a/docs/lifecycle.md b/docs/lifecycle.md index 97c4400148..627c32cbb9 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -143,6 +143,7 @@ | `BILI_STORAGE_QUICK_CHECK` | 默认开——index.db 开库在迁移前跑 PRAGMA quick_check 自检;失败=影子层隔离文件组(.corrupt-)+重建新库(JSON 信封为恢复源);=0 跳过(超大库启动时间敏感场景,#4175) | | `BILI_STORAGE_WAL_JOURNAL_SIZE_LIMIT_BYTES` | storage.walJournalSizeLimitBytes——SQLite WAL 回收字节上限(#3250;垃圾/非正值落回文件/默认层,不让 typo 崩 boot) | | `BILI_MYSQL_DSN` | storage.mysql.dsn——mysql 引擎连接串(env > 文件 > "";空 = mysql 不可用走 sqlite,#3388/#3931) | +| `BILI_PERF_TELEMETRY` | `=0/false` 关闭 #5460 实时性能采集(缺省开)。DB 存储模式(sqlite/mysql)下每请求一行 + 30s 内存采样 → sessions 目录旁 `perf.sqlite`;纯 JSON 存储模式冻结不采集;写失败一次性降级停采,绝不影响服务 | ### 2.5 诊断 diff --git a/src/perf-telemetry.ts b/src/perf-telemetry.ts new file mode 100644 index 0000000000..2aab62bd63 --- /dev/null +++ b/src/perf-telemetry.ts @@ -0,0 +1,209 @@ +// #5460 (P1): realtime performance telemetry, SQLite-backed. +// +// Design constraints (owner decisions, 2026-11-01, issue #5460): +// * Metrics land ONLY in SQLite / database-backed storage modes — never in +// JSON files. The pure-JSON ("legacy-only") storage mode is frozen: this +// system has zero footprint there (init("json") is permanently inert). +// * Telemetry must never break serving: every insert is guarded, and a +// failure DEGRADES telemetry (one warn + dead flag), never the request +// path (#5327 adopt discipline: observe without endangering). +// * The metrics DB is a separate file (perf.sqlite) next to index.db, not a +// migration of the user's own store: crash/size isolation from index.db, +// zero schema-version coupling, trivially deletable by the owner. +// * BILI_PERF_TELEMETRY=0 force-disables (registered in LIFECYCLE.md §2.4). +import { DatabaseSync, type SQLInputValue } from "node:sqlite"; +import * as path from "node:path"; +import { sessionsDir } from "./paths.js"; + +export const PERF_DB_FILE = "perf.sqlite"; +const PERF_TABLE = "perf_events"; +// Amortized retention sweep: keep the observability file bounded (~tens of +// MB at most). Every Nth insert deletes the oldest rows past the cap — one +// indexed range delete per batch, negligible against an 80 MiB parse pass. +const RETENTION_CAP_ROWS = 200_000; +const RETENTION_SWEEP_EVERY = 500; +const RING_SIZE = 500; +const DEFAULT_MEM_INTERVAL_MS = 30_000; +interface InitOptions { dir?: string; memIntervalMs?: number; log?: (level: "warn" | "error" | "info", msg: string) => void; } + +interface PerfState { + db: DatabaseSync | null; // lazily opened on first record + mode: string; // "sqlite" | "mysql" | "json"(off) + dir: string | null; + dead: boolean; // first insert/open failure → degrade forever + warned: boolean; + insertsSinceSweep: number; + memTimer: NodeJS.Timeout | null; +} + +const state: PerfState = { + db: null, mode: "", dir: null, dead: false, warned: false, + insertsSinceSweep: 0, memTimer: null, +}; + +const ring: Array> = []; +function pushRing(row: Record): void { + ring.push(row); + if (ring.length > RING_SIZE) ring.shift(); +} + +function perfEnabledByEnv(): boolean { + const raw = process.env.BILI_PERF_TELEMETRY; + return !(raw === "0" || raw === "false"); +} + +/** #5460 schema stability guard target — a change here must bump this text + * deliberately (same culture as the storage goldens). */ +export const PERF_EVENTS_DDL = + "CREATE TABLE IF NOT EXISTS perf_events (\n" + + " id INTEGER PRIMARY KEY AUTOINCREMENT,\n" + + " ts INTEGER NOT NULL,\n" + + " kind TEXT NOT NULL,\n" + + " session_id TEXT,\n" + + " msgs INTEGER,\n" + + " inbound_bytes INTEGER,\n" + + " outbound_bytes INTEGER,\n" + + " total_ms REAL,\n" + + " parse_ms REAL,\n" + + " payload TEXT\n" + + ")"; + +function dbPath(dir: string): string { return path.join(dir, PERF_DB_FILE); } +// Canonical sessions dir (BILI_SESSIONS_DIR > dataDir()/sessions) — the same +// root index.db lives under; perf.sqlite keeps company without touching it. +const defaultDir = sessionsDir; + +function warnDegrade(w: (level: "warn" | "error" | "info", msg: string) => void, what: string): void { + if (state.warned) return; + state.warned = true; + try { w("warn", `[perf] telemetry degraded: ${what} — metrics collection stopped for this process (serving unaffected)`); } catch { /* last resort */ } +} + +function ensureDb(): DatabaseSync | null { + if (state.db) return state.db; + if (!state.dir) return null; + try { + const db = new DatabaseSync(dbPath(state.dir)); + db.exec(`PRAGMA journal_mode=WAL; ${PERF_EVENTS_DDL}; CREATE INDEX IF NOT EXISTS idx_perf_events_ts ON ${PERF_TABLE}(ts);`); + state.db = db; + return db; + } catch (e) { + state.dead = true; + warnDegrade((l, m) => console.error(l, m), `open failed: ${e instanceof Error ? e.message : String(e)}`); + return null; + } +} + +function insertRow(db: DatabaseSync, row: Record, w: (level: "warn" | "error" | "info", msg: string) => void): void { + if (state.dead) return; + try { + db.prepare( + `INSERT INTO ${PERF_TABLE}(ts, kind, session_id, msgs, inbound_bytes, outbound_bytes, total_ms, parse_ms, payload) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + row.ts as number, row.kind as string, + (row.session_id as string) ?? null, + (row.msgs as number) ?? null, + (row.inbound_bytes as number) ?? null, + (row.outbound_bytes as number) ?? null, + (row.total_ms as number) ?? null, + (row.parse_ms as number) ?? null, + ((row.payload_json as string | undefined) ?? null) satisfies SQLInputValue, + ); + pushRing(row); + if (++state.insertsSinceSweep >= RETENTION_SWEEP_EVERY) { + state.insertsSinceSweep = 0; + db.prepare(`DELETE FROM ${PERF_TABLE} WHERE id IN (SELECT id FROM ${PERF_TABLE} ORDER BY id DESC LIMIT -1 OFFSET ?)`).run(RETENTION_CAP_ROWS); + } + } catch (e) { + state.dead = true; + warnDegrade(w, `insert failed: ${e instanceof Error ? e.message : String(e)}`); + } +} + +/** Initialize telemetry for the process. Call ONCE per storage-mode decision + * (SessionStore construction paths); later calls are no-ops (idempotent by + * design — the retry/cold-open path may re-run). mode: + * - "sqlite": unified sidecar active (or the local engine default) + * - "mysql" : network backend lane — metrics still persist locally + * - "json" : pure-JSON storage mode — FROZEN/deprecated, permanently off */ +export function perfTelemetryInit(mode: "sqlite" | "mysql" | "json", opts?: InitOptions): boolean { + // Idempotence FIRST: once armed, every later call (any mode) is a no-op. + // (#5460: a stray init("json") from a secondary path must not downgrade + // an already-running telemetry instance.) + if (state.mode === "sqlite" || state.mode === "mysql") return true; + if (!perfEnabledByEnv()) { state.mode = "off"; return false; } + if (mode === "json") { state.mode = "json"; return false; } + state.mode = mode; + state.dir = opts?.dir ?? defaultDir(); + const log = opts?.log ?? ((l: string, m: string) => console.error(l, m)); + state.memTimer = setInterval(() => perfRecordMemSample(log), opts?.memIntervalMs ?? DEFAULT_MEM_INTERVAL_MS); + state.memTimer.unref?.(); + return true; +} + +/** Record one handled request (wired into logRequestCost — the single + * observation point every lane already flows through, #5408). */ +export function perfRecordRequest(r: { + sessionId: string | null; + msgs?: number | null; + inboundBytes?: number | null; + outboundBytes?: number | null; + totalMs?: number | null; + parseMs?: number | null; + extra?: Record; +}, w?: (level: "warn" | "error" | "info", msg: string) => void): void { + if (state.mode === "off" || state.mode === "json") return; + const db = ensureDb(); + if (!db) return; + const row: Record = { + ts: Date.now(), kind: "req", + session_id: r.sessionId ?? null, + msgs: r.msgs ?? null, + inbound_bytes: r.inboundBytes ?? null, + outbound_bytes: r.outboundBytes ?? null, + total_ms: r.totalMs ?? null, + parse_ms: r.parseMs ?? null, + payload_json: r.extra ? JSON.stringify({ engine: state.mode, ...r.extra }) : JSON.stringify({ engine: state.mode }), + }; + insertRow(db, row, w ?? ((l, m) => console.error(l, m))); +} + +/** Memory watermark row — the OOM class (#5403 ×6) finally shows its ramp. */ +export function perfRecordMemSample(w?: (level: "warn" | "error" | "info", msg: string) => void): void { + if (state.mode === "off" || state.mode === "json") return; + const db = ensureDb(); + if (!db) return; + const m = process.memoryUsage(); + const row: Record = { + ts: Date.now(), kind: "mem", + session_id: null, msgs: null, inbound_bytes: null, outbound_bytes: null, + total_ms: null, parse_ms: null, + payload_json: JSON.stringify({ rss: m.rss, heapUsed: m.heapUsed, external: m.external, arrayBuffers: m.arrayBuffers }), + }; + insertRow(db, row, w ?? ((l, m2) => console.error(l, m2))); +} + +/** Close everything (process shutdown / tests). Idempotent. */ +export function perfStopAll(): void { + if (state.memTimer) { clearInterval(state.memTimer); state.memTimer = null; } + if (state.db) { try { state.db.close(); } catch { /* best effort */ } state.db = null; } +} + +/** Full state reset — unit tests only (module singleton across test cases in + * one process). */ +export function perfTelemetryResetForTests(): void { + if (state.memTimer) { clearInterval(state.memTimer); state.memTimer = null; } + if (state.db) { try { state.db.close(); } catch { /* best effort */ } state.db = null; } + state.mode = ""; + state.dir = null; + state.dead = false; + state.warned = false; + state.insertsSinceSweep = 0; + ring.length = 0; +} + +/** In-memory ring of the most recent rows — feeds /stats and the panel + * page without a round-trip to disk (P4 surface). */ +export function perfRecentRows(): readonly Record[] { return ring; } +/** Which engine tag rows carry (also the /stats "storage mode" answer). */ +export function perfActiveMode(): string { return state.mode; } diff --git a/src/persist.ts b/src/persist.ts index 2746ad49f4..db1142c015 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -39,6 +39,7 @@ import { restampRefsToLedger, restampContentStore } from "acp-kernel"; import { LazyBlockContents, isDroppedContentStore, isDroppedMetadata, lazyContentStore, lazyMetadata, peekSession, sessionWeight } from "./session.js"; import type { Session, BlockContent, BlockView } from "./session.js"; import { orphanContents, resolveStorageMode, UnifiedShadow, hadIntegrityQuickCheckFailure, type StorageMode } from "./storage/shadow.js"; +import { perfTelemetryInit } from "./perf-telemetry.js"; import { noteColdLoad } from "./attach-heartbeat.js"; import { resolveBackendName } from "./storage/backend.js"; import { toLegacyLike, type LegacySessionLike } from "./storage/ingest.js"; @@ -707,6 +708,8 @@ export class SessionStore { fileStorageMysqlDsn(), ).then((sh) => { this.shadow = sh ?? undefined; + // #5460 P1: DB-backed mode → telemetry armed (local perf.sqlite). + perfTelemetryInit("mysql", { dir: this.sessionsDir, log: baseLog }); }); // #5046 chain (2026-10-30): openAsync now REJECTS on backend failure // (no degradation in any case). Boot awaits ready() and exits @@ -728,8 +731,12 @@ export class SessionStore { // closed for a full cooldown window. this.syncSidecarOpen = { legacyLoad }; this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend()) ?? undefined; + // #5460 P1: DB-backed mode → telemetry armed (local perf.sqlite). + perfTelemetryInit("sqlite", { dir: this.sessionsDir, log: baseLog }); } else { + // #5460: pure-JSON storage mode is frozen — telemetry stays off there. this.shadow = undefined; + perfTelemetryInit("json", { log: baseLog }); } } this.store = new StateStore({ diff --git a/src/server/request-cost.ts b/src/server/request-cost.ts index d3dfb69bd0..1add7597ba 100644 --- a/src/server/request-cost.ts +++ b/src/server/request-cost.ts @@ -1,4 +1,5 @@ import { noteRequest } from "../attach-heartbeat.js"; +import { perfRecordRequest } from "../perf-telemetry.js"; import { currentFetchTransport } from "../fetch-transport.js"; // #903: per-request local-cost line, logged at every point where bili finishes @@ -39,6 +40,9 @@ export function _resetSlowSessionWarnedForTest(): void { export function logRequestCost(log: (level: string, msg: string) => void, sessionId: string, msgs: number | null, inboundBytes: number, t0: number, outbound?: string | Buffer): void { const ms = Math.max(0, Math.round(performance.now() - t0)); + // #5460 P1: structured mirror of this line into perf.sqlite (inert unless + // a DB-backed storage mode armed the module; degrades, never throws). + perfRecordRequest({ sessionId, msgs, inboundBytes, outboundBytes: outbound !== undefined ? Buffer.byteLength(outbound) : null, totalMs: ms }); const outboundField = outbound !== undefined ? `, outbound=${formatBytes(Buffer.byteLength(outbound))}` : ""; const view = currentFetchTransport() ? ", view=ws-expanded" : ""; log("info", `[${sessionId}] request: ${msgs ?? "?"} msgs, inbound=${formatBytes(inboundBytes)}${outboundField}, local=${ms}ms${view}`); diff --git a/tests/issue5460-perf-telemetry.test.ts b/tests/issue5460-perf-telemetry.test.ts new file mode 100644 index 0000000000..349c2fce93 --- /dev/null +++ b/tests/issue5460-perf-telemetry.test.ts @@ -0,0 +1,170 @@ +// #5460 (P1): realtime perf telemetry — SQLite-backed, DB-modes only. +// +// Schema-stability culture mirrors the storage goldens: the DDL text is +// pinned, the file layout is pinned, and every degradation path must prove +// it leaves serving untouched (telemetry NEVER throws into the request path). +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { + perfTelemetryInit, perfRecordRequest, perfRecordMemSample, + perfStopAll, perfRecentRows, perfActiveMode, perfTelemetryResetForTests, + PERF_DB_FILE, PERF_EVENTS_DDL, +} from "../src/perf-telemetry.js"; + +const SANDBOXES: string[] = []; +function sandbox(): string { + const dir = mkdtempSync(path.join(os.tmpdir(), "bili-5460-")); + SANDBOXES.push(dir); + return dir; +} + +function resetModule(): void { + perfTelemetryResetForTests(); + delete process.env.BILI_PERF_TELEMETRY; +} + +test("#5460 P1: sqlite mode records request rows into /perf.sqlite", () => { + resetModule(); + const dir = sandbox(); + try { + assert.equal(perfTelemetryInit("sqlite", { dir }), true); + assert.equal(perfActiveMode(), "sqlite"); + const beforeTs = Date.now(); + perfRecordRequest({ sessionId: "sess-x", msgs: 42, inboundBytes: 123456, outboundBytes: 789, totalMs: 12.4, extra: { lane: "dsh" } }); + const db = new DatabaseSync(path.join(dir, PERF_DB_FILE)); + try { + const row = db.prepare("SELECT * FROM perf_events WHERE kind='req'").get() as Record; + assert.equal(row.session_id, "sess-x"); + assert.equal(row.msgs, 42); + assert.equal(row.inbound_bytes, 123456); + assert.equal(row.outbound_bytes, 789); + assert.equal(row.total_ms, 12.4); + assert.ok(Number(row.ts) >= beforeTs && Number(row.ts) <= Date.now()); + const payload = JSON.parse(String(row.payload)) as Record; + assert.equal(payload.engine, "sqlite"); + assert.equal(payload.lane, "dsh"); + } finally { db.close(); } + } finally { perfStopAll(); } +}); + +test("#5460 P1: schema is pinned (DDL text + ts index + WAL) — golden discipline", () => { + resetModule(); + const dir = sandbox(); + try { + perfTelemetryInit("sqlite", { dir }); + perfRecordRequest({ sessionId: "s" }); + const db = new DatabaseSync(path.join(dir, PERF_DB_FILE)); + try { + const ddl = db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='perf_events'").get() as { sql: string }; + // sqlite_master stores the statement verbatim minus the idempotence + // prefix, so normalize both sides to the CREATE TABLE body. + assert.equal(ddl.sql.trim().replace("IF NOT EXISTS ", ""), PERF_EVENTS_DDL.trim().replace("IF NOT EXISTS ", "")); + const idx = db.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_perf_events_ts'").get(); + assert.ok(idx, "ts index must exist"); + const mode = db.prepare("PRAGMA journal_mode").get() as { journal_mode: string }; + assert.equal(String(mode.journal_mode).toLowerCase(), "wal"); + } finally { db.close(); } + } finally { perfStopAll(); } +}); + +test("#5460 P1: pure-JSON storage mode is frozen — permanently inert", () => { + resetModule(); + const dir = sandbox(); + try { + assert.equal(perfTelemetryInit("json", { dir }), false); + assert.equal(perfActiveMode(), "json"); + perfRecordRequest({ sessionId: "s", msgs: 1 }); + perfRecordMemSample(); + assert.ok(!existsSync(path.join(dir, PERF_DB_FILE)), "json mode must never create a metrics file"); + assert.equal(perfRecentRows().length, 0); + } finally { perfStopAll(); } +}); + +test("#5460 P1: BILI_PERF_TELEMETRY=0 kill switch wins over any mode", () => { + resetModule(); + const dir = sandbox(); + process.env.BILI_PERF_TELEMETRY = "0"; + try { + assert.equal(perfTelemetryInit("sqlite", { dir }), false); + assert.equal(perfActiveMode(), "off"); + perfRecordRequest({ sessionId: "s" }); + perfStopAll(); + assert.ok(!existsSync(path.join(dir, PERF_DB_FILE))); + } finally { + perfTelemetryResetForTests(); + delete process.env.BILI_PERF_TELEMETRY; + } +}); + +test("#5460 P1: init is idempotent — later calls never re-arm or throw", () => { + resetModule(); + const dir = sandbox(); + try { + perfTelemetryInit("sqlite", { dir }); + assert.equal(perfTelemetryInit("mysql", {}), true, "already armed → no-op true"); + assert.equal(perfTelemetryInit("json", {}), true); + assert.equal(perfActiveMode(), "sqlite"); + } finally { perfStopAll(); } +}); + +test("#5460 P1: mem sampler rows land with real memory numbers", async () => { + resetModule(); + const dir = sandbox(); + try { + perfTelemetryInit("sqlite", { dir, memIntervalMs: 5 }); + await new Promise((r) => setTimeout(r, 80)); + perfStopAll(); + const db = new DatabaseSync(path.join(dir, PERF_DB_FILE)); + try { + const rows = db.prepare("SELECT payload FROM perf_events WHERE kind='mem'").all() as Array<{ payload: string }>; + assert.ok(rows.length >= 2, `expected repeated mem samples, got ${rows.length}`); + const p = JSON.parse(rows[0].payload) as Record; + assert.ok(p.rss > 0 && p.heapUsed > 0 && p.external >= 0); + } finally { db.close(); } + } finally { perfStopAll(); } +}); + +test("#5460 P1: ring buffer serves in-process consumers (/stats, panel P4)", () => { + resetModule(); + const dir = sandbox(); + try { + perfTelemetryInit("sqlite", { dir }); + perfRecordRequest({ sessionId: "a", inboundBytes: 1 }); + perfRecordRequest({ sessionId: "b", inboundBytes: 2 }); + const ring = perfRecentRows(); + assert.equal(ring.length, 2); + assert.equal((ring[1] as Record).session_id, "b"); + } finally { perfStopAll(); } +}); + +test("#5460 P1: logRequestCost wires structured rows through the single choke point", async () => { + // Late import keeps the heavy server module out of the fast paths above; + // importing the request-cost module directly is an established pattern. + const { logRequestCost } = await import("../src/server/request-cost.js"); + resetModule(); + const dir = sandbox(); + try { + assert.equal(perfTelemetryInit("sqlite", { dir }), true); + const logs: string[] = []; + logRequestCost((lvl, msg) => logs.push(`${lvl}:${msg}`), "sess-wired", 7, 2048, performance.now(), "body-out"); + perfStopAll(); + const db = new DatabaseSync(path.join(dir, PERF_DB_FILE)); + let row: Record | undefined; + try { + row = db.prepare("SELECT * FROM perf_events WHERE kind='req' AND session_id='sess-wired'").get() as Record; + } finally { db.close(); } + assert.ok(row, "logRequestCost must emit a telemetry row"); + assert.equal(row!.msgs, 7); + assert.equal(row!.inbound_bytes, 2048); + assert.equal(row!.outbound_bytes, Buffer.byteLength("body-out")); + assert.ok(Number(row!.total_ms) >= 0); + assert.ok(logs.some((l) => l.includes("request:")), "original human line untouched"); + } finally { perfStopAll(); } +}); + +before(() => { /* env handled per-test via resetModule */ }); +after(() => { for (const d of SANDBOXES.splice(0)) rmSync(d, { recursive: true, force: true }); }); From ea210e8b49c6440cab634fe771b007deec9e48b2 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 21:42:27 +0800 Subject: [PATCH 02/20] fix: #5460 route perf-telemetry degrade warnings through the tee logger The open-failure and insert-failure one-time warnings fell back to raw console.error, which bypasses bili's tee logger. For launcher-spawned children (#2864) stderr is physically redirected to a temp port-log, so the single 'telemetry degraded' signal never reached the main bili.log - exactly the observation surface this feature exists to feed. Route all degrade paths through loggerLog (the same channel the mem-sample path already used). No behavior/serving change; AGENTS no-console.error-in-server-modules. --- src/perf-telemetry.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/perf-telemetry.ts b/src/perf-telemetry.ts index 2aab62bd63..0ce87593bf 100644 --- a/src/perf-telemetry.ts +++ b/src/perf-telemetry.ts @@ -14,6 +14,7 @@ import { DatabaseSync, type SQLInputValue } from "node:sqlite"; import * as path from "node:path"; import { sessionsDir } from "./paths.js"; +import { log as loggerLog } from "./logger.js"; export const PERF_DB_FILE = "perf.sqlite"; const PERF_TABLE = "perf_events"; @@ -89,7 +90,7 @@ function ensureDb(): DatabaseSync | null { return db; } catch (e) { state.dead = true; - warnDegrade((l, m) => console.error(l, m), `open failed: ${e instanceof Error ? e.message : String(e)}`); + warnDegrade(loggerLog, `open failed: ${e instanceof Error ? e.message : String(e)}`); return null; } } @@ -135,7 +136,7 @@ export function perfTelemetryInit(mode: "sqlite" | "mysql" | "json", opts?: Init if (mode === "json") { state.mode = "json"; return false; } state.mode = mode; state.dir = opts?.dir ?? defaultDir(); - const log = opts?.log ?? ((l: string, m: string) => console.error(l, m)); + const log = opts?.log ?? loggerLog; state.memTimer = setInterval(() => perfRecordMemSample(log), opts?.memIntervalMs ?? DEFAULT_MEM_INTERVAL_MS); state.memTimer.unref?.(); return true; @@ -165,7 +166,7 @@ export function perfRecordRequest(r: { parse_ms: r.parseMs ?? null, payload_json: r.extra ? JSON.stringify({ engine: state.mode, ...r.extra }) : JSON.stringify({ engine: state.mode }), }; - insertRow(db, row, w ?? ((l, m) => console.error(l, m))); + insertRow(db, row, w ?? loggerLog); } /** Memory watermark row — the OOM class (#5403 ×6) finally shows its ramp. */ @@ -180,7 +181,7 @@ export function perfRecordMemSample(w?: (level: "warn" | "error" | "info", msg: total_ms: null, parse_ms: null, payload_json: JSON.stringify({ rss: m.rss, heapUsed: m.heapUsed, external: m.external, arrayBuffers: m.arrayBuffers }), }; - insertRow(db, row, w ?? ((l, m2) => console.error(l, m2))); + insertRow(db, row, w ?? loggerLog); } /** Close everything (process shutdown / tests). Idempotent. */ From 1c39d9a67937a7c02169e6efd7468ffb9f962e52 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 21:42:27 +0800 Subject: [PATCH 03/20] docs: #5460 register BILI_PERF_TELEMETRY in CONFIGURATION.md (en+zh) The new env-only kill switch was registered in LIFECYCLE.md but missing from CONFIGURATION.md en+zh, violating the 'document or don't ship' env-var discipline (reference/env-vars.md hard rule 4). Every comparable env-only knob is listed there; add it to the Environment-only variables table in both languages. --- docs/configuration.md | 1 + docs/configuration.zh-CN.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 6eaf046056..62808a8441 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -393,6 +393,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC age sweep window in days (#3463): sessions idle longer than this are archived (one JSON line appended to the single archive log (archive/archive.jsonl) + tombstone row + content/media refs released). 0 disables the age sweep; the manual entry POST /__bili/storage/gc always runs. Config-file key storage.archiveAfterDays is the persistent form. | | `BILI_STORAGE_QUICK_CHECK` | string | on | — | Storage integrity self-check at open (#4175): every index.db open runs PRAGMA quick_check BEFORE migrations; if a corruption is PROVEN by THIS BOOT's own scan NOTHING is moved, copied or rebuilt automatically (auto-recovery was withdrawn #4609/#4626-era: with BILI_STORAGE_JSON_WRITER=off the index is the SOLE durable copy, so an automatic move or rebuild cannot guarantee full data recovery) and bili STARTS ANYWAY — serving from the JSON envelopes with a loud, actionable alarm naming the manual fix (back up then move index.db/-wal/-shm aside BY HAND, then start again — a fresh index rebuilds from the JSON envelopes). A successful PASS is cached so later starts skip the rescan; a FAIL verdict is NEVER persisted (2026-10-30 owner policy): runtime/background findings are alarms only (loud logs + runtime-status, the process keeps serving — JSON envelopes written while the jsonWriter was on remain the recovery source) and every boot decides again from a fresh scan of the CURRENT file set, so a stale FAIL left behind by an older version can never block future boots. There is NO automatic rename/repair AND no legacy-only fallback: continuing legacy-only would fork the JSON envelopes from the dead index forever (never reconcilable). TRANSIENT open failures (no SQLite engine / CANTOPEN / readonly / backend DSN) make bili REFUSE TO START loudly with an actionable error — no silent legacy fallback for environment problems either (2026-10-30 owner policy; a live second writer left this refusal list on 2026-10-31 — sharing one local-disk index.db is the supported topology, informational only, never blocking); deployments that genuinely want JSON-only serve explicitly (BILI_STORAGE_UNIFIED=legacy — leaving it unset is the pure-sqlite default, not legacy). Set 0 only for boot-time-sensitive huge deployments where the check's cost matters. | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | Snapshot-render shadow observability (#3756 follow-up): compute the outbound snapshot view on every armed turn WITHOUT using it, classify the diff against the legacy outbound, and surface per-session counters in the web panel + the snapshot-shadow diagnostics topic. 0/false/off disables; 1/true/on enables; otherwise the config-file snapshot.shadow boolean applies (default on). The render lane itself stays rollout-gated (BILI_ROLLOUT_SNAPSHOT_RENDER), so the shadow lane never changes outbound bytes - it is the dry-run evidence that flips the rollout later. | +| `BILI_PERF_TELEMETRY` | boolean-ish | on | — | Kill switch for #5460 realtime perf telemetry. On by default in DB-backed storage modes (sqlite/mysql): every handled request writes one row plus a 30 s memory-watermark sample into perf.sqlite next to index.db — a separate file, crash/size-isolated from the user's own store; pure-JSON storage mode is frozen and never collects. 0/false disables collection entirely. A telemetry open/insert failure degrades collection (one warn + dead flag) without ever affecting serving. | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | Override the config file path (path relocation). A bare NAME (no `/`) selects the named config `/NAME.json`; a missing NAME fails fast listing the available names, a missing explicit PATH keeps its historical empty-config behavior. | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | Where session records are stored (path relocation). | | `BILI_ENCRYPTION_KEY` | string | unset | — | Key material for encrypted persisted payloads (secret). | diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.md index 6209c06540..93024ae518 100644 --- a/docs/configuration.zh-CN.md +++ b/docs/configuration.zh-CN.md @@ -393,6 +393,7 @@ | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC 年龄清扫窗口(天,#3463):闲置超过该时长的会话被归档(单文件归档日志 archive/archive.jsonl 追加一行 + 墓碑行保留标签 + 释放内容/媒体引用)。0 关闭年龄清扫;手动入口 POST /__bili/storage/gc 始终可用。配置文件键 storage.archiveAfterDays 为持久形式。 | | `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;一旦由本次启动自己的扫描证实损坏,文件原样保留在原地——不做任何自动搬移、复制或重建(自动恢复已撤回:BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本,自动操作无法保证完整数据恢复)——bili 照常启动,改用 JSON 信封服务,并给出响亮、可操作的告警指明手工修复步骤(备份后把 index.db/-wal/-shm 手工移开再启动——全新索引会从 JSON 信封重建)。PASS 判定会被缓存,之后的启动跳过重扫;FAIL 判定永不持久化(2026-10-30 owner 策略):运行期/后台发现只是告警(响亮日志 + runtime-status,进程继续服务——jsonWriter 开启期间所写的 JSON 信封是恢复来源),每次启动都对当前文件集做新鲜扫描再判定,旧版本遗留的陈旧 FAIL 永远不能阻断后续启动。既没有自动改名/修复,也没有 legacy-only 兜底(继续以 legacy-only 运行会让 JSON 信封与死掉的索引永久分叉、再无对账可能)。瞬态打开失败(无引擎 / CANTOPEN / 只读 / 后端 DSN)仍响亮地拒绝启动并给出可操作的错误——环境类问题同样不允许静默走 legacy 兜底(活的第二写者已于 2026-10-31 移出该拒绝清单:多进程共享一个本地盘 index.db 是受支持拓扑,只提示、永不阻断启动);真正需要纯 JSON 服务的部署请显式设置 BILI_STORAGE_UNIFIED=legacy(注意:留空现在是纯 SQLite 缺省,不是 legacy)。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。 | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | 快照渲染影子对账(#3756 后续):每轮已折叠会话都用引擎现算一次出站快照视图但不使用,与 legacy 实际出站比对分类,逐会话计数进 web 面板与 snapshot-shadow 诊断主题。0/false/off 关闭;1/true/on 开启;否则读配置文件 snapshot.shadow 布尔(默认开)。渲染车道本身仍受灰度开关控制(BILI_ROLLOUT_SNAPSHOT_RENDER),影子车道绝不改变出站字节 —— 它是后续翻转灰度的干跑证据。 | +| `BILI_PERF_TELEMETRY` | boolean-ish | on(默认开) | — | #5460 实时性能采集总开关。DB 存储模式(sqlite/mysql)下默认开启:每处理一个请求落一行,外加每 30 秒一条内存水位采样,写入 index.db 旁边的独立小库 perf.sqlite —— 独立文件,与用户自己的库崩溃/体积隔离;纯 JSON 存储模式冻结、永不采集。0/false 整体关闭采集。任何打开/插入失败只降级采集(一次性告警 + 死标记),绝不影响服务。 | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | 覆盖配置文件路径(路径重定位)。不带 `/` 的纯名字 NAME 选择命名配置 `/NAME.json`;名字写错立即报错并列出可用名字,显式路径写错保持历史空配置行为。 | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | 会话记录的存放目录(路径重定位)。 | | `BILI_ENCRYPTION_KEY` | string | unset | — | 加密持久化载荷的密钥材料(机密)。 | From 664ffad93164ee5b03e1595bb6c65d85f93cdba6 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 21:45:31 +0800 Subject: [PATCH 04/20] =?UTF-8?q?fix(perf):=20#5460=20review=20=E2=80=94?= =?UTF-8?q?=20pure-JSON=20(legacy)=20storage=20mode=20stays=20frozen,=20ze?= =?UTF-8?q?ro=20footprint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sqlite arming branch runs for EVERY enabled store, including storage.unified=legacy deployments where maybeOpen() returns null — so a pure-JSON install would still create perf.sqlite on first record, violating owner constraint ② (#5460: JSON mode frozen). Gate both DB branch arming sites on the RESOLVED mode: legacy → init("json") (permanently inert), shadow/full → arm as before. A configured DB-mode store whose sidecar transiently failed to open still arms (observability is most needed while degraded); only an explicit legacy choice freezes. --- src/persist.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index db1142c015..54264a823d 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -709,7 +709,9 @@ export class SessionStore { ).then((sh) => { this.shadow = sh ?? undefined; // #5460 P1: DB-backed mode → telemetry armed (local perf.sqlite). - perfTelemetryInit("mysql", { dir: this.sessionsDir, log: baseLog }); + // Pure-JSON ("legacy") mode stays frozen even when it rides this + // branch (a mysql backend with storage.unified=legacy) — zero footprint. + perfTelemetryInit(storageMode === "legacy" ? "json" : "mysql", { dir: this.sessionsDir, log: baseLog }); }); // #5046 chain (2026-10-30): openAsync now REJECTS on backend failure // (no degradation in any case). Boot awaits ready() and exits @@ -732,7 +734,9 @@ export class SessionStore { this.syncSidecarOpen = { legacyLoad }; this.shadow = UnifiedShadow.maybeOpen(this.sessionsDir, process.env, legacyLoad, fileStorageUnified(), fileStorageWalJournalSizeLimit(), fileStorageBackend()) ?? undefined; // #5460 P1: DB-backed mode → telemetry armed (local perf.sqlite). - perfTelemetryInit("sqlite", { dir: this.sessionsDir, log: baseLog }); + // Pure-JSON ("legacy") mode flows through this branch too + // (maybeOpen returns null for it) but stays frozen — zero footprint. + perfTelemetryInit(storageMode === "legacy" ? "json" : "sqlite", { dir: this.sessionsDir, log: baseLog }); } else { // #5460: pure-JSON storage mode is frozen — telemetry stays off there. this.shadow = undefined; From 037d901a5b33e9f6e817bbb2950d8f3ea1eadaa7 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 21:51:28 +0800 Subject: [PATCH 05/20] =?UTF-8?q?fix(perf):=20#5460=20review=20=E2=80=94?= =?UTF-8?q?=20enforce=20the=20one-shot=20degrade=20contract=20on=20open=20?= =?UTF-8?q?failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureDb() set state.dead on open failure but never checked it back, so a permanently broken dir retried new DatabaseSync on EVERY request and every 30s mem tick — contradicting the module's documented contract (one warn + permanent stop, serving untouched). Add the dead check at the top. The open-failure warn keeps the tee-logger routing landed upstream (9438e4104) and additionally honors the caller-supplied logger (state.log) so tests can pin the one-shot behavior; production callers pass baseLog either way. Regression pins (both previously uncovered — the serving-immunity claim had no negative-direction test at all): * open failure (garbage file at the db path) → exactly ONE degrade warn, no throw through perfRecordRequest/perfRecordMemSample, no rows recorded; * storage.unified=legacy SessionStore → arms the frozen engine, creates NO perf.sqlite (wiring-level pin of the previous commit's constraint ② gate). Also drops the empty before() hook left in the test file. --- src/perf-telemetry.ts | 11 ++++-- tests/issue5460-perf-telemetry.test.ts | 47 ++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/perf-telemetry.ts b/src/perf-telemetry.ts index 0ce87593bf..1d7b2ed214 100644 --- a/src/perf-telemetry.ts +++ b/src/perf-telemetry.ts @@ -35,11 +35,12 @@ interface PerfState { warned: boolean; insertsSinceSweep: number; memTimer: NodeJS.Timeout | null; + log: ((level: "warn" | "error" | "info", msg: string) => void) | null; } const state: PerfState = { db: null, mode: "", dir: null, dead: false, warned: false, - insertsSinceSweep: 0, memTimer: null, + insertsSinceSweep: 0, memTimer: null, log: null, }; const ring: Array> = []; @@ -82,6 +83,10 @@ function warnDegrade(w: (level: "warn" | "error" | "info", msg: string) => void, function ensureDb(): DatabaseSync | null { if (state.db) return state.db; + // Degrade contract (#5327 discipline): one failure → dead forever. Without + // this check a broken dir would retry new DatabaseSync on EVERY request + // and every mem tick instead of stopping collection once. + if (state.dead) return null; if (!state.dir) return null; try { const db = new DatabaseSync(dbPath(state.dir)); @@ -90,7 +95,7 @@ function ensureDb(): DatabaseSync | null { return db; } catch (e) { state.dead = true; - warnDegrade(loggerLog, `open failed: ${e instanceof Error ? e.message : String(e)}`); + warnDegrade(state.log ?? loggerLog, `open failed: ${e instanceof Error ? e.message : String(e)}`); return null; } } @@ -137,6 +142,7 @@ export function perfTelemetryInit(mode: "sqlite" | "mysql" | "json", opts?: Init state.mode = mode; state.dir = opts?.dir ?? defaultDir(); const log = opts?.log ?? loggerLog; + state.log = log; state.memTimer = setInterval(() => perfRecordMemSample(log), opts?.memIntervalMs ?? DEFAULT_MEM_INTERVAL_MS); state.memTimer.unref?.(); return true; @@ -200,6 +206,7 @@ export function perfTelemetryResetForTests(): void { state.dead = false; state.warned = false; state.insertsSinceSweep = 0; + state.log = null; ring.length = 0; } diff --git a/tests/issue5460-perf-telemetry.test.ts b/tests/issue5460-perf-telemetry.test.ts index 349c2fce93..c14a059e6f 100644 --- a/tests/issue5460-perf-telemetry.test.ts +++ b/tests/issue5460-perf-telemetry.test.ts @@ -3,9 +3,9 @@ // Schema-stability culture mirrors the storage goldens: the DDL text is // pinned, the file layout is pinned, and every degradation path must prove // it leaves serving untouched (telemetry NEVER throws into the request path). -import { test, before, after } from "node:test"; +import { test, after } from "node:test"; import assert from "node:assert/strict"; -import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { DatabaseSync } from "node:sqlite"; @@ -100,6 +100,25 @@ test("#5460 P1: BILI_PERF_TELEMETRY=0 kill switch wins over any mode", () => { } }); +test("#5460 P1: open failure degrades ONCE — no throw into the request path, no retry", () => { + resetModule(); + const dir = sandbox(); + // A non-SQLite regular file at the db path makes the first exec throw + // deterministically ("file is not a database") — the open-failure path. + writeFileSync(path.join(dir, PERF_DB_FILE), "not a sqlite database"); + const warns: string[] = []; + try { + assert.equal(perfTelemetryInit("sqlite", { dir, log: (l, m) => warns.push(`${l}:${m}`) }), true); + perfRecordRequest({ sessionId: "s1" }); + perfRecordRequest({ sessionId: "s2" }); + perfRecordMemSample(); + const degraded = warns.filter((w) => w.startsWith("warn:") && w.includes("telemetry degraded")); + assert.equal(degraded.length, 1, `exactly ONE degrade warn, got ${degraded.length}`); + assert.ok(degraded[0].includes("open failed"), "warn must name the open failure"); + assert.equal(perfRecentRows().length, 0, "no rows may be recorded while dead"); + } finally { perfStopAll(); } +}); + test("#5460 P1: init is idempotent — later calls never re-arm or throw", () => { resetModule(); const dir = sandbox(); @@ -166,5 +185,27 @@ test("#5460 P1: logRequestCost wires structured rows through the single choke po } finally { perfStopAll(); } }); -before(() => { /* env handled per-test via resetModule */ }); +test("#5460 P1: legacy (pure-JSON) storage mode leaves ZERO telemetry footprint — owner constraint ②", async () => { + // The sqlite arming branch runs for every enabled store — including + // storage.unified=legacy ones where maybeOpen() returns null. This pins + // that such a store still never creates perf.sqlite. + const { SessionStore } = await import("../src/persist.js"); + resetModule(); + const savedUnified = process.env.BILI_STORAGE_UNIFIED; + const savedKey = process.env.BILI_ENCRYPTION_KEY; + delete process.env.BILI_ENCRYPTION_KEY; + process.env.BILI_STORAGE_UNIFIED = "legacy"; + const dir = sandbox(); + try { + new SessionStore({ enabled: true, dir }); + perfStopAll(); + assert.equal(perfActiveMode(), "json", "legacy mode must arm the frozen (inert) engine"); + assert.ok(!existsSync(path.join(dir, PERF_DB_FILE)), "pure-JSON mode must never create a metrics file"); + } finally { + if (savedUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = savedUnified; + if (savedKey === undefined) delete process.env.BILI_ENCRYPTION_KEY; else process.env.BILI_ENCRYPTION_KEY = savedKey; + perfTelemetryResetForTests(); + } +}); + after(() => { for (const d of SANDBOXES.splice(0)) rmSync(d, { recursive: true, force: true }); }); From f4a76fc8bae0742fdd72aee2be02b340954e3f18 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 21:52:10 +0800 Subject: [PATCH 06/20] =?UTF-8?q?fix(perf):=20#5460=20review=20=E2=80=94?= =?UTF-8?q?=20synchronous=3DNORMAL=20for=20perf.sqlite=20(hot-path=20fsync?= =?UTF-8?q?=20parity=20with=20index.db)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WAL was set but synchronous stayed at the SQLite default FULL: every request row pays an fsync per commit on the request path. index.db sets journal_mode=WAL + synchronous=NORMAL together (src/storage/schema.ts); match that pair and pin the pragma in the schema golden test. --- src/perf-telemetry.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/perf-telemetry.ts b/src/perf-telemetry.ts index 1d7b2ed214..5c6f512e93 100644 --- a/src/perf-telemetry.ts +++ b/src/perf-telemetry.ts @@ -90,7 +90,10 @@ function ensureDb(): DatabaseSync | null { if (!state.dir) return null; try { const db = new DatabaseSync(dbPath(state.dir)); - db.exec(`PRAGMA journal_mode=WAL; ${PERF_EVENTS_DDL}; CREATE INDEX IF NOT EXISTS idx_perf_events_ts ON ${PERF_TABLE}(ts);`); + // synchronous=NORMAL matches index.db (src/storage/schema.ts): under WAL + // the SQLite default (FULL) would fsync on EVERY commit — one per request + // row on the hot path. Telemetry data is re-collectable; batch the fsyncs. + db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; ${PERF_EVENTS_DDL}; CREATE INDEX IF NOT EXISTS idx_perf_events_ts ON ${PERF_TABLE}(ts);`); state.db = db; return db; } catch (e) { From f18e30e33416c72a90438bf817951b65f95049d7 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 21:56:12 +0800 Subject: [PATCH 07/20] =?UTF-8?q?refactor(perf):=20#5460=20review=20?= =?UTF-8?q?=E2=80=94=20compute=20outbound=20byte=20length=20once=20in=20lo?= =?UTF-8?q?gRequestCost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR's telemetry line and the pre-existing human-log line each ran Buffer.byteLength(outbound) — a second O(n) scan over potentially large response strings on every request. Hoist to one const. --- src/server/request-cost.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/request-cost.ts b/src/server/request-cost.ts index 1add7597ba..f165bb9a73 100644 --- a/src/server/request-cost.ts +++ b/src/server/request-cost.ts @@ -40,10 +40,11 @@ export function _resetSlowSessionWarnedForTest(): void { export function logRequestCost(log: (level: string, msg: string) => void, sessionId: string, msgs: number | null, inboundBytes: number, t0: number, outbound?: string | Buffer): void { const ms = Math.max(0, Math.round(performance.now() - t0)); + const outboundBytes = outbound !== undefined ? Buffer.byteLength(outbound) : null; // #5460 P1: structured mirror of this line into perf.sqlite (inert unless // a DB-backed storage mode armed the module; degrades, never throws). - perfRecordRequest({ sessionId, msgs, inboundBytes, outboundBytes: outbound !== undefined ? Buffer.byteLength(outbound) : null, totalMs: ms }); - const outboundField = outbound !== undefined ? `, outbound=${formatBytes(Buffer.byteLength(outbound))}` : ""; + perfRecordRequest({ sessionId, msgs, inboundBytes, outboundBytes, totalMs: ms }); + const outboundField = outboundBytes !== null ? `, outbound=${formatBytes(outboundBytes)}` : ""; const view = currentFetchTransport() ? ", view=ws-expanded" : ""; log("info", `[${sessionId}] request: ${msgs ?? "?"} msgs, inbound=${formatBytes(inboundBytes)}${outboundField}, local=${ms}ms${view}`); // #5408: this line is THE observation point — a hot-at-attach session that From 5482becbd7f6248cb2903f87f403d100f46866f9 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 22:12:52 +0800 Subject: [PATCH 08/20] =?UTF-8?q?test(perf):=20#5460=20review=20=E2=80=94?= =?UTF-8?q?=20pin=20the=20200k-row=20retention=20cap=20end-to-end?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The amortized sweep (DELETE ... ORDER BY id DESC LIMIT -1 OFFSET cap every 500th insert) is what keeps perf.sqlite bounded per owner constraint ① (crash/size isolation from index.db); an off-by-one in that SQL would let the file grow unbounded with zero other signal. Drive 200_001 real inserts through the public API and assert exactly 200_000 rows remain, newest kept. --- tests/issue5460-perf-telemetry.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/issue5460-perf-telemetry.test.ts b/tests/issue5460-perf-telemetry.test.ts index c14a059e6f..a3fab0e3c2 100644 --- a/tests/issue5460-perf-telemetry.test.ts +++ b/tests/issue5460-perf-telemetry.test.ts @@ -185,6 +185,25 @@ test("#5460 P1: logRequestCost wires structured rows through the single choke po } finally { perfStopAll(); } }); +test("#5460 P1: retention sweep caps perf_events at 200k rows — oldest drop first", () => { + resetModule(); + const dir = sandbox(); + try { + assert.equal(perfTelemetryInit("sqlite", { dir }), true); + for (let i = 0; i < 200_001; i++) { + perfRecordRequest({ sessionId: `r${i}`, inboundBytes: i }); + } + perfStopAll(); + const db = new DatabaseSync(path.join(dir, PERF_DB_FILE)); + try { + const n = db.prepare("SELECT COUNT(*) AS n FROM perf_events").get() as { n: number }; + assert.equal(Number(n.n), 200_000, "retention cap must hold"); + const newest = db.prepare("SELECT session_id FROM perf_events ORDER BY id DESC LIMIT 1").get() as { session_id: string }; + assert.equal(newest.session_id, "r200000", "newest rows survive, oldest drop"); + } finally { db.close(); } + } finally { perfStopAll(); } +}); + test("#5460 P1: legacy (pure-JSON) storage mode leaves ZERO telemetry footprint — owner constraint ②", async () => { // The sqlite arming branch runs for every enabled store — including // storage.unified=legacy ones where maybeOpen() returns null. This pins From 49b0ec7f38ecbf7951ed15a5c85c1d4d6424834d Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 22:20:30 +0800 Subject: [PATCH 09/20] =?UTF-8?q?test(perf):=20#5460=20review=20=E2=80=94?= =?UTF-8?q?=20correct=20retention=20expectation=20to=20the=20sweep=20bound?= =?UTF-8?q?ary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit asserted exactly 200_000 rows after 200_001 inserts, which lands one insert past the last sweep boundary: the amortized sweep (every 500th insert) legitimately leaves up to cap+period-1 rows between sweeps, so 200_001 is correct behavior, not a cap violation. Drive to an exact multiple of the sweep period past the cap (200_500) so the final sweep deterministically trims the table to precisely the 200k window. --- tests/issue5460-perf-telemetry.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/issue5460-perf-telemetry.test.ts b/tests/issue5460-perf-telemetry.test.ts index a3fab0e3c2..4f9009c149 100644 --- a/tests/issue5460-perf-telemetry.test.ts +++ b/tests/issue5460-perf-telemetry.test.ts @@ -185,21 +185,25 @@ test("#5460 P1: logRequestCost wires structured rows through the single choke po } finally { perfStopAll(); } }); -test("#5460 P1: retention sweep caps perf_events at 200k rows — oldest drop first", () => { +test("#5460 P1: retention sweep caps perf_events at the 200k-row window — oldest drop first", () => { resetModule(); const dir = sandbox(); try { assert.equal(perfTelemetryInit("sqlite", { dir }), true); - for (let i = 0; i < 200_001; i++) { + // Stop EXACTLY on a sweep boundary past the cap (a multiple of the + // 500-insert sweep period): the sweep that just fired must leave the + // table at precisely the cap, newest kept. Between sweeps the table + // may hold up to cap+period-1 (amortized by design). + for (let i = 0; i < 200_500; i++) { perfRecordRequest({ sessionId: `r${i}`, inboundBytes: i }); } perfStopAll(); const db = new DatabaseSync(path.join(dir, PERF_DB_FILE)); try { const n = db.prepare("SELECT COUNT(*) AS n FROM perf_events").get() as { n: number }; - assert.equal(Number(n.n), 200_000, "retention cap must hold"); + assert.equal(Number(n.n), 200_000, "retention cap must hold at the sweep boundary"); const newest = db.prepare("SELECT session_id FROM perf_events ORDER BY id DESC LIMIT 1").get() as { session_id: string }; - assert.equal(newest.session_id, "r200000", "newest rows survive, oldest drop"); + assert.equal(newest.session_id, "r200499", "newest rows survive, oldest drop"); } finally { db.close(); } } finally { perfStopAll(); } }); From a195ca00fcdbb371d07cedc44542184231b6dbb5 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 22:48:17 +0800 Subject: [PATCH 10/20] =?UTF-8?q?fix(perf):=20#5460=20resolve=20the=20SQLi?= =?UTF-8?q?te=20engine=20through=20the=20shared=20storage=20driver=20seam?= =?UTF-8?q?=20=E2=80=94=20CLI=20stays=20loadable=20on=20Node=20<22.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ src/perf-telemetry.ts | 20 +++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5122a633..2c9c5c4639 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,10 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag - **中断后/usage 丢失后的「幻影紧急压缩」:本轮拿不到计费样本时,压力档压缩延后一轮再决定**:中断的流(用户 ESC)或上游短 TTL 缓存把 usage 报告弄丢时,本轮的「计量表针」只能落回字符级本地估算——它量的是即将被改写丢掉的入站字节,实测比真实计费高约 3 倍(0.1.212 当天两起真实现场:ESC 的会话从 9478 跳成 228376、长跑会话被连续两轮注入 153% 幻影压力带,徒劳改写并烧掉前缀缓存)。此前这种坏表针直接按 EMERGENCY/OVER-LIMIT 扣扳机。现在:内核收到以「估算值」计量的压力带读数(宿主标记 tokenCountSource="estimate")时,本轮压力档注压延后一轮——下一次请求必然带着新鲜计费报告,拿真数再决定,代价≈0;计费级读数("usage")与不传 source 的老调用方字节不变,非压力档(增长/节拍)完全不受影响,硬溢出兜底臂全程照旧武装。宿主侧提前广告臂(emergencyNudge)同步镜像该位,不会拿着同一个坏读数抢跑。逃生阀:`compress.requireUsageSampleForPressure: false` 恢复旧的估算驱动行为。tests:kernel/tests/nudge-pressure-no-sample-hold.test.ts(6 例)、tests/nudge-pressure-no-sample-armed.test.ts(3 例);kernel 既有套件 1582/1582 绿。 - **External-summary mode no longer reports protected or already-folded ranges as failed compression.** An unchanged kernel preview now returns the normal idempotent no-op or kernel-error receipt without a paid summarizer call; validated internal probe summaries prevent an irrelevant submitted-hint length from blocking that receipt. Probe-created blocks with no recoverable content still refuse rather than folding an unreviewed hint. Regression: `tests/external-summary-runtime.test.ts` (24/24). + + + +- **#5460(P1 评审发现):Node <22.5 上整个 CLI 不再因性能采集找不到 SQLite 而启动即死。** P1 的 perf.sqlite 在模块顶层直接 `import "node:sqlite"`——该内置模块 Node ≥22.5 才有,而 package.json 仍声明 engines >=20;静态导入让**整个进程**在模块加载期崩溃(不可用的是全部 bili,不只是遥测)。现在与 index.db 走同一条引擎解析链(`openSqliteDatabase`:builtin node:sqlite → 宿主 better-sqlite3 → 自下载,#2833/#2845):老 Node 自动落到原生引擎,全缺时按既有契约一次性警告、只降级停用遥测,服务可用性不受影响;同时满足 dist 自包含门禁(零新增外部 specifier)。 * **#5412: 异步存储后端的「压缩冻结会话」现在也能自愈。** #5408 事故(一个会话连续 17 次摘要漂移后被保护机制冻住、8 小时无人知晓)的主修复 #5409 只教了同步存储路径(SQLite 直写)自救:漂移挂起期间保存会自动升级为全量重导入并清掉索引里的残留行。异步路径(opt-in 网络后端,如 mysql)那份孪生代码没搬——同样被冻住的会话在那上面永远救不回来,每次保存都重写不完整视图、下轮采样继续漂移。本 PR 在 `src/storage/shadow.ts` 的异步入队路径补上同一行对齐(`completeLog = hasRawTail || driftedSessions.has(id)`),行为与同步路径逐字一致;回归测试同时钉住正门(latch 挂起的折叠保存清掉外来行)与负门(未挂起保存不清扫)。 - **中断频繁的会话压缩提醒几乎不触发,积到几十万 token 才压一次(#5415,0.1.210 引入)**:#5293 的修复把每次用户中断后的增长参照点直接抬到当前读数,连「中断前已实测的真实增长」也一起清零;高频中断的客户端(实测一份日志 4.5 小时 85 次中断)永远攒不到增长台阶,30 万可压内容 idle 数小时,最后一次性压 33 万。现在吸收时把被清零的实测增长记入「欠账」(`nudge.absorbedDebt`)并累计吸收次数(`nudge.absorbCount`),两者都只在**第 2 次起**的中断后生效:单次中断字节级保持 #5293 的安静(不打扰用户刚改的主意——欠账再多也不当场弹提醒);第二次中断才证明会话是「高频中断型」而非「改主意」,欠账从此与新增量一起计入增长台阶,恢复 209 时代的节奏。另加「大积压排水」兑底:欠账>0 且可压量≥2×阈值且当前无未消化提醒时不再等增长台阶(与 #194 同哲学);无欠账/单次中断的会话决策字节不变。压缩成功时欠账与计数随其它基线一起清零;idle 日志同步显示 `growth X + absorbed Y across N aborts < floor Z`(单次中断显示 `held (single abort, #5293)`)便于排查。tests:kernel/tests/issue5415-absorb-debt-nudge.test.ts(欠账付台阶/单次中断钉死不弹/排水兑底/忽略提醒不重发/无欠账字节不变/压缩后清账+清计数);tests/issue5415-absorb-debt.test.ts(吸收记账算术、欠账+计数、累次累计、参照点已越过 anchor 不记账也计数、陈旧 flag 丢弃)。 - **长会话每回合把 Node 主线程卡死十几秒、网页面板冻结(#5337)**:每回合的压缩筛查把历史切成若干可折叠单元,逐个跑「能否折叠」判定——而每次判定都从零重扫全量消息数组(id 查找无索引,五处线性 `.find()`/`includes`),还重复重算全套整史统计(受保护/过期工具调用集合、工具配对表、末尾锚点、完整性预扫)。总工作量≈单元数×全史长度=历史规模平方级,且同步跑在主线程:2.3万条核心消息实测单回合约17秒(事件循环被占死,请求排队;#5304 同签名报告)。修复=把这批 O(n) 推导提升为每轮筛查**一次**构建(`buildProjectionShared`)、共享给同一轮所有单元;id→序号一次性建表替五处线性查找;「已丢弃」成员判断从 includes-in-some 改为 Set。判定逻辑本身一字未动(#3236 单一判定纪律),只是输入到达判定的方式变了。同机同语料实测:2万条消息完整推荐链 9028ms→约572ms(≈16×),回落到线性量级。单次调用路径(apply/report)保持原有内联计算,输出逐字节不变。kernel v0.0.175。tests/issue5337-shared-screening.test.ts(共享/非共享投影逐字段等价+降级边界一致+n=20000 耗时上界钉);既有 issue3406 曲线钉与 #3236 一致性回归不变。 diff --git a/src/perf-telemetry.ts b/src/perf-telemetry.ts index 5c6f512e93..8efb083db4 100644 --- a/src/perf-telemetry.ts +++ b/src/perf-telemetry.ts @@ -11,8 +11,14 @@ // migration of the user's own store: crash/size isolation from index.db, // zero schema-version coupling, trivially deletable by the owner. // * BILI_PERF_TELEMETRY=0 force-disables (registered in LIFECYCLE.md §2.4). -import { DatabaseSync, type SQLInputValue } from "node:sqlite"; +// * Engine acquisition goes through the shared storage driver seam +// (src/storage/driver.js, #2833 order: builtin node:sqlite → host +// better-sqlite3 → self-downloaded prebuilt) — NEVER a direct static +// node:sqlite import: that builtin is absent on Node <22.5 while +// package.json claims engines >=20, and a static import would crash the +// whole CLI at module load instead of degrading telemetry alone. import * as path from "node:path"; +import { openSqliteDatabase, type SqliteDatabase } from "./storage/driver.js"; import { sessionsDir } from "./paths.js"; import { log as loggerLog } from "./logger.js"; @@ -28,7 +34,7 @@ const DEFAULT_MEM_INTERVAL_MS = 30_000; interface InitOptions { dir?: string; memIntervalMs?: number; log?: (level: "warn" | "error" | "info", msg: string) => void; } interface PerfState { - db: DatabaseSync | null; // lazily opened on first record + db: SqliteDatabase | null; // lazily opened on first record mode: string; // "sqlite" | "mysql" | "json"(off) dir: string | null; dead: boolean; // first insert/open failure → degrade forever @@ -81,15 +87,15 @@ function warnDegrade(w: (level: "warn" | "error" | "info", msg: string) => void, try { w("warn", `[perf] telemetry degraded: ${what} — metrics collection stopped for this process (serving unaffected)`); } catch { /* last resort */ } } -function ensureDb(): DatabaseSync | null { +function ensureDb(): SqliteDatabase | null { if (state.db) return state.db; // Degrade contract (#5327 discipline): one failure → dead forever. Without - // this check a broken dir would retry new DatabaseSync on EVERY request + // this check a broken dir would retry opening the DB on EVERY request // and every mem tick instead of stopping collection once. if (state.dead) return null; if (!state.dir) return null; try { - const db = new DatabaseSync(dbPath(state.dir)); + const { db } = openSqliteDatabase(dbPath(state.dir)); // synchronous=NORMAL matches index.db (src/storage/schema.ts): under WAL // the SQLite default (FULL) would fsync on EVERY commit — one per request // row on the hot path. Telemetry data is re-collectable; batch the fsyncs. @@ -103,7 +109,7 @@ function ensureDb(): DatabaseSync | null { } } -function insertRow(db: DatabaseSync, row: Record, w: (level: "warn" | "error" | "info", msg: string) => void): void { +function insertRow(db: SqliteDatabase, row: Record, w: (level: "warn" | "error" | "info", msg: string) => void): void { if (state.dead) return; try { db.prepare( @@ -116,7 +122,7 @@ function insertRow(db: DatabaseSync, row: Record, w: (level: "w (row.outbound_bytes as number) ?? null, (row.total_ms as number) ?? null, (row.parse_ms as number) ?? null, - ((row.payload_json as string | undefined) ?? null) satisfies SQLInputValue, + (row.payload_json as string | undefined) ?? null, ); pushRing(row); if (++state.insertsSinceSweep >= RETENTION_SWEEP_EVERY) { From dfbb8b3e7eb0dbf103e27b5bb44c263c8d9a3df2 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 22:49:40 +0800 Subject: [PATCH 11/20] =?UTF-8?q?feat(perf):=20#5460(P2)=20bili=20bench=20?= =?UTF-8?q?+=20fixture-gen=20=E2=80=94=20hermetic=20record-only=20perf=20h?= =?UTF-8?q?arness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + src/bench.ts | 409 ++++++++++++++++++++++++++++++++++++++++++++ src/cli.ts | 160 ++++++++++++++++- src/perf-fixture.ts | 108 ++++++++++++ 4 files changed, 672 insertions(+), 6 deletions(-) create mode 100644 src/bench.ts create mode 100644 src/perf-fixture.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c9c5c4639..4eb4ca590f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Added +* **#5460(P2):性能基准夹具与 `bili bench`(只记录不判)**。`bili fixture-gen --seed K --msgs N --shape openai|anthropic` 把参数当纯函数吐出纯合成对话(同 seed 字节级相同、零隐私面;S-small 50 / S-med 2k / S-large 2万条≈80MiB 复刻 #5403 形状)。`bili bench ` 在一次性 XDG 目录+随机端口里拉起真代理、对进程内确定性假上游(复用 e2e-dsh hermetic 手法)按 dsh 插件车道的全量重发形状逐回合回放夹具,结束后读子进程 perf.sqlite 产出 JSON 工件:`total_ms` p50/p95/max/mean、进出站字节、内存 HWM、回合错误清单。**只记录不判**:指标再慢也 exit 0,只有 harness 失败(服务起不来/全回合失败/采集未武装)才 exit 1——容差带判定与基线 ratchet 留给 P3(`--rounds 3` 中位数支持已就位)。内置 6 个场景(core×5 + nightly×1 S-large);PR 级 CI 零 perf job(owner 定),FULL/夜间接线在 P3。tests/issue5460-perf-bench.test.ts(夹具确定性/哨兵不变量/S-large 尺寸锚/统计数学/CLI 解析;端到端 smoke 门控 ACP_TEST_BENCH=1,需先 build)。 * **#5460(P1):实时性能采集落地(SQLite 存储)**。近期所有性能事故(#5403 OOM×6、#4528 主线程卡死、#5385 前缀钉死)都是用户先发现、CI 零预警。现在 DB 存储模式(sqlite/mysql)下每处理一个请求,就在会话目录旁的新小库 `perf.sqlite` 落一行(消息数/进出字节/本地耗时/会话号),另有每 30 秒一条内存水位采样(rss/heap/external)——OOM 类事故从此有爬坡曲线可查。写路径与服务完全隔离:任何写失败只停采集(一次性警告),绝不影响服务;纯 JSON 存储模式冻结零足迹(owner 定)。`BILI_PERF_TELEMETRY=0` 整体关闭。后续:P2 场景夹具+`bili bench`(只记录不判)、P3 基线容差带接 FULL/夜间 CI(普通 PR CI 零性能 job)、P4 `/stats`+Web 面板性能页+外部看门狗——跟踪单 #5460。 ### Fixed diff --git a/src/bench.ts b/src/bench.ts new file mode 100644 index 0000000000..7bef81707d --- /dev/null +++ b/src/bench.ts @@ -0,0 +1,409 @@ +// #5460(P2): `bili bench ` — hermetic perf harness, RECORD-ONLY. +// Boots the real proxy (this very entry point) as a child process against an +// in-process deterministic fake upstream inside throwaway XDG dirs, replays a +// seeded fixture as sequential full-resend rounds (the dsh plugin lane's +// traffic shape), then reads the child's perf.sqlite (#5460 P1) and emits a +// JSON artifact with p50/p95/HWM stats. It NEVER judges metrics: slow numbers +// exit 0; only harness failures (server never ready, every round failed, +// telemetry did not arm) fail the run. Baseline tolerance judging is P3. + +import { spawn, type ChildProcess } from "node:child_process"; +import http from "node:http"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import fs from "node:fs"; +import { generateFixture, renderAnthropicBody, renderOpenaiBody, type FixtureShape } from "./perf-fixture.js"; +import { BUILD_COMMIT, VERSION } from "./version.js"; +import { openSqliteDatabase } from "./storage/driver.js"; + +export interface BenchScenario { + id: string; + desc: string; + shape: FixtureShape; + msgs: number; + seed: number; + /** Declared model context window (tokens) forcing compression density; + * undefined = the store's fallback window applies. */ + window?: number; + /** core = cheap enough for frequent runs; nightly = heavy (S-large). */ + tier: "core" | "nightly"; +} + +export const BENCH_SCENARIOS: readonly BenchScenario[] = [ + { id: "smoke-openai-tiny", desc: "harness smoke: 8 openai rounds, no compression pressure", shape: "openai", msgs: 8, seed: 0x546000, tier: "core" }, + { id: "proxy-append-openai-s-small", desc: "pure-proxy append, 50 rounds (~200 KB final history)", shape: "openai", msgs: 50, seed: 0x546001, tier: "core" }, + { id: "proxy-append-openai-s-med", desc: "pure-proxy append, 2000 rounds (~7 MB final history; late rounds cross the fallback window)", shape: "openai", msgs: 2000, seed: 0x546002, tier: "core" }, + { id: "proxy-append-anthropic-s-med", desc: "pure-proxy append on the anthropic wire, 2000 rounds", shape: "anthropic", msgs: 2000, seed: 0x546003, tier: "core" }, + { id: "proxy-compress-openai-s-med", desc: "16K declared window: preflight/nudge compression fires mid-run every turn", shape: "openai", msgs: 2000, seed: 0x546004, window: 16_000, tier: "core" }, + { id: "proxy-append-openai-s-large", desc: "#5403 shape: 20k rounds, ~80 MiB final history — nightly only", shape: "openai", msgs: 20_000, seed: 0x546005, tier: "nightly" }, +]; + +const BENCH_MODEL = "bench-model"; +const HEALTH_TIMEOUT_MS = 30_000; +const STOP_TIMEOUT_MS = 5_000; +const ROUND_ERROR_CAP = 5; + +export interface BenchStat { min: number; p50: number; p95: number; max: number; mean: number } + +/** Nearest-rank percentile on a sorted copy: idx = ceil(q*n)-1. Pure so the + * report math is unit-testable without booting anything. */ +export function computeStats(values: number[]): BenchStat | null { + if (values.length === 0) return null; + const s = [...values].sort((a, b) => a - b); + const pick = (q: number) => s[Math.min(s.length - 1, Math.max(0, Math.ceil(q * s.length) - 1))]; + const mean = s.reduce((acc, v) => acc + v, 0) / s.length; + return { min: s[0], p50: pick(0.5), p95: pick(0.95), max: s[s.length - 1], mean }; +} + +interface PerfRow { id: number; kind: string; session_id: string | null; inbound_bytes: number | null; outbound_bytes: number | null; total_ms: number | null; payload_json: string | null } + +/** Rows inserted after `minId` — each run slices its own window so run N's + * report never absorbs run N-1's rows. Engine acquisition goes through the + * shared storage driver seam (#5460 C1), so the bench degrades like the + * store itself instead of hard-requiring the Node builtin. */ +function readPerfRows(dbFile: string, minId: number): PerfRow[] { + const { db } = openSqliteDatabase(dbFile); + try { + return db.prepare("SELECT id, kind, session_id, inbound_bytes, outbound_bytes, total_ms, payload AS payload_json FROM perf_events WHERE id > ? ORDER BY id").all(minId) as unknown as PerfRow[]; + } finally { + db.close(); + } +} + +function maxPerfId(dbFile: string): number { + // The child creates perf.sqlite lazily at first insert — absent here is normal. + if (!fs.existsSync(dbFile)) return 0; + const { db } = openSqliteDatabase(dbFile); + try { + const row = db.prepare("SELECT COALESCE(MAX(id), 0) AS m FROM perf_events").get() as { m: number }; + return row.m; + } finally { + db.close(); + } +} + +export interface BenchRunReport { + run: number; + rounds_run: number; + rounds_failed: number; + round_errors: string[]; + client_wall_ms_total: number; + metrics: { + req_rows: number; + mem_samples: number; + total_ms: BenchStat | null; + inbound_bytes: { total: number; max_per_req: number } | null; + outbound_bytes: { total: number; max_per_req: number } | null; + mem_hwm: { rss_bytes: number; heap_used_bytes: number } | null; + }; +} + +export interface BenchArtifact { + schema_version: 1; + generated_at: string; + bili_version: string; + commit: string; + scenario: { id: string; desc: string; shape: FixtureShape; msgs: number; seed: number; window: number | null; tier: string }; + fixture: { bytes: number }; + env: { os: string; arch: string; node: string; storage_mode: string }; + runs: BenchRunReport[]; +} + +function buildRunReport(run: number, rows: PerfRow[], roundsRun: number, roundErrors: string[], wallMsTotal: number): BenchRunReport { + const reqs = rows.filter((r) => r.kind === "req"); + const mems = rows.filter((r) => r.kind === "mem"); + const totals = reqs.map((r) => r.total_ms).filter((v): v is number => typeof v === "number"); + const inb = reqs.map((r) => r.inbound_bytes).filter((v): v is number => typeof v === "number"); + const outb = reqs.map((r) => r.outbound_bytes).filter((v): v is number => typeof v === "number"); + let hwmRss = 0; + let hwmHeap = 0; + for (const m of mems) { + try { + const p = JSON.parse(m.payload_json ?? "{}") as Record; + if (typeof p.rss === "number") hwmRss = Math.max(hwmRss, p.rss); + if (typeof p.heapUsed === "number") hwmHeap = Math.max(hwmHeap, p.heapUsed); + } catch { /* keep scanning: one bad sample must not void the HWM */ } + } + return { + run, + rounds_run: roundsRun, + rounds_failed: roundErrors.length, + round_errors: roundErrors.slice(0, ROUND_ERROR_CAP), + client_wall_ms_total: Math.round(wallMsTotal), + metrics: { + req_rows: reqs.length, + mem_samples: mems.length, + total_ms: computeStats(totals), + inbound_bytes: inb.length > 0 ? { total: inb.reduce((a, b) => a + b, 0), max_per_req: Math.max(...inb) } : null, + outbound_bytes: outb.length > 0 ? { total: outb.reduce((a, b) => a + b, 0), max_per_req: Math.max(...outb) } : null, + mem_hwm: mems.length > 0 ? { rss_bytes: hwmRss, heap_used_bytes: hwmHeap } : null, + }, + }; +} + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.once("error", reject); + srv.listen(0, "127.0.0.1", () => { + const { port } = srv.address() as net.AddressInfo; + srv.close(() => resolve(port)); + }); + }); +} + +async function waitPortDead(port: number): Promise { + await new Promise((resolve) => { + const sock = net.connect({ port, host: "127.0.0.1" }); + sock.once("connect", () => { sock.destroy(); resolve(); }); + sock.once("error", () => resolve()); + }); +} + +/** Deterministic fake upstream speaking both wires the scenarios use. + * Recognizes bili's own summarization requests by their task text and + * answers them sentinel-preserving, exactly like tests/e2e/fake-upstream.mjs + * does for the codex lane, so compression stays deterministic. */ +function startFakeUpstream(port: number): Promise<{ close: () => Promise; requests: () => number }> { + let n = 0; + const estIn = (s: string) => Math.max(1, Math.round(s.length / 4)); + const estOut = (s: string) => Math.max(1, Math.round(s.length / 4)); + const SUMMARY_RE = /must be compressed because the session context exceeds|Write a \*?\*?(tier-1|\d+-tier) compression summary/; + const summaryFor = (raw: string): string | null => { + if (!SUMMARY_RE.test(raw)) return null; + const sentinels = [...new Set([...raw.matchAll(/sentinel value:\s*(\d+)/g)].map((m) => m[1]))]; + const docs = [...new Set(raw.match(/doc#\d+/g) || [])]; + let text = `[Compressed conversation section] — folded ${docs.length || "?"} archive fragment(s). Preserved sentinel values: ${sentinels.length ? sentinels.join(", ") : "(none)"}. Verbatim filler lines omitted to save tokens.`; + while (text.length < 50) text += " (context preserved)"; + return text; + }; + const chatSse = (res: http.ServerResponse, model: string, text: string, raw: string): void => { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" }); + const base = { id: `chatcmpl_${++n}`, object: "chat.completion.chunk", model, created: 0 }; + res.write(`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: "assistant", content: text }, finish_reason: null }] })}\n\n`); + res.write(`data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: estIn(raw), completion_tokens: estOut(text), total_tokens: estIn(raw) + estOut(text) } })}\n\n`); + res.write("data: [DONE]\n\n"); + res.end(); + }; + const anthropicSse = (res: http.ServerResponse, model: string, text: string, raw: string): void => { + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" }); + const ev = (event: string, data: unknown) => res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); + ev("message_start", { type: "message_start", message: { id: `msg_${++n}`, type: "message", role: "assistant", model, usage: { input_tokens: estIn(raw), output_tokens: 1 } } }); + ev("content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }); + ev("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text } }); + ev("content_block_stop", { type: "content_block_stop", index: 0 }); + ev("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: estOut(text) } }); + ev("message_stop", { type: "message_stop" }); + res.end(); + }; + const server = http.createServer((req, res) => { + try { + if (req.method === "GET" && /\/models$/.test(req.url ?? "")) { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ object: "list", data: [{ id: BENCH_MODEL, object: "model" }] })); + return; + } + if (req.method !== "POST") { res.writeHead(404); res.end(); return; } + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => { + n++; + const summary = summaryFor(raw); + const text = summary ?? `ack${n % 1000}`; + if (/\/chat\/completions$/.test(req.url ?? "")) chatSse(res, BENCH_MODEL, text, raw); + else if (/\/(v1\/)?messages$/.test(req.url ?? "")) anthropicSse(res, BENCH_MODEL, text, raw); + else { res.writeHead(404); res.end(); } + }); + } catch { + try { res.writeHead(500); res.end(); } catch { /* socket already gone */ } + } + }); + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", () => resolve({ close: () => new Promise((r) => server.close(() => r())), requests: () => n })); + }); +} + +interface HermeticEnv { env: NodeJS.ProcessEnv; configHome: string; dataHome: string } + +function hermetic(workdir: string): HermeticEnv { + const env: NodeJS.ProcessEnv = {}; + for (const [k, v] of Object.entries(process.env)) { + if (v === undefined) continue; + // Every bili side-channel leaves: the child must see a vanilla host, + // or the measurement inherits this process's posture. + if (/^BILI_/.test(k) || /^ACP_/.test(k)) continue; + if (k === "BILLION_CONTEXT_PROXY" || k === "NODE_OPTIONS" || k === "NODE_TEST_CONTEXT") continue; + env[k] = v; + } + const home = path.join(workdir, "home"); + const configHome = path.join(workdir, "xdg", "config"); + const cacheHome = path.join(workdir, "xdg", "cache"); + const stateHome = path.join(workdir, "xdg", "state"); + const dataHome = path.join(workdir, "xdg", "data"); + const tmpHome = path.join(workdir, "tmp"); + env.HOME = home; + env.XDG_CONFIG_HOME = configHome; + env.XDG_CACHE_HOME = cacheHome; + env.XDG_STATE_HOME = stateHome; + env.XDG_DATA_HOME = dataHome; + env.TMPDIR = tmpHome; + env.ACP_AUTO_UPDATE = "0"; + // Telemetry arms only in DB-backed modes (#5460 constraint 1); pin the + // current default explicitly so the baseline never silently shifts if the + // default ever changes. + env.BILI_STORAGE_UNIFIED = "full"; + for (const dir of [home, configHome, cacheHome, stateHome, dataHome, tmpHome, path.join(workdir, "log")]) fs.mkdirSync(dir, { recursive: true }); + return { env, configHome, dataHome }; +} + +async function waitForHealth(url: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const resp = await fetch(url, { signal: AbortSignal.timeout(2000) }); + if (resp.ok) return; + } catch { /* not up yet */ } + if (Date.now() > deadline) throw new Error(`proxy never became healthy at ${url}`); + await new Promise((r) => setTimeout(r, 250)); + } +} + +function stopChild(child: ChildProcess): Promise { + return new Promise((resolve) => { + if (child.exitCode !== null || child.killed) return resolve(); + let done = false; + const finish = () => { if (!done) { done = true; resolve(); } }; + child.once("exit", finish); + child.kill("SIGTERM"); + setTimeout(() => { if (!done) { try { child.kill("SIGKILL"); } catch { /* already gone */ } finish(); } }, STOP_TIMEOUT_MS).unref?.(); + }); +} + +function resolveDist(explicit?: string): string { + const candidate = explicit ?? process.env.E2E_BILI_DIST ?? process.argv[1]; + if (!candidate) throw new Error("cannot locate the bili entry point — pass --dist or set E2E_BILI_DIST when running from source"); + const abs = path.resolve(candidate); + if (!fs.existsSync(abs)) throw new Error(`bili entry point missing: ${abs} (run npm run build first)`); + return abs; +} + +export interface BenchOptions { + scenarioId: string; + rounds?: number; + dist?: string; + workdirRoot?: string; + keepWorkdir?: boolean; + log?: (msg: string) => void; +} + +export async function runBench(opts: BenchOptions): Promise<{ artifact: BenchArtifact; workdir: string }> { + const scenario = BENCH_SCENARIOS.find((s) => s.id === opts.scenarioId); + if (!scenario) throw new Error(`unknown bench scenario "${opts.scenarioId}" — valid: ${BENCH_SCENARIOS.map((s) => s.id).join(", ")}`); + const rounds = opts.rounds ?? 1; + if (!Number.isInteger(rounds) || rounds < 1) throw new Error(`bench rounds must be a positive integer, got ${rounds}`); + const log = opts.log ?? (() => { /* silent */ }); + const dist = resolveDist(opts.dist); + const fixture = generateFixture({ seed: scenario.seed, msgs: scenario.msgs, shape: scenario.shape }); + const workdir = path.join(opts.workdirRoot ?? os.tmpdir(), `bili-bench-${scenario.id}-${process.pid}-${Date.now()}`); + fs.mkdirSync(workdir, { recursive: true }); + const he = hermetic(workdir); + const fakePort = await freePort(); + const proxyPort = await freePort(); + await waitPortDead(proxyPort); + const fake = await startFakeUpstream(fakePort); + if (scenario.window !== undefined) { + const cfgDir = path.join(he.configHome, "billion-context"); + fs.mkdirSync(cfgDir, { recursive: true }); + fs.writeFileSync(path.join(cfgDir, "billion-context.json"), JSON.stringify({ providers: { [`http://127.0.0.1:${fakePort}`]: { models: { [BENCH_MODEL]: { context: scenario.window } } } } }, null, 1)); + } + const outLog = fs.openSync(path.join(workdir, "log", "proxy.out.log"), "w"); + const errLog = fs.openSync(path.join(workdir, "log", "proxy.err.log"), "w"); + const child = spawn(process.execPath, [dist, "start", "--port", String(proxyPort), "--no-auto-update"], { + cwd: workdir, + env: he.env, + stdio: ["ignore", outLog, errLog], + }); + const perfDb = perfDbPath(he.dataHome); + let lastRowId = maxPerfId(perfDb); + let harnessError: Error | null = null; + const runs: BenchRunReport[] = []; + try { + await waitForHealth(`http://127.0.0.1:${proxyPort}/__bili/health`, HEALTH_TIMEOUT_MS); + const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${fakePort}/v1/${scenario.shape === "openai" ? "chat/completions" : "messages"}`; + for (let run = 1; run <= rounds; run++) { + const sessionId = `bench-${scenario.id}-r${run}`; + const errors: string[] = []; + let wallMs = 0; + for (let k = 1; k <= fixture.count; k++) { + const body = JSON.stringify(scenario.shape === "openai" + ? renderOpenaiBody(BENCH_MODEL, fixture.messages.slice(0, k)) + : renderAnthropicBody(BENCH_MODEL, fixture.messages.slice(0, k))); + const t0 = performance.now(); + try { + const resp = await fetch(url, { method: "POST", headers: { "content-type": "application/json", "x-session-id": sessionId }, body }); + const text = await resp.text(); + if (!resp.ok) errors.push(`round ${k}: HTTP ${resp.status} ${text.slice(0, 160)}`); + } catch (e) { + errors.push(`round ${k}: ${String(e)}`); + } + wallMs += performance.now() - t0; + } + log(`run ${run}/${rounds}: ${fixture.count} rounds, ${errors.length} failed`); + const rows = readPerfRows(perfDb, lastRowId); + lastRowId = rows.reduce((m, r) => Math.max(m, r.id), lastRowId); + if (errors.length === fixture.count) harnessError = new Error(`every round failed in run ${run}; first: ${errors[0]}`); + else runs.push(buildRunReport(run, rows, fixture.count, errors, wallMs)); + if (harnessError) break; + } + } catch (e) { + harnessError = e instanceof Error ? e : new Error(String(e)); + } finally { + // Snapshot death state BEFORE our own SIGTERM: a 143 at teardown is + // normal, a non-null exitCode before teardown is a real mid-run crash. + const preDeath = child.exitCode; + await stopChild(child); + await fake.close(); + fs.closeSync(outLog); + fs.closeSync(errLog); + if (preDeath !== null && !harnessError) { + const tail = safeReadTail(path.join(workdir, "log", "proxy.err.log")); + harnessError = new Error(`proxy exited ${preDeath} during the run${tail ? `:\n${tail}` : ""}`); + } + } + if (harnessError) { + if (!opts.keepWorkdir) fs.rmSync(workdir, { recursive: true, force: true }); + throw harnessError; + } + if (!fs.existsSync(perfDb)) harnessError = new Error(`telemetry did not arm: ${perfDb} missing (storage mode?)`); + if (harnessError) { + if (!opts.keepWorkdir) fs.rmSync(workdir, { recursive: true, force: true }); + throw harnessError; + } + const artifact: BenchArtifact = { + schema_version: 1, + generated_at: new Date().toISOString(), + bili_version: VERSION, + commit: BUILD_COMMIT, + scenario: { id: scenario.id, desc: scenario.desc, shape: scenario.shape, msgs: scenario.msgs, seed: scenario.seed, window: scenario.window ?? null, tier: scenario.tier }, + fixture: { bytes: fixture.bytes }, + env: { os: os.platform(), arch: os.arch(), node: process.version, storage_mode: "full" }, + runs, + }; + if (!opts.keepWorkdir) fs.rmSync(workdir, { recursive: true, force: true }); + return { artifact, workdir }; +} + +function perfDbPath(dataHome: string): string { + return path.join(dataHome, "billion-context", "sessions", "perf.sqlite"); +} + +function safeReadTail(file: string): string { + try { + const buf = fs.readFileSync(file); + return buf.subarray(Math.max(0, buf.length - 2000)).toString("utf8"); + } catch { + return ""; + } +} diff --git a/src/cli.ts b/src/cli.ts index ae2e533fb3..6f1887dae6 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -30,6 +30,7 @@ * for the full config-file schema (which also supports `debug`, `port`, etc. * — flags are just convenient overrides). */ +import fs from "node:fs"; import { loadOptions, ensureConfigTemplate } from "./config.js"; import { startServer, restartForceCloseOf } from "./server.js"; import { serveStartupFailurePage } from "./startup-failure-page.js"; @@ -59,6 +60,8 @@ import { installPackFromNpm, removePackFile } from "./pack-install.js"; import { listResolvedPacks } from "./compress-settings.js"; import { renderJson, renderText, runDiff } from "./acp-cache-diff.js"; import { fixDshLedgers, renderDoctorReport, renderLedgerFixReport, runDoctor } from "./doctor.js"; +import { BENCH_SCENARIOS, runBench } from "./bench.js"; +import { generateFixture } from "./perf-fixture.js"; import { VERSION, PACKAGE_NAME, BUILD_COMMIT } from "./version.js"; const HELP = `bili ${VERSION} — billion-context proxy @@ -143,9 +146,19 @@ Usage: at their host's updater — never overwritten bili plugin list show install status + installed (on-disk) version for every host bili mcp run the bili MCP server standalone (stdio) - bili plugin-register pre-bind a conversation to the plugin mode - (--origin URL, --agent name) - bili --version print version + bili plugin-register pre-bind a conversation to the plugin mode + (--origin URL, --agent name) + bili bench [scenario] perf harness (#5460 P2): boots a hermetic proxy + against a deterministic fake upstream in throwaway + XDG dirs, replays a seeded fixture as full-resend + rounds, reports p50/p95/HWM from perf.sqlite — + RECORD ONLY, never judges metrics; no scenario + lists the catalog (--rounds N, --out FILE, + --keep-workdir, --dist PATH) + bili fixture-gen emit a deterministic perf fixture — pure function + of --seed K --msgs N --shape openai|anthropic + (--out FILE writes JSON instead of stdout) + bili --version print version bili --help show this help Launcher (bili pi / bili codex / bili claude / bili omp / bili opencode / bili hermes / bili dsh / bili codebuddy / bili qoder / bili trae / bili jcode / bili kimi / bili gemini / bili iflow / bili qwen / bili antigravity / bili mcode / bili aider / bili copilot / bili amp / bili crush / bili zed / bili goose / bili kiro): @@ -216,7 +229,7 @@ Docs: https://github.com/ranxianglei/billion-context `; type Parsed = { - command: "start" | "update" | "doctor" | "help" | "version" | "launch" | "test" | "export" | "plugin-register" | "mcp" | "plugin" | "acp-cache" | "sessions" | "storage" | "pack"; + command: "start" | "update" | "doctor" | "help" | "version" | "launch" | "test" | "export" | "plugin-register" | "mcp" | "plugin" | "acp-cache" | "sessions" | "storage" | "pack" | "bench" | "fixture-gen"; client?: ClientName; clientArgs: string[]; mitmDomains: string[]; @@ -248,6 +261,14 @@ type Parsed = { jsonOutput?: boolean; doctorJson?: boolean; doctorFix?: boolean; + benchScenario?: string; + benchRounds?: number; + benchKeepWorkdir?: boolean; + benchDist?: string; + outFile?: string; + fixtureSeed?: number; + fixtureMsgs?: number; + fixtureShape?: "openai" | "anthropic"; }; export function parseArgs(argv: string[]): Parsed { @@ -285,6 +306,14 @@ export function parseArgs(argv: string[]): Parsed { let jsonOutput = false; let doctorJson = false; let doctorFix = false; + let benchScenario: string | undefined; + let benchRounds: number | undefined; + let benchKeepWorkdir = false; + let benchDist: string | undefined; + let outFile: string | undefined; + let fixtureSeed: number | undefined; + let fixtureMsgs: number | undefined; + let fixtureShape: Parsed["fixtureShape"]; for (let i = 0; i < argv.length; i++) { const a = argv[i]!; @@ -443,6 +472,78 @@ export function parseArgs(argv: string[]): Parsed { acpCacheSession = val; break; } + case "--out": { + const val = argv[++i]; + if (val === undefined) { + console.error(`bili: ${a} requires a value`); + process.exit(2); + } + outFile = val; + break; + } + case "--rounds": { + const val = argv[++i]; + if (val === undefined) { + console.error(`bili: ${a} requires a value`); + process.exit(2); + } + const n = Number(val); + if (!Number.isInteger(n) || n < 1) { + console.error(`bili: --rounds expects a positive integer (got "${val}")`); + process.exit(2); + } + benchRounds = n; + break; + } + case "--keep-workdir": + benchKeepWorkdir = true; + break; + case "--dist": { + const val = argv[++i]; + if (val === undefined || val.length === 0) { + console.error(`bili: ${a} requires a non-empty value`); + process.exit(2); + } + benchDist = val; + break; + } + case "--seed": { + const val = argv[++i]; + if (val === undefined) { + console.error(`bili: ${a} requires a value`); + process.exit(2); + } + const n = Number(val); + if (!Number.isInteger(n) || n < 0) { + console.error(`bili: --seed expects a non-negative integer (got "${val}")`); + process.exit(2); + } + fixtureSeed = n; + break; + } + case "--msgs": { + const val = argv[++i]; + if (val === undefined) { + console.error(`bili: ${a} requires a value`); + process.exit(2); + } + const n = Number(val); + if (!Number.isInteger(n) || n < 1) { + console.error(`bili: --msgs expects a positive integer (got "${val}")`); + process.exit(2); + } + fixtureMsgs = n; + break; + } + case "--shape": { + const val = argv[++i]; + if (val !== "openai" && val !== "anthropic") { + console.error(`bili: --shape expects openai or anthropic (got "${val ?? ""}")`); + process.exit(2); + } + fixtureShape = val; + break; + } case "-F": case "-c": case "--port": @@ -580,12 +681,18 @@ export function parseArgs(argv: string[]): Parsed { console.error(`bili storage: unknown action "${positional[1] ?? ""}" (try "bili storage gc")`); process.exit(2); } + } else if (cmd === "bench") { + command = "bench"; + benchScenario = positional[1]; + } else if (cmd === "fixture-gen") { + command = "fixture-gen"; } else { console.error(`bili: unknown command "${cmd}" (try "bili --help")`); process.exit(2); } } + // -i/-s posture flags (#5428): translate onto the existing lanes so the // launcher needs no new plumbing — `-i pi` IS `bili pi-inline` (env scrub // + extension preflight + BILI_PI_INLINE=1, no proxy spawned), and @@ -612,11 +719,12 @@ export function parseArgs(argv: string[]): Parsed { } } - return { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, sessionsApply, sessionsRestoreTarget, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, packAction, packSpec, packProject, packForce }; + return { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, sessionsApply, sessionsRestoreTarget, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, packAction, packSpec, packProject, packForce, benchScenario, benchRounds, benchKeepWorkdir, benchDist, outFile, fixtureSeed, fixtureMsgs, fixtureShape }; + } export async function main(): Promise { - const { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, sessionsApply, sessionsRestoreTarget, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, packAction, packSpec, packProject, packForce } = parseArgs(process.argv.slice(2)); + const { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, sessionsApply, sessionsRestoreTarget, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, packAction, packSpec, packProject, packForce, benchScenario, benchRounds, benchKeepWorkdir, benchDist, outFile, fixtureSeed, fixtureMsgs, fixtureShape } = parseArgs(process.argv.slice(2)); // -c/--config fail-fast: a bare NAME that resolves to a missing // /NAME.json names the typo + the available siblings instead of // silently starting on an empty config (see paths.ts for path behavior). @@ -638,6 +746,46 @@ export async function main(): Promise { process.stdout.write(`${VERSION} (${BUILD_COMMIT})\n`); return; } + if (command === "fixture-gen") { + try { + if (fixtureSeed === undefined || fixtureMsgs === undefined) { + console.error("bili fixture-gen: --seed K and --msgs N are required (try one of: bili bench )"); + process.exit(2); + } + const fx = generateFixture({ seed: fixtureSeed, msgs: fixtureMsgs, shape: fixtureShape ?? "openai" }); + const text = JSON.stringify({ seed: fx.seed, shape: fx.shape, count: fx.count, bytes: fx.bytes, messages: fx.messages }, null, 1); + if (outFile) { + fs.writeFileSync(outFile, text); + } else { + process.stdout.write(text + "\n"); + } + } catch (error) { + console.error(`bili fixture-gen: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } + return; + } + if (command === "bench") { + if (benchScenario === undefined) { + const lines = BENCH_SCENARIOS.map((s) => ` ${s.id.padEnd(34)} [${s.tier.padEnd(7)}] ${s.desc}`); + process.stdout.write(`bench scenarios (#5460 P2 — record-only, never judges):\n${lines.join("\n")}\n\nrun: bili bench [--rounds N] [--out FILE] [--keep-workdir] [--dist PATH]\n`); + return; + } + try { + const { artifact } = await runBench({ scenarioId: benchScenario, rounds: benchRounds, dist: benchDist, keepWorkdir: benchKeepWorkdir, log: (m) => console.error(m) }); + const text = JSON.stringify(artifact, null, 1); + if (outFile) { + fs.writeFileSync(outFile, text); + console.error(`bench artifact written to ${outFile}`); + } else { + process.stdout.write(text + "\n"); + } + } catch (error) { + console.error(`bili bench: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } + return; + } if (command === "acp-cache") { try { const report = await runDiff(acpCacheDir!, { logFile: acpCacheLog, noLog: acpCacheNoLog, session: acpCacheSession }); diff --git a/src/perf-fixture.ts b/src/perf-fixture.ts new file mode 100644 index 0000000000..7568a51fec --- /dev/null +++ b/src/perf-fixture.ts @@ -0,0 +1,108 @@ +// Deterministic perf fixtures for the #5460(P2) bench layer. A fixture is a +// synthetic conversation history whose bytes are a PURE function of +// (seed, msgs, shape) — no Date.now, no Math.random, no host state — so any +// two runs (any machine, any time) replay byte-identical traffic. Content is +// ASCII-only technical filler carrying `sentinel value:` lines and doc# refs +// so that if a run triggers compression, the proxy's own summarization call +// gets a deterministic answer from the bench fake upstream. Zero privacy +// surface by construction: nothing here can contain user data. +// +// Sizes mirror the #5403 incident shape at the large end: ~4 KiB mean +// message → S-large (20k messages) lands near 80 MiB of full-resend history. + +export type FixtureShape = "openai" | "anthropic"; + +export interface FixtureSpec { + seed: number; + msgs: number; + shape: FixtureShape; +} + +export interface FixtureMessage { + role: "user" | "assistant"; + content: string; +} + +export interface Fixture { + seed: number; + shape: FixtureShape; + count: number; + /** Total UTF-8 bytes across all message contents. */ + bytes: number; + messages: FixtureMessage[]; +} + +/** mulberry32: small deterministic PRNG, period 2^32, good enough for + * filler text — reproducibility matters, statistical quality does not. */ +function mulberry32(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const WORDS = [ + "context", "window", "token", "session", "ledger", "snapshot", "anchor", + "prefix", "suffix", "fold", "block", "carrier", "coverage", "reconcile", + "baseline", "ratchet", "payload", "envelope", "journal", "wal", "index", + "cursor", "drift", "churn", "spike", "hwm", "percentile", "latency", + "throughput", "buffer", "stream", "chunk", "delta", "frame", "header", + "route", "relay", "probe", "sentinel", "fragment", "archive", "digest", + "mirror", "shadow", "unified", "legacy", "fallback", "degrade", "isolate", +]; + +function paragraph(rnd: () => number, words: number): string { + const out: string[] = []; + for (let i = 0; i < words; i++) out.push(WORDS[Math.floor(rnd() * WORDS.length)]); + return out.join(" "); +} + +/** One synthetic message: 2-4 short paragraphs, an occasional large + * tool-output-style dump block (~14 B per row), and every 10th message a + * sentinel line the fake summarizer must preserve. Mean ≈ 3.7 KiB. */ +function buildContent(i: number, seed: number, rnd: () => number): string { + const lines: string[] = []; + const paras = 2 + Math.floor(rnd() * 3); + for (let p = 0; p < paras; p++) { + let text = paragraph(rnd, 18 + Math.floor(rnd() * 43)); + if (p === 0) text += ` see doc#${i % 500}`; + lines.push(text); + } + if (i % 10 === 0) lines.push(`sentinel value: ${i * 7 + (seed % 997)}`); + if (rnd() < 0.3) { + const rows = 200 + Math.floor(rnd() * 1000); + for (let r = 0; r < rows; r++) { + lines.push(`row=${Math.floor(rnd() * 0x100000000).toString(16).padStart(8, "0")}`); + } + } + return lines.join("\n"); +} + +export function generateFixture(spec: FixtureSpec): Fixture { + if (!Number.isInteger(spec.seed) || spec.seed < 0) throw new Error(`fixture seed must be a non-negative integer, got ${spec.seed}`); + if (!Number.isInteger(spec.msgs) || spec.msgs < 1) throw new Error(`fixture msgs must be a positive integer, got ${spec.msgs}`); + if (spec.shape !== "openai" && spec.shape !== "anthropic") throw new Error(`fixture shape must be openai|anthropic, got ${String(spec.shape)}`); + const rnd = mulberry32(spec.seed); + const messages: FixtureMessage[] = []; + let bytes = 0; + for (let i = 0; i < spec.msgs; i++) { + const content = buildContent(i, spec.seed, rnd); + bytes += Buffer.byteLength(content, "utf8"); + messages.push({ role: i % 2 === 0 ? "user" : "assistant", content }); + } + return { seed: spec.seed, shape: spec.shape, count: spec.msgs, bytes, messages }; +} + +/** OpenAI chat-completions body for one append round (full-resend prefix). */ +export function renderOpenaiBody(model: string, history: readonly FixtureMessage[], stream = true): Record { + return { model, stream, messages: history.map((m) => ({ role: m.role, content: m.content })) }; +} + +/** Anthropic Messages-API body for one append round (string content blocks, + * fixed max_tokens so output cost stays flat across rounds). */ +export function renderAnthropicBody(model: string, history: readonly FixtureMessage[], stream = true): Record { + return { model, max_tokens: 1024, stream, messages: history.map((m) => ({ role: m.role, content: m.content })) }; +} From e3ee985f25dcaa37794228e823bb489dd45d4ac9 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 22:49:40 +0800 Subject: [PATCH 12/20] test(perf): #5460(P2) fixture determinism + bench report math + CLI parse coverage --- tests/issue5460-perf-bench.test.ts | 129 +++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 tests/issue5460-perf-bench.test.ts diff --git a/tests/issue5460-perf-bench.test.ts b/tests/issue5460-perf-bench.test.ts new file mode 100644 index 0000000000..882e07de46 --- /dev/null +++ b/tests/issue5460-perf-bench.test.ts @@ -0,0 +1,129 @@ +// #5460(P2): perf fixture determinism + bench report math + CLI wiring. +// The end-to-end harness boot (real proxy child + fake upstream) is gated +// behind ACP_TEST_BENCH=1 and needs a built dist (npm run build first) — +// same convention as the other e2e lanes (tests/e2e/README.md). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { generateFixture, renderAnthropicBody, renderOpenaiBody, type FixtureMessage } from "../src/perf-fixture.js"; +import { computeStats, BENCH_SCENARIOS } from "../src/bench.js"; +import { parseArgs } from "../src/cli.js"; + +const fixtureJson = (spec: { seed: number; msgs: number; shape: "openai" | "anthropic" }) => JSON.stringify(generateFixture(spec)); + +test("fixture: deterministic — same seed yields identical bytes", () => { + const a = fixtureJson({ seed: 42, msgs: 64, shape: "openai" }); + const b = fixtureJson({ seed: 42, msgs: 64, shape: "openai" }); + assert.equal(a, b); +}); + +test("fixture: different seeds diverge", () => { + const a = fixtureJson({ seed: 42, msgs: 64, shape: "openai" }); + const b = fixtureJson({ seed: 43, msgs: 64, shape: "openai" }); + assert.notEqual(a, b); +}); + +test("fixture: content carries compression sentinels and doc refs", () => { + const fx = generateFixture({ seed: 7, msgs: 128, shape: "anthropic" }); + assert.equal(fx.count, 128); + assert.ok(fx.bytes > 0); + let sentinelSeen = false; + let docSeen = false; + for (const m of fx.messages) { + if (/sentinel value:\s*\d+/.test(m.content)) sentinelSeen = true; + if (/doc#\d+/.test(m.content)) docSeen = true; + } + assert.ok(sentinelSeen, "at least one sentinel line"); + assert.ok(docSeen, "at least one doc# ref"); +}); + +test("fixture: S-large size matches the #5403 shape (~80 MiB)", () => { + const fx = generateFixture({ seed: 1, msgs: 20_000, shape: "openai" }); + const mb = fx.bytes / (1024 * 1024); + assert.ok(mb > 40 && mb < 160, `S-large should land near 80 MiB, got ${mb.toFixed(1)} MiB`); +}); + +test("fixture: renders both wires deterministically from the same history", () => { + const fx = generateFixture({ seed: 9, msgs: 12, shape: "openai" }); + const prefix = fx.messages.slice(0, 5); + const o1 = renderOpenaiBody("m", prefix); + const o2 = renderOpenaiBody("m", prefix); + assert.deepEqual(o1, o2); + assert.equal(o1.model, "m"); + assert.equal(o1.stream, true); + assert.deepEqual(o1.messages, prefix); + const a1 = renderAnthropicBody("m", prefix); + assert.equal(a1.max_tokens, 1024); + assert.deepEqual(a1.messages, prefix); +}); + +test("computeStats: nearest-rank percentiles", () => { + assert.equal(computeStats([]), null); + assert.deepEqual(computeStats([7]), { min: 7, p50: 7, p95: 7, max: 7, mean: 7 }); + const s = computeStats([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])!; + assert.equal(s.min, 1); + assert.equal(s.max, 10); + assert.equal(s.p50, 5); + assert.equal(s.p95, 10); + assert.equal(s.mean, 5.5); +}); + +test("scenarios: catalog is well-formed and covers the P2 matrix axes", () => { + const ids = BENCH_SCENARIOS.map((s) => s.id); + assert.equal(new Set(ids).size, ids.length); + assert.ok(ids.includes("smoke-openai-tiny")); + assert.ok(BENCH_SCENARIOS.some((s) => s.shape === "anthropic")); + assert.ok(BENCH_SCENARIOS.some((s) => s.window !== undefined)); + assert.ok(BENCH_SCENARIOS.some((s) => s.tier === "nightly" && s.msgs === 20_000)); + for (const s of BENCH_SCENARIOS) { + assert.ok(s.msgs >= 1, `${s.id}: msgs`); + assert.match(s.id, /^[a-z0-9-]+$/); + } +}); + +test("cli: bench flags parse", () => { + const p = parseArgs(["bench", "proxy-append-openai-s-med", "--rounds", "3", "--out", "x.json"]); + assert.equal(p.command, "bench"); + assert.equal(p.benchScenario, "proxy-append-openai-s-med"); + assert.equal(p.benchRounds, 3); + assert.equal(p.outFile, "x.json"); + const list = parseArgs(["bench"]); + assert.equal(list.command, "bench"); + assert.equal(list.benchScenario, undefined); +}); + +test("cli: fixture-gen flags parse", () => { + const p = parseArgs(["fixture-gen", "--seed", "42", "--msgs", "50", "--shape", "anthropic", "--out", "f.json"]); + assert.equal(p.command, "fixture-gen"); + assert.equal(p.fixtureSeed, 42); + assert.equal(p.fixtureMsgs, 50); + assert.equal(p.fixtureShape, "anthropic"); + assert.equal(p.outFile, "f.json"); +}); + +test("bench: end-to-end hermetic run (smoke scenario)", { timeout: 180_000 }, async (t) => { + if (!process.env.ACP_TEST_BENCH) { + t.skip("set ACP_TEST_BENCH=1 (and npm run build first) to run the hermetic harness"); + return; + } + const { runBench } = await import("../src/bench.js"); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-bench-test-")); + try { + const { artifact } = await runBench({ scenarioId: "smoke-openai-tiny", workdirRoot: dir, log: () => { /* quiet */ } }); + assert.equal(artifact.schema_version, 1); + assert.equal(artifact.scenario.id, "smoke-openai-tiny"); + assert.equal(artifact.env.storage_mode, "full"); + assert.equal(artifact.runs.length, 1); + const run = artifact.runs[0]!; + assert.equal(run.rounds_run, 8); + assert.equal(run.rounds_failed, 0); + assert.ok(run.metrics.req_rows >= 8, `expected >=8 req rows, got ${run.metrics.req_rows}`); + assert.ok(run.metrics.total_ms !== null, "total_ms stats present"); + assert.ok(run.metrics.inbound_bytes !== null && run.metrics.inbound_bytes.total > 0); + assert.ok(run.metrics.mem_hwm === null || run.metrics.mem_hwm.rss_bytes > 0); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); From 0fcf305588143b822dc45c8f59653c43d24817ff Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 23:40:24 +0800 Subject: [PATCH 13/20] docs(perf): document bili bench / fixture-gen surface for config-surface-guard (#5460 P2) The six new CLI flags (--rounds/--out/--keep-workdir/--dist/--seed/--msgs/--shape) had no mention in README.md or CONFIGURATION.md, which is exactly what the config-surface-guard CI job blocks on (AGENTS.md Configuration Surface Discipline). Adds both subcommands + their flags to the CLI reference tables (en+zh) and a short performance-tooling section in both READMEs. --- README.md | 9 +++++++++ README.zh-CN.md | 9 +++++++++ docs/configuration.md | 9 +++++++++ docs/configuration.zh-CN.md | 9 +++++++++ 4 files changed, 36 insertions(+) diff --git a/README.md b/README.md index b52b072107..77c2d88405 100644 --- a/README.md +++ b/README.md @@ -511,6 +511,15 @@ bili --bin /path/to/client # launcher: spawn this exact client binary instead Flags override env vars and the config file. `bili --help` lists them all. +### Performance tooling (`bili bench` / `bili fixture-gen`, #5460) + +Record-only performance harness: `bili bench ` boots the real proxy as a child process against an in-process deterministic fake upstream in throwaway XDG dirs (random ports, auto-update off), replays a seeded fixture as sequential full-resend rounds (the dsh plugin lane's traffic shape), then reads the child's `perf.sqlite` (#5460 P1) and prints a JSON artifact — `total_ms` p50/p95/max/min/mean, inbound/outbound byte totals, memory HWM, per-round errors. **It records, never judges:** slow metrics still exit 0; only harness failures (server never ready, every round failed, telemetry did not arm, mid-run crash) exit 1. Tolerance bands / baseline ratchet / FULL+nightly CI wiring are follow-ups (#5460 P3). `bili bench` with no scenario lists the catalog: `smoke-openai-tiny`, `proxy-append-openai-s-small`, `proxy-append-openai-s-med`, `proxy-append-anthropic-s-med`, `proxy-compress-openai-s-med` (declares a 16K window to force fold density), `proxy-append-openai-s-large` (~80 MiB fixture mirroring #5403 — nightly tier). + +```bash +bili bench proxy-append-openai-s-med --rounds 3 --out report.json # --keep-workdir keeps the throwaway dir; --dist PATH picks which dist/index.js to boot +bili fixture-gen --seed 42 --msgs 2000 --shape anthropic > fx.json # pure function of seed/msgs/shape: same input ⇒ byte-identical output; --out FILE writes instead of stdout +``` + ### Remote agents (`--host`) By default the proxy binds `127.0.0.1` and only accepts loopback connections. To serve agents on other machines: `bili --host 0.0.0.0` (or your LAN IP); remote agents point their model `baseURL` at `http://:/bili/…`. diff --git a/README.zh-CN.md b/README.zh-CN.md index ee02e832f4..cf890925a9 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -348,6 +348,15 @@ bili --no-auto-update # 本次启动禁用自动更新 参数优先级高于环境变量和配置文件。`bili --help` 列出全部。 +### 性能工具(`bili bench` / `bili fixture-gen`,#5460) + +只记录不评判的性能基准:`bili bench ` 把真实代理作为子进程拉起,对着一次性 XDG 目录里的进程内确定性假上游(随机端口、自动更新关闭)重放种子 fixture(顺序全量重发回合,即 dsh 插件车道的流量形状),再读子进程的 `perf.sqlite`(#5460 P1),输出 JSON 产物 —— `total_ms` p50/p95/max/min/mean、入站/出站字节总量、内存 HWM、逐回合错误。**它只记录、不评判**:指标慢照样 exit 0;只有 harness 故障(服务始终未就绪、全部回合失败、遥测未武装、运行中崩溃)才 exit 1。容差带 / 基线棘轮 / FULL+nightly CI 接线是后续工作(#5460 P3)。不带 scenario 的 `bili bench` 列出全部场景:`smoke-openai-tiny`、`proxy-append-openai-s-small`、`proxy-append-openai-s-med`、`proxy-append-anthropic-s-med`、`proxy-compress-openai-s-med`(声明 16K 窗口逼出折叠密度)、`proxy-append-openai-s-large`(约 80 MiB fixture,镜像 #5403 形状 —— nightly 档)。 + +```bash +bili bench proxy-append-openai-s-med --rounds 3 --out report.json # --keep-workdir 保留一次性目录;--dist PATH 指定拉起哪个 dist/index.js +bili fixture-gen --seed 42 --msgs 2000 --shape anthropic > fx.json # seed/msgs/shape 的纯函数:同输入 ⇒ 字节级一致;--out FILE 写文件而非 stdout +``` + ### 远程 agent(`--host`) 默认绑定 `127.0.0.1`,只接受本机连接。要给其他机器上的 agent 用:`bili --host 0.0.0.0`(或局域网 IP);远程 agent 把模型 `baseURL` 指向 `http://<本机>:<端口>/bili/…`。 diff --git a/docs/configuration.md b/docs/configuration.md index 62808a8441..4a062ee2c9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2276,6 +2276,8 @@ Full command surface (`bili --help` prints an abridged version). Precedence ever | `bili plugin list` | Show install status for every host | | `bili mcp` | Run the bili MCP server standalone on stdio | | `bili plugin-register [--origin URL] [--agent name]` | Pre-bind a conversation id to plugin mode (advanced) | +| `bili bench [scenario] [--rounds N] [--out FILE] [--keep-workdir] [--dist PATH]` | Record-only perf harness (#5460 P2): boots this proxy as a child against a hermetic deterministic fake upstream in throwaway XDG dirs, replays a seeded fixture as full-resend rounds, reports p50/p95/HWM from `perf.sqlite` — slow metrics never fail a run, only harness failures do; no scenario lists the catalog | +| `bili fixture-gen --seed K --msgs N [--shape openai\|anthropic] [--out FILE]` | Emit a deterministic synthetic conversation fixture — pure function of seed/msgs/shape (same input ⇒ byte-identical bytes); big fixtures are generated on demand, never committed | | `bili --version` / `bili --help` | Print version / help | Anything after `--` in a launcher command is passed through to the client verbatim (`bili pi -- print "hi"`). @@ -2292,6 +2294,13 @@ Anything after `--` in a launcher command is passed through to the client verbat | `--no-passthrough` | Force compression on (overrides config) | | `--no-auto-update` | Disable background self-update for this run | | `--mitm-domain ` | Extra MITM whitelist entry (repeatable; launcher only) | +| `--rounds ` | `bench`: replay runs per scenario (default `1`; each run lands as a separate `runs[]` entry in the artifact, so consumers can take a median over N) | +| `--out ` | `bench` / `fixture-gen`: write the JSON artifact / fixture to FILE instead of stdout | +| `--keep-workdir` | `bench`: keep the throwaway XDG workdir (logs, session state, `perf.sqlite`) for inspection | +| `--dist ` | `bench`: which `dist/index.js` to boot (default: the running entry point; test seam env `E2E_BILI_DIST`) | +| `--seed ` | `fixture-gen`: PRNG seed, non-negative integer (required with `--msgs`) | +| `--msgs ` | `fixture-gen`: message count, positive integer (required with `--seed`) | +| `--shape ` | `fixture-gen`: wire shape, `openai` (default) or `anthropic` | --- diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.md index 93024ae518..55871b5a33 100644 --- a/docs/configuration.zh-CN.md +++ b/docs/configuration.zh-CN.md @@ -2242,6 +2242,8 @@ ACP 原生 agent(当前为 `pi` 扩展)会在每个进程内向代理上报 | `bili plugin list` | 显示每个宿主的安装状态 | | `bili mcp` | 独立运行 bili MCP 服务器(stdio) | | `bili plugin-register [--origin URL] [--agent name]` | 预绑定会话 id 到插件模式(高级用法) | +| `bili bench [scenario] [--rounds N] [--out FILE] [--keep-workdir] [--dist PATH]` | 只记录不评判的性能基准(#5460 P2):把本代理作为子进程拉起,对着一次性 XDG 目录里的确定性假上游重放种子 fixture(全量重发回合),从 `perf.sqlite` 读 p50/p95/HWM 出 JSON 报告 —— 指标慢不会让运行失败,只有 harness 故障才失败;不带 scenario 时列出全部场景 | +| `bili fixture-gen --seed K --msgs N [--shape openai\|anthropic] [--out FILE]` | 生成确定性合成会话 fixture —— seed/msgs/shape 的纯函数(同输入 ⇒ 字节级一致);大数据按需生成,永不入库 | | `bili --version` / `bili --help` | 打印版本 / 帮助 | launcher 命令里 `--` 之后的参数原样透传给客户端(`bili pi -- print "hi"`)。 @@ -2258,6 +2260,13 @@ launcher 命令里 `--` 之后的参数原样透传给客户端(`bili pi -- pr | `--no-passthrough` | 强制开启压缩(覆盖配置文件) | | `--no-auto-update` | 本次运行禁用后台自动更新 | | `--mitm-domain ` | 追加 MITM 白名单域名(可重复;仅 launcher) | +| `--rounds ` | `bench`:每个场景的重放轮数(默认 `1`;每轮各自记入产物 `runs[]`,消费方可跨 N 轮取中位数) | +| `--out ` | `bench` / `fixture-gen`:JSON 产物 / fixture 写入 FILE,而非 stdout | +| `--keep-workdir` | `bench`:保留一次性 XDG 工作目录(日志、会话状态、`perf.sqlite`)供排查 | +| `--dist ` | `bench`:拉起哪个 `dist/index.js`(默认:当前运行入口;测试缝 env `E2E_BILI_DIST`) | +| `--seed ` | `fixture-gen`:PRNG 种子,非负整数(与 `--msgs` 同必填) | +| `--msgs ` | `fixture-gen`:消息条数,正整数(与 `--seed` 同必填) | +| `--shape ` | `fixture-gen`:线路形状,`openai`(默认)或 `anthropic` | --- From bc0e3c725c6ac961a7400393ee318c4024db1403 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 23:40:24 +0800 Subject: [PATCH 14/20] chore(cli): restore original help-block indentation on unchanged lines The help text for bili plugin-register / bili --version was shifted 2->3 spaces as an incidental whitespace edit; restored to master's bytes so the diff carries no whitespace churn on pre-existing lines (new bench/fixture-gen lines keep the subcommand-group indent used by the sessions/storage groups). --- src/cli.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 6f1887dae6..06da6526b2 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -146,8 +146,8 @@ Usage: at their host's updater — never overwritten bili plugin list show install status + installed (on-disk) version for every host bili mcp run the bili MCP server standalone (stdio) - bili plugin-register pre-bind a conversation to the plugin mode - (--origin URL, --agent name) + bili plugin-register pre-bind a conversation to the plugin mode + (--origin URL, --agent name) bili bench [scenario] perf harness (#5460 P2): boots a hermetic proxy against a deterministic fake upstream in throwaway XDG dirs, replays a seeded fixture as full-resend @@ -158,7 +158,7 @@ Usage: bili fixture-gen emit a deterministic perf fixture — pure function of --seed K --msgs N --shape openai|anthropic (--out FILE writes JSON instead of stdout) - bili --version print version + bili --version print version bili --help show this help Launcher (bili pi / bili codex / bili claude / bili omp / bili opencode / bili hermes / bili dsh / bili codebuddy / bili qoder / bili trae / bili jcode / bili kimi / bili gemini / bili iflow / bili qwen / bili antigravity / bili mcode / bili aider / bili copilot / bili amp / bili crush / bili zed / bili goose / bili kiro): From 07578ea424d011ba4f08693a0eed2af632cba541 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 23:57:47 +0800 Subject: [PATCH 15/20] docs(perf): register BILI_PERF_TELEMETRY in the config-docs seed so the table row is generated (#5460 P1) check-coverage requires every process.env read in src to have a seed entry in website/config-reference/*.yaml, and docs-sync regenerates the CONFIGURATION tables strictly from those seeds. P1 hand-wrote its row into both files instead, so regeneration deleted it -> drift -> red docs-coverage + docs-sync on the stacked head. Seed added with P1's exact wording; regenerated row lands at the same table position (default column normalized to the seed's "on"). EN file ends byte-identical to before; zh picks up the normalization. --- docs/configuration.ja.md | 1 + docs/configuration.zh-CN.md | 2 +- website/config-reference/env-only.yaml | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/configuration.ja.md b/docs/configuration.ja.md index 215a6ffc3e..a2c4826632 100644 --- a/docs/configuration.ja.md +++ b/docs/configuration.ja.md @@ -390,6 +390,7 @@ | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC 年齢スウィープウィンドウ (日) (#3463): これより長くアイドル状態のセッションはアーカイブされる (単一のアーカイブログ (archive/archive.jsonl) に 1 行の JSON が追記 + トombstone 行 + コンテンツ/メディア参照の解放)。0 で年齢スウィープを無効化; 手動エントリ POST /__bili/storage/gc は常に実行される。設定ファイルキー storage.archiveAfterDays が永続形式である。 | | `BILI_STORAGE_QUICK_CHECK` | string | on | — | オープン時のストレージ整合性セルフチェック (#4175): すべての index.db オープン時にマイグレーション前に PRAGMA quick_check を実行; このブートのスキャンによって破損が証明された場合、何も自動的に移動、コピー、再構築されない (自動回復は #4609/#4626 時代に撤回: BILI_STORAGE_JSON_WRITER=off ではインデックスが唯一の永続コピーであり、自動的な移動または再構築では完全なデータ回復を保証できないため) かつ bili はそれでも起動する — JSON エンベロープからサーブし、手動修正 (バックアップしてから index.db/-wal/-shm を手動で退避し、再起動 — 新しいインデックスは JSON エンベロープから再構築される) を名指しする大きくて実行可能なアラームを発する。成功した PASS はキャッシュされ、後の起動で再スキャンをスキップする; FAIL 判定は決して永続化されない (2026-10-30 オーナーポリシー): ランタイム/バックグラウンドの発見はアラームのみ (大きなログ + ランタイムステータス、プロセスはサーブを継続 — jsonWriter が on の間に書かれた JSON エンベロープが回復ソースのまま) であり、すべてのブートは現在のファイルセットの新しいスキャンから再決定するため、古いバージョンが残した古い FAIL が将来のブートをブロックすることは決してない。自動リネーム/修復もレガシー専用フォールバックもない: レガシー専用で続行すると、JSON エンベロープが死んだインデックスから永遠に分岐する (決して調整不能)。一時的なオープン失敗 (SQLite エンジンなし / CANTOPEN / 読み取り専用 / バックエンド DSN) は、実行可能なエラーで bili が大きく起動を拒否する — 環境問題に対するサイレントレガシーフォールバックもない (2026-10-30 オーナーポリシー; ライブな第二ライターがこの拒否リストを 2026-10-31 に残した — 1 つのローカルディスク index.db を共有することはサポートされるトポロジーであり、情報提供のみ、決してブロックしない); 真に JSON のみでサーブしたいデプロイは明示的に設定する (BILI_STORAGE_UNIFIED=legacy — 未設定はピュア SQLite のデフォルトであり legacy ではない)。チェックのコストが重要なブート時感度の高い巨大デプロイでのみ 0 を設定する。 | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | スナップショットレンダリングのシャドウ観測可能性(#3756 のフォローアップ): アームされたターンごとに送信スナップショットビューを計算するが使用せず、レガシー送信との差分を分類し、セッションごとのカウンターを Web パネルおよびスナップショットシャドウ診断トピックに表示する。0/false/off で無効化、1/true/on で有効化、それ以外の場合は設定ファイルの snapshot.shadow ブール値が適用される(デフォルトは on)。レンダリングレーン自体はロールアウトゲート(BILI_ROLLOUT_SNAPSHOT_RENDER)で制御されるため、シャドウレーンは送信バイト列を変更しない - これは後日のロールアウトを切り替えるためのドライラン証拠である。 | +| `BILI_PERF_TELEMETRY` | boolean-ish | on | — | #5460 リアルタイム性能テレメトリのキルスイッチ。DB ベースのストレージモード(sqlite/mysql)ではデフォルトで有効: 処理された各リクエストについて 1 行と 30 秒ごとのメモリ水位サンプルを index.db の隣にある perf.sqlite に書き込む —— 独立したファイルで、ユーザー自身のストアとクラッシュ/サイズで分離; 純 JSON ストレージモードは凍結されており、決して収集しない。0/false で収集を完全に無効化。テレメトリのオープン/挿入失敗は収集を劣化させる(警告 1 回 + dead フラグ)だけで、サービスには決して影響しない。 | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | 設定ファイルのパスを上書きする(パスの再配置)。スラッシュ `/` を含まない名前(NAME)のみは命名設定 `/NAME.json` を選択します。名前が誤っている場合は利用可能な名前を列挙して即座にエラーとなり、明示的なパスが誤っている場合は従来の空設定の挙動を維持します。 | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | セッションレコードの保存先(パスの再配置)。 | | `BILI_ENCRYPTION_KEY` | string | unset | — | 暗号化された永続化ペイロード用の鍵材料(シークレット)。 | diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.md index 55871b5a33..e042a8b57d 100644 --- a/docs/configuration.zh-CN.md +++ b/docs/configuration.zh-CN.md @@ -393,7 +393,7 @@ | `BILI_STORAGE_ARCHIVE_AFTER_DAYS` | integer | 365 | — | GC 年龄清扫窗口(天,#3463):闲置超过该时长的会话被归档(单文件归档日志 archive/archive.jsonl 追加一行 + 墓碑行保留标签 + 释放内容/媒体引用)。0 关闭年龄清扫;手动入口 POST /__bili/storage/gc 始终可用。配置文件键 storage.archiveAfterDays 为持久形式。 | | `BILI_STORAGE_QUICK_CHECK` | string | on | — | 存储开库自检(#4175):每次 index.db 打开在迁移之前先跑 PRAGMA quick_check;一旦由本次启动自己的扫描证实损坏,文件原样保留在原地——不做任何自动搬移、复制或重建(自动恢复已撤回:BILI_STORAGE_JSON_WRITER=off 时索引是唯一的持久副本,自动操作无法保证完整数据恢复)——bili 照常启动,改用 JSON 信封服务,并给出响亮、可操作的告警指明手工修复步骤(备份后把 index.db/-wal/-shm 手工移开再启动——全新索引会从 JSON 信封重建)。PASS 判定会被缓存,之后的启动跳过重扫;FAIL 判定永不持久化(2026-10-30 owner 策略):运行期/后台发现只是告警(响亮日志 + runtime-status,进程继续服务——jsonWriter 开启期间所写的 JSON 信封是恢复来源),每次启动都对当前文件集做新鲜扫描再判定,旧版本遗留的陈旧 FAIL 永远不能阻断后续启动。既没有自动改名/修复,也没有 legacy-only 兜底(继续以 legacy-only 运行会让 JSON 信封与死掉的索引永久分叉、再无对账可能)。瞬态打开失败(无引擎 / CANTOPEN / 只读 / 后端 DSN)仍响亮地拒绝启动并给出可操作的错误——环境类问题同样不允许静默走 legacy 兜底(活的第二写者已于 2026-10-31 移出该拒绝清单:多进程共享一个本地盘 index.db 是受支持拓扑,只提示、永不阻断启动);真正需要纯 JSON 服务的部署请显式设置 BILI_STORAGE_UNIFIED=legacy(注意:留空现在是纯 SQLite 缺省,不是 legacy)。仅当超大部署对启动耗时敏感、检测成本构成问题时才设 0 跳过。 | | `BILI_SNAPSHOT_SHADOW` | boolean | true (on) | — | 快照渲染影子对账(#3756 后续):每轮已折叠会话都用引擎现算一次出站快照视图但不使用,与 legacy 实际出站比对分类,逐会话计数进 web 面板与 snapshot-shadow 诊断主题。0/false/off 关闭;1/true/on 开启;否则读配置文件 snapshot.shadow 布尔(默认开)。渲染车道本身仍受灰度开关控制(BILI_ROLLOUT_SNAPSHOT_RENDER),影子车道绝不改变出站字节 —— 它是后续翻转灰度的干跑证据。 | -| `BILI_PERF_TELEMETRY` | boolean-ish | on(默认开) | — | #5460 实时性能采集总开关。DB 存储模式(sqlite/mysql)下默认开启:每处理一个请求落一行,外加每 30 秒一条内存水位采样,写入 index.db 旁边的独立小库 perf.sqlite —— 独立文件,与用户自己的库崩溃/体积隔离;纯 JSON 存储模式冻结、永不采集。0/false 整体关闭采集。任何打开/插入失败只降级采集(一次性告警 + 死标记),绝不影响服务。 | +| `BILI_PERF_TELEMETRY` | boolean-ish | on | — | #5460 实时性能采集总开关。DB 存储模式(sqlite/mysql)下默认开启:每处理一个请求落一行,外加每 30 秒一条内存水位采样,写入 index.db 旁边的独立小库 perf.sqlite —— 独立文件,与用户自己的库崩溃/体积隔离;纯 JSON 存储模式冻结、永不采集。0/false 整体关闭采集。任何打开/插入失败只降级采集(一次性告警 + 死标记),绝不影响服务。 | | `BILI_CONFIG_FILE` | string | unset (XDG config path) | — | 覆盖配置文件路径(路径重定位)。不带 `/` 的纯名字 NAME 选择命名配置 `/NAME.json`;名字写错立即报错并列出可用名字,显式路径写错保持历史空配置行为。 | | `BILI_SESSIONS_DIR` | string | unset (XDG state path) | — | 会话记录的存放目录(路径重定位)。 | | `BILI_ENCRYPTION_KEY` | string | unset | — | 加密持久化载荷的密钥材料(机密)。 | diff --git a/website/config-reference/env-only.yaml b/website/config-reference/env-only.yaml index b34c86ce13..1a443bbeba 100644 --- a/website/config-reference/env-only.yaml +++ b/website/config-reference/env-only.yaml @@ -83,6 +83,15 @@ entries: desc_en: "Snapshot-render shadow observability (#3756 follow-up): compute the outbound snapshot view on every armed turn WITHOUT using it, classify the diff against the legacy outbound, and surface per-session counters in the web panel + the snapshot-shadow diagnostics topic. 0/false/off disables; 1/true/on enables; otherwise the config-file snapshot.shadow boolean applies (default on). The render lane itself stays rollout-gated (BILI_ROLLOUT_SNAPSHOT_RENDER), so the shadow lane never changes outbound bytes - it is the dry-run evidence that flips the rollout later." desc_zh: "快照渲染影子对账(#3756 后续):每轮已折叠会话都用引擎现算一次出站快照视图但不使用,与 legacy 实际出站比对分类,逐会话计数进 web 面板与 snapshot-shadow 诊断主题。0/false/off 关闭;1/true/on 开启;否则读配置文件 snapshot.shadow 布尔(默认开)。渲染车道本身仍受灰度开关控制(BILI_ROLLOUT_SNAPSHOT_RENDER),影子车道绝不改变出站字节 —— 它是后续翻转灰度的干跑证据。" desc_ja: "スナップショットレンダリングのシャドウ観測可能性(#3756 のフォローアップ): アームされたターンごとに送信スナップショットビューを計算するが使用せず、レガシー送信との差分を分類し、セッションごとのカウンターを Web パネルおよびスナップショットシャドウ診断トピックに表示する。0/false/off で無効化、1/true/on で有効化、それ以外の場合は設定ファイルの snapshot.shadow ブール値が適用される(デフォルトは on)。レンダリングレーン自体はロールアウトゲート(BILI_ROLLOUT_SNAPSHOT_RENDER)で制御されるため、シャドウレーンは送信バイト列を変更しない - これは後日のロールアウトを切り替えるためのドライラン証拠である。" + - key: BILI_PERF_TELEMETRY + type: boolean-ish + default: "on" + kind: env-only + audience: user + group: envonly + desc_en: "Kill switch for #5460 realtime perf telemetry. On by default in DB-backed storage modes (sqlite/mysql): every handled request writes one row plus a 30 s memory-watermark sample into perf.sqlite next to index.db — a separate file, crash/size-isolated from the user's own store; pure-JSON storage mode is frozen and never collects. 0/false disables collection entirely. A telemetry open/insert failure degrades collection (one warn + dead flag) without ever affecting serving." + desc_zh: "#5460 实时性能采集总开关。DB 存储模式(sqlite/mysql)下默认开启:每处理一个请求落一行,外加每 30 秒一条内存水位采样,写入 index.db 旁边的独立小库 perf.sqlite —— 独立文件,与用户自己的库崩溃/体积隔离;纯 JSON 存储模式冻结、永不采集。0/false 整体关闭采集。任何打开/插入失败只降级采集(一次性告警 + 死标记),绝不影响服务。" + desc_ja: "#5460 リアルタイム性能テレメトリのキルスイッチ。DB ベースのストレージモード(sqlite/mysql)ではデフォルトで有効: 処理された各リクエストについて 1 行と 30 秒ごとのメモリ水位サンプルを index.db の隣にある perf.sqlite に書き込む —— 独立したファイルで、ユーザー自身のストアとクラッシュ/サイズで分離; 純 JSON ストレージモードは凍結されており、決して収集しない。0/false で収集を完全に無効化。テレメトリのオープン/挿入失敗は収集を劣化させる(警告 1 回 + dead フラグ)だけで、サービスには決して影響しない。" - key: BILI_CONFIG_FILE type: string default: "unset (XDG config path)" From 7c35967808555945cdf13948ce3887ed31662843 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 1 Nov 2026 23:57:47 +0800 Subject: [PATCH 16/20] refactor(perf): drop internal-only exports flagged by check-unused-exports (#5460 P2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six types (BenchScenario/BenchStat/BenchRunReport/BenchArtifact/ BenchOptions/FixtureSpec) are exported but never imported outside their own file — the gate classifies that INTERNAL_ONLY. Tests consume only the value exports (computeStats, BENCH_SCENARIOS, generateFixture, render*Body); no public API surface changes. --- src/bench.ts | 10 +++++----- src/perf-fixture.ts | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/bench.ts b/src/bench.ts index 7bef81707d..1e4103a843 100644 --- a/src/bench.ts +++ b/src/bench.ts @@ -17,7 +17,7 @@ import { generateFixture, renderAnthropicBody, renderOpenaiBody, type FixtureSha import { BUILD_COMMIT, VERSION } from "./version.js"; import { openSqliteDatabase } from "./storage/driver.js"; -export interface BenchScenario { +interface BenchScenario { id: string; desc: string; shape: FixtureShape; @@ -44,7 +44,7 @@ const HEALTH_TIMEOUT_MS = 30_000; const STOP_TIMEOUT_MS = 5_000; const ROUND_ERROR_CAP = 5; -export interface BenchStat { min: number; p50: number; p95: number; max: number; mean: number } +interface BenchStat { min: number; p50: number; p95: number; max: number; mean: number } /** Nearest-rank percentile on a sorted copy: idx = ceil(q*n)-1. Pure so the * report math is unit-testable without booting anything. */ @@ -83,7 +83,7 @@ function maxPerfId(dbFile: string): number { } } -export interface BenchRunReport { +interface BenchRunReport { run: number; rounds_run: number; rounds_failed: number; @@ -99,7 +99,7 @@ export interface BenchRunReport { }; } -export interface BenchArtifact { +interface BenchArtifact { schema_version: 1; generated_at: string; bili_version: string; @@ -289,7 +289,7 @@ function resolveDist(explicit?: string): string { return abs; } -export interface BenchOptions { +interface BenchOptions { scenarioId: string; rounds?: number; dist?: string; diff --git a/src/perf-fixture.ts b/src/perf-fixture.ts index 7568a51fec..ea3be1d5d9 100644 --- a/src/perf-fixture.ts +++ b/src/perf-fixture.ts @@ -12,7 +12,7 @@ export type FixtureShape = "openai" | "anthropic"; -export interface FixtureSpec { +interface FixtureSpec { seed: number; msgs: number; shape: FixtureShape; From ee49ab553b2c8177ddca274d7b9d02f512960ffd Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 14 Nov 2026 01:06:11 +0800 Subject: [PATCH 17/20] fix(perf): release perf.sqlite handle on store close (Windows EBUSY, #5460 P1) Windows cannot unlink a file with an open SQLite handle. P1 telemetry opens perf.sqlite in the store's sessions dir; a short-lived store closing without releasing it pins the caller's rmrf (EBUSY, win22 shards). Release the handle in close/closeQuietly/closeAsync via perfTelemetryReleaseIfBound (same contract index.db already follows). No-op for a foreign dir, idempotent. --- src/perf-telemetry.ts | 18 +++++++++++++ src/persist.ts | 9 ++++--- tests/issue5460-perf-telemetry.test.ts | 37 ++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/src/perf-telemetry.ts b/src/perf-telemetry.ts index 8efb083db4..ce87f414e9 100644 --- a/src/perf-telemetry.ts +++ b/src/perf-telemetry.ts @@ -199,6 +199,24 @@ export function perfRecordMemSample(w?: (level: "warn" | "error" | "info", msg: insertRow(db, row, w ?? loggerLog); } +/** The directory this process's telemetry is bound to (null = unbound). + * Store close paths use it to release the handle they armed. */ +export function perfBoundDir(): string | null { return state.dir; } + +/** Release the handle if this process's telemetry is bound to `dir` — + * the store that armed it is closing. Windows cannot delete a directory + * while perf.sqlite is open, so a store's close path must release the + * handle it caused to be opened (same contract as index.db — see + * SessionStore.closeQuietly). The armed mode is kept: the singleton is + * one-per-process by design; a later store in the same process finds no + * binding and records nothing (no degrade, no retry). */ +export function perfTelemetryReleaseIfBound(dir: string): boolean { + if (state.dir !== dir) return false; + if (state.db) { try { state.db.close(); } catch { /* best effort */ } state.db = null; } + state.dir = null; + return true; +} + /** Close everything (process shutdown / tests). Idempotent. */ export function perfStopAll(): void { if (state.memTimer) { clearInterval(state.memTimer); state.memTimer = null; } diff --git a/src/persist.ts b/src/persist.ts index 54264a823d..2e08d90372 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -39,7 +39,7 @@ import { restampRefsToLedger, restampContentStore } from "acp-kernel"; import { LazyBlockContents, isDroppedContentStore, isDroppedMetadata, lazyContentStore, lazyMetadata, peekSession, sessionWeight } from "./session.js"; import type { Session, BlockContent, BlockView } from "./session.js"; import { orphanContents, resolveStorageMode, UnifiedShadow, hadIntegrityQuickCheckFailure, type StorageMode } from "./storage/shadow.js"; -import { perfTelemetryInit } from "./perf-telemetry.js"; +import { perfTelemetryInit, perfTelemetryReleaseIfBound } from "./perf-telemetry.js"; import { noteColdLoad } from "./attach-heartbeat.js"; import { resolveBackendName } from "./storage/backend.js"; import { toLegacyLike, type LegacySessionLike } from "./storage/ingest.js"; @@ -4025,6 +4025,7 @@ export class SessionStore { await this.flushScheduledSaves(); await this.store.flushAll(); this.shadow?.close(); + perfTelemetryReleaseIfBound(this.sessionsDir); } /** #3406: drain the host-owned debounced saves before the kernel store's @@ -4065,11 +4066,12 @@ export class SessionStore { /** Close the unified sidecar without flushing pending writes. Intended for * short-lived stores (CLI export, one-shot loaders): cancels debounced * writes first so nothing builds against a closed handle. Windows cannot - * delete a directory while index.db is open, so leaking the handle pins - * the caller's cleanup. Never throws. */ + * delete a directory while index.db (or perf.sqlite, #5460) is open, so + * leaking the handle pins the caller's cleanup. Never throws. */ closeQuietly(): void { this.cancelAll(); this.shadow?.close(); + perfTelemetryReleaseIfBound(this.sessionsDir); } /** #3931 step 3: async twin of closeQuietly for network backends — @@ -4081,6 +4083,7 @@ export class SessionStore { await this.ready(); this.cancelAll(); await this.shadow?.closeAsync(); + perfTelemetryReleaseIfBound(this.sessionsDir); } /** Whether a write is currently pending (debounce timer armed) for a id. */ diff --git a/tests/issue5460-perf-telemetry.test.ts b/tests/issue5460-perf-telemetry.test.ts index 4f9009c149..e1bdf0c91b 100644 --- a/tests/issue5460-perf-telemetry.test.ts +++ b/tests/issue5460-perf-telemetry.test.ts @@ -12,6 +12,7 @@ import { DatabaseSync } from "node:sqlite"; import { perfTelemetryInit, perfRecordRequest, perfRecordMemSample, perfStopAll, perfRecentRows, perfActiveMode, perfTelemetryResetForTests, + perfBoundDir, perfTelemetryReleaseIfBound, PERF_DB_FILE, PERF_EVENTS_DDL, } from "../src/perf-telemetry.js"; @@ -231,4 +232,40 @@ test("#5460 P1: legacy (pure-JSON) storage mode leaves ZERO telemetry footprint } }); +test("#5460 P1: releaseIfBound releases only the bound store's handle; released telemetry never reopens", () => { + resetModule(); + const dir = sandbox(); + const other = sandbox(); + try { + perfTelemetryInit("sqlite", { dir }); + perfRecordRequest({ sessionId: "s" }); + assert.equal(perfBoundDir(), dir, "init binds the store's dir"); + assert.equal(perfTelemetryReleaseIfBound(other), false, "foreign dir → no-op"); + assert.equal(perfTelemetryReleaseIfBound(dir), true, "bound dir → released"); + assert.equal(perfTelemetryReleaseIfBound(dir), false, "already released → false"); + assert.equal(perfBoundDir(), null, "binding cleared"); + rmSync(path.join(dir, PERF_DB_FILE), { force: true }); + perfRecordRequest({ sessionId: "s2" }); + assert.equal(existsSync(path.join(dir, PERF_DB_FILE)), false, "released handle must not be reopened"); + } finally { perfStopAll(); } +}); + +test("#5460 P1: SessionStore.closeQuietly releases the perf.sqlite handle it armed (Windows EBUSY)", async () => { + // Windows refuses to unlink a file whose SQLite handle is still open: + // a short-lived store must release the telemetry handle it armed, or + // the caller's rmrf of the store's dir dies with EBUSY (win22 shards — + // issue3769/issue4241/issue4732 tests). + const { SessionStore } = await import("../src/persist.js"); + resetModule(); + const dir = sandbox(); + try { + const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + perfRecordRequest({ sessionId: "s" }); + assert.equal(perfBoundDir(), dir, "store construction arms telemetry on its own dir"); + store.closeQuietly(); + assert.equal(perfBoundDir(), null, "closeQuietly released the binding"); + assert.equal(perfTelemetryReleaseIfBound(dir), false, "no double release"); + } finally { perfStopAll(); } +}); + after(() => { for (const d of SANDBOXES.splice(0)) rmSync(d, { recursive: true, force: true }); }); From c7a640e76d818085914a0229fe6ca60da0fc067d Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 16 Nov 2026 04:23:04 +0800 Subject: [PATCH 18/20] test(perf): make closeQuietly-release test deterministic against leaked BILI_STORAGE_UNIFIED/BILI_ENCRYPTION_KEY (#5460 P1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test asserts the store arms sqlite telemetry on its own dir, but its resetModule() only cleared BILI_PERF_TELEMETRY — not BILI_STORAGE_UNIFIED or BILI_ENCRYPTION_KEY. When a prior test file in the same shard leaks one of those vars, the store arms in inert json mode (or is blocked by encryption) and perfBoundDir() returns null. Save+delete both vars before constructing the store, restore in finally — same pattern the legacy-footprint test already uses. --- tests/issue5460-perf-telemetry.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/issue5460-perf-telemetry.test.ts b/tests/issue5460-perf-telemetry.test.ts index e1bdf0c91b..bee5ffa508 100644 --- a/tests/issue5460-perf-telemetry.test.ts +++ b/tests/issue5460-perf-telemetry.test.ts @@ -257,6 +257,10 @@ test("#5460 P1: SessionStore.closeQuietly releases the perf.sqlite handle it arm // issue3769/issue4241/issue4732 tests). const { SessionStore } = await import("../src/persist.js"); resetModule(); + const savedUnified = process.env.BILI_STORAGE_UNIFIED; + const savedKey = process.env.BILI_ENCRYPTION_KEY; + delete process.env.BILI_ENCRYPTION_KEY; + delete process.env.BILI_STORAGE_UNIFIED; const dir = sandbox(); try { const store = new SessionStore({ enabled: true, dir, debounceMs: 0 }); @@ -265,7 +269,11 @@ test("#5460 P1: SessionStore.closeQuietly releases the perf.sqlite handle it arm store.closeQuietly(); assert.equal(perfBoundDir(), null, "closeQuietly released the binding"); assert.equal(perfTelemetryReleaseIfBound(dir), false, "no double release"); - } finally { perfStopAll(); } + } finally { + if (savedUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; else process.env.BILI_STORAGE_UNIFIED = savedUnified; + if (savedKey === undefined) delete process.env.BILI_ENCRYPTION_KEY; else process.env.BILI_ENCRYPTION_KEY = savedKey; + perfStopAll(); + } }); after(() => { for (const d of SANDBOXES.splice(0)) rmSync(d, { recursive: true, force: true }); }); From fc115a967a5d81778df97d776c6aff533283d4f4 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 16 Nov 2026 04:42:53 +0800 Subject: [PATCH 19/20] docs(arch): register bench.ts / perf-fixture.ts / perf-telemetry.ts in the file map (#5460) The architecture-docs guard (tools/check-architecture-docs.mjs) requires every tracked src/ file to be listed in reference/architecture.md. The three new perf modules landed without map entries, tripping the guard. --- reference/architecture.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reference/architecture.md b/reference/architecture.md index 484cd70bf1..d6afe9f1ea 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -145,6 +145,9 @@ billion-context/ │ ├── session-gc.ts # Session-file GC, opt-in (sessions.gc.* / BILI_SESSION_GC*) (#1082) │ ├── persist.ts # On-disk session persistence (kernel StateStore) │ ├── persist-eperm.ts # EPERM/EACCES write-failure alert layer (#362) +│ ├── bench.ts # Performance noise-floor bench runner (record-only, #5460 P2) +│ ├── perf-fixture.ts # Deterministic fixture generator for the bench runner (#5460 P2) +│ ├── perf-telemetry.ts # Per-request perf telemetry: sqlite ring buffer + memory sampler (#5460 P1) │ ├── storage/ # Unified storage layer (schema, SQLite engine ladder, backends) (#2671) │ │ ├── backend.ts # Storage backend contract: sqlite + mysql operation seam (#3931) │ │ ├── canonical.ts # Shared canonicalization for storage digests (#2671) From e3998e673df5e8a622cfc17848e334c4718410c0 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 16 Nov 2026 05:34:04 +0800 Subject: [PATCH 20/20] =?UTF-8?q?fix:=20#5460(P2)=20review=20=E2=80=94=20h?= =?UTF-8?q?elp=20indentation,=20stray=20blank=20lines,=20fixture-gen=20hin?= =?UTF-8?q?t,=20gated=20bench=20dist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Small review nits on the P2 bench/fixture surface: - src/cli.ts: the two new help lines (bili bench / bili fixture-gen) were indented 3 spaces while every other command line uses 2 — realigned. - src/cli.ts: removed a doubled blank line before the -i/-s posture block and a stray blank line after parseArgs' return. - src/cli.ts: the fixture-gen missing-args error suggested 'bili bench ', which is not an alternative for missing fixture-gen args — replaced with a concrete 'bili fixture-gen --seed 42 --msgs 50' example. - tests/issue5460-perf-bench.test.ts: the gated e2e smoke called runBench without a dist, so resolveDist fell back to process.argv[1] (the test file itself under 'node --test') and the proxy could never boot. Pass the built dist explicitly (E2E_BILI_DIST ?? ../dist/index.js), matching the e2e lanes' convention. Verified: ACP_TEST_BENCH=1 now goes 10/10 without E2E_BILI_DIST. - CHANGELOG.md: collapsed three consecutive blank lines to one. --- CHANGELOG.md | 2 -- src/cli.ts | 8 +++----- tests/issue5460-perf-bench.test.ts | 4 +++- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eb4ca590f..9018ed6da4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,8 +60,6 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag - **中断后/usage 丢失后的「幻影紧急压缩」:本轮拿不到计费样本时,压力档压缩延后一轮再决定**:中断的流(用户 ESC)或上游短 TTL 缓存把 usage 报告弄丢时,本轮的「计量表针」只能落回字符级本地估算——它量的是即将被改写丢掉的入站字节,实测比真实计费高约 3 倍(0.1.212 当天两起真实现场:ESC 的会话从 9478 跳成 228376、长跑会话被连续两轮注入 153% 幻影压力带,徒劳改写并烧掉前缀缓存)。此前这种坏表针直接按 EMERGENCY/OVER-LIMIT 扣扳机。现在:内核收到以「估算值」计量的压力带读数(宿主标记 tokenCountSource="estimate")时,本轮压力档注压延后一轮——下一次请求必然带着新鲜计费报告,拿真数再决定,代价≈0;计费级读数("usage")与不传 source 的老调用方字节不变,非压力档(增长/节拍)完全不受影响,硬溢出兜底臂全程照旧武装。宿主侧提前广告臂(emergencyNudge)同步镜像该位,不会拿着同一个坏读数抢跑。逃生阀:`compress.requireUsageSampleForPressure: false` 恢复旧的估算驱动行为。tests:kernel/tests/nudge-pressure-no-sample-hold.test.ts(6 例)、tests/nudge-pressure-no-sample-armed.test.ts(3 例);kernel 既有套件 1582/1582 绿。 - **External-summary mode no longer reports protected or already-folded ranges as failed compression.** An unchanged kernel preview now returns the normal idempotent no-op or kernel-error receipt without a paid summarizer call; validated internal probe summaries prevent an irrelevant submitted-hint length from blocking that receipt. Probe-created blocks with no recoverable content still refuse rather than folding an unreviewed hint. Regression: `tests/external-summary-runtime.test.ts` (24/24). - - - **#5460(P1 评审发现):Node <22.5 上整个 CLI 不再因性能采集找不到 SQLite 而启动即死。** P1 的 perf.sqlite 在模块顶层直接 `import "node:sqlite"`——该内置模块 Node ≥22.5 才有,而 package.json 仍声明 engines >=20;静态导入让**整个进程**在模块加载期崩溃(不可用的是全部 bili,不只是遥测)。现在与 index.db 走同一条引擎解析链(`openSqliteDatabase`:builtin node:sqlite → 宿主 better-sqlite3 → 自下载,#2833/#2845):老 Node 自动落到原生引擎,全缺时按既有契约一次性警告、只降级停用遥测,服务可用性不受影响;同时满足 dist 自包含门禁(零新增外部 specifier)。 * **#5412: 异步存储后端的「压缩冻结会话」现在也能自愈。** #5408 事故(一个会话连续 17 次摘要漂移后被保护机制冻住、8 小时无人知晓)的主修复 #5409 只教了同步存储路径(SQLite 直写)自救:漂移挂起期间保存会自动升级为全量重导入并清掉索引里的残留行。异步路径(opt-in 网络后端,如 mysql)那份孪生代码没搬——同样被冻住的会话在那上面永远救不回来,每次保存都重写不完整视图、下轮采样继续漂移。本 PR 在 `src/storage/shadow.ts` 的异步入队路径补上同一行对齐(`completeLog = hasRawTail || driftedSessions.has(id)`),行为与同步路径逐字一致;回归测试同时钉住正门(latch 挂起的折叠保存清掉外来行)与负门(未挂起保存不清扫)。 - **中断频繁的会话压缩提醒几乎不触发,积到几十万 token 才压一次(#5415,0.1.210 引入)**:#5293 的修复把每次用户中断后的增长参照点直接抬到当前读数,连「中断前已实测的真实增长」也一起清零;高频中断的客户端(实测一份日志 4.5 小时 85 次中断)永远攒不到增长台阶,30 万可压内容 idle 数小时,最后一次性压 33 万。现在吸收时把被清零的实测增长记入「欠账」(`nudge.absorbedDebt`)并累计吸收次数(`nudge.absorbCount`),两者都只在**第 2 次起**的中断后生效:单次中断字节级保持 #5293 的安静(不打扰用户刚改的主意——欠账再多也不当场弹提醒);第二次中断才证明会话是「高频中断型」而非「改主意」,欠账从此与新增量一起计入增长台阶,恢复 209 时代的节奏。另加「大积压排水」兑底:欠账>0 且可压量≥2×阈值且当前无未消化提醒时不再等增长台阶(与 #194 同哲学);无欠账/单次中断的会话决策字节不变。压缩成功时欠账与计数随其它基线一起清零;idle 日志同步显示 `growth X + absorbed Y across N aborts < floor Z`(单次中断显示 `held (single abort, #5293)`)便于排查。tests:kernel/tests/issue5415-absorb-debt-nudge.test.ts(欠账付台阶/单次中断钉死不弹/排水兑底/忽略提醒不重发/无欠账字节不变/压缩后清账+清计数);tests/issue5415-absorb-debt.test.ts(吸收记账算术、欠账+计数、累次累计、参照点已越过 anchor 不记账也计数、陈旧 flag 丢弃)。 diff --git a/src/cli.ts b/src/cli.ts index 06da6526b2..17bc06b947 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -148,14 +148,14 @@ Usage: bili mcp run the bili MCP server standalone (stdio) bili plugin-register pre-bind a conversation to the plugin mode (--origin URL, --agent name) - bili bench [scenario] perf harness (#5460 P2): boots a hermetic proxy + bili bench [scenario] perf harness (#5460 P2): boots a hermetic proxy against a deterministic fake upstream in throwaway XDG dirs, replays a seeded fixture as full-resend rounds, reports p50/p95/HWM from perf.sqlite — RECORD ONLY, never judges metrics; no scenario lists the catalog (--rounds N, --out FILE, --keep-workdir, --dist PATH) - bili fixture-gen emit a deterministic perf fixture — pure function + bili fixture-gen emit a deterministic perf fixture — pure function of --seed K --msgs N --shape openai|anthropic (--out FILE writes JSON instead of stdout) bili --version print version @@ -692,7 +692,6 @@ export function parseArgs(argv: string[]): Parsed { } } - // -i/-s posture flags (#5428): translate onto the existing lanes so the // launcher needs no new plumbing — `-i pi` IS `bili pi-inline` (env scrub // + extension preflight + BILI_PI_INLINE=1, no proxy spawned), and @@ -720,7 +719,6 @@ export function parseArgs(argv: string[]): Parsed { } return { command, client, clientArgs, mitmDomains, overrides, exportSelector, exportOutput, exportFull, exportJsonl, registerConversationId, pluginAction, pluginAgent, pluginWithMcp, acpCacheDir, acpCacheLog, acpCacheNoLog, acpCacheSession, sessionsSub, sessionsTarget, sessionsNoVacuum, sessionsApply, sessionsRestoreTarget, storageDays, storageVacuum, storageDryRun, jsonOutput, doctorJson, doctorFix, packAction, packSpec, packProject, packForce, benchScenario, benchRounds, benchKeepWorkdir, benchDist, outFile, fixtureSeed, fixtureMsgs, fixtureShape }; - } export async function main(): Promise { @@ -749,7 +747,7 @@ export async function main(): Promise { if (command === "fixture-gen") { try { if (fixtureSeed === undefined || fixtureMsgs === undefined) { - console.error("bili fixture-gen: --seed K and --msgs N are required (try one of: bili bench )"); + console.error("bili fixture-gen: --seed K and --msgs N are required (e.g. bili fixture-gen --seed 42 --msgs 50)"); process.exit(2); } const fx = generateFixture({ seed: fixtureSeed, msgs: fixtureMsgs, shape: fixtureShape ?? "openai" }); diff --git a/tests/issue5460-perf-bench.test.ts b/tests/issue5460-perf-bench.test.ts index 882e07de46..9627d9d8e9 100644 --- a/tests/issue5460-perf-bench.test.ts +++ b/tests/issue5460-perf-bench.test.ts @@ -11,6 +11,8 @@ import { generateFixture, renderAnthropicBody, renderOpenaiBody, type FixtureMes import { computeStats, BENCH_SCENARIOS } from "../src/bench.js"; import { parseArgs } from "../src/cli.js"; +const DIST = process.env.E2E_BILI_DIST ?? path.resolve(import.meta.dirname, "../dist/index.js"); + const fixtureJson = (spec: { seed: number; msgs: number; shape: "openai" | "anthropic" }) => JSON.stringify(generateFixture(spec)); test("fixture: deterministic — same seed yields identical bytes", () => { @@ -111,7 +113,7 @@ test("bench: end-to-end hermetic run (smoke scenario)", { timeout: 180_000 }, as const { runBench } = await import("../src/bench.js"); const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-bench-test-")); try { - const { artifact } = await runBench({ scenarioId: "smoke-openai-tiny", workdirRoot: dir, log: () => { /* quiet */ } }); + const { artifact } = await runBench({ scenarioId: "smoke-openai-tiny", workdirRoot: dir, dist: DIST, log: () => { /* quiet */ } }); assert.equal(artifact.schema_version, 1); assert.equal(artifact.scenario.id, "smoke-openai-tiny"); assert.equal(artifact.env.storage_mode, "full");