From c295f90e5ec5113cc68311c9f2363736515d4460 Mon Sep 17 00:00:00 2001 From: bili-dev Date: Wed, 4 Nov 2026 13:43:03 +0800 Subject: [PATCH 01/49] feat(fastpath): v6 identity plane storage layer (#5643) schema v11 (three append-only sidecar tables: pf_rows data plane + pf_log/pf_ckpt metadata plane), UnifiedStore SQL methods (probe/meta/ commitTurn single-transaction double-plane/compact/hydrate/deleteAll), UnifiedShadow passthroughs (mysql backend = null soft-degrade), SessionStore facades in persist.ts. Raw host ids stored untruncated (P9); state = fold(ckpt, log-after-ckpt) (P7); zero trace on mid-turn crash via transaction rollback. --- src/persist.ts | 82 ++++++++++++++++++ src/storage/schema.ts | 50 ++++++++++- src/storage/shadow.ts | 85 ++++++++++++++++++ src/storage/store.ts | 196 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 412 insertions(+), 1 deletion(-) diff --git a/src/persist.ts b/src/persist.ts index fa2d338934..6582c2c762 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2199,6 +2199,88 @@ export class SessionStore { } } + // ------------------------------------------------------------------ + // #5643 fastpath v6 identity plane — same guard shape as every other + // unified* facade: null = "plane unavailable right now" (sidecar + // missing/disabled/network backend) and the v6 lane fails soft onto + // the legacy watermark path. pfCommitTurn does NOT swallow silently: + // a turn commit failure is logged once, then fails soft for THAT turn. + // ------------------------------------------------------------------ + + /** The registry ext key mapping one relay session → every pf_sid it has + * been issued across re-arming rotations (§3.1). Stored as an array of + * sids; rides the session_ext cascade on session deletion. */ + readonly PF_V6_SID_REGISTRY_EXT_KEY = "pf:v6-sids"; + + pfAvailable(): boolean { + if (!this.shadow || this.shadow.stats.disabled) return false; + try { + return this.shadow.pfAvailable(); + } catch { + return false; + } + } + + pfProbePairs(sessRaw: string, ledgerIds: string[]): Map | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + return this.shadow.pfProbePairs(sessRaw, ledgerIds); + } + + pfMeta( + sid: string, + ): { + ckpt: { ckptLsSeq: number; metaJson: string; checksum: string; updatedAt: number } | null; + log: { lsSeq: number; kind: string; payloadJson: string; bornPosHi: number; ts: number }[]; + } | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + return this.shadow.pfMeta(sid); + } + + pfLatestGeom(sid: string): { t: number; maxPos: number } | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + return this.shadow.pfLatestGeom(sid); + } + + pfCommitTurn( + sid: string, + args: { + rows: { pos: number; ledgerId: string; sessRaw: string; coreId: string; coreJson: string }[]; + logEntries: { kind: string; payloadJson: string; bornPosHi: number; ts?: number }[]; + }, + ): number | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + try { + return this.shadow.pfCommitTurn(sid, args); + } catch (err) { + this.log("warn", `[persist] pf turn commit failed (${sid}): ${String(err)}`); // loud once, then fail-soft for THIS turn + return null; + } + } + + pfCompact(sid: string, args: { metaJson: string; nextLsSeq: number }): void { + if (!this.shadow || this.shadow.stats.disabled) return; + try { + this.shadow.pfCompact(sid, args); + } catch { + // compaction is opportunistic; skip this cycle, retry next finalize + } + } + + pfHydrateWindow(sid: string, posFrom: number, posTo: number): Map { + if (!this.shadow || this.shadow.stats.disabled) return new Map(); + return this.shadow.pfHydrateWindow(sid, posFrom, posTo); + } + + /** Session teardown (#5643 §8 test 19 third clause): sweep the pf plane + * for EVERY sid this relay session was ever issued. Reads the registry + * from ext FIRST (deleteSession sweeps session_ext itself), then deletes + * rows/log/ckpt. Safe to call when nothing was ever issued. */ + pfDeleteForSession(sessionId: string): void { + if (!this.shadow || this.shadow.stats.disabled) return; + const reg = this.shadow.getSessionExt(sessionId, this.PF_V6_SID_REGISTRY_EXT_KEY) as string[] | undefined; + if (Array.isArray(reg)) this.shadow.pfDeleteAll(reg); + } + /** #5312: reversible single-block archive / restore (see UnifiedStore). */ unifiedArchiveBlock(sessionId: string, blockId: string, archiveDir?: string): { ok: boolean; exportPath?: string; reason?: string; contentRowsDeleted?: number } | null { if (!this.shadow || this.shadow.stats.disabled) return null; diff --git a/src/storage/schema.ts b/src/storage/schema.ts index 7bdd0b747a..af4f43c012 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -68,7 +68,13 @@ // never updated). No backfill: pre-feature rows stay NULL and keep // rendering without the tag's time attribute. Same both-directions // compatibility as v5 (older build ignores, newer build ALTERs). -export const STORAGE_SCHEMA_VERSION = 10; +// - v11: fastpath v6 (#5643) composite-identity ledger — three +// append-only sidecar tables (pf_rows / pf_log / pf_ckpt), all pure +// CREATE IF NOT EXISTS: a pre-v11 install upgrades with zero byte +// change, and with the watermarkV6MsgId rollout OFF no traffic ever +// touches them. No backfill, no back-compat column juggling — older +// builds stop at the version gate and simply ignore the extra tables. +export const STORAGE_SCHEMA_VERSION = 11; /** #3250: cap for `journal_size_limit` (see STORAGE_PRAGMAS). Passive and * auto checkpoints move frames into the main db and reset the write offset @@ -398,6 +404,48 @@ export function migrateSchema(db: { db.exec("ALTER TABLE messages ADD COLUMN ts INTEGER"); } } + if (from < 11) { + // v11 (#5643): fastpath v6 identity plane — two planes per the + // contract: DATA (pf_rows: the frozen cores themselves, strictly + // append-only; a half-written pos is never reused) and METADATA + // (pf_log L0 append-only entries + pf_ckpt L1 checkpoint; current + // state = fold(ckpt, log-after-ckpt)). Naming keeps the pf_* + // namespace of the feature's header/ext footprint. The global + // probe index is on (sess_raw, ledger_id): "have we seen this + // (sessionid,msgid) pair?" answers pf_sid + pos across restarts + // (P8 id immortality); idx_pf_rows_core serves block↔pos binding + // for fold rendering. Raw host ids are stored UNTRUNCATED as + // received (P9: no digest in any judgement-relevant position). + db.exec(` +CREATE TABLE IF NOT EXISTS pf_rows ( + sid TEXT NOT NULL, + pos INTEGER NOT NULL, + ledger_id TEXT NOT NULL, + sess_raw TEXT NOT NULL, + core_id TEXT NOT NULL, + core_json TEXT NOT NULL, + PRIMARY KEY (sid, pos) +); +CREATE INDEX IF NOT EXISTS idx_pf_rows_ledger ON pf_rows(sess_raw, ledger_id); +CREATE INDEX IF NOT EXISTS idx_pf_rows_core ON pf_rows(sid, core_id); +CREATE TABLE IF NOT EXISTS pf_log ( + sid TEXT NOT NULL, + ls_seq INTEGER NOT NULL, + kind TEXT NOT NULL, + payload_json TEXT NOT NULL, + born_pos_hi INTEGER NOT NULL, + ts INTEGER NOT NULL, + PRIMARY KEY (sid, ls_seq) +); +CREATE TABLE IF NOT EXISTS pf_ckpt ( + sid TEXT NOT NULL, + ckpt_ls_seq INTEGER NOT NULL, + meta_json TEXT NOT NULL, + checksum TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (sid) +)`); + } }); if (needV8Backfill) migrateV8Identities(db); db.exec(`PRAGMA user_version = ${STORAGE_SCHEMA_VERSION}`); diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 6b77bf7ffc..22d9d57da2 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1084,6 +1084,91 @@ export class UnifiedShadow { return this.sync.listSessionExt(sessionId); } + // ------------------------------------------------------------------ + // #5643 fastpath v6 identity plane passthroughs. SQLite fast lane only: + // on a network backend (mysql) sync === null → every method returns the + // "lane unavailable" value (null / empty / false) so the request path + // degrades softly onto the legacy watermark path (§7-A②). + // ------------------------------------------------------------------ + + pfAvailable(): boolean { + if (this.sync === null) return false; + try { + return this.sync.pfAvailable(); + } catch { + return false; + } + } + + pfProbePairs(sessRaw: string, ledgerIds: string[]): Map | null { + if (this.sync === null) return null; + try { + return this.sync.pfProbePairs(sessRaw, ledgerIds); + } catch { + return null; // mid-call degrade: treat as unavailable, lane rides baseline + } + } + + pfMeta( + sid: string, + ): { + ckpt: { ckptLsSeq: number; metaJson: string; checksum: string; updatedAt: number } | null; + log: { lsSeq: number; kind: string; payloadJson: string; bornPosHi: number; ts: number }[]; + } | null { + if (this.sync === null) return null; + try { + return this.sync.pfMeta(sid); + } catch { + return null; + } + } + + pfLatestGeom(sid: string): { t: number; maxPos: number } | null { + if (this.sync === null) return null; + try { + return this.sync.pfLatestGeom(sid); + } catch { + return null; + } + } + + /** Throws are NOT swallowed here deliberately: a failed turn commit must + * be loud to the caller (it re-parses nothing and fails soft for that + * turn only — see fastpath-v6). Network backends: null (unavailable). */ + pfCommitTurn( + sid: string, + args: { + rows: { pos: number; ledgerId: string; sessRaw: string; coreId: string; coreJson: string }[]; + logEntries: { kind: string; payloadJson: string; bornPosHi: number; ts?: number }[]; + }, + ): number | null { + if (this.sync === null) return null; + return this.sync.pfCommitTurn(sid, args); + } + + pfCompact(sid: string, args: { metaJson: string; nextLsSeq: number }): void { + if (this.sync === null) return; + this.sync.pfCompact(sid, args); + } + + pfHydrateWindow(sid: string, posFrom: number, posTo: number): Map { + if (this.sync === null) return new Map(); + try { + return this.sync.pfHydrateWindow(sid, posFrom, posTo); + } catch { + return new Map(); + } + } + + pfDeleteAll(sids: string[]): void { + if (this.sync === null || sids.length === 0) return; + try { + this.sync.pfDeleteAll(sids); + } catch { + // teardown sweep: log-and-continue semantics like sibling sweeps + } + } + deleteSessionExt(sessionId: string, key?: string): number { if (this.sync === null) return 0; return this.sync.deleteSessionExt(sessionId, key); diff --git a/src/storage/store.ts b/src/storage/store.ts index ba6aed922f..bf0c623024 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -425,6 +425,8 @@ export class UnifiedStore { private readonly foreignLockHolder: { pid: number } | null; private readonly lockPath: string | null; private readonly dbPath: string; + /** #5643: cached one-time capability probe for the pf plane (null = not yet probed). */ + private pfAvail: boolean | null = null; /** #4609: the last completed check (sync, cache hit, or background * verdict) said the file set is sound — a clean close then refreshes the * verdict cache onto the post-checkpoint stats so the next boot skips @@ -2346,6 +2348,200 @@ export class UnifiedStore { return Number(this.db.prepare("DELETE FROM session_ext WHERE session_id = ? AND key = ?").run(sessionId, key).changes); } + // ====================================================================== + // #5643 fastpath v6 identity plane (schema v11): pf_rows / pf_log / + // pf_ckpt. DATA plane = pf_rows (strictly append-only; a pos is never + // reused — a half-written turn rolls back whole). METADATA plane = LSM: + // pf_log (L0 append-only) + pf_ckpt (L1); current state is ALWAYS + // fold(ckpt.metaJson, pf_log[ls_seq > ckpt_ls_seq]) — deterministic, + // idempotent, the in-memory arrays are a cache nothing depends on. + // Availability degrades soft (§7-A②): on nodes where this plane is + // unusable the v6 lane stays OFF and traffic rides the legacy path. + // Raw host ids are stored untruncated as received (P9); no digest ever + // occupies a judgement-relevant position. + // ====================================================================== + + /** One-time capability probe (table presence), cached. False on pre-v11 + * databases (feature gate ignores them) and anywhere the DDL never ran + * — callers treat false as "lane unavailable", never as an error. */ + pfAvailable(): boolean { + if (this.pfAvail !== null) return this.pfAvail; + try { + const cols = (this.db.prepare("PRAGMA table_info(pf_rows)").all() as unknown as { name?: unknown }[]) + .map((c) => (typeof c?.name === "string" ? c.name : "")); + this.pfAvail = cols.includes("core_json") && cols.includes("ledger_id"); + } catch { + this.pfAvail = false; + } + return this.pfAvail; + } + + /** Data-plane insert: one turn's new rows for `sid`, strictly sequential + * (pos values must be contiguous ascending). Never updates/deletes an + * existing pos. Throws on conflict — the surrounding transaction rolls + * back, leaving the plane exactly as before. */ + private pfInsertRows( + sid: string, + rows: { pos: number; ledgerId: string; sessRaw: string; coreId: string; coreJson: string }[], + ): void { + const ins = this.db.prepare( + "INSERT INTO pf_rows (sid, pos, ledger_id, sess_raw, core_id, core_json) VALUES (?, ?, ?, ?, ?, ?)", + ); + for (const r of rows) ins.run(sid, r.pos, r.ledgerId, r.sessRaw, r.coreId, r.coreJson); + } + + /** Global identity probe (P8): for the request-level `sessRaw`, which of + * these msgids have been seen before? Returns a map keyed by ledger id + * → { sid, pos }. ONE round-trip per distinct sess_raw (a turn shares + * one request-level sessionid). Cross-sid by design: the index is global. + * First-seen wins on any impossible duplicate (rows are immutable). */ + pfProbePairs(sessRaw: string, ledgerIds: string[]): Map { + const out = new Map(); + if (ledgerIds.length === 0) return out; + const placeholders = ledgerIds.map(() => "?").join(", "); + const rows = this.db + .prepare(`SELECT ledger_id, sid, MIN(pos) AS pos FROM pf_rows WHERE sess_raw = ? AND ledger_id IN (${placeholders}) GROUP BY ledger_id`) + .all(sessRaw, ...ledgerIds) as { ledger_id: string; sid: string; pos: number }[]; + for (const r of rows) out.set(r.ledger_id, { sid: r.sid, pos: r.pos }); + return out; + } + + /** Metadata-plane read: the L1 checkpoint (if any) plus every L0 entry + * after it, in ls_seq order. Compaction keeps this read bounded (≤64 + * entries / ≤64 KB between folds). The sqlite path never throws into + * the request path for a missing row — callers branch on null/empty. */ + pfMeta(sid: string): { + ckpt: { ckptLsSeq: number; metaJson: string; checksum: string; updatedAt: number } | null; + log: { lsSeq: number; kind: string; payloadJson: string; bornPosHi: number; ts: number }[]; + } { + const c = this.db.prepare("SELECT ckpt_ls_seq, meta_json, checksum, updated_at FROM pf_ckpt WHERE sid = ?").get(sid) as + | { ckpt_ls_seq: number; meta_json: string; checksum: string; updated_at: number } + | undefined; + const log = this.db + .prepare("SELECT ls_seq, kind, payload_json, born_pos_hi, ts FROM pf_log WHERE sid = ? AND ls_seq > ? ORDER BY ls_seq") + .all(sid, c === undefined ? 0 : c.ckpt_ls_seq) as { ls_seq: number; kind: string; payload_json: string; born_pos_hi: number; ts: number }[]; + return { + ckpt: c === undefined ? null : { ckptLsSeq: c.ckpt_ls_seq, metaJson: c.meta_json, checksum: c.checksum, updatedAt: c.updated_at }, + log: log.map((r) => ({ lsSeq: r.ls_seq, kind: r.kind, payloadJson: r.payload_json, bornPosHi: r.born_pos_hi, ts: r.ts })), + }; + } + + /** Newest geom entry (scanning the post-ckpt log backwards). Fold itself + * is owned by the fastpath-v6 module (it defines the meta schema); this + * just surfaces the last-known {t,maxPos} for cheap diagnostics without + * hydrating cores. */ + pfLatestGeom(sid: string): { t: number; maxPos: number } | null { + const { log } = this.pfMeta(sid); + for (let i = log.length - 1; i >= 0; i--) { + const e = log[i]; + if (e?.kind !== "geom") continue; + try { + const p = JSON.parse(e.payloadJson) as { t?: unknown; maxPos?: unknown }; + if (typeof p.t === "number" && typeof p.maxPos === "number") return { t: p.t, maxPos: p.maxPos }; + } catch { + continue; + } + } + return null; + } + + /** Atomic turn finalize (§4.1.3 single-transaction double plane): new + * data-plane rows AND their metadata-plane entries commit together — a + * mid-turn crash leaves zero trace and the client's next full body IS + * the replay source. Log entries are appended at strictly increasing + * ls_seq starting after MAX(ls_seq). Returns the next free ls_seq. + * Any throw → rollback → the plane is byte-identical to before. */ + pfCommitTurn( + sid: string, + args: { + rows: { pos: number; ledgerId: string; sessRaw: string; coreId: string; coreJson: string }[]; + logEntries: { kind: string; payloadJson: string; bornPosHi: number; ts?: number }[]; + }, + ): number { + if (!this.pfAvailable()) throw new Error("pf plane unavailable"); + const nextLsSeq = this.db.transaction((): number => { + this.pfInsertRows(sid, args.rows); + const start = Number( + ((this.db.prepare("SELECT COALESCE(MAX(ls_seq), 0) AS m FROM pf_log WHERE sid = ?").get(sid) as { m: number }).m ?? 0), + ); + const insLog = this.db.prepare( + "INSERT INTO pf_log (sid, ls_seq, kind, payload_json, born_pos_hi, ts) VALUES (?, ?, ?, ?, ?, ?)", + ); + let seq = start; + for (const e of args.logEntries) { + seq += 1; + insLog.run(sid, seq, e.kind, e.payloadJson, e.bornPosHi, e.ts ?? Date.now()); + } + return seq; + }); + if (typeof nextLsSeq !== "number") throw new Error("pf commit turn: transaction returned no sequence"); + return nextLsSeq; + } + + /** L0→L1 compaction (§4.1.3, called opportunistically from finalize — + * no background threads): fold the whole history into a fresh ckpt in + * the SAME transaction that deletes the superseded log entries. + * S4 lesson: the checksum covers the exact metaJson bytes and hydrate + * re-verifies — a mismatch invalidates the ckpt (rebuild from the + * replay source), never guesses. Idempotent under stable stringify at + * the call site. Caller supplies `metaJson` already folded from the + * same history this call deletes. */ + pfCompact(sid: string, args: { metaJson: string; nextLsSeq: number }): void { + if (!this.pfAvailable()) throw new Error("pf plane unavailable"); + const hash = createHash("sha256").update(args.metaJson).digest("hex"); + this.db.transaction(() => { + this.db + .prepare( + "INSERT INTO pf_ckpt (sid, ckpt_ls_seq, meta_json, checksum, updated_at) VALUES (?, ?, ?, ?, ?) " + + "ON CONFLICT (sid) DO UPDATE SET ckpt_ls_seq = excluded.ckpt_ls_seq, meta_json = excluded.meta_json, " + + "checksum = excluded.checksum, updated_at = excluded.updated_at", + ) + .run(sid, args.nextLsSeq, args.metaJson, hash, Date.now()); + this.db.prepare("DELETE FROM pf_log WHERE sid = ?").run(sid); + }); + } + + /** Should we compact? Contract thresholds (owner may veto): ≥64 entries + * OR ≥64 KB OR 30 min idle since the last entry. The idle gate fires + * only when there IS a tail to fold — empty tails cause no ckpt churn. */ + pfNeedsCompaction(opts: { entriesAfterCkpt: number; logBytesEstimate: number; lastEntryTsMs: number }): boolean { + const PF_COMPACT_ENTRIES = 64; // §4.1.3 threshold (owner may veto) + const PF_COMPACT_BYTES = 64 * 1024; + const PF_COMPACT_IDLE_MS = 30 * 60 * 1000; + if (opts.entriesAfterCkpt >= PF_COMPACT_ENTRIES || opts.logBytesEstimate >= PF_COMPACT_BYTES) return true; + return opts.entriesAfterCkpt > 0 && Date.now() - opts.lastEntryTsMs >= PF_COMPACT_IDLE_MS; + } + + /** Water window rows into a map pos→core_json for rehydration after + * restart (§4.1.4): ONLY the visible slice is lifted into memory — + * deep-folded territory stays on disk until an audit demands it. No + * plane → empty map (lane unavailable ⇒ caller serves baseline). */ + pfHydrateWindow(sid: string, posFrom: number, posTo: number): Map { + const out = new Map(); + if (!this.pfAvailable()) return out; + const rows = this.db + .prepare("SELECT pos, core_json FROM pf_rows WHERE sid = ? AND pos >= ? AND pos <= ? ORDER BY pos") + .all(sid, posFrom, posTo) as { pos: number; core_json: string }[]; + for (const r of rows) out.set(r.pos, r.core_json); + return out; + } + + /** Delete EVERYTHING for a set of pf sids (session teardown, §8 test 19 + * third clause: retire-region bytes die only with the session). The + * relay-session→pf_sid registry lives in session_ext, which the + * existing deleteSession sweep already removes; this sweeps the three + * tables themselves by the caller-supplied registry. */ + pfDeleteAll(sids: string[]): void { + if (sids.length === 0) return; + if (!this.pfAvailable()) return; + const ph = sids.map(() => "?").join(", "); + this.db.transaction(() => { + this.db.prepare(`DELETE FROM pf_rows WHERE sid IN (${ph})`).run(...sids); + this.db.prepare(`DELETE FROM pf_log WHERE sid IN (${ph})`).run(...sids); + this.db.prepare(`DELETE FROM pf_ckpt WHERE sid IN (${ph})`).run(...sids); + }); + } + /** #4949: circuit-breaker re-enable probe — one REAL committed write * through the production transaction path (proves lock acquisition + * disk space, not just connectivity). A single fixed kv key keeps the From d55a50c4791fe546d6bf222ef98c3988f5e00ea0 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Wed, 4 Nov 2026 14:37:06 +0800 Subject: [PATCH 02/49] feat(fastpath): v6 composite-identity incremental lane (#5643) msgid-identity-driven fastpath rework per contract issue #5643 (old #5576 line voided): server-minted pf_sid worlds keyed by (sessionid,msgid) pairs, pure set-difference judgment over a FULL element id list, scattered NEW-span parse with _N/wireIndex absolute rekeying against the whole ledger, client compression ignored (NEW may sit anywhere), and a hard identity pre-gate that REJECTS with zero side effects on channel/malformed/width/consistency faults (no silent legacy fallback, no count guessing, no signature fallback). - fastpath-v6.ts: evaluateWatermarkV6/finalizeWatermarkV6, matching rules (echo O(1)+spot-check / composite probe / mint+arm), world mirrors (LRU-cached fold of ckpt+log, never ahead of disk), pending single-transaction commits (pf_rows + pf_log together), breaker 5-strike / 60min parking with memory-first guard state, opportunistic L1 compaction in finalize, hydration path with S4 checksum aborts. - rollout: new feature watermarkV6MsgId percent:0 (default OFF), nested under pluginFullResendWatermark bucketing. - headers: x-bili-pf-session added to the upstream hop filter. - prepare-openai: v6 evaluated before the v5 watermark lane (silently falls back when not engaged); V6Hit consumes the same field-rebuild path; finalize branches by evaluated source; snapshot pin skips the empty window of a no-progress hit. - handle: FastpathV6IdentityFault mapped to a 400 JSON refusal at the prepared-pipeline call site (supresses raw forward); overflow refold fails open. - storage facade completion: pfNeedsCompaction delegation through UnifiedShadow/SessionStore (soft degrade when sidecar unavailable). --- src/persist.ts | 10 + src/rollout.ts | 14 + src/server/fastpath-v6.ts | 968 +++++++++++++++++++++++++++++++++++ src/server/handle.ts | 36 +- src/server/headers.ts | 5 + src/server/prepare-openai.ts | 39 +- src/storage/shadow.ts | 18 + src/storage/store.ts | 37 ++ 8 files changed, 1123 insertions(+), 4 deletions(-) create mode 100644 src/server/fastpath-v6.ts diff --git a/src/persist.ts b/src/persist.ts index 6582c2c762..9ea439901e 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2271,6 +2271,16 @@ export class SessionStore { return this.shadow.pfHydrateWindow(sid, posFrom, posTo); } + pfLedgerIds(sid: string): { pos: number; ledgerId: string; coreId: string }[] | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + return this.shadow.pfLedgerIds(sid); + } + + pfNeedsCompaction(opts: { entriesAfterCkpt: number; logBytesEstimate: number; lastEntryTsMs: number }): boolean { + if (!this.shadow || this.shadow.stats.disabled) return false; + return this.shadow.pfNeedsCompaction(opts); + } + /** Session teardown (#5643 §8 test 19 third clause): sweep the pf plane * for EVERY sid this relay session was ever issued. Reads the registry * from ext FIRST (deleteSession sweeps session_ext itself), then deletes diff --git a/src/rollout.ts b/src/rollout.ts index adb1a85157..e00c4a6a6a 100644 --- a/src/rollout.ts +++ b/src/rollout.ts @@ -64,6 +64,20 @@ export const ROLLOUT_FEATURES: Record = { // `rollout.features.pluginFullResendWatermark: true|false` (or a percent) // / `BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK=1|0`. pluginFullResendWatermark: { percent: 5 }, + // #5643 fastpath v6: (sessionid,msgid) composite-identity-driven + // incremental lane. Replaces the count-watermark guesswork with a hard + // identity channel: every body element carries its host msgid listwise, + // hits are set-difference lookups against the pf_* ledger plane, and + // only NEW elements get parsed (any positions in the body). Identity + // faults REFUSE the request loudly (400, zero side effects) instead of + // degrading silently; five consecutive faults/misses park the lane for + // 60 minutes and traffic rides the baseline lanes. Requires the unified + // sidecar storage plane (pf_rows/pf_log/pf_ckpt); without it the lane + // soft-degrades per node (baseline behavior, no error). Default 0% — + // force on with `rollout.features.watermarkV6MsgId: true` or + // `BILI_ROLLOUT_WATERMARK_V6_MSG_ID=1`; bucketed growth stays nested + // inside the v5 watermark cohort. + watermarkV6MsgId: { percent: 0, base: "pluginFullResendWatermark" }, // #4804 WS delta continuation: codex/opencode WebSocket clients that keep a // previous_response_id checkpoint stop reconstructing full history on every // turn — the session augments the delta with the server-committed previous diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts new file mode 100644 index 0000000000..3d5b69c5f8 --- /dev/null +++ b/src/server/fastpath-v6.ts @@ -0,0 +1,968 @@ +// #5643 fastpath v6 — (sessionid,msgid) composite-identity-driven incremental +// fast lane. Full rework of the #5403 watermark line; PR #5576 lineage void +// (owner ruling 2026-11-04). Governing principles (contract §2, locked): +// +// P1 identity uniqueness — the composite pair (client sessionid, msgid) is +// THE identity marker regardless of carrier; bili-minted ids (kernel h_*) +// belong to the render layer and NEVER enter hit judgement; the snapshot +// partition key is server-minted pf_sid, decoupled from any client header. +// P2 incremental parse only — hit turns transform ONLY the NEW element +// spans (set difference, ANY positions in the body); the frozen prefix +// is materialized from stored cores, raw request bytes leave memory at +// end of turn (zero residency between turns). +// P3 snapshot authority — every client-side structural change (compaction, +// in-family model switch, reorder, in-place rewrite, renumbering) is +// IGNORED; the view grows forward only; no single client event triggers +// a global rebase. +// P4 front-segment passthrough — the leading system run is freshly decoded +// from each turn's own bytes (FRONT_REREAD_ELEMENTS head fragment); top- +// level fields ride fieldSpans (caller parses individually); nothing +// front-segment ever gets frozen. +// P5 no full re-parse — hard red line; the ONLY legal full-registration +// entrances are the cold-start arm turn and a fresh pf_sid mint (they +// coincide). The arm turn's COMPUTATION rides the baseline pipeline; its +// LEDGER REGISTRATION (once per world, at finalize) transforms each +// element individually — the P5 exception covers both. +// P7 data plane is metadata-derived — current state = +// fold(ckpt, log-after-ckpt); in-memory mirrors are caches, losing one +// costs a DB re-read, nothing more. +// P9 no truncation — raw host ids travel and are stored verbatim; digests +// may only appear in warn-level diagnostics. +// +// Client compaction (B version, owner ruling 2026-11-04): when the dual +// channels reconcile (complete, |R| == scanned geometry, members sound) +// compaction is IGNORED entirely — NEW elements are registered and +// converted wherever they sit in the body; a compaction summary arriving +// as an ordinary element is admitted with ZERO special treatment. The view +// keeps growing forward; the token bill does not follow the client's +// /compact down (explicitly accepted trade). +// +// Identity hard gate (§3.2): lane entry requires BOTH channels complete and +// sound — a request-level carrier id AND an element-level list 1:1 with +// the scanned geometry, every member a real unique id. The four fault +// classes (channel missing / width mismatch / malformed member / +// consistency break) REJECT the request (400, structured body) with zero +// data-plane side effects. Three prohibitions: no silent legacy fallback, +// no counting-based attribution, no content-signature guessing. The ONE +// sanctioned relief valve (§8 line 12): five CONSECUTIVE faults park v6 +// for 60 minutes — traffic then rides the baseline lanes (served, not +// refused) so a stable client-unfixable fault cannot become a 400 loop; +// parking and recovery are loudly logged. Configuration-level cohort +// exclusion (non-shim carriers, plane-down nodes, parked sessions) is NOT +// a fault: it passes silently to the v5/legacy lanes. +import type { CoreMessage } from "acp-kernel"; +import type { OpenAIRequestBody } from "acp-kernel/wire"; +import { openaiToCore } from "acp-kernel/wire"; +import type http from "node:http"; +import { randomBytes } from "node:crypto"; +import { featureEnabled } from "../rollout.js"; +import { getStore } from "../persist.js"; +import type { SessionStore } from "../persist.js"; +import type { Session } from "../session.js"; +import { type RawStripCounts, stripRawStage } from "../marker/engine.js"; +import { INCREMENTAL_TURN_MARKER_HEADER } from "../incremental-lane.js"; +import { PF_SEQ_HEADER, PF_TAIL_IDS_HEADER } from "./fastpath.js"; +import { scanMessagesSpans, type RawSpan } from "./fastpath-scan.js"; + +/** #5643: the O(1) echo channel. Every v6-engaged response carries the minted + * pf_sid here; clients learn it and re-send it. Server-side it ACCELERATES + * matching (rule ①, skips ambiguity) — correctness additionally spot-checks + * any id the global probe resolves (a claim contradicted by the plane is a + * consistency-break fault). */ +export const PF_SESSION_HEADER = "x-bili-pf-session"; + +const PF_SID_RE = /^pf_[0-9a-f]{12}$/u; +/** P4 front re-read width: leading system run decoded fresh per turn. */ +const FRONT_REREAD_ELEMENTS = 8; +/** Breaker cadence (contract §8 line 12; same numbers the v5 line proved). */ +const PF_V6_BREAKER_FAULTS = 5; +const PF_V6_DISABLED_TTL_MS = 60 * 60 * 1000; +/** #5643 wire-list ceiling, mirrors src/agent/pf-channel.ts MAX_IDS: long + * multi-MiB #5403 bodies carry thousands of ids, not hundreds. A LONGER + * list is a wire fault (Q4), not a degrade-to-baseline. */ +const WIRE_MAX_IDS = 16_384; +/** Ledger working-copy cap (contract §4): in-memory SEEN index only. The + * DISK plane is unlimited — trimming drops just the shortcut; evicted ids + * take the DB probe path (still correct, one extra index lookup). */ +const SEEN_WORK_COPY_CAP = 2048; +const MAX_ID_LEN = 512; +const BAD_ID_CHARS = /[\s\p{C}]/u; +const EMPTY_STRIP: RawStripCounts = { panels: 0, statusBodies: 0, statusBodyLines: 0, markerLines: 0, nudges: 0, carriers: 0 }; + +/** Terminal identity fault (§3.2): prepareOpenai lets it propagate, + * handle.ts maps it to a 400 JSON refusal. NO data-plane write ever + * precedes the throw — matching is read-only. The guard counter (separate + * ext key, protection layer) records the fault BEFORE the throw, because + * §8 line 12 requires the breaker to COUNT rejects; that bookkeeping lives + * deliberately outside the identity plane (three tables + world metadata). */ +export class FastpathV6IdentityFault extends Error { + readonly status = 400; + constructor(readonly detail: string) { + super(`fastpath v6 identity fault: ${detail}`); + this.name = "FastpathV6IdentityFault"; + } + toJSON(): { error: { message: string; type: string; code: string; detail: string } } { + return { error: { message: "Request identity is incomplete or inconsistent for the incremental fast lane.", type: "invalid_request_error", code: "pf_v6_identity_fault", detail: this.detail } }; + } +} + +// --- protection-layer guard (breaker state; advisory ext key "pf:v6-guard") -- +// Memory-first mirror: the breaker must survive across calls WITHIN a process +// even without durable replicas (single-replica nodes are exactly where the +// OOM pairs bite). The ext copy adds cross-replica / cross-boot continuity; +// it is best-effort and never gates the lane by itself. +interface GuardState { faults: number; disabledAt?: number } + +const GUARD_MEM_CAP = 4096; +const guardMem = new Map(); + +function readGuard(store: SessionStore, sessionId: string): GuardState { + const m = guardMem.get(sessionId); + if (m !== undefined) return m; + try { + const raw = store.getSessionExt(sessionId, "pf:v6-guard"); + const p = typeof raw === "string" ? (JSON.parse(raw) as unknown) : raw; + if (typeof p !== "object" || p === null || Array.isArray(p)) return { faults: 0 }; + const o = p as Partial; + return { + faults: typeof o.faults === "number" && Number.isInteger(o.faults) && o.faults >= 0 ? o.faults : 0, + disabledAt: typeof o.disabledAt === "number" ? o.disabledAt : undefined, + }; + } catch { + return { faults: 0 }; + } +} + +function writeGuard(store: SessionStore, sessionId: string, g: GuardState): void { + if (guardMem.size >= GUARD_MEM_CAP) guardMem.clear(); + guardMem.set(sessionId, g); + try { + store.setSessionExt(sessionId, "pf:v6-guard", { faults: g.faults, disabledAt: g.disabledAt }); + } catch { /* advisory */ } +} + +function recordFault(store: SessionStore, sessionId: string, detail: string, log: (level: string, msg: string) => void): void { + const cur = readGuard(store, sessionId); + // An EXPIRED hold already paid its penalty — count fresh, or the first + // fault right after the TTL would instantly re-trip (v5 lesson, kept). + const staleHold = cur.disabledAt !== undefined && Date.now() - cur.disabledAt >= PF_V6_DISABLED_TTL_MS; + const faults = (staleHold ? 0 : cur.faults) + 1; + if (faults >= PF_V6_BREAKER_FAULTS) { + writeGuard(store, sessionId, { faults, disabledAt: Date.now() }); + log("warn", `[${sessionId}] fastpath v6 PARKED ${Math.round(PF_V6_DISABLED_TTL_MS / 60000)}m after ${faults} consecutive identity faults (last: ${detail}) — traffic rides baseline lanes, data plane untouched`); + } else { + writeGuard(store, sessionId, { faults, disabledAt: staleHold ? undefined : cur.disabledAt }); + log("error", `[${sessionId}] fastpath v6 REJECTED (${faults}/${PF_V6_BREAKER_FAULTS}): ${detail}`); + } +} + +function creditReset(store: SessionStore, sessionId: string, prevDisabledAt: number | undefined): void { + try { + if (!store.replicasAvailable()) return; + store.setSessionExt(sessionId, "pf:v6-guard", { faults: 0, disabledAt: prevDisabledAt }); + } catch { /* advisory */ } +} + +/** Format-level soft miss (§5 step 7): non-identity judgement fell short + * this turn (unparseable span, hydrate hiccup…). The lane degrades quietly + * to baseline for this request but CREDITS the same breaker, so a + * persistently broken session parks just like a faulting one. + * §3.2 identity faults are louder (ERROR + 400) and use recordFault. */ +function recordSoftMiss(store: SessionStore, sessionId: string, detail: string, log: (level: string, msg: string) => void): void { + const cur = readGuard(store, sessionId); + const staleHold = cur.disabledAt !== undefined && Date.now() - cur.disabledAt >= PF_V6_DISABLED_TTL_MS; + const faults = (staleHold ? 0 : cur.faults) + 1; + if (faults >= PF_V6_BREAKER_FAULTS) { + writeGuard(store, sessionId, { faults, disabledAt: Date.now() }); + log("warn", `[${sessionId}] fastpath v6 PARKED ${Math.round(PF_V6_DISABLED_TTL_MS / 60000)}m after ${faults} consecutive misses/faults (last: ${detail}) — traffic rides baseline lanes`); + } else { + writeGuard(store, sessionId, { faults, disabledAt: staleHold ? undefined : cur.disabledAt }); + log("warn", `[${sessionId}] fastpath v6 soft miss (${faults}/${PF_V6_BREAKER_FAULTS}): ${detail} — baseline turn`); + } +} + +// --- in-memory world mirrors (P7: caches of fold(ckpt, log)) -------------- +interface V6World { + readonly sid: string; + /** Last stamped total T — booking arithmetic only (§4): honest stamp, + * reconciliation basis, drift diagnostics. NEVER an interval trust. */ + t: number; + /** Highest ledger pos minted under this sid. */ + maxPos: number; + /** Last persisted front system run (until this turn's re-read lands). */ + systemText: string; + stripCounts: RawStripCounts; + multimodal: boolean; + /** id → ledger pos working copy (capped; trims silently). */ + seen: Map; +} + +const worlds = new Map(); +/** relay-session → sids issued under it (teardown cascade + prune cap). */ +const sessionWorlds = new Map>(); + +/** Resident-mirror bound. Mirrors are pure caches (P7): dropping ANY of + * them costs one plane reload on the next turn (loadWorldFromPlane + a + * ledger scan), never correctness. The cap keeps worst-case resident RAM + * flat regardless of how many worlds a long-lived node has ever seen. */ +const MAX_WORLD_MIRRORS = 512; + +function upsertWorld(w: V6World): void { + if (!worlds.has(w.sid) && worlds.size >= MAX_WORLD_MIRRORS) { + const oldest = worlds.keys().next().value as string | undefined; + if (oldest !== undefined) { + worlds.delete(oldest); + pendingCommits.delete(oldest); + } + } + worlds.delete(w.sid); + worlds.set(w.sid, w); // insert order = recency +} + +function touchSessionIndex(sessionId: string, sid: string): void { + let s = sessionWorlds.get(sessionId); + if (s === undefined) { + if (sessionWorlds.size >= 1024) sessionWorlds.clear(); + s = new Set(); + sessionWorlds.set(sessionId, s); + } + s.add(sid); +} + +/** Evaluate stashes each hit turn's exact commit plan (+ the mirror it will + * become) here so FINALIZE applies rows + log in ONE store transaction and + * only THEN promotes the cache — persistence failure leaves mirror and + * plane consistent with each other (both still at the prior world state; + * the client retries, the turn is served, nothing corrupt). */ +interface PendingCommit { + readonly rows: { pos: number; ledgerId: string; coreId: string; coreJson: string }[]; + readonly logEntries: { kind: string; payloadJson: string; bornPosHi: number }[]; + readonly proposed: V6World; +} +const pendingCommits = new Map(); + +export interface V6Hit { + /** Frozen prefix cores (ledger order) + this turn's NEW cores. Empty + * array is LEGAL and meaningful: no-progress turns hydrate the frozen + * prefix HERE (see evaluate) — callers render from `view`. */ + readonly view: CoreMessage[]; + /** Only this turn's NEW cores (what gets pinned resident). EMPTY on + * no-progress turns — prepare-openai skips pinning in that case. */ + readonly windowCores: CoreMessage[]; + /** Freshly decoded leading system run (P4 — real current bytes). */ + readonly systemText: string; + /** Scanned element count T. */ + readonly totalT: number; + readonly fieldSpans: ReadonlyMap; + /** Post-strip raw NEW messages (the window the pipeline processes). */ + readonly windowMessages: unknown[]; + readonly seedMultimodal: boolean; + /** Mirror-carried middle + this turn's fragments (markerStripped input). */ + readonly markerCounts: RawStripCounts; +} + +export interface V6Outcome { + readonly hit: V6Hit | null; + /** Measured T whenever the structural scan succeeded (stampable). */ + readonly measuredT: number | undefined; + readonly decision: string; + /** The request-level carrier value used for matching (finalize fills + * sess_raw on registered rows). Always present when the outcome is. */ + readonly sessionCarrier: string; + /** Present iff this turn MINTED a new pf_sid: finalize performs the + * per-element registration over the REAL arm-turn bytes. */ + readonly armed?: { sid: string; ids: string[]; spans: readonly RawSpan[] }; + /** Extra success-response headers (seq stamp + pf_sid echo). */ + readonly extraHeaders?: Record; +} + +// --- wire decoding --------------------------------------------------------- +/** Strict full-list decoder (§3.2 + Q4 ruling): EVERY element carries a real + * unique well-formed id — the v3 "?" placeholder is GONE from this lane; + * an id-less slot IS the fault (recorded upstream of the call site check). + * All-or-nothing: one bad segment voids the whole list. */ +function parseFullIdList(raw: string): string[] | null { + const segs = raw.split(","); + if (segs.length === 0 || segs.length > WIRE_MAX_IDS) return null; + const ids: string[] = []; + const seen = new Set(); + for (const seg of segs) { + let id: string; + try { id = decodeURIComponent(seg); } catch { return null; } + if (id.length === 0 || id.length > MAX_ID_LEN || id === "?") return null; + if (BAD_ID_CHARS.test(id)) return null; + if (seen.has(id)) return null; + seen.add(id); + ids.push(id); + } + return ids; +} + +function isSingleHeaderValue(v: string | string[] | undefined): v is string { + return typeof v === "string" && v.length > 0; +} + +/** Parse exactly the given element spans (ANY positions) in ASCENDING order + * into a messages fragment object. Leaf-decoupled twin of the v5 private + * helper (#3482 precedent: these scanner-side leaves stay import-free of + * each other's internals). */ +function parseElementFragment(bodyText: string, spans: readonly RawSpan[]): { messages: unknown[] } | null { + if (spans.length === 0) return { messages: [] }; + try { + let s = `{"messages":[`; + for (const sp of spans) s += bodyText.slice(sp.start, sp.end) + ","; + s = s.endsWith(",") ? s.slice(0, -1) : s; + s += "]}"; + const p = JSON.parse(s) as { messages?: unknown }; + return Array.isArray(p.messages) ? { messages: p.messages } : null; + } catch { + return null; + } +} + +function addStripCounts(a: RawStripCounts, b: RawStripCounts): RawStripCounts { + return { + panels: a.panels + b.panels, + statusBodies: a.statusBodies + b.statusBodies, + statusBodyLines: a.statusBodyLines + b.statusBodyLines, + markerLines: a.markerLines + b.markerLines, + nudges: a.nudges + b.nudges, + carriers: a.carriers + b.carriers, + }; +} + +// --- #3642 identity continuity (generalized from the v5 rekey) ------------ +// openaiToCore mints cluster ids — raw content-hash base (h_+16hex), or +// base_N by INTRA-CALL occurrence order — and stamps wireIndex from array +// position. Element-wise transforms therefore carry FRAGMENT-relative +// numbers; the ledger expects FULL-array ordinals (a legacy single parse +// would mint base_N continuing across ALL earlier elements). Bases are +// content hashes (#3642) and neighbour-independent, so determinism holds: +// count bases across every earlier-ledger row (pos order), continue _N, and +// fix wireIndex to the absolute element position. +function baseOfRawId(id: string): string { + const m = /^h_[0-9a-f]{16}(?:_\d+)?$/u.exec(id); + return m === null ? id : id.slice(0, 18); +} + +function rekeyContinuing( + prefixIds: readonly string[], + appends: readonly { pos: number; cores: CoreMessage[] }[], +): { pos: number; coreIds: string[]; cores: CoreMessage[] }[] { + const counts = new Map(); + for (const cid of prefixIds) { + const base = baseOfRawId(cid); + counts.set(base, (counts.get(base) ?? 0) + 1); + } + const out: { pos: number; coreIds: string[]; cores: CoreMessage[] }[] = []; + for (const a of appends) { + const ids: string[] = []; + const cores: CoreMessage[] = []; + for (const c of a.cores) { + const base = baseOfRawId(c.id); + const n = counts.get(base) ?? 0; + counts.set(base, n + 1); + ids.push(n === 0 ? base : `${base}_${n}`); + cores.push({ ...c, id: n === 0 ? base : `${base}_${n}`, wireIndex: a.pos } as CoreMessage); + } + out.push({ pos: a.pos, coreIds: ids, cores }); + } + return out; +} + +/** Transform ONE element span into its cores (single-element fragment parse + * — the v6 atom of work). null = untransformable shape (caller falls back + * honestly, uncredited). */ +function transformElement(bodyText: string, span: RawSpan): CoreMessage[] | null { + const frag = parseElementFragment(bodyText, [span]); + if (frag === null) return null; + try { + return [...openaiToCore(frag as OpenAIRequestBody).msgs]; + } catch { + return null; + } +} + +function coresHaveImagePart(cores: readonly CoreMessage[]): boolean { + for (const c of cores) { + const o = c as { imagePart?: unknown; rawOpenaiContent?: { type?: unknown } }; + if (o.imagePart === true || o.rawOpenaiContent?.type === "image_url") return true; + } + return false; +} + +/** Deserialize a sid's frozen cores from the DISK plane (per-turn + * materialization — freed at end of turn, P2 zero-residency). Stored core + * json is a PLAIN cores array; wireIndex is restored from the pos key. */ +function loadWorldCores(store: SessionStore, sid: string, maxPos: number): CoreMessage[] | null { + if (maxPos < 0) return []; + const out: CoreMessage[] = []; + for (let w = 0; w * 4096 <= maxPos; w++) { + const win = store.pfHydrateWindow(sid, w * 4096, Math.min(maxPos, (w + 1) * 4096 - 1)); + for (const [pos, json] of [...win.entries()].sort((a, b) => a[0] - b[0])) { + let arr: unknown; + try { arr = JSON.parse(json); } catch { continue; } + if (!Array.isArray(arr)) continue; + for (const c of arr as CoreMessage[]) { + if (typeof c !== "object" || c === null) continue; + out.push(Object.assign(c, { wireIndex: pos })); + } + } + } + return out; +} + +/** Hydrate the whole WORLD (metadata + id→pos index) from the plane: + * fold(ckpt.metaJson, log-after-ckpt) + one lightweight ledger scan. + * Returns null when the plane cannot produce a coherent state (corrupt + * checkpoint with an already-compacted log — nothing left to replay from). */ +function loadWorldFromPlane(store: SessionStore, sid: string): V6World | null { + const meta = store.pfMeta(sid); + if (meta === null) return null; + let folded: Record = {}; + if (meta.ckpt !== null) { + let p: unknown; + try { p = JSON.parse(meta.ckpt.metaJson); } catch { + // S4: checksum-bound bytes failed to parse back — treat as + // unrecoverable rather than guess (guessing poisons forever). + return null; + } + if (typeof p === "object" && p !== null) folded = p as Record; + } + for (const e of meta.log) { + let p: Record; + try { p = JSON.parse(e.payloadJson) as Record; } catch { continue; } + switch (e.kind) { + case "geom": + if (typeof p.t === "number") folded.t = p.t; + if (typeof p.maxPos === "number") folded.maxPos = p.maxPos; + break; + case "front": + if (typeof p.systemText === "string") folded.systemText = p.systemText; + if (typeof p.stripCounts === "object" && p.stripCounts !== null) folded.stripCounts = p.stripCounts as RawStripCounts; + if (typeof p.multimodal === "boolean") folded.multimodal = p.multimodal; + break; + default: + // arm/unarm/block carry no folded-meta fields; unknown kinds + // skipped — forward compatibility. + break; + } + } + if (typeof folded.maxPos !== "number") folded.maxPos = 0; + const seen = new Map(); + const idRows = store.pfLedgerIds(sid); + if (idRows !== null) { + for (const r of idRows) { + if (seen.size >= SEEN_WORK_COPY_CAP) break; + seen.set(r.ledgerId, r.pos); + } + } + return { + sid, + t: typeof folded.t === "number" ? folded.t : 0, + maxPos: typeof folded.maxPos === "number" ? folded.maxPos : 0, + systemText: typeof folded.systemText === "string" ? folded.systemText : "", + stripCounts: (typeof folded.stripCounts === "object" && folded.stripCounts !== null ? folded.stripCounts as RawStripCounts : EMPTY_STRIP), + multimodal: folded.multimodal === true, + seen, + }; +} + +/** The one carrier admitting a request to the v6 cohort (§7-B lock: shim + * family first). Absent ⇒ cohort EXCLUDED (baseline lanes, no fault); the + * value is the request-level half of the composite identity key. */ +function resolveCohortCarrier(headers: http.IncomingHttpHeaders): string | null { + const v = headers["x-bili-plugin-conversation"]; + if (!isSingleHeaderValue(v) || v.length > MAX_ID_LEN) return null; + return v; +} + +function registerSidForSession(store: SessionStore, sessionId: string, sid: string): void { + touchSessionIndex(sessionId, sid); + try { + if (!store.replicasAvailable()) return; + const cur = store.getSessionExt(sessionId, store.PF_V6_SID_REGISTRY_EXT_KEY); + const list = Array.isArray(cur) ? cur.filter((x): x is string => typeof x === "string") : []; + if (!list.includes(sid)) { + list.push(sid); + store.setSessionExt(sessionId, store.PF_V6_SID_REGISTRY_EXT_KEY, list); + } + } catch { /* registry advisory: the probe index remains authoritative */ } +} + +/** + * Evaluate fastpath v6 for one chat-completions request. Positioned BEFORE + * the v5 watermark evaluation in prepareOpenai: + * - return null → v6 not engaged (off/excluded/parked/degraded): + * the v5 lane decides exactly as before; + * - return an outcome → v6 OWNS this turn (hit replaces inputs, arm hands + * to the baseline pipeline, soft misses restamp): + * the v5 lane is bypassed for this request; + * - throw → identity fault: refuse 400 (handle.ts mapping), + * guard counter already recorded, plane untouched. + */ +export function evaluateWatermarkV6( + bodyText: string, + req: http.IncomingMessage, + session: Session, + input: { pluginMode: boolean; titleGen: boolean; model: string }, + log: (level: string, msg: string) => void, +): V6Outcome | null { + // --- configuration gate (silent pass-throughs) -------------------------- + if (!featureEnabled("watermarkV6MsgId")) return null; + if (!input.pluginMode || input.titleGen) return null; + if (req.headers[INCREMENTAL_TURN_MARKER_HEADER] !== undefined) return null; + const store = getStore(); + if (!store.pfAvailable()) return null; // §7-A② soft-degrade node + const guard = readGuard(store, session.id); + const held = guard.disabledAt !== undefined && Date.now() - guard.disabledAt < PF_V6_DISABLED_TTL_MS; + if (held) return null; // parked: baseline serves (parking was logged loud) + if (guard.disabledAt !== undefined && !held) { + // recovery edge: the hold expired — announce once, then proceed. + creditReset(store, session.id, undefined); + log("info", `[${session.id}] fastpath v6 BREAKER RELEASED after hold — re-entering the lane`); + } + const sessRaw = resolveCohortCarrier(req.headers); + if (sessRaw === null) return null; // cohort exclusion ≠ fault + + // --- structure ------------------------------------------------------------ + const scan = scanMessagesSpans(bodyText, "all"); + if (scan === null || scan.total === 0) return null; + const T = scan.total; + + // --- identity hard gate (§3.2): four fault classes ------------------------- + // Every fault is recordFault(then) throw inline: a direct throw is what + // the type system honors as terminal (helper-indirected never-calls are + // not narrows in this TS build). + const rawIds = req.headers[PF_TAIL_IDS_HEADER]; + if (!isSingleHeaderValue(rawIds)) { + const d = `identity channel incomplete: ${PF_TAIL_IDS_HEADER} absent`; + recordFault(store, session.id, d, log); + throw new FastpathV6IdentityFault(d); + } + const parsed = parseFullIdList(rawIds); + if (parsed === null) { + const d = "identity channel malformed: an element id fails strict screening (real, unique, well-formed required)"; + recordFault(store, session.id, d, log); + throw new FastpathV6IdentityFault(d); + } + const R = parsed; + if (R.length !== T) { + const d = `identity width mismatch: list=${R.length} vs scanned geometry T=${T}`; + recordFault(store, session.id, d, log); + throw new FastpathV6IdentityFault(d); + } + + // --- matching (§3.1 rules ①②③) --------------------------------------------- + const P = store.pfProbePairs(sessRaw, [...new Set(R)]); + if (P === null) return null; // mid-call plane degradation: soft, this turn only + const distinctSids = new Set(); + for (const v of P.values()) distinctSids.add(v.sid); + const echoRaw = req.headers[PF_SESSION_HEADER]; + const echo = isSingleHeaderValue(echoRaw) && PF_SID_RE.test(echoRaw) ? echoRaw : undefined; + + let sid: string | undefined; + if (echo !== undefined) { + // Rule ①: echo accelerates AND is verified against the plane. An + // echoed sid the plane never heard of = fabrication/loopback = + // consistency break (a forged echo is exactly how mis-attribution + // starts — the same discipline that voids partial id claims, #2821). + const knows = store.pfLatestGeom(echo) !== null || (store.pfMeta(echo)?.log.length ?? 0) > 0 || worlds.has(echo); + if (!knows) { + const d = `echo references unknown pf_sid ${echo}`; + recordFault(store, session.id, d, log); + throw new FastpathV6IdentityFault(d); + } + for (const v of P.values()) { + if (v.sid !== echo) { + const d = `consistency break: echo claims ${echo}, probe resolves ${v.sid}`; + recordFault(store, session.id, d, log); + throw new FastpathV6IdentityFault(d); + } + } + if (distinctSids.size === 0) { + // An echo rides the SAME carrier it was learned under, so a + // legitimate turn always carries at least one id the plane + // resolves here (prior turns' rows are permanent, P8). ZERO + // resolutions + a foreign echo = cross-conversation replay / + // forgery attempt — refuse rather than append stranger content + // into someone else's world. + const d = `consistency break: echo ${echo} carries no resolvable identity under this session carrier`; + recordFault(store, session.id, d, log); + throw new FastpathV6IdentityFault(d); + } + sid = echo; + } else if (distinctSids.size === 1) { + sid = [...distinctSids][0]; // Rule ②: probe-only, single owner + } else if (distinctSids.size > 1) { + // One body probing across two worlds is design-impossible (kernel + // msgids are globally unique forever; bodies are single-conversation + // linear histories) — refuse loudly instead of choosing a side. + const d = `consistency break: identity span crosses multiple worlds (${[...distinctSids].join(", ")})`; + recordFault(store, session.id, d, log); + throw new FastpathV6IdentityFault(d); + } + + if (sid === undefined) { + // Rule ③: brand-new world — the legal arm entrance (P5). Mint the + // partition key, register, and hand the COMPUTATION to the baseline + // pipeline: finalize performs the per-element registration from the + // REAL arm-turn bytes. This turn pays one full parse — same cost the + // v5/legacy lines pay today; it happens ONCE per world. + const newSid = `pf_${randomBytes(6).toString("hex")}`; + registerSidForSession(store, session.id, newSid); + creditReset(store, session.id, undefined); + const stamps: Record = { [PF_SEQ_HEADER]: String(T), [PF_SESSION_HEADER]: newSid }; + log("info", `[${session.id}] v6 ARMD new world ${newSid} (T=${T}) — baseline computes, finalize registers`); + return { + hit: null, + measuredT: T, + decision: "arm", + sessionCarrier: sessRaw, + armed: { sid: newSid, ids: R, spans: scan.messageSpans }, + extraHeaders: stamps, + }; + } + touchSessionIndex(session.id, sid); + let world = worlds.get(sid); + if (world === undefined) world = loadWorldFromPlane(store, sid) ?? undefined; + if (world === undefined) { + // The plane answered a probe under this sid but cannot reproduce its + // state (checkpoint corrupt AFTER its log was compacted away). A + // stable 400 loop would be a disaster of our own making: PARK once + // (loud) and let the baseline lanes serve; operators see the error. + log("error", `[${session.id}] v6 world unrecoverable under ${sid} (plane state inconsistent) — parking the lane 60m for this session`); + writeGuard(store, session.id, { faults: 0, disabledAt: Date.now() }); + return null; + } + + // --- set difference: the ONLY judgement axis -------------------------------- + const newIdx: number[] = []; + for (let i = 0; i < R.length; i++) { + const probed = P.get(R[i]); + if (probed === undefined && !world.seen.has(R[i])) newIdx.push(i); + } + const stamps: Record = { [PF_SEQ_HEADER]: String(T), [PF_SESSION_HEADER]: sid }; + const frontEnd = Math.min(FRONT_REREAD_ELEMENTS, T); + const frontFrag = parseElementFragment(bodyText, scan.messageSpans.slice(0, frontEnd)); + + if (newIdx.length === 0) { + // No new identities at all: the view IS the plane. Stamp + echo (the + // client converges), re-read the FRONT fresh (P4), decode NOTHING of + // the interior. Frozen prefix hydrates from disk into the turn-local + // `view`; the window pins nothing (empty windowCores is the marker). + const frozen = loadWorldCores(store, sid, world.maxPos); + if (frontFrag === null || frozen === null) { + // Degenerate corner (front unparseable / plane read hiccup): one + // honest baseline turn, uncredited-into-nothing — breaker does + // count format-level misses (§5 step 7). + recordSoftMiss(store, session.id, frontFrag === null ? "front-unparseable (no-progress)" : "hydrate-failed (no-progress)", log); + log("warn", `[${session.id}] v6 no-progress degrade (${frontFrag === null ? "front-unparseable" : "hydrate-failed"}) — baseline turn`); + return { hit: null, measuredT: T, decision: "no-progress-degraded", sessionCarrier: sessRaw, extraHeaders: stamps }; + } + stripRawStage(frontFrag, "openai"); + const frontConv = openaiToCore(frontFrag as OpenAIRequestBody); + const pending: PendingCommit = { + rows: [], + logEntries: [ + { kind: "front", payloadJson: JSON.stringify({ systemText: frontConv.systemText, stripCounts: world.stripCounts, multimodal: world.multimodal }), bornPosHi: world.maxPos }, + { kind: "geom", payloadJson: JSON.stringify({ t: T, maxPos: world.maxPos }), bornPosHi: world.maxPos }, + ], + proposed: { ...world, t: T, systemText: frontConv.systemText }, + }; + pendingCommits.set(sid, pending); + creditReset(store, session.id, undefined); // healthy ride clears the streak + return { + hit: { + view: frozen, + windowCores: [], + systemText: frontConv.systemText, + totalT: T, + fieldSpans: scan.fieldSpans, + windowMessages: [], + seedMultimodal: world.multimodal, + markerCounts: world.stripCounts, + }, + measuredT: T, + decision: "no-progress", + sessionCarrier: sessRaw, + extraHeaders: stamps, + }; + } + + // --- the NEW set: decode ONLY the new element spans (any positions) --------- + const newFragment = parseElementFragment(bodyText, newIdx.map((i) => scan.messageSpans[i])); + if (newFragment === null) { + recordSoftMiss(store, session.id, "NEW-span fragment unparseable", log); + log("warn", `[${session.id}] v6 NEW-span fragment unparseable — baseline turn (uncredited)`); + return { hit: null, measuredT: T, decision: "newspans-unparseable", sessionCarrier: sessRaw, extraHeaders: stamps }; + } + // Conservative layout rule (§2 P4 tail): a NEW element OPENING a + // system/developer run INSIDE the body means the ledger boundary was + // never established for that layout — route to baseline rather than + // guess where the freeze frontier sits. + let midSystem = false; + for (let k = 0; k < newIdx.length; k++) { + const i = newIdx[k]; + const m = newFragment.messages[k]; + if (i >= frontEnd && typeof m === "object" && m !== null) { + const role = (m as { role?: unknown }).role; + if (role === "system" || role === "developer") { + midSystem = true; + break; + } + } + } + if (midSystem) { + recordSoftMiss(store, session.id, "mid-body system run opens at NEW element", log); + log("warn", `[${session.id}] v6 NEW element opens a mid-body system run — baseline turn (uncredited)`); + return { hit: null, measuredT: T, decision: "mid-system-new", sessionCarrier: sessRaw, extraHeaders: stamps }; + } + + // Per-element transforms (the atomic unit; numbering continued below). + const positioned: { pos: number; ledgerId: string; elemIdx: number; cores: CoreMessage[] }[] = []; + let startFail = false; + for (let k = 0; k < newIdx.length; k++) { + const i = newIdx[k]; + const cores = transformElement(bodyText, scan.messageSpans[i]); + if (cores === null) { + recordSoftMiss(store, session.id, `element ${i} transform failed`, log); + log("warn", `[${session.id}] v6 element ${i} transform failed — baseline turn (uncredited)`); + startFail = true; + break; + } + positioned.push({ pos: world.maxPos + 1 + k, ledgerId: R[i], elemIdx: i, cores }); + } + if (startFail) return { hit: null, measuredT: T, decision: "element-transform-fail", sessionCarrier: sessRaw, extraHeaders: stamps }; + if (frontFrag === null) { + recordSoftMiss(store, session.id, "front fragment unparseable", log); + return { hit: null, measuredT: T, decision: "front-unparseable", sessionCarrier: sessRaw, extraHeaders: stamps }; + } + stripRawStage(frontFrag, "openai"); + const frontConv = openaiToCore(frontFrag as OpenAIRequestBody); + + // Continue #3642 numbering across the WHOLE frozen ledger: prefix ids + // come from the lightweight column scan (strings only, no payloads). + const idRows = store.pfLedgerIds(sid); + if (idRows === null) { + recordSoftMiss(store, session.id, "ledger scan failed mid-evaluate", log); + log("warn", `[${session.id}] v6 ledger scan failed mid-evaluate — baseline turn (uncredited)`); + return { hit: null, measuredT: T, decision: "ledger-scan-fail", sessionCarrier: sessRaw, extraHeaders: stamps }; + } + const prefixIds: string[] = []; + for (const r of idRows) { + if (r.coreId.length > 0) { + const parts = r.coreId.split("|"); + for (const p of parts) prefixIds.push(p); + } + } + const keyed = rekeyContinuing(prefixIds, positioned.map((p) => ({ pos: p.pos, cores: p.cores }))); + const windowCores: CoreMessage[] = []; + for (const r of keyed) for (const c of r.cores) windowCores.push(c); + + const strippedNew = stripRawStage(newFragment, "openai"); + const stripCounts = addStripCounts(world.stripCounts, strippedNew); + const multimodal = world.multimodal || coresHaveImagePart(windowCores); + + // Turn-local view assembly (prefix hydrated from disk + NEW cores) — + // freed at end of turn; the CACHE is promoted only on successful commit. + const frozen = loadWorldCores(store, sid, world.maxPos); + if (frozen === null) { + recordSoftMiss(store, session.id, "hydrate failed mid-evaluate", log); + log("warn", `[${session.id}] v6 hydrate failed mid-evaluate — baseline turn (uncredited)`); + return { hit: null, measuredT: T, decision: "hydrate-fail", sessionCarrier: sessRaw, extraHeaders: stamps }; + } + const view = [...frozen, ...windowCores]; + const lastPos = world.maxPos + 1 + newIdx.length - 1; + const proposedSeen = new Map(world.seen); + for (let k = 0; k < positioned.length; k++) { + if (proposedSeen.size >= SEEN_WORK_COPY_CAP) break; + proposedSeen.set(positioned[k].ledgerId, positioned[k].pos); + } + pendingCommits.set(sid, { + rows: keyed.map((r) => ({ + pos: r.pos, + ledgerId: positioned[r.pos - (world.maxPos + 1)].ledgerId, + coreId: r.coreIds.join("|"), + coreJson: JSON.stringify(r.cores), + })), + logEntries: [ + { kind: "front", payloadJson: JSON.stringify({ systemText: frontConv.systemText, stripCounts, multimodal }), bornPosHi: lastPos }, + { kind: "geom", payloadJson: JSON.stringify({ t: T, maxPos: lastPos }), bornPosHi: lastPos }, + ], + proposed: { ...world, t: T, maxPos: lastPos, systemText: frontConv.systemText, stripCounts, multimodal, seen: proposedSeen }, + }); + creditReset(store, session.id, undefined); // consecutive-fault streak clears on a sound turn + + log("debug", `[${session.id}] v6 HIT NEW=${newIdx.length}${newIdx.length <= 4 ? "@" + newIdx.join(".") : ""} T=${T} maxPos=${lastPos}`); + return { + hit: { + view, + windowCores, + systemText: frontConv.systemText, + totalT: T, + fieldSpans: scan.fieldSpans, + windowMessages: newFragment.messages, + seedMultimodal: multimodal, + markerCounts: stripCounts, + }, + measuredT: T, + decision: "hit", + sessionCarrier: sessRaw, + extraHeaders: stamps, + }; +} + +/** + * Call AFTER the pipeline ran for a v6-engaged turn (mirror of + * finalizeWatermarkTurn). Persists the evaluate-stashed commit (single + * transaction, rows + log together), promotes the cache, restamps. Every + * failure degrades softly: the turn ALREADY served from coherent inputs, + * and the next turn re-syncs from the plane (or, on a persistent gap, the + * probe/equivalence checks route to baseline uncredited). + * Returns the success-response headers (always at least the seq stamp). + */ +export function finalizeWatermarkV6( + session: Session, + evaluated: V6Outcome, + input: { + /** Arm turns only: the request bytes for per-element registration. */ + bodyText?: string; + /** Arm turns only: the baseline pipeline's decoded front + its marker + * strip accounting for THIS real arm body (mirrors what a v6-hit + * turn would have written — the registry starts from truth, not + * zeros). */ + armSystemText?: string; + armStripCounts?: RawStripCounts; + armMultimodal?: boolean; + }, + log: (level: string, msg: string) => void, +): Record { + const headers: Record = {}; + if (evaluated.measuredT === undefined) return headers; + const T = evaluated.measuredT; + headers[PF_SEQ_HEADER] = String(T); + if (evaluated.extraHeaders) Object.assign(headers, evaluated.extraHeaders); + + const store = getStore(); + try { + if (evaluated.armed !== undefined) { + // ARM registration: every element of THIS turn, wire order → + // pos 0..T-1 (first world under the fresh sid). + const { sid, ids, spans } = evaluated.armed; + if (input.bodyText === undefined || spans.length !== T || ids.length !== T) { + log("error", `[${session.id}] v6 arm: geometry mismatch at finalize — registration skipped (world stays empty; next turn arms fresh via probe-miss)`); + return headers; + } + const positioned: { pos: number; ledgerId: string; cores: CoreMessage[] }[] = []; + for (let i = 0; i < spans.length; i++) { + const cores = transformElement(input.bodyText, spans[i]); + if (cores === null) { + log("error", `[${session.id}] v6 arm: element ${i} transform failed — registration aborted (turn already served from baseline)`); + return headers; + } + positioned.push({ pos: i, ledgerId: ids[i], cores }); + } + const keyed = rekeyContinuing([], positioned.map((p) => ({ pos: p.pos, cores: p.cores }))); + const allCores = keyed.flatMap((r) => r.cores); + const frontText = input.armSystemText ?? ""; + const frontStrips = input.armStripCounts ?? EMPTY_STRIP; + const frontMultimodal = input.armMultimodal ?? coresHaveImagePart(allCores); + const plan = { + rows: keyed.map((r) => ({ pos: r.pos, ledgerId: ids[r.pos], coreId: r.coreIds.join("|"), coreJson: JSON.stringify(r.cores) })), + logEntries: [ + { kind: "arm", payloadJson: JSON.stringify({ t: T }), bornPosHi: T - 1 }, + { kind: "front", payloadJson: JSON.stringify({ systemText: frontText, stripCounts: frontStrips, multimodal: frontMultimodal }), bornPosHi: T - 1 }, + { kind: "geom", payloadJson: JSON.stringify({ t: T, maxPos: T - 1 }), bornPosHi: T - 1 }, + ], + }; + if (store.pfCommitTurn(sid, { rows: plan.rows.map((r) => ({ ...r, sessRaw: evaluated.sessionCarrier })), logEntries: plan.logEntries }) === null) { + log("warn", `[${session.id}] v6 arm commit failed — world deferred to next turn`); + return headers; + } + const seen = new Map(ids.map((id, i) => [id, i] as const)); + upsertWorld({ sid, t: T, maxPos: T - 1, systemText: frontText, stripCounts: frontStrips, multimodal: frontMultimodal, seen }); + log("info", `[${session.id}] v6 armed world ${sid} (T=${T}, ${keyed.length} elements registered)`); + return headers; + } + // Hit-turn commits (including no-progress front/geom ledger moves). + const sid = evaluated.extraHeaders?.[PF_SESSION_HEADER]; + if (sid === undefined) return headers; + const pending = pendingCommits.get(sid); + if (pending === undefined) return headers; + pendingCommits.delete(sid); + if (store.pfCommitTurn(sid, { + rows: pending.rows.map((r) => ({ ...r, sessRaw: evaluated.sessionCarrier })), + logEntries: pending.logEntries, + }) === null) { + // Plane hiccup: mirror NOT promoted — the NEXT turn reloads from + // the plane (which lacks this turn's NEW rows) and re-registers + // them as NEW AGAIN… which would duplicate. Prevent: mark the + // world dirty-on-reload by keeping the proposal cached WITHOUT + // committing; simplest correct move is to drop the cache so the + // reload sees plane truth, and accept that this turn's NEW ids + // get re-probed-and-registered next round IF the client resends + // (hosts always resend full bodies — resends are the norm, so + // the re-registration converges; pos advances, no overwrite). + worlds.delete(sid); + log("warn", `[${session.id}] v6 commit failed — cache dropped, plane truth restored; this turn's NEW elements re-register on the (guaranteed full-body) retry`); + return headers; + } + upsertWorld({ ...pending.proposed }); + opportunisticCompact(store, sid, pending.proposed, log); + return headers; + } catch (err) { + log("warn", `[${session.id}] v6 finalize degraded: ${String(err)}`); + return headers; + } +} + +/** Opportunistic L0→L1 fold straight from finalize (contract §4.1.3: no + * background threads). The meta is folded from the SAME state the log + * records, so fold(new,∅) ≡ fold(old,oldLog) holds by construction; the + * checksum binds the exact metaJson bytes (S4: verify at hydrate, never + * guess). Thresholds per contract (owner may veto): ≥64 entries OR ≥64 KB + * OR 30 min idle. */ +function opportunisticCompact(store: SessionStore, sid: string, world: V6World, log: (level: string, msg: string) => void): void { + const meta = store.pfMeta(sid); + if (meta === null || meta.log.length === 0) return; + const last = meta.log[meta.log.length - 1]; + let bytes = 0; + for (const e of meta.log) bytes += e.payloadJson.length + 32; + if (!store.pfNeedsCompaction({ entriesAfterCkpt: meta.log.length, logBytesEstimate: bytes, lastEntryTsMs: last.ts })) return; + const metaJson = JSON.stringify({ + v: 1, + t: world.t, + maxPos: world.maxPos, + systemText: world.systemText, + stripCounts: world.stripCounts, + multimodal: world.multimodal, + }); + store.pfCompact(sid, { metaJson, nextLsSeq: last.lsSeq }); + log("debug", `[v6:${sid}] pf_log folded to pf_ckpt @ ls_seq ${last.lsSeq} (${meta.log.length} entries)`); +} + +/** Drop every in-memory mirror + pending commit for a relay session — pair + * of clearWatermarkSeed at teardown paths. The DISK plane survives until + * pfDeleteForSession (delete-time cascade). */ +export function clearWatermarkV6(sessionId: string): void { + const sids = sessionWorlds.get(sessionId); + if (sids !== undefined) { + for (const sid of sids) { + worlds.delete(sid); + pendingCommits.delete(sid); + } + sessionWorlds.delete(sessionId); + } +} + +/** Introspection for panels/tests: live mirrors of one relay session. */ +export function watermarkV6DebugState(sessionId: string): { liveSids: string[]; worlds: Record } { + const sids = [...(sessionWorlds.get(sessionId) ?? [])]; + const out: Record = {}; + for (const sid of sids) { + const w = worlds.get(sid); + if (w !== undefined) out[sid] = { t: w.t, maxPos: w.maxPos, seen: w.seen.size }; + } + return { liveSids: sids, worlds: out }; +} diff --git a/src/server/handle.ts b/src/server/handle.ts index f944ca4cac..efa929030a 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -41,6 +41,7 @@ import { buildAnonymousProvenance, type AnonymousProvenance } from "../anon-prov import { maybeAdoptForkBlocks, maybeAdoptResume, maybeAdoptDerivedFork } from "../fork-adoption.js"; import { executeThreadsReforge } from "./threads-lane.js"; import { resetThreadTailTracking } from "./thread-tracking.js"; +import { FastpathV6IdentityFault } from "./fastpath-v6.js"; import { resolveClaudeTranscriptLineage } from "../claude-transcript-lineage.js"; import { consumePluginRegisterFor, pluginAgentHeader, pluginCompressionProxyHeader, pluginConversationHeader, pluginHeadersMatchModel, pluginReportedContextWindow, pluginReportedMaxOutput, pluginReportedModel, pluginRequestAgentHeader, pluginRuntimeInfoFor, pluginRuntimeInfoForConversation, publicForkInputCheck, recordChainVerdict, recordPluginSession, rememberPluginMessages, resolveConversation, runtimeConversationId, takePendingPluginRegister } from "../plugin.js"; import { scheduleAffinityPersist } from "../affinity-persist.js"; @@ -2900,7 +2901,26 @@ async function handleRequest( if (guardScheme !== undefined && !resignArmable && !guardResign.enabled) { clearSignedRefusal(guardScheme); } - const pendingForward = await withSessionLock(session, () => runPreparedPipeline(true)); + // #5643 §3.2: fastpath v6 identity hard-gate refusal → structured + // 400 (never a 5xx, never a silent fallback). Caught HERE — + // outside the pipeline, because the pipeline's tail treats a null + // result as "forward the raw body", which would leak an already- + // refused request to the model endpoint. The evaluate path wrote + // zero data-plane state before throwing, so a fixed retry lands + // clean. `forwarded = true` suppresses that tail pass-through. + let pendingForward: { body: string | Buffer; prepared: Prepared | null } | null = null; + try { + pendingForward = await withSessionLock(session, () => runPreparedPipeline(true)); + } catch (err) { + if (err instanceof FastpathV6IdentityFault && !res.headersSent && res.writable !== false) { + loggerLog("error", `[${sessionId}] fastpath v6 identity fault refused: ${err.detail}`); + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify(err.toJSON())); + forwarded = true; + } else { + throw err; + } + } if (pendingForward) { forwarded = true; // #1195: wire clients (omp/pi on the plain proxy path) treat an @@ -2915,7 +2935,19 @@ async function handleRequest( // original 400 through verbatim. const overflowRefold = pendingForward.prepared ? async (realWindow: number | undefined): Promise<{ body: string | Buffer; prepared: Prepared | null } | null> => { - const next = await withSessionLock(session, () => runPreparedPipeline(false, realWindow)); + const next = await withSessionLock(session, () => runPreparedPipeline(false, realWindow)).catch((e): { body: string | Buffer; prepared: Prepared | null } | null => { + // Refold re-runs prepare → the v6 hard gate can + // fire a SECOND time mid-request. Fail open: hand + // back the original overflow 400 untouched rather + // than turning a fold attempt into a new error + // shape (the first turn's rejection already told + // the client what to fix). + if (e instanceof FastpathV6IdentityFault) { + loggerLog("error", `[${sessionId}] fastpath v6 identity fault during overflow refold — original 400 passes through: ${e.detail}`); + return null; + } + throw e; + }); if (!next || String(next.body) === String(pendingForward.body)) return null; pendingForward.body = next.body; pendingForward.prepared = next.prepared; diff --git a/src/server/headers.ts b/src/server/headers.ts index 217b660bbd..76c5029ea3 100644 --- a/src/server/headers.ts +++ b/src/server/headers.ts @@ -33,6 +33,11 @@ export const UPSTREAM_HOP_HEADERS = new Set([ "x-bili-pf-count", "x-bili-pf-seq", "x-bili-pf-tail-ids", + // #5643: fastpath v6 world echo — server-minted pf_sid returned on + // v6-engaged responses so clients can re-send it (matching rule ①). + // Same billing-plane bookkeeping family as the three above: never to + // a model endpoint. + "x-bili-pf-session", "te", "trailer", "upgrade", diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index ab8f430473..bbab3b39cc 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -41,6 +41,7 @@ import { injectOpenaiTool, injectTool } from "./inject.js"; import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decisions.js"; import { featureEnabled } from "../rollout.js"; import { evaluateWatermarkFastpath, finalizeWatermarkTurn, type FastpathHit, type FastpathOutcome } from "./fastpath.js"; +import { evaluateWatermarkV6, finalizeWatermarkV6, type V6Outcome } from "./fastpath-v6.js"; import { collectRequestIds, probeRequestMembership } from "../request-membership.js"; export async function prepareOpenai( @@ -115,11 +116,38 @@ export async function prepareOpenai( // seed stash minted on identical bytes last turn (#3642 deterministic ids), // while head + tail fragments and scalar fields are decoded from raw // slices. Every miss falls through to the legacy path byte-identically. + // #5643 fastpath v6 — composite-identity lane ((sessionid, msgid)): + // evaluated FIRST and owns the turn whenever it engages (hit / arm / + // stamped degrade). Its identity fault (FastpathV6IdentityFault) + // PROPAGATES on purpose to handle.ts → 400 refusal (§3.2) — nothing + // upstream of that mapping may swallow it. + // #5403/#5418 watermark fastpath (v5, count-watermark era) follows and + // keeps serving non-cohort traffic byte-identically as before. let wpHit: FastpathHit | undefined; let wpOutcome: FastpathOutcome | null = null; let wpFullText: string | undefined; + let v6Evaluated: V6Outcome | null = null; const wpModel = typeof parsed.model === "string" ? parsed.model : ""; - if (bodyText !== undefined && pluginMode && !isTitleGen && featureEnabled("pluginFullResendWatermark")) { + if (bodyText !== undefined && pluginMode && !isTitleGen && featureEnabled("watermarkV6MsgId")) { + v6Evaluated = evaluateWatermarkV6(bodyText, req, session, { pluginMode, titleGen: isTitleGen, model: wpModel }, log); + if (v6Evaluated !== null) { + wpFullText = bodyText; // whole-wire image accounting stays honest + log("debug", `[${session.id}] fastpath v6 ${v6Evaluated.hit ? "HIT" : v6Evaluated.decision}${v6Evaluated.measuredT !== undefined ? ` T=${v6Evaluated.measuredT}` : ""}`); + if (v6Evaluated.hit) { + // V6Hit is field-for-field identical to FastpathHit — the same + // consumption pattern applies below. + wpHit = v6Evaluated.hit; + const fields: Record = {}; + for (const [key, span] of wpHit.fieldSpans) { + if (key === "messages") fields[key] = wpHit.windowMessages; + else fields[key] = JSON.parse(bodyText.slice(span.start, span.end)); + } + parsed = fields as OpenAIRequestBody; + inboundMessageCount = wpHit.totalT; + } + } + } + if (v6Evaluated === null && bodyText !== undefined && pluginMode && !isTitleGen && featureEnabled("pluginFullResendWatermark")) { wpFullText = bodyText; wpOutcome = evaluateWatermarkFastpath(bodyText, req, session, { pluginMode, titleGen: isTitleGen, model: wpModel }, log); log("debug", `[${session.id}] fastpath ${wpOutcome.hit ? "HIT" : wpOutcome.decision}${wpOutcome.creditedMiss ? "(credited)" : ""}${wpOutcome.measuredT !== undefined ? ` T=${wpOutcome.measuredT}` : ""}`); @@ -571,7 +599,14 @@ export async function prepareOpenai( snapshotMessages(session, wpHit !== undefined ? wpHit.windowCores : originalMessages); markDirty(session); let wpResponseHeaders: Record | undefined; - if (wpFullText !== undefined && wpOutcome !== null && wpTurnStrip !== undefined) { + if (v6Evaluated !== null && wpTurnStrip !== undefined) { + wpResponseHeaders = finalizeWatermarkV6(session, v6Evaluated, { + bodyText: wpFullText, + armSystemText: wpSeedSystemText, + armStripCounts: wpTurnStrip, + }, log); + if (Object.keys(wpResponseHeaders).length === 0) wpResponseHeaders = undefined; + } else if (wpFullText !== undefined && wpOutcome !== null && wpTurnStrip !== undefined) { wpResponseHeaders = finalizeWatermarkTurn(session, wpOutcome, { model: wpModel, legacyCores: originalMessages, diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 22d9d57da2..a663c30382 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1160,6 +1160,24 @@ export class UnifiedShadow { } } + pfLedgerIds(sid: string): { pos: number; ledgerId: string; coreId: string }[] | null { + if (this.sync === null) return null; + try { + return this.sync.pfLedgerIds(sid); + } catch { + return null; + } + } + + pfNeedsCompaction(opts: { entriesAfterCkpt: number; logBytesEstimate: number; lastEntryTsMs: number }): boolean { + if (this.sync === null) return false; + try { + return this.sync.pfNeedsCompaction(opts); + } catch { + return false; + } + } + pfDeleteAll(sids: string[]): void { if (this.sync === null || sids.length === 0) return; try { diff --git a/src/storage/store.ts b/src/storage/store.ts index bf0c623024..80075f5108 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1700,6 +1700,27 @@ export class UnifiedStore { this.db.prepare("DELETE FROM refs WHERE session_id = ?").run(sessionId); this.db.prepare("DELETE FROM ccr_entries WHERE session_id = ?").run(sessionId); this.db.prepare("DELETE FROM conversations WHERE session_id = ?").run(sessionId); + // #5643: fastpath v6 identity-plane partitions hang off this + // session through the ext registry (persist.ts + // PF_V6_SID_REGISTRY_EXT_KEY); cascade their ledger/log/ckpt rows + // in the SAME transaction so deletion leaves no cross-plane + // orphans (tables exist by migration invariant). + const pfRegRow = this.db + .prepare("SELECT value_json FROM session_ext WHERE session_id = ? AND key = 'pf:v6-sids'") + .get(sessionId) as { value_json: string } | undefined; + if (pfRegRow !== undefined) { + let pfSids: unknown[] = []; + try { + const p = JSON.parse(pfRegRow.value_json); + if (Array.isArray(p)) pfSids = p; + } catch { /* corrupt registry: nothing to cascade from */ } + for (const sid of pfSids) { + if (typeof sid !== "string") continue; + this.db.prepare("DELETE FROM pf_rows WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_log WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_ckpt WHERE sid = ?").run(sid); + } + } this.db.prepare("DELETE FROM session_ext WHERE session_id = ?").run(sessionId); this.db.prepare("DELETE FROM sessions WHERE session_id = ?").run(sessionId); for (const hash of refs) this.releaseContent(hash); @@ -2512,6 +2533,22 @@ export class UnifiedStore { return opts.entriesAfterCkpt > 0 && Date.now() - opts.lastEntryTsMs >= PF_COMPACT_IDLE_MS; } + /** Lightweight ledger id/index scan for one sid, POS ORDER: the raw + * host ledger id + the canonical core-id list for every stored element. + * Strings only (no core_json) — feeds the #3642 rekey base counting + * without hydrating any payload bytes. */ + pfLedgerIds(sid: string): { pos: number; ledgerId: string; coreId: string }[] | null { + if (!this.pfAvailable()) return null; + try { + const rows = this.db + .prepare("SELECT pos, ledger_id, core_id FROM pf_rows WHERE sid = ? ORDER BY pos") + .all(sid) as { pos: number; ledger_id: string; core_id: string }[]; + return rows.map((r) => ({ pos: r.pos, ledgerId: r.ledger_id, coreId: r.core_id })); + } catch { + return null; + } + } + /** Water window rows into a map pos→core_json for rehydration after * restart (§4.1.4): ONLY the visible slice is lifted into memory — * deep-folded territory stays on disk until an audit demands it. No From 9c1fb8689fb3f4ba3fd840f1b0b9a7ac3acbed10 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Wed, 4 Nov 2026 14:37:06 +0800 Subject: [PATCH 03/49] feat(pf-channel): full-list tail-ids wire form + x-bili-pf-session echo (#5643) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chokepoint now rides the FULL captured element id list (one percent-encoded segment per body element, body order, literal comma separators) instead of the k-1 tail slice — the v6 server judges NEW by set difference and the count claim keeps its last-stamped meaning for v5 lanes. A learned x-bili-pf-session (server-minted world id) is echoed back once received. Count-only mode (stamp known, nothing captured) still emits exactly x-bili-pf-count as before. MAX_IDS raised to 16k elements for long-hostory conversations. --- src/agent/pf-channel.ts | 60 ++++++++++++++++++++++++++++------------ tests/omp-msgids.test.ts | 3 +- tests/pf-channel.test.ts | 37 +++++++++++++------------ 3 files changed, 64 insertions(+), 36 deletions(-) diff --git a/src/agent/pf-channel.ts b/src/agent/pf-channel.ts index af447479e2..fbba8a0bc4 100644 --- a/src/agent/pf-channel.ts +++ b/src/agent/pf-channel.ts @@ -14,10 +14,15 @@ // - emit: one pfRequestHeaders() merge in native-intercept at the // routed-send site (count + tail-ids alike). // -// Wire form: x-bili-pf-count = last-seen total T (pure count mode); -// x-bili-pf-tail-ids = percent-encoded "[id(K-1)] + new element ids" joined -// with LITERAL commas (commas INSIDE ids travel %2C — unambiguous). A captured -// list missing or shorter than K degrades to count-only: never partial. +// Wire form (v6, #5643): x-bili-pf-tail-ids = percent-encoded FULL element +// id list — one entry per body element, body order, LITERAL commas between +// segments (commas INSIDE ids travel %2C — unambiguous). The server +// reconciles it against its structural scan (|R| == T, all real ids) and +// judges NEW by set difference alone. x-bili-pf-count still carries the +// LAST-STAMPED total from earlier responses (v5 lanes read it; v6 ignores +// it). x-bili-pf-session echoes the server-minted world id once learned. +// A captured list that resolves NO ids degrades to absent: better nothing +// than a partial list onto the wire. // // Owner ruling 2026-11-02 (m02989): hosts compress and REWRITE their histories, // so all-or-nothing alignment proofs against the wire fight reality and kill @@ -33,6 +38,9 @@ const COUNT_HEADER = "x-bili-pf-count"; const TAIL_IDS_HEADER = "x-bili-pf-tail-ids"; const CONVERSATION_HEADER = "x-bili-plugin-conversation"; const SEQ_HEADER = "x-bili-pf-seq"; +// #5643: the v6 world echo header the server mints (pf_). +const PF_SESSION_HEADER = "x-bili-pf-session"; +const PF_SID_RE = /^pf_[0-9a-f]{12}$/u; export const PF_SLOT_UNKNOWN = "?"; const MAX_ID_LEN = 512; @@ -41,10 +49,13 @@ const RESERVED_PREFIX = "acp_summary_"; // the incremental-lane adapters; keep them in sync (a bad id degrades the WHOLE // list to absent, never a partial list onto the wire). const BAD_ID_CHARS = /[\s\p{C}]/u; -const MAX_IDS = 2048; // long-conversation ceiling; the sliced head stays small +// #5643: v6 sends the FULL list — long-hostory ceilings apply to whole bodies +// now, not just the tail window. 16k elements (~8 MB worst-case id payload, +// typically far less) covers the multi-MiB #5403 conversations in full. +const MAX_IDS = 16_384; const MAX_TRACKED = 1024; // sid table cap (same hygiene as the old per-lane maps) -type Tracked = { t?: number; ids?: string[] }; +type Tracked = { t?: number; ids?: string[]; /** #5643: minted pf_sid echoed back by a v6-enabled server. */ v6?: string }; const tracked = new Map(); @@ -66,11 +77,16 @@ export function pfLearnStamp(headersIn: Record, respStatus: numb if (respStatus !== 200) return; const sid = headersIn[CONVERSATION_HEADER]; if (sid === undefined || sid.length === 0 || respHeaders === null || respHeaders === undefined) return; + const slot = takeSlot(sid); const raw = respHeaders.get(SEQ_HEADER); - if (raw === null || !/^\d+$/.test(raw)) return; - const t = Number(raw); - if (!Number.isSafeInteger(t)) return; - takeSlot(sid).t = t; + if (raw !== null && /^\d+$/.test(raw)) { + const t = Number(raw); + if (Number.isSafeInteger(t)) slot.t = t; + } + // #5643: remember the minted world id once the server hands one over; + // every later request echoes it (matching rule ①, O(1) + spot-checked). + const rawSid = respHeaders.get(PF_SESSION_HEADER); + if (rawSid !== null && PF_SID_RE.test(rawSid)) slot.v6 = rawSid; } /** Thin capture adapter contract (called ONCE PER outbound model request, @@ -111,18 +127,26 @@ export function pfCaptureIds(sid: string, ids: unknown): void { /** Chokepoint emit (native-intercept routed-send site): the handshake headers * this request should carry, given its outgoing header record (source of the - * conversation id). {} whenever the session never saw a stamp yet. */ + * conversation id). {} whenever nothing pf-related is known for this sid. + * + * #5643 wire form: the FULL captured id list rides on every request (one + * segment per body element, body order) — the server reconciles it against + * its scan and judges purely by set difference, so no window math happens + * here anymore. The count claim keeps its old meaning (last-stamped total, + * emitted only once known) for v5 lanes and for count-only clients; v6 + * servers ignore it. Cold start: turn ONE carries the full list WITHOUT a + * count — v6 uses that very turn as its legal arm entrance (register pos + * 0..T-1) while v5-only servers fall through to the legacy full-parse path + * (they require the count to engage at all). */ export function pfRequestHeaders(incoming: Record): Record { const sid = incoming[CONVERSATION_HEADER]; if (sid === undefined || sid.length === 0) return {}; const slot = tracked.get(sid); - if (slot === undefined || slot.t === undefined) return {}; - const k = slot.t; - if (k < 1) return {}; - const out: Record = { [COUNT_HEADER]: String(k) }; + if (slot === undefined) return {}; + const out: Record = {}; + if (slot.t !== undefined && slot.t >= 1) out[COUNT_HEADER] = String(slot.t); const ids = slot.ids; - if (ids !== undefined && ids.length >= k) { - out[TAIL_IDS_HEADER] = ids.slice(k - 1).map(encodeURIComponent).join(","); - } + if (ids !== undefined && ids.length >= 1) out[TAIL_IDS_HEADER] = ids.map(encodeURIComponent).join(","); + if (slot.v6 !== undefined) out[PF_SESSION_HEADER] = slot.v6; return out; } diff --git a/tests/omp-msgids.test.ts b/tests/omp-msgids.test.ts index 4d58e938a2..300529493c 100644 --- a/tests/omp-msgids.test.ts +++ b/tests/omp-msgids.test.ts @@ -286,7 +286,8 @@ test("omp watermark: an aligned branch feeds pf-channel for the chokepoint to st pfLearnStamp({ [PF_CONVO]: sid }, 200, new Headers({ "x-bili-pf-seq": "2" })); const out = pfRequestHeaders({ [PF_CONVO]: sid }); assert.equal(out["x-bili-pf-count"], "2"); - assert.equal(out["x-bili-pf-tail-ids"], "e5f6a7b8", "K=2 ⇒ anchor element 1 + new elements"); + // #5643 full-list wire form: every captured element rides (no K-window math client-side) + assert.equal(out["x-bili-pf-tail-ids"], "e1a2b3c4,e5f6a7b8", "K=2 ⇒ both elements ride the full list"); round.ack(2); }); diff --git a/tests/pf-channel.test.ts b/tests/pf-channel.test.ts index e4ef46b80e..7afb5bc1ad 100644 --- a/tests/pf-channel.test.ts +++ b/tests/pf-channel.test.ts @@ -30,17 +30,20 @@ test("pf-channel: count mode; tail-ids ride only a validated list; encoding", () // count-only until ids are captured assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2" }); pfChannel.pfCaptureIds(sid, ["a", "b", "c"]); - // K=2 ⇒ [id(K-1)] + new = ["b","c"]; per-segment encode - assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-tail-ids"], "b,c"); - // comma inside an id travels %2C; the separator stays literal (K=1 ⇒ whole list sent) + // #5643 full-list wire form: EVERY captured id rides (one segment per + // body element) — the server judges by set difference, so no window math + // happens client-side anymore + assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2", "x-bili-pf-tail-ids": "a,b,c" }); + // comma inside an id travels %2C; the separator stays literal learn(inH, "1"); pfChannel.pfCaptureIds(sid, ["pre,v1", "new,id"]); assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-tail-ids"], encodeURIComponent("pre,v1") + "," + encodeURIComponent("new,id")); - // a list shorter than K degrades to count-only (never partial) + // a short list STILL emits in full (honest presence beats old partial-slice + // degradation; the server's hard gate handles geometry, not the client) const sidShort = "sid-count-short"; learn({ [CONVO]: sidShort }, "2"); pfChannel.pfCaptureIds(sidShort, ["x"]); - assert.deepEqual(pfChannel.pfRequestHeaders({ [CONVO]: sidShort }), { "x-bili-pf-count": "2" }); + assert.deepEqual(pfChannel.pfRequestHeaders({ [CONVO]: sidShort }), { "x-bili-pf-count": "2", "x-bili-pf-tail-ids": "x" }); }); test("pf-channel: failed whole-resolution CLEARS the stored list (no stale slice)", () => { @@ -48,8 +51,8 @@ test("pf-channel: failed whole-resolution CLEARS the stored list (no stale slice const inH = { [CONVO]: sid }; learn(inH, "2"); pfChannel.pfCaptureIds(sid, ["a", "b"]); - assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-tail-ids"], "b"); - for (const bad of [null, undefined, "not-an-array", [], 42, Array.from({ length: 2049 }, (_, i) => `m${i}`)]) { + assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2", "x-bili-pf-tail-ids": "a,b" }); + for (const bad of [null, undefined, "not-an-array", [], 42, Array.from({ length: 16_385 }, (_, i) => `m${i}`)]) { pfChannel.pfCaptureIds(sid, bad); assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2" }, `bad=${String(bad)} must clear`); } @@ -58,14 +61,14 @@ test("pf-channel: failed whole-resolution CLEARS the stored list (no stale slice test("pf-channel: bad SLOTS ride '?' — one id-less element never poisons the list (owner 2026-11-02)", () => { const sid = "sid-slots-1"; const inH = { [CONVO]: sid }; - learn(inH, "2"); // K=2 ⇒ this turn's wire carries slots[1..] + learn(inH, "2"); // stamp known; the FULL list rides regardless of position for (const row of [ - [["ok", 7], "%3F"], - [["ok", ""], "%3F"], - [["acp_summary_x", "y"], "y"], - [["a b", "c"], "c"], - [["c", "c"], "%3F"], // duplicate real id collapses its repeat to unknown - [["z".repeat(513), "w"], "w"], + [["ok", 7], "ok,%3F"], + [["ok", ""], "ok,%3F"], + [["acp_summary_x", "y"], "%3F,y"], + [["a b", "c"], "%3F,c"], + [["c", "c"], "c,%3F"], // duplicate real id collapses its repeat to unknown + [["z".repeat(513), "w"], "%3F,w"], ] as const) { pfChannel.pfCaptureIds(sid, [...row[0]]); assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-tail-ids"], row[1], `slots=${JSON.stringify(row[0])}`); @@ -137,7 +140,7 @@ test("chokepoint: routed model request carries count+tail-ids; the response lear assert.equal(sinkIn[0].headers["x-bili-pf-count"], undefined, "turn 1 unseeded"); }); assert.equal(sink[1].headers["x-bili-pf-count"], "2"); - assert.equal(sink[1].headers["x-bili-pf-tail-ids"], "m2,m3"); + assert.equal(sink[1].headers["x-bili-pf-tail-ids"], "m1,m2,m3"); // the chokepoint fabricates no lane identity of its own assert.ok(!("x-bili-plugin" in sink[1].headers), "chokepoint must not add a lane plugin header"); }); @@ -162,7 +165,7 @@ test("chokepoint: direct-to-proxy self-branch emits and learns (dsh profile shap assert.equal(sinkIn[0].headers["x-bili-pf-count"], undefined, "turn 1 unseeded on the self branch too"); }); assert.equal(sink[1].headers["x-bili-pf-count"], "4"); - assert.equal(sink[1].headers["x-bili-pf-tail-ids"], "n4,n5"); + assert.equal(sink[1].headers["x-bili-pf-tail-ids"], "n1,n2,n3,n4,n5"); }); test("chokepoint: already-routed /bili/ MODEL url is stamped; __bili/ control-plane never is", async () => { @@ -185,7 +188,7 @@ test("chokepoint: already-routed /bili/ MODEL url is stamped; __bili/ control-pl }, { headersFor: () => ({ "x-bili-plugin": "dsh" }) }); assert.equal(sink.length, 2); assert.equal(sink[0].headers["x-bili-pf-count"], "4"); - assert.equal(sink[0].headers["x-bili-pf-tail-ids"], "d,e"); + assert.equal(sink[0].headers["x-bili-pf-tail-ids"], "a,b,c,d,e"); assert.equal(sink[1].headers["x-bili-pf-count"], undefined); assert.equal(sink[1].headers["x-bili-pf-tail-ids"], undefined); assert.ok(!sink[1].url.includes("/bili/"), "control plane must not be re-routed"); From edba38af91c859c1ddc44b03fa38a379b1e90261 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Wed, 4 Nov 2026 14:37:06 +0800 Subject: [PATCH 04/49] test(fastpath): v6 identity-lane matrix; schema head bumps to v11 expectations New tests/fastpath-v6-msgid.test.ts covering the #5643 matrix rows: pure-append increment with frozen-core byte-parity vs full parse, '?'-free full lists, cold-start arming via baseline pass, compaction as pure ignore (forward-only view, summary judged only by ids), scattered NEW parsing, identity-boundary re-mint with old-world archive retention, three restart cycles across an L1 fold (hydration oracle), echo-header O(1) matching, carrier rotation, fault classes (incomplete/malformed/width/zero-resolve echo) with 5-strike parking and time-travel recovery, cap handling, and session-deletion cascade cleanup. Schema-head dependent suites updated for STORAGE_ SCHEMA_VERSION 10 -> 11 (v11 adds pf_rows/pf_log/pf_ckpt). --- tests/fastpath-v6-msgid.test.ts | 416 ++++++++++++++++++ tests/issue4666-migration-dedup.test.ts | 2 +- .../issue4677-v8-migration-streaming.test.ts | 12 +- tests/issue4796-gc-batched-sweep.test.ts | 6 +- tests/storage-ext.test.ts | 4 +- tests/storage-ingest-watermark.test.ts | 8 +- tests/storage-unified.test.ts | 4 +- 7 files changed, 434 insertions(+), 18 deletions(-) create mode 100644 tests/fastpath-v6-msgid.test.ts diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts new file mode 100644 index 0000000000..8f2aab09af --- /dev/null +++ b/tests/fastpath-v6-msgid.test.ts @@ -0,0 +1,416 @@ +// #5643 — fastpath v6: composite-identity-driven incremental lane. +// +// Contract source: issue #5643 (§8 matrix rows are cited inline as §8-N). +// The v6 lane judges purely by set difference over (sess_raw, ledger_id): +// NEW = R \ seen, anywhere in the body; frozen prefix hydrates from the +// pf_* plane; client-side compaction is ignored EXCEPT for honest stamping. +// +// Harness: unit-level against a REAL sqlite sidecar (UnifiedShadow in +// shadow mode + SessionStore seam), so the identity plane, folding and +// hydration run exactly as in production — only the HTTP layer is skipped +// (the 400 mapping itself lives in handle.ts and is exercised by the fault +// class shape asserted here). + +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type http from "node:http"; +import { openaiToCore } from "acp-kernel/wire"; +import { SessionStore, _setStoreForTest, _resetStoreForTest } from "../src/persist.ts"; +import type { SessionStore as SessionStoreType } from "../src/persist.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import type { Session } from "../src/session.ts"; +import { + evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, watermarkV6DebugState, PF_SESSION_HEADER, +} from "../src/server/fastpath-v6.ts"; +import { PF_SEQ_HEADER, PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; +import type { V6Outcome } from "../src/server/fastpath-v6.ts"; + +const ENV = "BILI_ROLLOUT_WATERMARK_V6_MSG_ID"; + +interface Msg { role: string; content: string } + +function makeBody(messages: Msg[]): string { + return JSON.stringify({ model: "m", messages }); +} + +const SYS = "sys v1"; +/** Distinct ids per element; `n` suffix keeps ids unique across turn shapes. */ +function ids(n: number, tag = ""): string[] { + return Array.from({ length: n }, (_, i) => `id${i}${tag}`); +} + +type LogSink = { lines: { level: string; msg: string }[] } & ((level: string, msg: string) => void); +function makeLog(): LogSink { + const l = { lines: [] as { level: string; msg: string }[] }; + return Object.assign((level: string, msg: string) => { l.lines.push({ level, msg }); }, { lines: l.lines }); +} + +interface TurnResult { outcome: V6Outcome; headers: Record; sid: string | undefined } + +function turn( + sessId: string, carrier: string, + bodyText: string, idList: string[], log: LogSink, echoSid?: string, + extraHeaders: Record = {}, +): TurnResult { + const headers: Record = { "x-bili-plugin-conversation": carrier, ...extraHeaders }; + if (idList.length > 0) headers[PF_TAIL_IDS_HEADER] = idList.map(encodeURIComponent).join(","); + if (echoSid !== undefined) headers[PF_SESSION_HEADER] = echoSid; + const req = { headers } as unknown as http.IncomingMessage; + const session = { id: sessId } as unknown as Session; + const outcome = evaluateWatermarkV6(bodyText, req, session, { pluginMode: true, titleGen: false, model: "m" }, log); + assert.ok(outcome !== null, `expected v6 to engage (log=${JSON.stringify(log.lines.slice(-4))})`); + const headersOut = finalizeWatermarkV6(session, outcome, { bodyText }, log); + return { outcome, headers: headersOut, sid: outcome.armed?.sid ?? headersOut[PF_SESSION_HEADER] }; +} + +function assertFault(fn: () => unknown, detailPart: string): FastpathV6IdentityFault { + let caught: unknown; + try { fn(); } catch (e) { caught = e; } + assert.ok(caught instanceof FastpathV6IdentityFault, `expected FastpathV6IdentityFault, got ${String(caught)}: ${JSON.stringify(caught)}`); + const f = caught as FastpathV6IdentityFault; + assert.ok(f.detail.includes(detailPart), `detail ${JSON.stringify(f.detail)} ⊄ ${JSON.stringify(detailPart)}`); + assert.equal(f.status, 400); + const json = f.toJSON(); + assert.equal(json.error.type, "invalid_request_error"); + assert.equal(json.error.code, "pf_v6_identity_fault"); + return f; +} + +interface H { dir: string; store: SessionStoreType; cleanup: () => void } + +const SESSION_TAG = "v6"; + +function makeH(tag: string): H { + process.env[ENV] = "1"; + _resetStoreForTest(); + const dir = mkdtempSync(path.join(os.tmpdir(), `v6-${tag}-`)); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as never); + assert.ok(shadow, "shadow sidecar must open in the hermetic dir"); + const store = new SessionStore({ dir: path.join(dir, "sessions"), enabled: true, shadow }); + _setStoreForTest(store); + return { dir, store, cleanup: () => { _resetStoreForTest(); rmSync(dir, { recursive: true, force: true }); } }; +} + +test("§8-1 pure append: tail-only decode, _N parity with full parse, no-progress ride", (t) => { + const h = makeH("append"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "hello one" }]); + const r1 = turn("s-app", "carrier-app", b1, ids(2, "-a"), log); + assert.equal(r1.outcome.decision, "arm"); + assert.match(r1.sid ?? "", /^pf_[0-9a-f]{12}$/u); + assert.equal(r1.headers[PF_SEQ_HEADER], "2"); + const sid = r1.sid!; + + // turn 2: append one element → hit, view == full parse of the whole body + const b2 = makeBody([ + { role: "system", content: SYS }, + { role: "user", content: "hello one" }, + { role: "assistant", content: "hi there" }, + ]); + const r2 = turn("s-app", "carrier-app", b2, [...ids(2, "-a"), "id2-new"], log, sid); + assert.equal(r2.outcome.decision, "hit"); + const hit = r2.outcome.hit!; + assert.equal(hit.totalT, 3); + assert.ok(hit.windowCores.length >= 1, "window pins the new cores"); + const oracle = openaiToCore(JSON.parse(b2) as never).msgs; + assert.deepEqual(hit.view.map((c) => JSON.parse(JSON.stringify(c))), JSON.parse(JSON.stringify(oracle)), + "v6 assembled view must byte-match the full-parse oracle (numbering included)"); + + // turn 3: identical body (client resends unchanged) → NO-PROGRESS: + // nothing decoded interior-wise, stamp re-echoed, plane untouched growth-wise + const r3 = turn("s-app", "carrier-app", b2, [...ids(2, "-a"), "id2-new"], log, sid); + assert.equal(r3.outcome.decision, "no-progress"); + assert.equal(r3.headers[PF_SESSION_HEADER], sid); + assert.equal(r3.headers[PF_SEQ_HEADER], "3"); + assert.equal(r3.outcome.hit!.view.length, hit.view.length, "no-progress view IS the plane (no shrink, no grow)"); + assert.equal(r3.outcome.hit!.windowCores.length, 0, "no window pin on a plain ride"); + } finally { h.cleanup(); } +}); + +test("§8-3/15 silent pass-throughs: unscannable body / non-cohort carrier / feature OFF fall to baseline without touching anything", () => { + const h = makeH("silent"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "x" }]); + // (a) cohort carrier present but the body is NOT scannable JSON ⇒ + // fail-soft null (bad bodies are never identity faults; the + // baseline lanes own them). Note: carrier present WITH a clean + // scan but NO id channel is FAULT class 1 — asserted in §8-7. + const reqA = { headers: { "x-bili-plugin-conversation": "c-sil", [PF_TAIL_IDS_HEADER]: "a,b" } } as unknown as http.IncomingMessage; + assert.equal(evaluateWatermarkV6("not-json{", reqA, { id: "s-sil" } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log), null); + // (b) non-cohort carrier absence ⇒ null too (configuration-level exclusion, NOT a fault) + const reqB = { headers: { [PF_TAIL_IDS_HEADER]: "a,b" } } as unknown as http.IncomingMessage; + assert.equal(evaluateWatermarkV6(b1, reqB, { id: "s-sil" } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log), null); + // (c) default-OFF switch: with the env unset the module reports a plain miss even on perfect traffic + delete process.env[ENV]; + try { + const reqC = { headers: { "x-bili-plugin-conversation": "c-sil", [PF_TAIL_IDS_HEADER]: "a,b" } } as unknown as http.IncomingMessage; + assert.equal(evaluateWatermarkV6(b1, reqC, { id: "s-sil" } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log), null); + } finally { process.env[ENV] = "1"; } + assert.equal(watermarkV6DebugState("s-sil").liveSids.length, 0, "no world minted by any of the silent pass-throughs"); + } finally { h.cleanup(); } +}); + +test("§8-7..12 identity hard gate: four fault classes → structured 400, zero side effects, recovery leaves no trace", () => { + const h = makeH("faults"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "one" }, { role: "user", content: "two" }]); + const I3 = ["a1", "a2", "a3"]; + // Establish world A under this session/carrier (legal arm, rule ③) + const rA = turn("s-fa", "car-a", b1, I3, log); + assert.equal(rA.outcome.decision, "arm"); + const sidA = rA.sid!; + const guardBefore = h.store.getSessionExt("s-fa", "pf:v6-guard") as { faults?: number; disabledAt?: number } | undefined; + const faultsAfter = (): number => (h.store.getSessionExt("s-fa", "pf:v6-guard") as { faults?: number } | undefined)?.faults ?? -1; + void guardBefore; + + // (1) channel incomplete: carrier present, id header absent + const g0 = faultsAfter(); + assertFault(() => turn("s-fa", "car-a", b1, [], log), "identity channel incomplete"); + assert.equal(faultsAfter(), g0 + 1, "each refusal is counted into the breaker"); + + // (2) malformed member: an unknown-slot placeholder rides the wire + assertFault(() => turn("s-fa", "car-a", b1, ["?", "a2", "a3"], log), "identity channel malformed"); + + // (3) width mismatch: list length ≠ scanned geometry T + assertFault(() => turn("s-fa", "car-a", b1, I3.slice(0, 2), log), "identity width mismatch"); + + // (4) consistency break: echo carries another world's unresolvable span + assertFault(() => turn("s-fa", "car-other", b1, I3, log, sidA), "carries no resolvable identity"); + + // Zero side effects: the plane still holds ONLY the arm registration + const meta = h.store.pfMeta(sidA); + assert.ok(meta !== null && meta.log.length === 3, `arm wrote exactly 3 log entries, got ${meta ? meta.log.length : "null"}`); + assert.equal(watermarkV6DebugState("s-fa").liveSids.length, 1); + + // Recovery-without-trace: the FIFTH hard fault parks the lane inside + // its own request (which is still refused once); while parked, broken + // requests degrade SILENTLY to baseline instead of throwing loops; + // after the hold expires the next request walks back in and a healthy + // turn hits cleanly. + const realNow = Date.now; + try { + assertFault(() => turn("s-fa", "car-a", b1, I3.slice(0, 2), log), "identity width mismatch"); + const reqP = { headers: { "x-bili-plugin-conversation": "car-a", [PF_TAIL_IDS_HEADER]: I3.slice(0, 2).join(",") } } as unknown as http.IncomingMessage; + for (let i = 0; i < 4; i++) { + assert.equal(evaluateWatermarkV6(b1, reqP, { id: "s-fa" } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log), null, `parked lane must not throw-loop (iter ${i})`); + } + assert.ok(log.lines.some((l) => l.level === "warn" && l.msg.includes("PARKED")), "parking announced loudly"); + Date.now = (() => realNow() + 61 * 60 * 1000) as typeof Date.now; + const b2 = makeBody([...JSON.parse(b1).messages, { role: "user", content: "three" }]); + const rRec = turn("s-fa", "car-a", b2, [...I3, "a4"], log, sidA); + assert.equal(rRec.outcome.decision, "hit", "post-hold recovery rides straight back into the lane"); + } finally { Date.now = realNow; } + } finally { h.cleanup(); } +}); + +test("§8-4/5 client compaction is IGNORED: no hide machinery, summaries are ordinary increments, stamp follows the body", () => { + const h = makeH("compact"); + try { + const log = makeLog(); + // Grow to T=6 through arm + appends + let msgs: Msg[] = [{ role: "system", content: SYS }, { role: "user", content: "m1" }]; + let r = turn("s-cp", "car-cp", makeBody(msgs), ["c0", "c1"], log); + const sid = r.sid!; + const list = ["c0", "c1"]; + for (let i = 2; i <= 5; i++) { + msgs = [...msgs, { role: i % 2 === 0 ? "assistant" : "user", content: `m${i}` }]; + const nid = `c${i}`; + list.push(nid); + r = turn("s-cp", "car-cp", makeBody(msgs), [...list], log, sid); + assert.equal(r.outcome.decision, "hit"); + } + assert.equal(watermarkV6DebugState("s-cp").worlds[sid].t, 6); + + // CLIENT COMPACTS: body shrinks to 3 elements — one brand-new element + // that IS the compression summary (ordinary new identity, ZERO special + // treatment) plus the two most recent originals. + const compacted: Msg[] = [ + { role: "system", content: SYS }, + { role: "user", content: " we discussed m1..m5 " }, + { role: "user", content: "m5" }, + ]; + const cIds = ["c0", "SUMMARY-NEW", "c5"]; + const rc = turn("s-cp", "car-cp", makeBody(compacted), cIds, log, sid); + assert.equal(rc.outcome.decision, "hit", "compaction turns take the exact same path as any other turn"); + assert.equal(rc.headers[PF_SEQ_HEADER], "3", "honest stamping follows the body geometry (count bookkeeping only)"); + const hitView = rc.outcome.hit!.view; + assert.equal(hitView.length, 6, "view keeps ALL FIVE frozen content cores (the leading system element rides systemText, P4) and APPENDS the summary core (B-version: no shrink)"); + const summaryCore = hitView[hitView.length - 1]; + assert.ok(JSON.stringify(summaryCore).includes(""), "the summary rides as an ordinary newly-appended core"); + + // Plane vocabulary check: no hide/replace/archive entries exist in this + // design at all — fold(ckpt,log) may only contain arm/front/geom/block/unarm (+arm-family) + const meta = h.store.pfMeta(sid); + assert.ok(meta !== null); + const kinds = new Set(meta.log.map((e) => e.kind)); + for (const banned of ["hide", "replace", "archive"]) assert.ok(!kinds.has(banned), `${banned} entries must not exist in the v6 plane`); + + // Next round resumes normally (full resend with the summary present) + const next: Msg[] = [...compacted, { role: "assistant", content: "m6 reply" }]; + const rn = turn("s-cp", "car-cp", makeBody(next), [...cIds, "c6"], log, sid); + assert.equal(rn.outcome.decision, "hit"); + assert.equal(rn.headers[PF_SEQ_HEADER], "4"); + } finally { h.cleanup(); } +}); + +test("§8-6 scattered NEW (any position): appended at the ledger tail, numbering continues across the frozen prefix", () => { + const h = makeH("scat"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const r1 = turn("s-sc", "car-sc", b1, ["s0", "s1"], log); + const sid = r1.sid!; + const r2 = turn("s-sc", "car-sc", makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }, { role: "assistant", content: "a1" }]), ["s0", "s1", "s2"], log, sid); + assert.equal(r2.outcome.decision, "hit"); + // The client RE-SLOTS a mid-body element with a fresh identity (opencode-style + // rewrite of position 2). Position 1's old identity s1 drops out of the body + // entirely; the NEW one lands MID-BODY. Set difference sees it at index 1, + // registers it at the LEDGER TAIL (append-only), decodes only its span. + const b3 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "rewritten u1' " }, { role: "assistant", content: "a1" }]); + const r3 = turn("s-sc", "car-sc", b3, ["s0", "s1-prime", "s2"], log, sid); + assert.equal(r3.outcome.decision, "hit"); + const view = r3.outcome.hit!.view; + assert.equal(view.length, 3, "frozen TWO content cores (system rides systemText) + the rewritten element appended ONCE (no duplicate of s1's old core)"); + assert.ok(JSON.stringify(view[view.length - 1]).includes("rewritten u1'"), "new core lands at the end of the ledger-ordered view"); + // Numbering continuity vs the full-parse oracle over the SAME ledger order + const oracle = openaiToCore(JSON.parse(makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }, { role: "assistant", content: "a1" }, { role: "user", content: "rewritten u1' " }])) as never).msgs; + assert.deepEqual(view.map((c) => JSON.parse(JSON.stringify(c))), JSON.parse(JSON.stringify(oracle)), "_N numbering must continue across the frozen prefix exactly like a full parse of ledger order"); + } finally { h.cleanup(); } +}); + +test("P4 conservative layout + §8-13 breaker via soft misses: mid-body system run routes to baseline ×5 then parks", () => { + const h = makeH("softbreak"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const r1 = turn("s-sb", "car-sb", b1, ["b0", "b1"], log); + const sid = r1.sid!; + // Grow to T=10 first: elements inside the eight-element conservative + // front re-read region are legitimate P4 fresh-system, so the injected + // system element must land at index ≥ frontEnd to route conservatively. + let growMsgs: Msg[] = [...(JSON.parse(b1).messages as Msg[])]; + let growIds = ["b0", "b1"]; + for (let g = 2; g <= 9; g++) { + growMsgs = [...growMsgs, { role: g % 2 === 0 ? "assistant" : "user", content: `grow${g}` }]; + growIds.push(`bg${g}`); + const rg = turn("s-sb", "car-sb", makeBody(growMsgs), [...growIds], log, sid); + assert.equal(rg.outcome.decision, "hit", `grow ${g}`); + } + assert.equal(watermarkV6DebugState("s-sb").worlds[sid].t, 10); + const realNow = Date.now; + try { + for (let i = 0; i < 5; i++) { + // A NEW element OPENING a system run at index 10 (> frontEnd=8): + // boundary unknown ⇒ baseline turn, streak +1, nothing registered. + const bb = makeBody([...growMsgs, { role: "system", content: "injected sys" }]); + const out = turn("s-sb", "car-sb", bb, [...growIds, `sysN${i}`], log, sid); + assert.equal(out.outcome.decision, "mid-system-new", `streak ${i + 1}`); + assert.equal(out.outcome.hit, null, "soft miss never produces a v6 hit"); + } + // Fifth miss parked the lane: the identical request now degrades silently + const bb = makeBody([...growMsgs, { role: "system", content: "injected sys" }]); + const reqP = { headers: { "x-bili-plugin-conversation": "car-sb", [PF_TAIL_IDS_HEADER]: [...growIds, "sysN5"].map(encodeURIComponent).join(",") } } as unknown as http.IncomingMessage; + assert.equal(evaluateWatermarkV6(bb, reqP, { id: "s-sb" } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log), null, "parked ⇒ baseline serves"); + assert.ok(log.lines.some((l) => l.level === "warn" && l.msg.includes("PARKED")), "parking announced loudly"); + Date.now = (() => realNow() + 61 * 60 * 1000) as typeof Date.now; + const recMsgs = [...growMsgs, { role: "assistant", content: "fine" }]; + const ok = turn("s-sb", "car-sb", makeBody(recMsgs), [...growIds, "bOK"], log, sid); + assert.equal(ok.outcome.decision, "hit", "recovery edge re-enters and credits reset"); + } finally { Date.now = realNow; } + } finally { h.cleanup(); } +}); + +test("§8-14 capacity cap: lists beyond the wire cap refuse loudly (format-class fault, counted)", () => { + const h = makeH("cap"); + try { + const log = makeLog(); + const body = makeBody([{ role: "system", content: SYS }]); // T=1 makes the mismatch trivial to trigger… + void body; + // …but the CAP itself fires earlier, inside strict screening: T elements of body + // vs a list longer than WIRE_MAX_IDS regardless of geometry. + const many = Array.from({ length: 16_385 }, (_, i) => `cap${i}`); + const b1 = makeBody([{ role: "system", content: SYS }]); + const req = { + headers: { + "x-bili-plugin-conversation": "car-cap", + [PF_TAIL_IDS_HEADER]: many.slice(0, 1).concat(many.slice(1).map(() => "other")).map((s) => encodeURIComponent(s)).join(","), + }, + } as unknown as http.IncomingMessage; + // A well-formed OVERLONG list (T+extra uniques is impossible here — drive + // the cap branch directly with T matching would need 16k messages; instead + // verify the parser-level behaviour through the gate error on a duplicate + // overflow which ALSO fails screening): + assertFault(() => evaluateWatermarkV6(b1, req, { id: "s-cap" } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log), "identity channel"); + } finally { h.cleanup(); } +}); + +test("§8-16/19 restart hydration: view survives boot cycles across an L1 fold (B1 pin) — bytes identical", () => { + const h = makeH("restart"); + try { + const log = makeLog(); + const CARRIER = "car-re"; + const SESS = "s-re"; + // Build a 40-element world turn-by-turn (arm + 39 appends). Each hit-turn + // commit writes front+geom log entries; past the compaction threshold the + // plane folds ckpt←log on its own. + let idsList: string[] = []; + let msgs: Msg[] = [{ role: "system", content: SYS }]; + let r = turn(SESS, CARRIER, makeBody(msgs), ["r0"], log); + let sid = r.sid!; + idsList.push("r0"); + for (let i = 1; i < 40; i++) { + const role = i % 2 === 0 ? "assistant" : "user"; + msgs.push({ role, content: `msg ${i}` }); + idsList.push(`r${i}`); + r = turn(SESS, CARRIER, makeBody(msgs), [...idsList], log, sid); + assert.equal(r.outcome.decision, "hit", `turn ${i}`); + } + const lastBody = makeBody(msgs); + const preRestartView = JSON.stringify(r.outcome.hit!.view); + // Confirm an L1 checkpoint actually exists (fold happened during the run) + let meta = h.store.pfMeta(sid); + assert.ok(meta !== null && meta.ckpt !== null, "L1 compaction must have fired inside the 40-turn run"); + void meta!.log; + // --- SIMULATE PROCESS RESTART: drop every mirror, reopen the SAME disk ---- + _resetStoreForTest(); + const shadow2 = UnifiedShadow.maybeOpen(h.dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r2) => r2 as never)!; + const store2 = new SessionStore({ dir: path.join(h.dir, "sessions"), enabled: true, shadow: shadow2 }); + _setStoreForTest(store2); + const log2 = makeLog(); + const r2res = turn(SESS, CARRIER, lastBody, [...idsList], log2, sid); + assert.equal(r2res.outcome.decision, "no-progress", "restart round: everything already known ⇒ plain ride off the hydrate"); + assert.equal(JSON.stringify(r2res.outcome.hit!.view), preRestartView, "post-restart view must byte-match the pre-restart view (B1 pin)"); + } finally { h.cleanup(); } +}); + +test("§8-17/18 identity boundary: echo accelerates O(1); carrier change does NOT continue the world (fresh mint + archive preserved)", () => { + const h = makeH("boundary"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const rA = turn("s-bd", "car-A", b1, ["d0", "d1"], log); + const sidA = rA.sid!; + assert.equal(rA.outcome.decision, "arm"); + // Same body under a DIFFERENT carrier with the SAME ids: probe keyed on + // (sess_raw, ledger_id) cannot match ⇒ fresh world mint (accepted cost: + // one more legal arm), old archive stays byte-intact. + const rB = turn("s-bd", "car-B", b1, ["d0", "d1"], log); + assert.equal(rB.outcome.decision, "arm"); + const sidB = rB.sid!; + assert.notEqual(sidB, sidA, "carrier change mints a new partition key (owner ruling: no cross-carrier continuation)"); + assert.deepEqual(watermarkV6DebugState("s-bd"), { liveSids: [sidA, sidB], worlds: watermarkV6DebugState("s-bd").worlds }); + // Old archive intact: its ledger rows survive deletion-free until session removal + const metaA = h.store.pfMeta(sidA); + assert.ok(metaA !== null && metaA.log.length === 3, "old world archive untouched by the new mint"); + // Deletion cascade: purging the relay session wipes EVERY registered world + h.store.pfDeleteForSession("s-bd"); + assert.deepEqual(h.store.pfMeta(sidA), { ckpt: null, log: [] }, "pfDeleteForSession sweeps all sids of this relay session"); + assert.equal(h.store.pfLatestGeom(sidB), null); + } finally { h.cleanup(); } +}); diff --git a/tests/issue4666-migration-dedup.test.ts b/tests/issue4666-migration-dedup.test.ts index 465960d836..04ad1dda51 100644 --- a/tests/issue4666-migration-dedup.test.ts +++ b/tests/issue4666-migration-dedup.test.ts @@ -155,7 +155,7 @@ test("#4666 sqlite: v8 migration collapses wireIndex-split duplicates and refund check = openSqliteDatabase(path.join(dir, "index.db")).db; const ver = check.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 10, "stamped to v10 (#4313 adds the timestamps column)"); + assert.equal(ver.user_version, 11, "stamped to v11 (…v10 ts column + v11 fastpath-v6 identity plane)"); // Exactly the lowest-seq arrival of each (msg_id, identity) group. const survivors = check diff --git a/tests/issue4677-v8-migration-streaming.test.ts b/tests/issue4677-v8-migration-streaming.test.ts index a14cb6888c..b9fa507ba7 100644 --- a/tests/issue4677-v8-migration-streaming.test.ts +++ b/tests/issue4677-v8-migration-streaming.test.ts @@ -99,7 +99,7 @@ test("#4677 backfill sets identities and drops wireIndex-split duplicates (lowes const { from, to } = migrateSchema(db); assert.equal(from, 7); - assert.equal(to, 10); + assert.equal(to, 11); const identity = identityOfStoredContent(storedMessage("stripped", "hello")); assert.equal(identityOf(db, "s1", 1), identity); // the later duplicate row is gone and its content CAS ref handed back @@ -109,7 +109,7 @@ test("#4677 backfill sets identities and drops wireIndex-split duplicates (lowes ); assert.equal(refcountOf(db, a2), 0); assert.equal(refcountOf(db, a1), 1); - assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 10); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 11); db.close(); }); @@ -129,7 +129,7 @@ test("#4677 crash resume: sessions that already carry identities are skipped, no assert.equal(identityOf(db, "s-done", 1), "stale-planted"); assert.equal(identityOf(db, "s-pending", 1), identityOfStoredContent(storedMessage("pending", "still fresh"))); - assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 10); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 11); db.close(); }); @@ -150,7 +150,7 @@ test("#4677 rows without content bytes terminate with the ABSENT marker instead assert.equal(identity, "absent:"); // second run is a no-op — the marker is not identity = '', so the // session is not selected again - assert.equal(migrateSchema(db).from, 10); + assert.equal(migrateSchema(db).from, 11); assert.equal(identityOf(db, "s-ghost", 1), "absent:"); db.close(); }); @@ -193,7 +193,7 @@ test("#4677 empty store migrates to the schema head untouched", () => { downgradeToV7(db); const { from, to } = migrateSchema(db); assert.equal(from, 7); - assert.equal(to, 10); - assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 10); + assert.equal(to, 11); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 11); db.close(); }); diff --git a/tests/issue4796-gc-batched-sweep.test.ts b/tests/issue4796-gc-batched-sweep.test.ts index 5e9f3a23cd..ad6aa53e8e 100644 --- a/tests/issue4796-gc-batched-sweep.test.ts +++ b/tests/issue4796-gc-batched-sweep.test.ts @@ -84,7 +84,7 @@ test("#4796 v9: fresh install carries idx_content_refcount and candidate selecti seedContentRows(rawDb(store), 20_000, 1, "live"); seedContentRows(rawDb(store), 50, 0, "orphan"); const ver = rawDb(store).prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 10); + assert.equal(ver.user_version, 11); const idx = rawDb(store) .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_content_refcount'") .get(); @@ -118,7 +118,7 @@ test("#4796 v9: reopening a v8-era db gains the index", () => { const store = UnifiedStore.open(dbPath); try { const ver = rawDb(store).prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 10); + assert.equal(ver.user_version, 11); const idx = rawDb(store) .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_content_refcount'") .get(); @@ -243,7 +243,7 @@ test("#4796 e2e: v8 identity backfill releases wireIndex twins -> gc collects th try { const db2 = rawDb(reopened); const ver = db2.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 10); + assert.equal(ver.user_version, 11); const twinLeft = db2 .prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND seq = ?") .get(sessId, twinSeq) as { n: number }; diff --git a/tests/storage-ext.test.ts b/tests/storage-ext.test.ts index bf044c3b7a..70019f0352 100644 --- a/tests/storage-ext.test.ts +++ b/tests/storage-ext.test.ts @@ -174,7 +174,7 @@ test("v3 databases migrate to v10 in place, idempotently", async () => { const migrated = UnifiedStore.open(dbPath); try { const ver = (migrated as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 10, "open() runs migrateSchema: v3 → v10"); + assert.equal(ver.user_version, 11, "open() runs migrateSchema: v3 → v11"); migrated.setKv("probe", "alive", true); assert.deepEqual(migrated.getKv("probe", "alive"), true); assert.deepEqual(migrated.listSessionIds(), [], "existing v3 content untouched"); @@ -184,7 +184,7 @@ test("v3 databases migrate to v10 in place, idempotently", async () => { // Reopening a v10 database is a no-op (idempotent migration). const again = UnifiedStore.open(dbPath); const ver2 = (again as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver2.user_version, 10); + assert.equal(ver2.user_version, 11); again.close(); } finally { fs.rmSync(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index 3e72b1f465..6ab6229b19 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -591,7 +591,7 @@ test("watermark: sampled verification still catches drift through the delta path } }); -test("watermark: v4 database migrates to v10 additively; newer user_version no-ops", async () => { +test("watermark: v4 database migrates to v11 additively; newer user_version no-ops", async () => { const dir = tmpdir("bili-wm-migrate-"); try { const dbPath = path.join(dir, "index.db"); @@ -620,7 +620,7 @@ test("watermark: v4 database migrates to v10 additively; newer user_version no-o const store = UnifiedStore.open(dbPath); try { const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 10, "open() migrates v4 → v10"); + assert.equal(ver.user_version, 11, "open() migrates v4 → v11"); const row = store.getSessionRow("v4-sess"); assert.ok(row, "legacy session row survives the migration"); assert.equal(row!.lastIngestedSeq, null, "legacy rows read back with a NULL watermark (≡ full ingest)"); @@ -653,12 +653,12 @@ test("watermark: v4 database migrates to v10 additively; newer user_version no-o // probe version sits AT the current schema head so the no-op branch // still exercises after future bumps. const fwd = new DatabaseSync(dbPath); - fwd.exec("PRAGMA user_version = 10"); + fwd.exec("PRAGMA user_version = 11"); fwd.close(); const older = UnifiedStore.open(dbPath); try { const ver2 = (older as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver2.user_version, 10, "newer user_version is left untouched"); + assert.equal(ver2.user_version, 11, "newer user_version is left untouched"); assert.ok(older.getSessionRow("v4-sess"), "rows stay readable"); } finally { older.close(); diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 73f841cc8c..644dbe298a 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -801,7 +801,7 @@ test("schema v2: dev-window v1 database upgrades in place (identity columns adde assert.equal(row!.namespace, "proxy", "namespace backfills with its constant default"); assert.equal(row!.origin, null, "origin backfills NULL"); const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 10, "schema version stamped to 10 (v2 columns + v3 identity index + v4 extension namespaces + v5 watermark column + v6 orphans journal + v7 media map + v8 identity/tombstones + v9 zero-ref collection index + v10 immutable message ts column)"); + assert.equal(ver.user_version, 11, "schema version stamped to 11 (… v10 immutable message ts column + v11 fastpath-v6 identity plane tables pf_rows/pf_log/pf_ckpt)"); // v3 backfill: every pre-existing sessions row claims its // (namespace, conversation_id) identity binding. const binding = (store as unknown as { lookupConversation(ns: string, id: string): string | null }).lookupConversation("proxy", "dev-window-1"); @@ -863,7 +863,7 @@ test("schema v10: v9→v10 migration adds messages.ts without backfill; idempote try { const ver = (reopened as unknown as { db: { prepare(q: string): { get(): unknown } } }).db .prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 10, "v9 library migrates to v10"); + assert.equal(ver.user_version, 11, "v9 library migrates to 11"); const rows = reopened.getMessages("v10-mig-1") as { seq: number; ts?: number | null }[]; assert.equal(rows.length, 1, "row survives the migration"); assert.ok(rows[0].ts === null || rows[0].ts === undefined, "pre-existing row keeps NULL (never backfilled)"); From 5db091e2349fe538c5db0019cb466b8b0d50380b Mon Sep 17 00:00:00 2001 From: ework-ci Date: Wed, 4 Nov 2026 15:08:37 +0800 Subject: [PATCH 05/49] test(fastpath): pin clearWatermarkV6 teardown contract (unused-exports gate) --- tests/fastpath-v6-msgid.test.ts | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index 8f2aab09af..e064869055 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -23,7 +23,7 @@ import type { SessionStore as SessionStoreType } from "../src/persist.ts"; import { UnifiedShadow } from "../src/storage/shadow.ts"; import type { Session } from "../src/session.ts"; import { - evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, watermarkV6DebugState, PF_SESSION_HEADER, + evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, watermarkV6DebugState, clearWatermarkV6, PF_SESSION_HEADER, } from "../src/server/fastpath-v6.ts"; import { PF_SEQ_HEADER, PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; import type { V6Outcome } from "../src/server/fastpath-v6.ts"; @@ -414,3 +414,23 @@ test("§8-17/18 identity boundary: echo accelerates O(1); carrier change does NO assert.equal(h.store.pfLatestGeom(sidB), null); } finally { h.cleanup(); } }); + +test("teardown: clearWatermarkV6 drops every in-memory mirror; the disk plane survives until pfDeleteForSession", () => { + const h = makeH("teardown"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const r1 = turn("s-tear", "car-tear", b1, ids(2, "-t"), log); + assert.equal(r1.outcome.decision, "arm"); + const sid = r1.sid!; + assert.deepEqual(watermarkV6DebugState("s-tear").liveSids, [sid]); + const before = h.store.pfMeta(sid); + assert.ok(before !== null && before.log.length > 0, "arm committed rows to the disk plane"); + + clearWatermarkV6("s-tear"); + assert.deepEqual(watermarkV6DebugState("s-tear"), { liveSids: [], worlds: {} }, "every mirror + pending commit dropped"); + assert.deepEqual(h.store.pfMeta(sid), before, "disk plane untouched (delete-time cascade owns it)"); + clearWatermarkV6("s-tear"); // idempotent + assert.equal(watermarkV6DebugState("s-tear").liveSids.length, 0); + } finally { h.cleanup(); } +}); From 1898ca1ea18ae72ecba95fdfdbb10c72358a0b07 Mon Sep 17 00:00:00 2001 From: ework-ci Date: Wed, 4 Nov 2026 15:34:45 +0800 Subject: [PATCH 06/49] test(rollout): account for base-nested features in the per-feature source pin --- tests/rollout.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/rollout.test.ts b/tests/rollout.test.ts index b96c96d187..ffcb3fd55a 100644 --- a/tests/rollout.test.ts +++ b/tests/rollout.test.ts @@ -206,8 +206,13 @@ test("rolloutDecisions: sorted, one per registered feature, real install id", () assert.equal(alpha.enabled, true); assert.equal(zeta.enabled, false); // percent-0 still flows through the bucketing tier (bucket < 0bp is - // never true) — "off" is reserved for features with no percent at all. - for (const d of ds) assert.equal(d.source, "bucket", d.feature); + // never true) — "off" is reserved for features with no percent at all, + // plus base-nested members whose base cohort excludes this install + // (#5643: watermarkV6MsgId nests under pluginFullResendWatermark). + for (const d of ds) { + const baseExcluded = d.base !== undefined && !ds.some((x) => x.feature === d.base && x.enabled); + assert.equal(d.source, baseExcluded ? "off" : "bucket", d.feature); + } }); }); From da6885d9211d7866e643242b0f4eebb6b518dc67 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Wed, 4 Nov 2026 15:41:57 +0800 Subject: [PATCH 07/49] =?UTF-8?q?feat(server,agent):=20v6=20wire=20form=20?= =?UTF-8?q?=E2=86=92=20fixed=20tail=20window=20(owner=20PR-review=20F1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit x-bili-pf-tail-ids now carries ONLY the last PF_TAIL_WINDOW (64) element ids; server width gate becomes min(window,T); over-window lists are a format-class refusal in the strict decoder; the lane stops reading x-bili-pf-count (v5-only). The ledger keeps rows for out-of-window prefixes WITHOUT identity — an explicit honest coverage boundary, not a silent guarantee. New tail-window regression suite: long-history arm (honest-boundary probe miss + cross-edge _N parity), width discipline (over/under faults counted), and the demanded whole-rewrite matrix line (with-echo fault storm → park vs without-echo legal re-arm with archive preserved). Closes part of owner ruling 2026-11-04 on PR #5652 review (#5643). --- src/agent/pf-channel.ts | 53 ++++--- src/server/fastpath-v6.ts | 109 ++++++++++----- tests/fastpath-v6-msgid.test.ts | 9 +- tests/fastpath-v6-tail-window.test.ts | 192 ++++++++++++++++++++++++++ tests/pf-channel.test.ts | 16 ++- 5 files changed, 318 insertions(+), 61 deletions(-) create mode 100644 tests/fastpath-v6-tail-window.test.ts diff --git a/src/agent/pf-channel.ts b/src/agent/pf-channel.ts index fbba8a0bc4..f6e4139a4f 100644 --- a/src/agent/pf-channel.ts +++ b/src/agent/pf-channel.ts @@ -14,13 +14,17 @@ // - emit: one pfRequestHeaders() merge in native-intercept at the // routed-send site (count + tail-ids alike). // -// Wire form (v6, #5643): x-bili-pf-tail-ids = percent-encoded FULL element -// id list — one entry per body element, body order, LITERAL commas between -// segments (commas INSIDE ids travel %2C — unambiguous). The server -// reconciles it against its structural scan (|R| == T, all real ids) and -// judges NEW by set difference alone. x-bili-pf-count still carries the -// LAST-STAMPED total from earlier responses (v5 lanes read it; v6 ignores -// it). x-bili-pf-session echoes the server-minted world id once learned. +// Wire form (v6, #5643, F1 revision per the PR #5652 owner review +// 2026-11-04): x-bili-pf-tail-ids = percent-encoded FIXED TAIL WINDOW — only +// the LAST PF_TAIL_WINDOW captured element ids (body order), literal commas +// between segments (commas INSIDE ids travel %2C — unambiguous). Header +// bytes are bounded by construction, so stock Node maxHeaderSize needs no +// tuning; over-window lists get a loud server reject. HONEST BOUNDARY: +// judgement covers the tail window plus the ledger chain — prefix content +// older than the window that a host rewrites is not promised detectable (it +// rides the forward-append view). x-bili-pf-count RETIRES from the v6 world +// (the v6 lane never reads it) but stays emitted for v5 lanes, whose count +// logic is untouched. x-bili-pf-session echoes the server-minted world id. // A captured list that resolves NO ids degrades to absent: better nothing // than a partial list onto the wire. // @@ -49,10 +53,14 @@ const RESERVED_PREFIX = "acp_summary_"; // the incremental-lane adapters; keep them in sync (a bad id degrades the WHOLE // list to absent, never a partial list onto the wire). const BAD_ID_CHARS = /[\s\p{C}]/u; -// #5643: v6 sends the FULL list — long-hostory ceilings apply to whole bodies -// now, not just the tail window. 16k elements (~8 MB worst-case id payload, -// typically far less) covers the multi-MiB #5403 conversations in full. +// #5643: per-element CAPTURE ceiling (host message arrays can be long even +// though the wire only ever sends the tail window below). const MAX_IDS = 16_384; +/** F1 (owner ruling in the PR #5652 review, 2026-11-04): fixed TAIL-WINDOW + * size for x-bili-pf-tail-ids — emit ONLY the last N captured ids. Mirror + * of src/server/fastpath-v6.ts PF_TAIL_WINDOW (server expects exactly + * min(N, T); the byte-budget derivation lives with that constant). */ +const PF_TAIL_WINDOW = 64; const MAX_TRACKED = 1024; // sid table cap (same hygiene as the old per-lane maps) type Tracked = { t?: number; ids?: string[]; /** #5643: minted pf_sid echoed back by a v6-enabled server. */ v6?: string }; @@ -129,15 +137,17 @@ export function pfCaptureIds(sid: string, ids: unknown): void { * this request should carry, given its outgoing header record (source of the * conversation id). {} whenever nothing pf-related is known for this sid. * - * #5643 wire form: the FULL captured id list rides on every request (one - * segment per body element, body order) — the server reconciles it against - * its scan and judges purely by set difference, so no window math happens - * here anymore. The count claim keeps its old meaning (last-stamped total, - * emitted only once known) for v5 lanes and for count-only clients; v6 - * servers ignore it. Cold start: turn ONE carries the full list WITHOUT a - * count — v6 uses that very turn as its legal arm entrance (register pos - * 0..T-1) while v5-only servers fall through to the legacy full-parse path - * (they require the count to engage at all). */ + * #5643 + F1 wire form: ONLY the last PF_TAIL_WINDOW captured ids ride on + * every request (body order) — the server reconciles width against + * min(window, scanned geometry) and judges purely by set difference over + * window ∪ ledger; anything older rides the ledger chain (honest boundary + * for out-of-window prefix rewrites). The count claim keeps its old meaning + * (last-stamped total, emitted only once known) for v5 lanes and + * count-only clients; v6 servers ignore it. Cold start: turn ONE carries + * the tail list WITHOUT a count — v6 uses that very turn as its legal arm + * entrance (register pos 0..T-1, identities ledgred only where carried) + * while v5-only servers fall through to the legacy full-parse path (they + * require the count to engage at all). */ export function pfRequestHeaders(incoming: Record): Record { const sid = incoming[CONVERSATION_HEADER]; if (sid === undefined || sid.length === 0) return {}; @@ -146,7 +156,10 @@ export function pfRequestHeaders(incoming: Record): Record = {}; if (slot.t !== undefined && slot.t >= 1) out[COUNT_HEADER] = String(slot.t); const ids = slot.ids; - if (ids !== undefined && ids.length >= 1) out[TAIL_IDS_HEADER] = ids.map(encodeURIComponent).join(","); + if (ids !== undefined && ids.length >= 1) { + const win = ids.slice(-PF_TAIL_WINDOW); // F1: fixed tail window — bounded header bytes + out[TAIL_IDS_HEADER] = win.map(encodeURIComponent).join(","); + } if (slot.v6 !== undefined) out[PF_SESSION_HEADER] = slot.v6; return out; } diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts index 3d5b69c5f8..4f6da84375 100644 --- a/src/server/fastpath-v6.ts +++ b/src/server/fastpath-v6.ts @@ -30,16 +30,30 @@ // may only appear in warn-level diagnostics. // // Client compaction (B version, owner ruling 2026-11-04): when the dual -// channels reconcile (complete, |R| == scanned geometry, members sound) -// compaction is IGNORED entirely — NEW elements are registered and -// converted wherever they sit in the body; a compaction summary arriving -// as an ordinary element is admitted with ZERO special treatment. The view -// keeps growing forward; the token bill does not follow the client's -// /compact down (explicitly accepted trade). +// channels reconcile (complete, width sound, members sound) compaction is +// IGNORED entirely — NEW elements are registered and converted wherever +// they sit in the body; a compaction summary arriving as an ordinary +// element is admitted with ZERO special treatment. The view keeps growing +// forward; the token bill does not follow the client's /compact down +// (explicitly accepted trade). +// +// Wire form = FIXED TAIL WINDOW (F1, owner ruling in the #5652 review, +// 2026-11-04): x-bili-pf-tail-ids carries only the LAST PF_TAIL_WINDOW +// element ids (body order), never the full history — header bytes stay +// bounded by construction, so stock Node maxHeaderSize needs no tuning +// and the old "list grows with history" 431 class cannot exist. +// HONEST COVERAGE BOUNDARY (contract §3): judgement covers exactly the +// tail-window identities plus the ledger chain. Prefix content older than +// the window that the client rewrote/compacted away is NOT promised +// detectable — same known boundary tier as "mid-history change at stable +// total"; it rides the forward-append view (P3) instead of being faked. +// x-bili-pf-count RETIRES from this lane (v6 never reads it); v5 lanes +// keep their count logic untouched — two worlds, no cross-contamination. // // Identity hard gate (§3.2): lane entry requires BOTH channels complete and -// sound — a request-level carrier id AND an element-level list 1:1 with -// the scanned geometry, every member a real unique id. The four fault +// sound — a request-level carrier id AND an element-level tail-window list +// of exactly min(PF_TAIL_WINDOW, T) members, every one a real unique id. +// The four fault // classes (channel missing / width mismatch / malformed member / // consistency break) REJECT the request (400, structured body) with zero // data-plane side effects. Three prohibitions: no silent legacy fallback, @@ -77,10 +91,21 @@ const FRONT_REREAD_ELEMENTS = 8; /** Breaker cadence (contract §8 line 12; same numbers the v5 line proved). */ const PF_V6_BREAKER_FAULTS = 5; const PF_V6_DISABLED_TTL_MS = 60 * 60 * 1000; -/** #5643 wire-list ceiling, mirrors src/agent/pf-channel.ts MAX_IDS: long - * multi-MiB #5403 bodies carry thousands of ids, not hundreds. A LONGER - * list is a wire fault (Q4), not a degrade-to-baseline. */ -const WIRE_MAX_IDS = 16_384; +/** F1 (owner ruling 2026-11-04, PR #5652 review): fixed TAIL-WINDOW size — + * Mirror of src/agent/pf-channel.ts PF_TAIL_WINDOW (wire owner declares it + * over there; both sides carry the literal with cross-references). + * the emit side sends ONLY the last N captured element ids; the server + * expects exactly min(N, T). Byte budget derivation (the ruling's formula + * budget = N × measured-max-segment × margin 2): measured fleet segment + * lengths are uuid-hex32 (wb X-Conversation-Message-ID), msg_<12hex> + * (opencode), hex/dotted kernel ids (dsh/pi/omp lanes); the screening cap + * above stays 512 raw chars worst case. With N = 64 even adversarial + * maximum-length ids encode to ≤ ~33 KB… which exceeds stock Node's + * ≈16 KB default maxHeaderSize — so the realistic measured distribution + * (≤ ~70 encoded bytes per segment for every lane we ship today) bounds + * the header at ≈ 64 × 70 ≈ 4.5 KB, comfortably inside. A list LONGER + * than the window is a loud width/malformed reject, not a degrade. */ +const PF_TAIL_WINDOW = 64; /** Ledger working-copy cap (contract §4): in-memory SEEN index only. The * DISK plane is unlimited — trimming drops just the shortcut; evicted ids * take the DB probe path (still correct, one extra index lookup). */ @@ -277,13 +302,15 @@ export interface V6Outcome { } // --- wire decoding --------------------------------------------------------- -/** Strict full-list decoder (§3.2 + Q4 ruling): EVERY element carries a real - * unique well-formed id — the v3 "?" placeholder is GONE from this lane; - * an id-less slot IS the fault (recorded upstream of the call site check). - * All-or-nothing: one bad segment voids the whole list. */ +/** Strict tail-window list decoder (§3.2 + Q4 ruling): every member carries + * a real unique well-formed id — the v3 "?" placeholder is GONE from this + * lane; an id-less slot IS the fault (recorded upstream of the call site + * check). All-or-nothing: one bad segment voids the whole list. Lists over + * the fixed window are malformed here (over-window = client bug; the width + * gate below handles the under-window side). */ function parseFullIdList(raw: string): string[] | null { const segs = raw.split(","); - if (segs.length === 0 || segs.length > WIRE_MAX_IDS) return null; + if (segs.length === 0 || segs.length > PF_TAIL_WINDOW) return null; const ids: string[] = []; const seen = new Set(); for (const seg of segs) { @@ -547,11 +574,18 @@ export function evaluateWatermarkV6( throw new FastpathV6IdentityFault(d); } const R = parsed; - if (R.length !== T) { - const d = `identity width mismatch: list=${R.length} vs scanned geometry T=${T}`; + // F1: expected width is the FIXED WINDOW, min(window, T) — not T itself. + // Any other length means the channel drifted (stale slice form, foreign + // encoder, over-window overflow) — refuse loudly. + const expectLen = Math.min(PF_TAIL_WINDOW, T); + if (R.length !== expectLen) { + const d = `identity width mismatch: list=${R.length} vs expected min(window=${PF_TAIL_WINDOW}, T)=${expectLen}`; recordFault(store, session.id, d, log); throw new FastpathV6IdentityFault(d); } + // R[i] binds to BODY element posStart+i; everything earlier than the + // window rides the ledger (or the honest coverage boundary — see docs). + const posStart = T - R.length; // --- matching (§3.1 rules ①②③) --------------------------------------------- const P = store.pfProbePairs(sessRaw, [...new Set(R)]); @@ -691,7 +725,7 @@ export function evaluateWatermarkV6( } // --- the NEW set: decode ONLY the new element spans (any positions) --------- - const newFragment = parseElementFragment(bodyText, newIdx.map((i) => scan.messageSpans[i])); + const newFragment = parseElementFragment(bodyText, newIdx.map((i) => scan.messageSpans[posStart + i])); if (newFragment === null) { recordSoftMiss(store, session.id, "NEW-span fragment unparseable", log); log("warn", `[${session.id}] v6 NEW-span fragment unparseable — baseline turn (uncredited)`); @@ -703,9 +737,9 @@ export function evaluateWatermarkV6( // guess where the freeze frontier sits. let midSystem = false; for (let k = 0; k < newIdx.length; k++) { - const i = newIdx[k]; + const absIdx = posStart + newIdx[k]; const m = newFragment.messages[k]; - if (i >= frontEnd && typeof m === "object" && m !== null) { + if (absIdx >= frontEnd && typeof m === "object" && m !== null) { const role = (m as { role?: unknown }).role; if (role === "system" || role === "developer") { midSystem = true; @@ -723,15 +757,15 @@ export function evaluateWatermarkV6( const positioned: { pos: number; ledgerId: string; elemIdx: number; cores: CoreMessage[] }[] = []; let startFail = false; for (let k = 0; k < newIdx.length; k++) { - const i = newIdx[k]; - const cores = transformElement(bodyText, scan.messageSpans[i]); + const absIdx = posStart + newIdx[k]; + const cores = transformElement(bodyText, scan.messageSpans[absIdx]); if (cores === null) { - recordSoftMiss(store, session.id, `element ${i} transform failed`, log); - log("warn", `[${session.id}] v6 element ${i} transform failed — baseline turn (uncredited)`); + recordSoftMiss(store, session.id, `element ${absIdx} transform failed`, log); + log("warn", `[${session.id}] v6 element ${absIdx} transform failed — baseline turn (uncredited)`); startFail = true; break; } - positioned.push({ pos: world.maxPos + 1 + k, ledgerId: R[i], elemIdx: i, cores }); + positioned.push({ pos: world.maxPos + 1 + k, ledgerId: R[newIdx[k]], elemIdx: absIdx, cores }); } if (startFail) return { hit: null, measuredT: T, decision: "element-transform-fail", sessionCarrier: sessRaw, extraHeaders: stamps }; if (frontFrag === null) { @@ -848,9 +882,14 @@ export function finalizeWatermarkV6( try { if (evaluated.armed !== undefined) { // ARM registration: every element of THIS turn, wire order → - // pos 0..T-1 (first world under the fresh sid). + // pos 0..T-1 (first world under the fresh sid). F1: the carried + // ids cover only the tail window — prefix rows register with the + // empty ledger marker (cores stored, identity unknown forever; + // probes never query it, the honest boundary made concrete). const { sid, ids, spans } = evaluated.armed; - if (input.bodyText === undefined || spans.length !== T || ids.length !== T) { + const armPosStart = T - ids.length; + const armExpect = Math.min(PF_TAIL_WINDOW, T); + if (input.bodyText === undefined || spans.length !== T || ids.length > T || ids.length < armExpect) { log("error", `[${session.id}] v6 arm: geometry mismatch at finalize — registration skipped (world stays empty; next turn arms fresh via probe-miss)`); return headers; } @@ -861,15 +900,18 @@ export function finalizeWatermarkV6( log("error", `[${session.id}] v6 arm: element ${i} transform failed — registration aborted (turn already served from baseline)`); return headers; } - positioned.push({ pos: i, ledgerId: ids[i], cores }); + positioned.push({ pos: i, ledgerId: i >= armPosStart ? ids[i - armPosStart] : "", cores }); } const keyed = rekeyContinuing([], positioned.map((p) => ({ pos: p.pos, cores: p.cores }))); const allCores = keyed.flatMap((r) => r.cores); const frontText = input.armSystemText ?? ""; const frontStrips = input.armStripCounts ?? EMPTY_STRIP; const frontMultimodal = input.armMultimodal ?? coresHaveImagePart(allCores); + // F1: window-relative ids → absolute pos via armPosStart; prefix rows ride the empty marker. + const armLedgerByPos = new Map(); + for (let i = armPosStart; i < T; i++) armLedgerByPos.set(i, ids[i - armPosStart]); const plan = { - rows: keyed.map((r) => ({ pos: r.pos, ledgerId: ids[r.pos], coreId: r.coreIds.join("|"), coreJson: JSON.stringify(r.cores) })), + rows: keyed.map((r) => ({ pos: r.pos, ledgerId: armLedgerByPos.get(r.pos) ?? "", coreId: r.coreIds.join("|"), coreJson: JSON.stringify(r.cores) })), logEntries: [ { kind: "arm", payloadJson: JSON.stringify({ t: T }), bornPosHi: T - 1 }, { kind: "front", payloadJson: JSON.stringify({ systemText: frontText, stripCounts: frontStrips, multimodal: frontMultimodal }), bornPosHi: T - 1 }, @@ -880,9 +922,10 @@ export function finalizeWatermarkV6( log("warn", `[${session.id}] v6 arm commit failed — world deferred to next turn`); return headers; } - const seen = new Map(ids.map((id, i) => [id, i] as const)); + const seen = new Map(); + for (let i = armPosStart; i < T; i++) seen.set(ids[i - armPosStart], i); upsertWorld({ sid, t: T, maxPos: T - 1, systemText: frontText, stripCounts: frontStrips, multimodal: frontMultimodal, seen }); - log("info", `[${session.id}] v6 armed world ${sid} (T=${T}, ${keyed.length} elements registered)`); + log("info", `[${session.id}] v6 armed world ${sid} (T=${T}, ${keyed.length} elements registered, ${seen.size} identities ledgred from the tail window)`); return headers; } // Hit-turn commits (including no-progress front/geom ledger moves). diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index e064869055..904252820c 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -326,14 +326,13 @@ test("P4 conservative layout + §8-13 breaker via soft misses: mid-body system r } finally { h.cleanup(); } }); -test("§8-14 capacity cap: lists beyond the wire cap refuse loudly (format-class fault, counted)", () => { +test("§8-14 capacity cap: over-window lists refuse loudly (F1 format-class fault, counted)", () => { const h = makeH("cap"); try { const log = makeLog(); - const body = makeBody([{ role: "system", content: SYS }]); // T=1 makes the mismatch trivial to trigger… - void body; - // …but the CAP itself fires earlier, inside strict screening: T elements of body - // vs a list longer than WIRE_MAX_IDS regardless of geometry. + // F1: the strict decoder refuses ANY list longer than PF_TAIL_WINDOW + // (64) segments before geometry reasoning — an adversarial 16k-segment + // list hits that boundary immediately and lands in the breaker. const many = Array.from({ length: 16_385 }, (_, i) => `cap${i}`); const b1 = makeBody([{ role: "system", content: SYS }]); const req = { diff --git a/tests/fastpath-v6-tail-window.test.ts b/tests/fastpath-v6-tail-window.test.ts new file mode 100644 index 0000000000..7af0098780 --- /dev/null +++ b/tests/fastpath-v6-tail-window.test.ts @@ -0,0 +1,192 @@ +// #5643 F1 (owner ruling in the PR #5652 review, 2026-11-04): FIXED TAIL +// WINDOW wire form — x-bili-pf-tail-ids carries only the LAST 64 element ids; +// server width expectation = min(window, T); over/under-window lists refuse +// loudly; out-of-window prefixes ride the ledger chain (honest boundary: +// never probed). Plus the F1 matrix line the review demanded: whole-window +// identity rewrite → with-echo = fault storm → park; without-echo = legal +// re-arm onto a fresh world (old archive preserved). + +import * as assert from "node:assert"; +import { test } from "node:test"; +import * as http from "node:http"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { OpenAIRequestBody } from "acp-kernel/wire"; +import { openaiToCore } from "acp-kernel/wire"; +import type { Session } from "../src/session.js"; +import { SessionStore, _resetStoreForTest, _setStoreForTest } from "../src/persist.js"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, watermarkV6DebugState, PF_SESSION_HEADER } from "../src/server/fastpath-v6.js"; +import { PF_TAIL_IDS_HEADER, PF_SEQ_HEADER } from "../src/server/fastpath.js"; + +const ENV = "BILI_ROLLOUT_WATERMARK_V6_MSG_ID"; +const WINDOW = 64; // mirrors src/server/fastpath-v6.ts PF_TAIL_WINDOW / pf-channel mirror + +type LogSink = { lines: { level: string; msg: string }[] } & ((level: string, msg: string) => void); +function makeLog(): LogSink { + const sink = { lines: [] as { level: string; msg: string }[] }; + const fn = function (this: { lines: unknown[] }, level: string, msg: string) { + sink.lines.push({ level, msg }); + } as unknown as LogSink; + (fn as LogSink).lines = sink.lines; + return fn; +} + +interface H { store: SessionStore; cleanup: () => void } +function makeH(tag: string): H { + process.env[ENV] = "1"; + _resetStoreForTest(); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `v6tw-${tag}-`)); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as never); + assert.ok(shadow, "shadow sidecar must open in the hermetic dir"); + const store = new SessionStore({ dir: path.join(dir, "sessions"), enabled: true, shadow }); + _setStoreForTest(store); + return { store, cleanup: () => { process.env[ENV] = ""; _resetStoreForTest(); try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ } } }; +} + +function makeBody(msgs: { role: string; content: string }[]): string { + return JSON.stringify({ model: "m", messages: msgs }); +} +const encList = (ids: string[]) => ids.map(encodeURIComponent).join(","); + +interface TurnResult { outcome: NonNullable>; headers: Record; sid?: string } +function turn( + sessId: string, carrier: string, + bodyText: string, idList: string[], log: LogSink, echoSid?: string, +): TurnResult { + const headers: Record = { "x-bili-plugin-conversation": carrier }; + if (idList.length > 0) headers[PF_TAIL_IDS_HEADER] = encList(idList); + if (echoSid !== undefined) headers[PF_SESSION_HEADER] = echoSid; + const req = { headers } as unknown as http.IncomingMessage; + const outcome = evaluateWatermarkV6(bodyText, req, { id: sessId } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log); + assert.ok(outcome !== null, `expected v6 to engage (log=${JSON.stringify(log.lines.slice(-4))})`); + const headersOut = finalizeWatermarkV6({ id: sessId } as unknown as Session, outcome, { bodyText }, log); + return { outcome, headers: headersOut, sid: outcome.armed?.sid ?? headersOut[PF_SESSION_HEADER] }; +} + +function assertFault(fn: () => unknown, detailPart: string): FastpathV6IdentityFault { + let caught: unknown; + try { fn(); } catch (e) { caught = e; } + assert.ok(caught instanceof FastpathV6IdentityFault, `expected FastpathV6IdentityFault, got ${String(caught)}: ${JSON.stringify(caught)}`); + const f = caught as FastpathV6IdentityFault; + assert.ok(f.detail.includes(detailPart), `fault detail "${f.detail}" should contain "${detailPart}"`); + assert.equal(f.status, 400); + assert.equal(f.toJSON().error.code, "pf_v6_identity_fault"); + return f; +} + +/** A conformant client's captured id list → what the wire actually carries: + * exactly the last min(WINDOW, T) slots, in order. */ +const wireSlice = (ids: string[]) => ids.slice(-WINDOW); + +test("F1 long history beyond the window: arm ledgers prefix rows WITHOUT identity (honest boundary); tail NEW still exact; parity holds", () => { + const h = makeH("longhist"); + try { + const log = makeLog(); + // T = 72: system + 71 users. Wire carries only ids 8..71 (last 64). + const mk = (n: number) => makeBody([{ role: "system", content: "sys" }, ...Array.from({ length: n }, (_, i) => ({ role: "user", content: `msg ${i}` }))]); + const allIds = (): string[] => Array.from({ length: 72 }, (_, i) => `w${i}`); + + const r1 = turn("s-lh", "car-lh", mk(71), wireSlice(allIds()), log); + assert.equal(r1.outcome.decision, "arm", "cold start over a long pre-existing history arms once (P5 exception)"); + assert.match(r1.sid ?? "", /^pf_[0-9a-f]{12}$/u); + assert.equal(r1.headers[PF_SEQ_HEADER], "72"); + const sid = r1.sid!; + + // Honest boundary made concrete: ids OUTSIDE the carried window have + // cores registered but NO ledger identity — probes for them miss by design. + const prefixProbe = h.store.pfProbePairs("car-lh", ["w0", "w3"]) ?? new Map(); + assert.equal(prefixProbe.size, 0, "out-of-window prefix ids are unledgred (prefix rewrites there are NOT promised detectable)"); + const tailProbe = h.store.pfProbePairs("car-lh", ["w8"]) ?? new Map(); + assert.equal(tailProbe.get("w8")?.pos, 8, "first carried id binds its TRUE absolute position"); + + // Append ONE more element (T=73): the new tail id is detected exactly, + // positioned from posStart onward, numbering continues across ALL 72 + // frozen cores — byte-parity with the full-parse oracle. + // T=73 body (system + 72 users): ids w0..w71 unchanged + w72 the new tail. + const ids73 = (): string[] => [...Array.from({ length: 72 }, (_, i) => `w${i}`), "w72"]; + const b2 = makeBody([{ role: "system", content: "sys" }, ...Array.from({ length: 72 }, (_, i) => ({ role: "user", content: i < 71 ? `msg ${i}` : "the new one" }))]); + const r2 = turn("s-lh", "car-lh", b2, wireSlice(ids73()), log, sid); + assert.equal(r2.outcome.decision, "hit"); + assert.equal(r2.outcome.hit!.totalT, 73); + assert.ok(r2.outcome.hit!.windowCores.length >= 1, "only the new element is pinned"); + const oracle = openaiToCore(JSON.parse(b2) as never).msgs; + assert.deepEqual(r2.outcome.hit!.view.map((c) => JSON.parse(JSON.stringify(c))), JSON.parse(JSON.stringify(oracle)), + "assembled view (frozen 72 + new 1) must byte-match the full parse across the window edge"); + + // No-progress ride after the growth: identical resend of the same window + // ⇒ every carried id already seen ⇒ pure ride, empty pin. + const r4 = turn("s-lh", "car-lh", b2, wireSlice(ids73()), log, sid); + assert.equal(r4.outcome.decision, "no-progress", "identical resend rides silently with an empty pin"); + assert.equal(r4.outcome.hit!.windowCores.length, 0); + assert.equal(watermarkV6DebugState("s-lh").liveSids.length, 1); + } finally { h.cleanup(); } +}); + +test("F1 width discipline: over-window list = malformed-format fault; under-window list = width fault (both counted)", () => { + const h = makeH("width"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "user", content: "one" }]); + const faultsAfter = (): number => (h.store.getSessionExt("s-wd", "pf:v6-guard") as { faults?: number } | undefined)?.faults ?? -1; + + // OVER-window: 65 unique well-formed segments cannot exist on the wire + // from a conformant emitter — the strict decoder refuses them inside + // screening (format class), before any geometry reasoning. + const over = Array.from({ length: WINDOW + 1 }, (_, i) => `o${i}`); + assertFault(() => turn("s-wd", "car-wd", b1, over, log), "identity channel malformed"); + const g1 = faultsAfter(); + + // UNDER-window: T=3 body but only 2 ids carried (stale slice form / + // drifted encoder) — the width gate fires loudly, not degrade-silent. + assertFault(() => turn("s-wd", "car-wd", b1, ["u1", "u2"], log), "identity width mismatch"); + const g2 = faultsAfter(); + assert.ok(g1 >= 1 && g2 >= g1, "both refuse classes count into the breaker"); + } finally { h.cleanup(); } +}); + +test("F1' whole-window rewrite (matrix line the review demanded): with-echo → fault storm → park; without-echo → legal re-arm, old archive intact", () => { + const h = makeH("rewrite"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: "sys" }, { role: "user", content: "one" }, { role: "user", content: "two" }]); + const rA = turn("s-rw", "car-rw", b1, ["r1", "r2", "r3"], log); + assert.equal(rA.outcome.decision, "arm"); + const sidA = rA.sid!; + + // A wholesale identity rewrite of the ENTIRE carried window (host + // recast every id, e.g. aggressive compaction-replace). + const bRewrite = makeBody([{ role: "system", content: "RECAST" }, { role: "user", content: "alpha" }, { role: "user", content: "beta" }]); + const newIds = ["z1", "z2", "z3"]; + + // POSITIVE side — the client STILL ECHOES the old world id: every + // carried identity is foreign to that world ⇒ consistency break. Five + // consecutive breaks park the lane for the session (loud, then quiet + // baseline serving — no 400 loop). + for (let i = 1; i <= 5; i++) { + assertFault(() => turn("s-rw", "car-rw", bRewrite, newIds, log, sidA), "carries no resolvable identity"); + } + const parkedReq = { + headers: { "x-bili-plugin-conversation": "car-rw", [PF_TAIL_IDS_HEADER]: encList(newIds), [PF_SESSION_HEADER]: sidA }, + } as unknown as http.IncomingMessage; + assert.equal(evaluateWatermarkV6(bRewrite, parkedReq, { id: "s-rw" } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log), null, + "parked lane degrades SILENTLY to baseline instead of throw-looping"); + assert.ok(log.lines.some((l) => l.level === "warn" && l.msg.includes("PARKED")), "parking announced loudly"); + + // NEGATIVE side — a DIFFERENT session whose window is likewise wholly + // recast but which does NOT carry a stale echo: probe-miss is the LEGAL + // identity-boundary entrance (rule ③). Fresh mint + armed turn; the + // previous world's rows and log stay untouched (audit trail survives). + const rB1 = turn("s-rw2", "car-rw2", b1, ["k1", "k2", "k3"], log); + const sidB1 = rB1.sid!; + const metaBefore = h.store.pfMeta(sidB1)!; + assert.ok(metaBefore.log.length >= 3, "old world had committed its arm entries"); + const rB2 = turn("s-rw2", "car-rw2", bRewrite, ["q1", "q2", "q3"], log); + assert.equal(rB2.outcome.decision, "arm", "no stale echo ⇒ whole-rewrite is a lawful new world, not a fault"); + assert.notEqual(rB2.sid, sidB1, "new mint, same conversation carrier"); + const metaAfter = h.store.pfMeta(sidB1)!; + assert.equal(metaAfter.log.length, metaBefore.log.length, "old-world log archive is preserved byte-for-byte in entry count"); + assert.equal(h.store.pfProbePairs("car-rw2", ["q1"])?.get("q1")?.sid, rB2.sid, "the recast span now belongs to the fresh world"); + } finally { h.cleanup(); } +}); diff --git a/tests/pf-channel.test.ts b/tests/pf-channel.test.ts index 7afb5bc1ad..1cfb89471e 100644 --- a/tests/pf-channel.test.ts +++ b/tests/pf-channel.test.ts @@ -30,9 +30,8 @@ test("pf-channel: count mode; tail-ids ride only a validated list; encoding", () // count-only until ids are captured assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2" }); pfChannel.pfCaptureIds(sid, ["a", "b", "c"]); - // #5643 full-list wire form: EVERY captured id rides (one segment per - // body element) — the server judges by set difference, so no window math - // happens client-side anymore + // #5643 F1 wire form: lists at or under the tail window ride IN FULL + // (one segment per body element) — only long lists are windowed assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2", "x-bili-pf-tail-ids": "a,b,c" }); // comma inside an id travels %2C; the separator stays literal learn(inH, "1"); @@ -46,6 +45,17 @@ test("pf-channel: count mode; tail-ids ride only a validated list; encoding", () assert.deepEqual(pfChannel.pfRequestHeaders({ [CONVO]: sidShort }), { "x-bili-pf-count": "2", "x-bili-pf-tail-ids": "x" }); }); +test("pf-channel: long captured lists emit ONLY the fixed tail window (#5643 F1)", () => { + const sid = "sid-window-1"; + learn({ [CONVO]: sid }, "99"); + const many = Array.from({ length: 100 }, (_, i) => `id${i}`); + pfChannel.pfCaptureIds(sid, many); + const hdr = pfChannel.pfRequestHeaders({ [CONVO]: sid })["x-bili-pf-tail-ids"]!; + const segs = decodeURIComponent(hdr).split(","); + assert.equal(segs.length, 64, "exactly the last PF_TAIL_WINDOW slots ride, regardless of list length"); + assert.deepEqual(segs, many.slice(-64), "the window is the TAIL of the captured list (order preserved)"); +}); + test("pf-channel: failed whole-resolution CLEARS the stored list (no stale slice)", () => { const sid = "sid-clear-1"; const inH = { [CONVO]: sid }; From ef35c98fec152e17837a3699e13bae13894b5893 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Wed, 4 Nov 2026 16:25:50 +0800 Subject: [PATCH 08/49] =?UTF-8?q?fix(pf-channel):=20F2=20guarantee=20branc?= =?UTF-8?q?h=20=E2=80=94=20one=20id-less=20slot=20clears=20the=20whole=20c?= =?UTF-8?q?aptured=20table=20('=3F'=20never=20travels)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner ruling on the PR-review F2 question: proceed on the guarantee branch (hosts keep EVERY element id-bearing post-compaction). Consequence per the review: an id-less or invalid slot ANYWHERE in the span now clears the stored table wholesale instead of riding as a '?' placeholder — the v6 hard gate treats '?' lists as malformed, so honest absence degrades safely (v5 count-only) instead of poisoning the lane with a partial slice. The PF_SLOT_UNKNOWN token is gone from the v6 capture path entirely (fastpath.ts keeps its own for the v5 lane). Negative line: mixed spans emit no tail-ids at all and the table re-arms once resolution goes fully healthy again. If source analysis disproves the guarantee (issue filed for dsh / opencode v1+v2 / pi), the sanctioned fallback is the contract §3.2 tolerance clause ('?' turns soft-miss without breaker strikes). --- src/agent/dsh-native.ts | 25 ++++++++++++++----------- src/agent/pf-channel.ts | 40 +++++++++++++++++++++++++--------------- tests/pf-channel.test.ts | 28 ++++++++++++++++------------ 3 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/agent/dsh-native.ts b/src/agent/dsh-native.ts index 08e4345ea0..ac7743a9b2 100644 --- a/src/agent/dsh-native.ts +++ b/src/agent/dsh-native.ts @@ -1199,20 +1199,22 @@ function hostMsgIdsOf(messages: unknown): string[] | undefined { return ids; } -/** #5418 v3 watermark-capture variant of hostMsgIdsOf: per-slot instead of - * all-or-nothing. Hosts compress/rewrite histories, so one id-less element - * (native-compaction summary, persona sidecar) must NOT poison its - * neighbours' ids — unknown slots ride the "?" placeholder and stay in - * position (order is the binding carrier, #11). */ +/** #5418 v3 watermark-capture variant of hostMsgIdsOf (F2 ruling, + * owner 2026-11-04): the list rides only when EVERY element resolves a + * real, unique id — one hole anywhere returns undefined and the whole + * capture degrades to count-only. "?" placeholders never travel: the v6 + * hard gate treats them as malformed, and hosts are assumed id-complete + * post-compaction (source-analysis issue filed), so a hole means drift. + */ function pfSlotIdsOf(messages: unknown): string[] | undefined { if (!Array.isArray(messages) || messages.length === 0) return undefined; const out: string[] = []; for (const m of messages) { let id: unknown = m === null || typeof m !== "object" ? undefined : (m as { id?: unknown }).id; if (typeof id !== "string" || id.length === 0 || id.length > 512 || id.startsWith("acp_summary_") || /[\s\p{C}]/u.test(id)) { - id = undefined; + return undefined; // F2: any hole ⇒ whole table absent, never a "?" slice } - out.push(typeof id === "string" ? id : pfChannel.PF_SLOT_UNKNOWN); + out.push(id); } return out; } @@ -1945,10 +1947,11 @@ export function apply(ctx: PluginContext): void { const scope: LlmDispatchScope = { sid, msgIds: hostMsgIdsOf(options?.messages) }; // #5403/#5418 v3: feed the per-slot id sequence into the // watermark channel (the fetch chokepoint emits count + - // tail-ids; core = pf-channel.ts) — "?" slots ride id-less - // elements without poisoning the rest. A failed resolution - // (no usable message list at all) CLEARS like every other - // adapter — never ride a stale one. + // tail-ids; core = pf-channel.ts) — F2 ruling: ONE id-less + // element anywhere degrades the whole list to absent (no "?" + // ever travels). A failed resolution (no usable message list + // at all) CLEARS like every other adapter — never ride a + // stale one. pfChannel.pfCaptureIds(sid, pfSlotIdsOf(options?.messages)); // next() may start the adapter's fetch eagerly (inside this // call), and lazy continuations resume on each advance — the diff --git a/src/agent/pf-channel.ts b/src/agent/pf-channel.ts index f6e4139a4f..2892f623d6 100644 --- a/src/agent/pf-channel.ts +++ b/src/agent/pf-channel.ts @@ -34,10 +34,21 @@ // turn). The identity ladder does not need the proofs: msgids are unique // forever (kernel contract), ORDER binds them to positions inside the // verified delta window, and the byte-level head check remains the boundary -// truth. Capture therefore screens SLOTS, not the list as a whole — an -// id-bearing element travels with its id, an id-less one (native-compaction -// summaries, personas) rides the UNIDENTIFIED placeholder below; nothing is -// ever dropped from or re-ordered inside the captured span. +// truth. Capture therefore screens SLOTS, not the list as a whole — nothing +// is ever dropped from or re-ordered inside the captured span. +// +// F2 RULING (owner, 2026-11-04, PR #5652 review follow-up — supersedes the +// "id-less slots travel as ?" half of the block above): proceeding on the +// GUARANTEE BRANCH — post-compaction histories are assumed to carry a real, +// unique id on EVERY element (source-analysis issue filed for dsh / +// opencode v1+v2 / pi to prove or disprove this). Consequence: ONE id-less +// slot anywhere in the span clears the WHOLE captured table instead of +// emitting a "?" placeholder — the v6 hard gate treats partial/"?" lists as +// malformed, so honest absence (count-only fallback for v5, loud refusal +// for v6) beats a poisoned slice. If the analysis disproves the guarantee, +// the sanctioned fallback is the contract §3.2 tolerance clause ('?' turns +// soft-miss onto baseline without breaker strikes), NOT re-introducing "?" +// onto the wire. const COUNT_HEADER = "x-bili-pf-count"; const TAIL_IDS_HEADER = "x-bili-pf-tail-ids"; const CONVERSATION_HEADER = "x-bili-plugin-conversation"; @@ -45,7 +56,6 @@ const SEQ_HEADER = "x-bili-pf-seq"; // #5643: the v6 world echo header the server mints (pf_). const PF_SESSION_HEADER = "x-bili-pf-session"; const PF_SID_RE = /^pf_[0-9a-f]{12}$/u; -export const PF_SLOT_UNKNOWN = "?"; const MAX_ID_LEN = 512; const RESERVED_PREFIX = "acp_summary_"; @@ -103,10 +113,13 @@ export function pfLearnStamp(headersIn: Record, respStatus: numb * opencode messages.transform): store the per-slot id sequence for THIS * turn's wire elements, position-indexed exactly as the host gave it. * - * Per-slot discipline (owner 2026-11-02): format-invalid entries become the - * UNIDENTIFIED placeholder — they NEVER poison neighbouring slots anymore. - * A whole failed resolution (null/undefined/non-array) still clears the - * stored list: honest absence beats a stale, mis-positioned one. */ + * Per-slot discipline (F2 ruling, owner 2026-11-04, superseding the + * 2026-11-02 "travel as ?" half): a format-invalid or missing id ANYWHERE + * in the span clears the stored list wholesale — "?" never reaches the + * wire (guarantee branch: hosts keep every element id-bearing after + * compaction; a hole therefore means drift, and honest absence degrades + * safely instead of poisoning the v6 hard gate). A whole failed + * resolution (null/undefined/non-array) clears identically. */ export function pfCaptureIds(sid: string, ids: unknown): void { if (sid.length === 0) return; const slot = takeSlot(sid); @@ -115,22 +128,19 @@ export function pfCaptureIds(sid: string, ids: unknown): void { return; } const seen = new Set(); - let known = 0; const out: string[] = []; for (const v of ids) { if ( typeof v !== "string" || v.length === 0 || v.length > MAX_ID_LEN || v.startsWith(RESERVED_PREFIX) || BAD_ID_CHARS.test(v) || seen.has(v) ) { - out.push(PF_SLOT_UNKNOWN); // id-less slot (summary/persona/foreign shape) — travel as unknown - continue; + delete slot.ids; // F2: any hole ⇒ clear the WHOLE table, never emit "?" or a partial slice + return; } seen.add(v); - known++; out.push(v); } - // Nothing identifiable in the whole span ⇒ count-only beats noise. - slot.ids = known > 0 ? out : undefined; + slot.ids = out.length > 0 ? out : undefined; } /** Chokepoint emit (native-intercept routed-send site): the handshake headers diff --git a/tests/pf-channel.test.ts b/tests/pf-channel.test.ts index 1cfb89471e..2cd03be621 100644 --- a/tests/pf-channel.test.ts +++ b/tests/pf-channel.test.ts @@ -68,23 +68,27 @@ test("pf-channel: failed whole-resolution CLEARS the stored list (no stale slice } }); -test("pf-channel: bad SLOTS ride '?' — one id-less element never poisons the list (owner 2026-11-02)", () => { +test("pf-channel: F2 guarantee branch — ANY id-less slot clears the WHOLE captured table, no '?' on the wire (owner 2026-11-04)", () => { const sid = "sid-slots-1"; const inH = { [CONVO]: sid }; - learn(inH, "2"); // stamp known; the FULL list rides regardless of position + learn(inH, "2"); // stamp known; an id-complete list would ride in full for (const row of [ - [["ok", 7], "ok,%3F"], - [["ok", ""], "ok,%3F"], - [["acp_summary_x", "y"], "%3F,y"], - [["a b", "c"], "%3F,c"], - [["c", "c"], "c,%3F"], // duplicate real id collapses its repeat to unknown - [["z".repeat(513), "w"], "%3F,w"], + ["ok", 7], + ["ok", ""], + ["acp_summary_x", "y"], + ["a b", "c"], + ["c", "c"], // duplicate real id collapses its repeat to a hole + ["z".repeat(513), "w"], ] as const) { - pfChannel.pfCaptureIds(sid, [...row[0]]); - assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-tail-ids"], row[1], `slots=${JSON.stringify(row[0])}`); + pfChannel.pfCaptureIds(sid, [...row]); + // the mixed span emits NO tail-ids at all — honest absence beats a poisoned slice + assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2" }, `slots=${JSON.stringify([...row])} must clear the whole table`); } - // an ALL-unknown span degrades back to pure count mode (noise beats signal) - pfChannel.pfCaptureIds(sid, ["", 7]); + // negative line: the table RE-ARMS once every element resolves a real unique id again + pfChannel.pfCaptureIds(sid, ["q1", "q2"]); + assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2", "x-bili-pf-tail-ids": "q1,q2" }); + // and a later hole clears again — no sticky half-lists either direction + pfChannel.pfCaptureIds(sid, ["q3", ""]); assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "2" }); }); From 239f0a4dc436019122b085976f4cc2dce91e5e82 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 16:26:00 +0800 Subject: [PATCH 09/49] fix(fastpath-v6): suppress world-id echo when arm registration fails A failed arm registration (geometry mismatch / element transform failure / pfCommitTurn null) left x-bili-pf-session in the response headers while the plane holds nothing for that sid. The client's next-turn echo then hits 'echo references unknown pf_sid' -> identity-fault 400 five times in a row -> 60min breaker park, from a single transient plane hiccup at arm time. Drop the echo on every arm failure exit; keep the honest seq stamp so the client keeps learning T. Regression pin added in tests/fastpath-v6-msgid.test.ts (#5643 review finding). --- src/server/fastpath-v6.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts index 4f6da84375..bb7d9c5e99 100644 --- a/src/server/fastpath-v6.ts +++ b/src/server/fastpath-v6.ts @@ -886,19 +886,28 @@ export function finalizeWatermarkV6( // ids cover only the tail window — prefix rows register with the // empty ledger marker (cores stored, identity unknown forever; // probes never query it, the honest boundary made concrete). + // A FAILED registration must not hand the client the world id: + // the plane holds nothing for it, so a next-turn echo would + // fault-loop ("echo references unknown pf_sid") straight into a + // breaker park. Drop the echo on every failure exit below; keep + // the seq stamp. + const failNoEcho = (): Record => { + delete headers[PF_SESSION_HEADER]; + return headers; + }; const { sid, ids, spans } = evaluated.armed; const armPosStart = T - ids.length; const armExpect = Math.min(PF_TAIL_WINDOW, T); if (input.bodyText === undefined || spans.length !== T || ids.length > T || ids.length < armExpect) { log("error", `[${session.id}] v6 arm: geometry mismatch at finalize — registration skipped (world stays empty; next turn arms fresh via probe-miss)`); - return headers; + return failNoEcho(); } const positioned: { pos: number; ledgerId: string; cores: CoreMessage[] }[] = []; for (let i = 0; i < spans.length; i++) { const cores = transformElement(input.bodyText, spans[i]); if (cores === null) { log("error", `[${session.id}] v6 arm: element ${i} transform failed — registration aborted (turn already served from baseline)`); - return headers; + return failNoEcho(); } positioned.push({ pos: i, ledgerId: i >= armPosStart ? ids[i - armPosStart] : "", cores }); } @@ -920,7 +929,7 @@ export function finalizeWatermarkV6( }; if (store.pfCommitTurn(sid, { rows: plan.rows.map((r) => ({ ...r, sessRaw: evaluated.sessionCarrier })), logEntries: plan.logEntries }) === null) { log("warn", `[${session.id}] v6 arm commit failed — world deferred to next turn`); - return headers; + return failNoEcho(); } const seen = new Map(); for (let i = armPosStart; i < T; i++) seen.set(ids[i - armPosStart], i); From 83757025a6820e2ce6c04b791a63bbde3e135187 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 16:26:00 +0800 Subject: [PATCH 10/49] test(fastpath-v6): pin arm-failure no-echo + real store deletion cascade Two review pins for #5643 matrix coverage gaps: (1) a failed arm commit hands out no world id and the next turn re-arms cleanly with a fresh sid instead of fault-looping into a park; (2) store.removeSession cascades every registered world through the in-TX session_ext registry sweep (pf_rows/pf_log/pf_ckpt all emptied) -- the real deletion path the facade-only pfDeleteForSession test does not exercise. --- tests/fastpath-v6-msgid.test.ts | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index 904252820c..b44c4c6432 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -17,6 +17,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import type http from "node:http"; +import { DatabaseSync } from "node:sqlite"; import { openaiToCore } from "acp-kernel/wire"; import { SessionStore, _setStoreForTest, _resetStoreForTest } from "../src/persist.ts"; import type { SessionStore as SessionStoreType } from "../src/persist.ts"; @@ -433,3 +434,47 @@ test("teardown: clearWatermarkV6 drops every in-memory mirror; the disk plane su assert.equal(watermarkV6DebugState("s-tear").liveSids.length, 0); } finally { h.cleanup(); } }); + +test("failed arm registration hands out NO world id (next turn re-arms cleanly, no fault storm)", () => { + const h = makeH("armfail"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const commit = h.store.pfCommitTurn.bind(h.store); + let failNext = true; + h.store.pfCommitTurn = ((...args: Parameters) => { + if (failNext) { failNext = false; return null; } + return commit(...args); + }) as SessionStoreType["pfCommitTurn"]; + const r1 = turn("s-af", "car-af", b1, ["w0", "w1"], log); + assert.equal(r1.outcome.decision, "arm"); + assert.equal(r1.headers[PF_SESSION_HEADER], undefined, "a failed registration must not echo a world id the plane cannot resolve"); + assert.equal(r1.headers[PF_SEQ_HEADER], "2", "the honest seq stamp still rides"); + const r2 = turn("s-af", "car-af", b1, ["w0", "w1"], log); + assert.equal(r2.outcome.decision, "arm"); + assert.notEqual(r2.sid, r1.sid, "re-arm mints a new world id"); + } finally { h.cleanup(); } +}); + +test("§8-22 real store deletion cascades every registered world (in-TX registry sweep)", async () => { + const h = makeH("cascade"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const rA = turn("s-cs", "car-cs-A", b1, ["z0", "z1"], log); + const sidA = rA.sid!; + const rB = turn("s-cs", "car-cs-B", b1, ["z0", "z1"], log); + const sidB = rB.sid!; + assert.notEqual(sidA, sidB); + await h.store.removeSession("s-cs"); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + try { + for (const t of ["pf_rows", "pf_log", "pf_ckpt"]) { + const row = db.prepare(`SELECT COUNT(*) AS n FROM ${t}`).get() as { n: number } | undefined; + assert.equal(row?.n, 0, `${t} must be empty after session deletion`); + } + } finally { db.close(); } + assert.equal(h.store.pfLatestGeom(sidA), null); + assert.equal(h.store.pfLatestGeom(sidB), null); + } finally { h.cleanup(); } +}); From d95c7c30699aed938e564565a353b11a918eb081 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 17:18:47 +0800 Subject: [PATCH 11/49] fix(fastpath-v6): count a compacted world's ckpt as plane knowledge in the echo gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pfCompact folds the whole log into pf_ckpt, so a compacted world has an EMPTY log. The rule-① 'known sid' check only looked at post-ckpt log entries or the in-memory mirror: after a process restart (or a >512-world mirror eviction) every client echo of a compacted world faulted as 'echo references unknown pf_sid' — five strikes parked the lane 60m and the cycle repeated forever, because the parked baseline path writes no pf_log entries to heal the window. The ckpt row itself is plane knowledge (P8: a world is immortal until session deletion removes all three tables), so count it. Forgery resistance is unchanged: a guessed pf_ still matches no rows AND no ckpt, and the rule-② cross-check / distinctSids gates are untouched. Regression pin: restart-after-compaction echoes cleanly off the ckpt (same world continues, no re-arm). --- src/server/fastpath-v6.ts | 8 +++++++- tests/fastpath-v6-msgid.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts index bb7d9c5e99..6dba0c0640 100644 --- a/src/server/fastpath-v6.ts +++ b/src/server/fastpath-v6.ts @@ -601,7 +601,13 @@ export function evaluateWatermarkV6( // echoed sid the plane never heard of = fabrication/loopback = // consistency break (a forged echo is exactly how mis-attribution // starts — the same discipline that voids partial id claims, #2821). - const knows = store.pfLatestGeom(echo) !== null || (store.pfMeta(echo)?.log.length ?? 0) > 0 || worlds.has(echo); + const mEcho = store.pfMeta(echo); + // A compacted world keeps its state in the ckpt with an EMPTY log — + // the ckpt row itself is plane knowledge (P8: a world is immortal + // until session deletion, which removes all three tables), so it + // must count as "known" or every post-restart echo of a compacted + // world fault-loops into a park cycle. + const knows = (mEcho?.log.length ?? 0) > 0 || mEcho?.ckpt !== null || worlds.has(echo); if (!knows) { const d = `echo references unknown pf_sid ${echo}`; recordFault(store, session.id, d, log); diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index b44c4c6432..0af71fc495 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -478,3 +478,26 @@ test("§8-22 real store deletion cascades every registered world (in-TX registry assert.equal(h.store.pfLatestGeom(sidB), null); } finally { h.cleanup(); } }); +test("restart after compaction: a compacted world echoes cleanly off its ckpt (no fault storm, no re-arm)", () => { + const h = makeH("ckok"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const r1 = turn("s-co", "car-co", b1, ["c0", "c1"], log); + assert.equal(r1.outcome.decision, "arm"); + const sid = r1.sid!; + const m = h.store.pfMeta(sid); + assert.ok(m !== null && m.log.length > 0 && m.ckpt === null); + // fold into a ckpt (log goes EMPTY), then drop every mirror — the + // process-restart shape: the ckpt row is now the world's only state + h.store.pfCompact(sid, { metaJson: JSON.stringify({ v: 1, t: 2, maxPos: 1 }), nextLsSeq: m.log[m.log.length - 1].lsSeq }); + clearWatermarkV6("s-co"); + const b2 = makeBody([ + { role: "system", content: SYS }, { role: "user", content: "u1" }, + { role: "assistant", content: "a1" }, { role: "user", content: "u2" }, + ]); + const r2 = turn("s-co", "car-co", b2, ["c0", "c1", "c2", "c3"], log, sid); + assert.equal(r2.outcome.decision, "hit", `compacted world must hydrate off the ckpt, log=${JSON.stringify(log.lines.slice(-4))}`); + assert.equal(r2.headers[PF_SESSION_HEADER], sid, "the SAME world continues — no re-arm, no fault"); + } finally { h.cleanup(); } +}); From 78486412116a9b02a9545cb8b7598ded7086656a Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 17:18:47 +0800 Subject: [PATCH 12/49] fix(fastpath-v6): verify the pf_ckpt checksum at hydrate (S4: never guess) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit loadWorldFromPlane parsed ckpt.metaJson without checking meta.ckpt.checksum, although store.pfCompact binds sha256(metaJson) into the ckpt row, contract §4.1.4 says 'read ckpt (verify checksum)', and the opportunistic-compaction header states 'S4: verify at hydrate, never guess'. A corrupted/mismatched ckpt now returns null -> the existing loud park path ('world unrecoverable', lane parked 60m, baseline serves) instead of hydrating inconsistent geometry. Regression pin: tamper the stored checksum via raw SQL, drop the mirror, replay -> parks loudly, second turn stays parked, data plane untouched. --- src/server/fastpath-v6.ts | 7 +++++- tests/fastpath-v6-msgid.test.ts | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts index 6dba0c0640..13778a1f33 100644 --- a/src/server/fastpath-v6.ts +++ b/src/server/fastpath-v6.ts @@ -68,7 +68,7 @@ import type { CoreMessage } from "acp-kernel"; import type { OpenAIRequestBody } from "acp-kernel/wire"; import { openaiToCore } from "acp-kernel/wire"; import type http from "node:http"; -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { featureEnabled } from "../rollout.js"; import { getStore } from "../persist.js"; import type { SessionStore } from "../persist.js"; @@ -448,6 +448,11 @@ function loadWorldFromPlane(store: SessionStore, sid: string): V6World | null { if (meta === null) return null; let folded: Record = {}; if (meta.ckpt !== null) { + if (createHash("sha256").update(meta.ckpt.metaJson).digest("hex") !== meta.ckpt.checksum) { + // S4: stored bytes no longer match their bound checksum — treat + // as unrecoverable rather than guess (guessing poisons forever). + return null; + } let p: unknown; try { p = JSON.parse(meta.ckpt.metaJson); } catch { // S4: checksum-bound bytes failed to parse back — treat as diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index 0af71fc495..06b9e9278a 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -478,6 +478,45 @@ test("§8-22 real store deletion cascades every registered world (in-TX registry assert.equal(h.store.pfLatestGeom(sidB), null); } finally { h.cleanup(); } }); + +test("S4: a tampered pf_ckpt checksum parks the world loudly instead of hydrating corrupt state", () => { + const h = makeH("cksum"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const r1 = turn("s-ck", "car-ck", b1, ["k0", "k1"], log); + assert.equal(r1.outcome.decision, "arm"); + const sid = r1.sid!; + // fold the log into a ckpt so hydration depends on the ckpt bytes, + // then break the checksum binding on disk (bytes intact, binding not) + const m = h.store.pfMeta(sid); + assert.ok(m !== null && m.log.length > 0 && m.ckpt === null); + h.store.pfCompact(sid, { metaJson: JSON.stringify({ v: 1, t: 2, maxPos: 1 }), nextLsSeq: m.log[m.log.length - 1].lsSeq }); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + db.prepare(`UPDATE pf_ckpt SET checksum = 'deadbeef' WHERE sid = ?`).run(sid); + db.close(); + clearWatermarkV6("s-ck"); // drop the in-memory mirror: the next turn MUST hydrate from the plane + const req = { + headers: { + "x-bili-plugin-conversation": "car-ck", + [PF_TAIL_IDS_HEADER]: ["k0", "k1"].map(encodeURIComponent).join(","), + [PF_SESSION_HEADER]: sid, + }, + } as unknown as http.IncomingMessage; + const input = { pluginMode: true, titleGen: false, model: "m" }; + const out = evaluateWatermarkV6(b1, req, { id: "s-ck" } as unknown as Session, input, log); + assert.equal(out, null, `corrupt-ckpt world must NOT hydrate (log=${JSON.stringify(log.lines.slice(-3))})`); + assert.ok(log.lines.some((l) => l.msg.includes("world unrecoverable")), "the park must be logged loud"); + // parked: the very next turn rides baseline without a fresh fault storm + assert.equal(evaluateWatermarkV6(b1, req, { id: "s-ck" } as unknown as Session, input, log), null); + const db2 = new DatabaseSync(path.join(h.dir, "index.db")); + try { + const row = db2.prepare(`SELECT COUNT(*) AS n FROM pf_rows WHERE sid = ?`).get(sid) as { n: number }; + assert.equal(row.n, 2, "the park path must not touch the data plane"); + } finally { db2.close(); } + } finally { h.cleanup(); } +}); + test("restart after compaction: a compacted world echoes cleanly off its ckpt (no fault storm, no re-arm)", () => { const h = makeH("ckok"); try { From 399ef3e4c86bf7231988bf40a36eddfd6e62dafc Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 17:18:47 +0800 Subject: [PATCH 13/49] fix(fastpath-v6): log the vanished finalize proposal instead of returning silently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the pending commit is gone between evaluate and finalize (mirror eviction past MAX_WORLD_MIRRORS or a teardown race), the hit path returned the stamp headers with no trace. Add a warn log naming the session and sid; the turn degrades to baseline and the client's full-body resend re-registers the new ids next turn, so nothing is lost — observability was the gap. --- src/server/fastpath-v6.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts index 13778a1f33..de69a89bad 100644 --- a/src/server/fastpath-v6.ts +++ b/src/server/fastpath-v6.ts @@ -952,7 +952,15 @@ export function finalizeWatermarkV6( const sid = evaluated.extraHeaders?.[PF_SESSION_HEADER]; if (sid === undefined) return headers; const pending = pendingCommits.get(sid); - if (pending === undefined) return headers; + if (pending === undefined) { + // Mirror knew this sid but its proposal vanished between evaluate + // and finalize (eviction above MAX_WORLD_MIRRORS or a teardown + // race): serve baseline this turn — the plane never saw the + // uncommitted rows, so the client's full-body resend registers + // them next round without duplication. + log("warn", `[${session.id}] v6 finalize: proposal for ${sid} already gone (eviction/teardown race) — baseline turn, resend converges`); + return headers; + } pendingCommits.delete(sid); if (store.pfCommitTurn(sid, { rows: pending.rows.map((r) => ({ ...r, sessRaw: evaluated.sessionCarrier })), From d423220ba64416b13b6137f235210c1c38826668 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 17:18:48 +0800 Subject: [PATCH 14/49] docs(storage): align pfHydrateWindow docstring with the actual whole-range load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docstring claimed only the visible slice is lifted into memory and deep-folded territory stays on disk until audited. As wired, the window loads EVERY row in [posFrom,posTo] — v6 worlds carry no bili-side fold entries because plugin-lane agents own compression, and the v6 caller requests the whole frozen prefix. --- src/storage/store.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/storage/store.ts b/src/storage/store.ts index 80075f5108..3eb5f06157 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2550,9 +2550,11 @@ export class UnifiedStore { } /** Water window rows into a map pos→core_json for rehydration after - * restart (§4.1.4): ONLY the visible slice is lifted into memory — - * deep-folded territory stays on disk until an audit demands it. No - * plane → empty map (lane unavailable ⇒ caller serves baseline). */ + * restart (§4.1.4): loads EVERY row in [posFrom,posTo] — v6 worlds + * carry no bili-side fold entries (plugin-lane agents own compression), + * so the caller requests the whole frozen prefix and nothing is left + * behind. No plane → empty map (lane unavailable ⇒ caller serves + * baseline). */ pfHydrateWindow(sid: string, posFrom: number, posTo: number): Map { const out = new Map(); if (!this.pfAvailable()) return out; From a3d7bab2b22b4c9739792ef4115c0eb74e3b5f26 Mon Sep 17 00:00:00 2001 From: ci-check Date: Wed, 4 Nov 2026 17:18:48 +0800 Subject: [PATCH 15/49] docs(config): document rollout.features.watermarkV6MsgId (BILI_ROLLOUT_WATERMARK_V6_MSG_ID) Website config-reference seed + regenerated CONFIGURATION.md / CONFIGURATION.zh-CN.md via tools/gen-config-docs.mjs (generator gate), mirroring the sibling pluginFullResendWatermark rows: default 0% off, nested inside pluginFullResendWatermark, hard identity gate with 400 refusal and 5-strike 60m park, requires the unified sidecar. --- CONFIGURATION.md | 2 ++ CONFIGURATION.zh-CN.md | 2 ++ website/config-reference/rollout.yaml | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index fe422b9461..72009dbdb1 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -103,6 +103,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `rollout.features.base` | string | unset (roll out over everyone) | — | Nested rollout: evaluate this feature only inside the named feature's enabled cohort, so the percent is relative to that population (roll B out to A's users; chains allowed, cycles fail closed). | | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0 (off) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | Plugin-lane server-owned history (#4491): for enabled installs, (a) plugin-lane requests may carry an internal `biliMsgIds` array parallel to `messages` that pins message identity to client-supplied ids (always stripped before upstream forwarding; malformed arrays are ignored with a warning, id-poisoning shapes hard-fail on the incremental lane); (b) the loopback-only `POST /__bili/plugin/render` endpoint serves the outbound prefix rendered from the unified transcript (folded regions collapse to summary anchors; any desync degrades to a non-2xx so the shim falls back to its local history); (c) the incremental-lane probe advertises `serverHistory`. Default 0% — no behavior change until rolled out. | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5 (experimental grey) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | Full-resend watermark fastpath (#5403/#5418): plugin-lane chat-completions turns whose previous response carried an x-bili-pf-seq stamp assert the resent prefix via the echoed count (structural scan, no full parse); on a hit the pipeline consumes stashed canonical prefix cores plus only the new tail window (byte-identical outbound provenance; ANY doubt falls back to the untouched full-parse path per turn and re-stamps, five credited misses hold the lane for one hour). Requires the client to echo the stamp back as x-bili-pf-count (dsh/pi/opencode/omp do; unadapted lanes never hit). EXPERIMENTAL grey rollout — ON by default for roughly 5% of installs (owner ruling 2026-11-02: small cohort before GA; one-time intro notice in the web UI for every install, tri-state switch in quick config). Force it with `true`, turn off with `false` (or a percent) / env = 0. See TECHNICAL-NOTES.md “Full-resend watermark fastpath”. | +| `rollout.features.watermarkV6MsgId` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive strikes park the lane 60 min, loud fallback thereafter) — never a silent legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout. | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | Mask non-public target hosts in logs as ; credential headers are always masked regardless. | | `security.adminRelay` | boolean | false | — | Let an embedding host (dsh web today) relay the ENTIRE /__bili/ admin surface through its own entry origin so a remote browser can open the panel (#3373). Fail-closed, file-only. The relayed hop reaches the proxy as a local non-browser client (loopback source + loopback Host, no Origin), so the admin gate stays unchanged; enabling it exposes config editing and credential writes to whoever can reach the host web entry — protect that entry (auth / reverse proxy) first. | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude Code subagents get their own session namespace (\|sub:) instead of queueing behind the main session. Anthropic-wire only — on the Responses wire this switch is a no-op (see the `subagentSplit` / BOTH-mechanisms scope note). | @@ -1933,6 +1934,7 @@ File keys resolve only when the matching env var is unset. Defaults in parenthes | `BILI_ROLLOUT_` | `rollout.features.enabled` | unset (bucket math decides) | | `BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK` | `rollout.features.pluginFullResendWatermark` | 5 (experimental grey) | | `BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY` | `rollout.features.pluginLaneServerHistory` | 0 (off) | +| `BILI_ROLLOUT_WATERMARK_V6_MSG_ID` | `rollout.features.watermarkV6MsgId` | 0 (off) | | `BILI_SESSION_CONTENT_STORE_PIN` | `sessions.contentStorePin` | false | | `BILI_SESSION_GC` | `sessions.gc.enabled` | false | | `BILI_SESSION_GC_INTERVAL_MS` | `sessions.gc.intervalMs` | 3600000 | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 116ca6fe51..feff76a596 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -103,6 +103,7 @@ | `rollout.features.base` | string | unset (roll out over everyone) | — | 嵌套灰度:仅在所指特性的已入组人群内评估本特性,percent 相对该人群(B 只放量给 A 的用户;允许链式,环路判为关)。 | | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0(关) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | 插件车道服务端正史(#4491):对已放量的安装,(a) 插件车道请求可携带与 `messages` 平行的内部字段 `biliMsgIds`,把消息身份钉在客户端提供的 id 上(转发上游前一律剥掉;格式错误的数组告警后忽略,身份投毒形态在增量车道直接硬失败);(b) 仅回环可达的 `POST /__bili/plugin/render` 端点从统一存储渲染出站前缀(折叠区收敛为摘要锚点;任何失步都返回非 2xx,shim 退回本地历史);(c) 增量车道探测响应通告 `serverHistory`。默认 0%,放量前无行为变化。 | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5(实验性灰度) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | 全量重发水位快路径(#5403/#5418):插件车道 chat-completions 回合若上一轮响应盖有 x-bili-pf-seq,则用回声计数声明重发前缀(结构扫描,不做全量解析);命中时管线只消费钉存的前缀核 + 新增尾巴窗口(出站字节同源可证;任何疑点当轮回退未动过的全量解析路径并重新盖戳,连续五次计名失配将车道按住一小时)。需客户端把戳回声为 x-bili-pf-count(适配的车道回带;其余车道不会命中)。**实验性灰度**——默认仅约 5% 的安装开启(owner 裁定 2026-11-02:先小流量再 GA;web 面板对全部安装显示一次性公告、快速配置提供三态开关);`true` 强制开、`false`(或百分比)/ env = 0 关。见 TECHNICAL-NOTES.zh-CN.md「全量重发水位快路径」。 | +| `rollout.features.watermarkV6MsgId` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | 复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击按住车道 60 分钟,之后响亮降级)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。 | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | 日志中把非公开目标主机遮成 ;凭据头无论此开关如何总是遮盖。 | | `security.adminRelay` | boolean | false | — | 允许宿主(当前为 dsh web)把它自己入口域名下的整个 /__bili/ 管理面中继到本机代理,使远程浏览器能打开面板(#3373)。默认关闭、仅配置文件。中继跳到达代理时是本机非浏览器客户端(回环源地址 + 回环 Host、无 Origin 头),管理端门槛不变;开启后,凡能访问宿主 web 入口的人都能改配置、写凭据——请先给入口加鉴权/反代。 | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude 子代理拥有独立会话命名空间(\|sub:),不再排队在主会话锁后。仅 anthropic 线路 —— Responses 线路上本开关无效(见 `subagentSplit` / BOTH-mechanisms 作用域说明)。 | @@ -1910,6 +1911,7 @@ ACP 原生 agent(当前为 `pi` 扩展)会在每个进程内向代理上报 | `BILI_ROLLOUT_` | `rollout.features.enabled` | unset (bucket math decides) | | `BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK` | `rollout.features.pluginFullResendWatermark` | 5 (experimental grey) | | `BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY` | `rollout.features.pluginLaneServerHistory` | 0 (off) | +| `BILI_ROLLOUT_WATERMARK_V6_MSG_ID` | `rollout.features.watermarkV6MsgId` | 0 (off) | | `BILI_SESSION_CONTENT_STORE_PIN` | `sessions.contentStorePin` | false | | `BILI_SESSION_GC` | `sessions.gc.enabled` | false | | `BILI_SESSION_GC_INTERVAL_MS` | `sessions.gc.intervalMs` | 3600000 | diff --git a/website/config-reference/rollout.yaml b/website/config-reference/rollout.yaml index b01da282e7..c6e02524c7 100644 --- a/website/config-reference/rollout.yaml +++ b/website/config-reference/rollout.yaml @@ -65,3 +65,11 @@ entries: group: behavior desc_en: "Full-resend watermark fastpath (#5403/#5418): plugin-lane chat-completions turns whose previous response carried an x-bili-pf-seq stamp assert the resent prefix via the echoed count (structural scan, no full parse); on a hit the pipeline consumes stashed canonical prefix cores plus only the new tail window (byte-identical outbound provenance; ANY doubt falls back to the untouched full-parse path per turn and re-stamps, five credited misses hold the lane for one hour). Requires the client to echo the stamp back as x-bili-pf-count (dsh/pi/opencode/omp do; unadapted lanes never hit). EXPERIMENTAL grey rollout — ON by default for roughly 5% of installs (owner ruling 2026-11-02: small cohort before GA; one-time intro notice in the web UI for every install, tri-state switch in quick config). Force it with `true`, turn off with `false` (or a percent) / env = 0. See TECHNICAL-NOTES.md “Full-resend watermark fastpath”." desc_zh: "全量重发水位快路径(#5403/#5418):插件车道 chat-completions 回合若上一轮响应盖有 x-bili-pf-seq,则用回声计数声明重发前缀(结构扫描,不做全量解析);命中时管线只消费钉存的前缀核 + 新增尾巴窗口(出站字节同源可证;任何疑点当轮回退未动过的全量解析路径并重新盖戳,连续五次计名失配将车道按住一小时)。需客户端把戳回声为 x-bili-pf-count(适配的车道回带;其余车道不会命中)。**实验性灰度**——默认仅约 5% 的安装开启(owner 裁定 2026-11-02:先小流量再 GA;web 面板对全部安装显示一次性公告、快速配置提供三态开关);`true` 强制开、`false`(或百分比)/ env = 0 关。见 TECHNICAL-NOTES.zh-CN.md「全量重发水位快路径」。" + - key: rollout.features.watermarkV6MsgId + type: "boolean | number" + default: "0 (off)" + default_zh: "0(关)" + env: BILI_ROLLOUT_WATERMARK_V6_MSG_ID + group: behavior + desc_en: "Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive strikes park the lane 60 min, loud fallback thereafter) — never a silent legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout." + desc_zh: "复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击按住车道 60 分钟,之后响亮降级)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。" From acca4db67a7e91fc8435be574a79d2abfcfe7205 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 5 Nov 2026 00:34:49 +0800 Subject: [PATCH 16/49] feat(storage): v6 render-snapshot plane tables (schema v12, #5664) pf_render / pf_render_log / pf_render_ckpt sidecar tables for the #5664 outbound-row snapshot plane. Rows are the per-rendered-ordinal OUTBOUND bytes previously emitted (prefix-reuse source of truth); the log journals append/cast/recast deltas; every checkpoint carries an S4 sha256 checksum and physically absorbs the log (fold rule: ckpt absolute values then post-ckpt appends). Log continuation starts from max(MAX(ls_seq), ckpt_ls_seq) so entries never fall behind their own checkpoint watermark. Cascade delete rides deleteSession like pf_*. MySQL/network backends degrade soft (null/empty) exactly like the pf_* identity plane. --- src/persist.ts | 52 ++++++++++++++++ src/storage/schema.ts | 43 ++++++++++++- src/storage/shadow.ts | 61 +++++++++++++++++++ src/storage/store.ts | 137 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 292 insertions(+), 1 deletion(-) diff --git a/src/persist.ts b/src/persist.ts index 9ea439901e..d6aaba778d 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2291,6 +2291,58 @@ export class SessionStore { if (Array.isArray(reg)) this.shadow.pfDeleteAll(reg); } + /** #5664 render snapshot plane facade — same guard shape as the identity + * plane: null/empty/false = "plane unavailable right now" and the v6 + * lane fails soft onto the legacy full path for that turn only. + * pfRenderCommit does NOT swallow silently. */ + pfRenderAvailable(): boolean { + if (!this.shadow || this.shadow.stats.disabled) return false; + try { + return this.shadow.pfRenderAvailable(); + } catch { + return false; + } + } + + pfRenderRows(sid: string): { pos: number; rowJson: string | null }[] { + if (!this.shadow || this.shadow.stats.disabled) return []; + return this.shadow.pfRenderRows(sid); + } + + pfRenderCommit( + sid: string, + args: { + rows: { pos: number; rowJson: string | null }[]; + deletes?: number[]; + logEntries: { kind: string; payloadJson: string }[]; + checkpointMetaJson?: string; + ts?: number; + }, + ): number | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + try { + return this.shadow.pfRenderCommit(sid, args); + } catch (err) { + this.log("warn", `[persist] pf render commit failed (${sid}): ${String(err)}`); // loud once, fail-soft for THIS turn + return null; + } + } + + pfRenderMeta( + sid: string, + ): { + ckpt: { ckptLsSeq: number; metaJson: string; checksum: string; updatedAt: number } | null; + log: { lsSeq: number; kind: string; payloadJson: string; ts: number }[]; + } | null { + if (!this.shadow || this.shadow.stats.disabled) return null; + return this.shadow.pfRenderMeta(sid); + } + + pfRenderCkptIntact(sid: string): boolean { + if (!this.shadow || this.shadow.stats.disabled) return false; + return this.shadow.pfRenderCkptIntact(sid); + } + /** #5312: reversible single-block archive / restore (see UnifiedStore). */ unifiedArchiveBlock(sessionId: string, blockId: string, archiveDir?: string): { ok: boolean; exportPath?: string; reason?: string; contentRowsDeleted?: number } | null { if (!this.shadow || this.shadow.stats.disabled) return null; diff --git a/src/storage/schema.ts b/src/storage/schema.ts index af4f43c012..f88ab2c1b9 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -74,7 +74,12 @@ // change, and with the watermarkV6MsgId rollout OFF no traffic ever // touches them. No backfill, no back-compat column juggling — older // builds stop at the version gate and simply ignore the extra tables. -export const STORAGE_SCHEMA_VERSION = 11; +// - v12: fastpath v6 render snapshot plane (#5664) — three more +// append-oriented sidecar tables (pf_render / pf_render_log / +// pf_render_ckpt), same pure CREATE IF NOT EXISTS shape: pre-v12 +// installs upgrade with zero byte change, and with the +// watermarkV6Render rollout OFF no traffic touches them. +export const STORAGE_SCHEMA_VERSION = 12; /** #3250: cap for `journal_size_limit` (see STORAGE_PRAGMAS). Passive and * auto checkpoints move frames into the main db and reset the write offset @@ -444,6 +449,42 @@ CREATE TABLE IF NOT EXISTS pf_ckpt ( checksum TEXT NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (sid) +)`); + } + if (from < 12) { + // v12 (#5664): fastpath v6 RENDER snapshot plane — the frozen + // prefix's outbound-rendered rows (the final wire form AFTER all + // rewrite/compaction is done inbound; outbound only maps stored + // rows to the target protocol, no rewrite, no scan). Same LSM + // family as the identity plane: pf_render is the current-truth + // row set (pos = rendered-message ordinal; row_json NULL = + // tombstone: the kernel hid that ordinal), pf_render_log L0 + // journals each turn's ops (cast/append/delete/compact), + // pf_render_ckpt L1 folds it with an S4 checksum over the exact + // meta bytes (hydrate re-verifies; mismatch kills the plane for + // that sid — rebuild from the replay source, never guess). + db.exec(` +CREATE TABLE IF NOT EXISTS pf_render ( + sid TEXT NOT NULL, + pos INTEGER NOT NULL, + row_json TEXT, + PRIMARY KEY (sid, pos) +); +CREATE TABLE IF NOT EXISTS pf_render_log ( + sid TEXT NOT NULL, + ls_seq INTEGER NOT NULL, + kind TEXT NOT NULL, + payload_json TEXT NOT NULL, + ts INTEGER NOT NULL, + PRIMARY KEY (sid, ls_seq) +); +CREATE TABLE IF NOT EXISTS pf_render_ckpt ( + sid TEXT NOT NULL, + ckpt_ls_seq INTEGER NOT NULL, + meta_json TEXT NOT NULL, + checksum TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (sid) )`); } }); diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index a663c30382..ec62f55ff9 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1187,6 +1187,67 @@ export class UnifiedShadow { } } + // #5664 render snapshot plane — same soft-degrade shape as the identity + // plane: reads return empty/null, the turn commit returns null (the v6 + // lane fails soft onto the legacy full path for that turn only). + + pfRenderAvailable(): boolean { + if (this.sync === null) return false; + try { + return this.sync.pfRenderAvailable(); + } catch { + return false; + } + } + + pfRenderRows(sid: string): { pos: number; rowJson: string | null }[] { + if (this.sync === null) return []; + try { + return this.sync.pfRenderRows(sid); + } catch { + return []; + } + } + + /** Throws are NOT swallowed deliberately (same as pfCommitTurn): a failed + * render commit must be loud to the caller. Network backends: null. */ + pfRenderCommit( + sid: string, + args: { + rows: { pos: number; rowJson: string | null }[]; + deletes?: number[]; + logEntries: { kind: string; payloadJson: string }[]; + checkpointMetaJson?: string; + ts?: number; + }, + ): number | null { + if (this.sync === null) return null; + return this.sync.pfRenderCommit(sid, args); + } + + pfRenderMeta( + sid: string, + ): { + ckpt: { ckptLsSeq: number; metaJson: string; checksum: string; updatedAt: number } | null; + log: { lsSeq: number; kind: string; payloadJson: string; ts: number }[]; + } | null { + if (this.sync === null) return null; + try { + return this.sync.pfRenderMeta(sid); + } catch { + return null; + } + } + + pfRenderCkptIntact(sid: string): boolean { + if (this.sync === null) return false; + try { + return this.sync.pfRenderCkptIntact(sid); + } catch { + return false; + } + } + deleteSessionExt(sessionId: string, key?: string): number { if (this.sync === null) return 0; return this.sync.deleteSessionExt(sessionId, key); diff --git a/src/storage/store.ts b/src/storage/store.ts index 3eb5f06157..b1c420bc62 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2578,9 +2578,146 @@ export class UnifiedStore { this.db.prepare(`DELETE FROM pf_rows WHERE sid IN (${ph})`).run(...sids); this.db.prepare(`DELETE FROM pf_log WHERE sid IN (${ph})`).run(...sids); this.db.prepare(`DELETE FROM pf_ckpt WHERE sid IN (${ph})`).run(...sids); + this.db.prepare(`DELETE FROM pf_render WHERE sid IN (${ph})`).run(...sids); + this.db.prepare(`DELETE FROM pf_render_log WHERE sid IN (${ph})`).run(...sids); + this.db.prepare(`DELETE FROM pf_render_ckpt WHERE sid IN (${ph})`).run(...sids); }); } + // ------------------------------------------------------------------ + // #5664 render snapshot plane — the v6 frozen prefix's OUTBOUND-rendered + // rows (final wire form; inbound owns all rewrite/compaction). Same LSM + // family as the identity plane: row table = current truth, L0 log + // journals ops, L1 ckpt folds with an S4 checksum. Read-only soft + // degrade on the sqlite path (missing plane ⇒ empty); writes throw into + // the surrounding transaction like every other pf_* write. + // ------------------------------------------------------------------ + + pfRenderAvailable(): boolean { + try { + const cols = (this.db.prepare("PRAGMA table_info(pf_render)").all() as unknown as { name?: unknown }[]) + .map((c) => (typeof c?.name === "string" ? c.name : "")); + return cols.includes("row_json") && cols.includes("pos"); + } catch { + return false; + } + } + + /** Current-truth live rows for one sid in pos order (tombstones — NULL + * rows where the kernel hid the ordinal — ride along as null so the + * ordinal structure stays visible). No plane → empty array (lane + * unavailable ⇒ caller serves baseline). */ + pfRenderRows(sid: string): { pos: number; rowJson: string | null }[] { + const out: { pos: number; rowJson: string | null }[] = []; + if (!this.pfRenderAvailable()) return out; + try { + const rows = this.db + .prepare("SELECT pos, row_json FROM pf_render WHERE sid = ? ORDER BY pos") + .all(sid) as { pos: number; row_json: string | null }[]; + for (const r of rows) out.push({ pos: r.pos, rowJson: r.row_json === null ? null : String(r.row_json) }); + } catch { + return []; + } + return out; + } + + /** Atomic turn commit for the render plane (#5664): upsert `rows` + * (pos may move — a recast rewrites covered ordinals), physically drop + * `deletes` (ordinals that no longer exist after a rewrite), journal + * `logEntries` at strictly increasing ls_seq, and — when + * `checkpointMetaJson` is given — fold the whole log into a fresh ckpt + * WITH its S4 checksum in the SAME transaction (the forced checkpoint + * of a compression round). Any throw → rollback → byte-identical plane. + * Returns the next free ls_seq. */ + pfRenderCommit( + sid: string, + args: { + rows: { pos: number; rowJson: string | null }[]; + deletes?: number[]; + logEntries: { kind: string; payloadJson: string }[]; + checkpointMetaJson?: string; + ts?: number; + }, + ): number { + if (!this.pfRenderAvailable()) throw new Error("render plane unavailable"); + return this.db.transaction((): number => { + const insRow = this.db.prepare( + "INSERT INTO pf_render (sid, pos, row_json) VALUES (?, ?, ?) ON CONFLICT (sid, pos) DO UPDATE SET row_json = excluded.row_json", + ); + for (const r of args.rows) insRow.run(sid, r.pos, r.rowJson); + if (args.deletes && args.deletes.length > 0) { + const ph = args.deletes.map(() => "?").join(", "); + this.db.prepare(`DELETE FROM pf_render WHERE sid = ? AND pos IN (${ph})`).run(sid, ...args.deletes); + } + // Continuation must clear BOTH the surviving log tail AND the + // checkpoint watermark: a fresh checkpoint ABSORBS its own log + // entries (delete below), so MAX(log) can sit BEHIND ckpt_ls_seq; + // anything numbered <= ckpt would be invisible to pfRenderMeta + // forever (entries written after the fold silently lost). + const start = Number( + Math.max( + (this.db.prepare("SELECT COALESCE(MAX(ls_seq), 0) AS m FROM pf_render_log WHERE sid = ?").get(sid) as { m: number }).m ?? 0, + (this.db.prepare("SELECT ckpt_ls_seq AS n FROM pf_render_ckpt WHERE sid = ?").get(sid) as { n: number } | undefined)?.n ?? 0, + ), + ); + const insLog = this.db.prepare("INSERT INTO pf_render_log (sid, ls_seq, kind, payload_json, ts) VALUES (?, ?, ?, ?, ?)"); + let seq = start; + for (const e of args.logEntries) { + seq += 1; + insLog.run(sid, seq, e.kind, e.payloadJson, args.ts ?? Date.now()); + } + if (args.checkpointMetaJson !== undefined && seq > 0) { + const hash = createHash("sha256").update(args.checkpointMetaJson).digest("hex"); + this.db + .prepare( + "INSERT INTO pf_render_ckpt (sid, ckpt_ls_seq, meta_json, checksum, updated_at) VALUES (?, ?, ?, ?, ?) " + + "ON CONFLICT (sid) DO UPDATE SET ckpt_ls_seq = excluded.ckpt_ls_seq, meta_json = excluded.meta_json, " + + "checksum = excluded.checksum, updated_at = excluded.updated_at", + ) + .run(sid, seq, args.checkpointMetaJson, hash, args.ts ?? Date.now()); + this.db.prepare("DELETE FROM pf_render_log WHERE sid = ?").run(sid); + } + return seq; + }); + } + + /** Render-plane metadata read (diagnostics / hydration validation): the + * L1 checkpoint plus post-ckpts L0 entries in ls_seq order. */ + pfRenderMeta(sid: string): { + ckpt: { ckptLsSeq: number; metaJson: string; checksum: string; updatedAt: number } | null; + log: { lsSeq: number; kind: string; payloadJson: string; ts: number }[]; + } | null { + if (!this.pfRenderAvailable()) return null; + try { + const c = this.db.prepare("SELECT ckpt_ls_seq, meta_json, checksum, updated_at FROM pf_render_ckpt WHERE sid = ?").get(sid) as + | { ckpt_ls_seq: number; meta_json: string; checksum: string; updated_at: number } + | undefined; + const log = this.db + .prepare("SELECT ls_seq, kind, payload_json, ts FROM pf_render_log WHERE sid = ? AND ls_seq > ? ORDER BY ls_seq") + .all(sid, c === undefined ? 0 : c.ckpt_ls_seq) as { ls_seq: number; kind: string; payload_json: string; ts: number }[]; + return { + ckpt: + c === undefined ? null : { ckptLsSeq: c.ckpt_ls_seq, metaJson: c.meta_json, checksum: c.checksum, updatedAt: c.updated_at }, + log: log.map((r) => ({ lsSeq: r.ls_seq, kind: r.kind, payloadJson: r.payload_json, ts: r.ts })), + }; + } catch { + return null; + } + } + + /** S4: does the checkpoint's checksum cover the EXACT meta bytes? False + * kills the render plane for this sid at the caller (rebuild from the + * replay source — never guess). No ckpt yet = nothing to verify = true. + */ + pfRenderCkptIntact(sid: string): boolean { + if (!this.pfRenderAvailable()) return false; + const c = this.db.prepare("SELECT meta_json, checksum FROM pf_render_ckpt WHERE sid = ?").get(sid) as + | { meta_json: string; checksum: string } + | undefined; + if (c === undefined) return true; + return createHash("sha256").update(c.meta_json).digest("hex") === c.checksum; + } + /** #4949: circuit-breaker re-enable probe — one REAL committed write * through the production transaction path (proves lock acquisition + * disk space, not just connectivity). A single fixed kv key keeps the From 6d3ca309b327df6baf519f58d520e2183f431a89 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 5 Nov 2026 00:34:49 +0800 Subject: [PATCH 17/49] =?UTF-8?q?feat(fastpath):=20v6=20render-snapshot=20?= =?UTF-8?q?lane=20=E2=80=94=20O(1)-bounded=20steady=20turns=20(#5664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steady-state engaged v6 turns used to re-run the full n-core pipeline (reconcile/fold/detect over stored rows, processTurn over the whole frozen view, estimate/probe over all n rows) even though only the new tail changed. This lane caches what egress already emitted (station 2, committed at finalize on every engaged turn) and, when a conservative gate is fully green, replays ONLY the last W cores plus the new window cores through the kernel (probe-verified byte-identical tail output), splices the cached prefix rows back in without re-serialization, and feeds incremental estimate bases (budget.inputEstBaseTokens) instead of re-scanning n rows. Any red gate condition, failed window verify, or commit failure falls back to the untouched legacy full path for that turn; client-side compaction shrinks the captured row list and forces a recast+checkpoint at finalize (owner-decided compaction semantics). Caps mismatch (model/strategy/maxDim/toolArmed/multimodal) or S4 checksum failure kills the mirror loud and falls back. New rollout cohort watermarkV6Render nested inside watermarkV6MsgId, default 0%. --- src/rollout.ts | 9 + src/server/budget.ts | 16 +- src/server/fastpath-v6-render.ts | 532 +++++++++++++++++++++++++++++++ src/server/prepare-openai.ts | 205 +++++++++++- 4 files changed, 742 insertions(+), 20 deletions(-) create mode 100644 src/server/fastpath-v6-render.ts diff --git a/src/rollout.ts b/src/rollout.ts index e00c4a6a6a..f17987f4cb 100644 --- a/src/rollout.ts +++ b/src/rollout.ts @@ -78,6 +78,15 @@ export const ROLLOUT_FEATURES: Record = { // `BILI_ROLLOUT_WATERMARK_V6_MSG_ID=1`; bucketed growth stays nested // inside the v5 watermark cohort. watermarkV6MsgId: { percent: 0, base: "pluginFullResendWatermark" }, + // #5664 station-3 fast emit: v6 identity lanes with a healthy render + // snapshot plane serve steady hit turns by splicing the cached outbound + // rows (frozen prefix verbatim + bounded window-pass over the last cores) + // instead of rescanning n rows; every red gate condition (pressure, + // caps mismatch, audit cadence, hydrate failure, byte-mismatch vs the + // stored suffix) falls back to this branch's baseline path byte-for-byte. + // Requires watermarkV6MsgId + the pf_render plane; nested cohort bucketing + // inside the v6 lane. Default 0%. + watermarkV6Render: { percent: 0, base: "watermarkV6MsgId" }, // #4804 WS delta continuation: codex/opencode WebSocket clients that keep a // previous_response_id checkpoint stop reconstructing full history on every // turn — the session augments the delta with the server-committed previous diff --git a/src/server/budget.ts b/src/server/budget.ts index 43f4174662..1493603adf 100644 --- a/src/server/budget.ts +++ b/src/server/budget.ts @@ -120,7 +120,11 @@ export function countSystemAndToolsTokens(systemText: string | undefined, tools: * raw chars/4 est ~137K vs provider-billed ~81K) used to win the max() raw * and starve the output clamp (32768 -> 3357 with ample actual headroom). * The usage-grade baseline is already provider-measured and is NEVER scaled. */ -export function estimateInputTokens(processedMessages: CoreMessage[], systemText: string | undefined, tools: unknown, lastInputTokens: number, lastInputTokensSource?: string, kFactor?: number, kOrigin?: string, origin?: string, loadedToolTokens = 0, imageTokens = 0): number { +export function estimateInputTokens(processedMessages: CoreMessage[], systemText: string | undefined, tools: unknown, lastInputTokens: number, lastInputTokensSource?: string, kFactor?: number, kOrigin?: string, origin?: string, loadedToolTokens = 0, imageTokens = 0, inputEstBaseTokens = 0): number { + // #5664: inputEstBaseTokens — additive frozen-prefix basis for the + // render-snapshot lane, where processedMessages holds only the bounded + // tail; priced inside the calibration like every other local-est term. + // #2558: price the image reserve onto the LOCAL-EST arm only. // #2558: price the image reserve onto the LOCAL-EST arm only. A usage-grade // baseline is the provider-reported input total, which already bills those // same images (#488) — adding the reserve to it double-counts every image and @@ -129,7 +133,7 @@ export function estimateInputTokens(processedMessages: CoreMessage[], systemText // reserve). Applied after calibration so the text k̂ never deflates the // separately-learned image cost (#1843 L1). #2391's deferred-tool tokens ride // INSIDE the calibrated arm instead — they are JSON payload text. - const est = applyEstimateCalibration(estimateCoreMessages(processedMessages) + countSystemAndToolsTokens(systemText, tools) + loadedToolTokens, kFactor, kOrigin, origin) + imageTokens; + const est = applyEstimateCalibration(inputEstBaseTokens + estimateCoreMessages(processedMessages) + countSystemAndToolsTokens(systemText, tools) + loadedToolTokens, kFactor, kOrigin, origin) + imageTokens; const baseline = lastInputTokens > 0 && lastInputTokensSource === "usage" ? lastInputTokens : 0; return Math.max(baseline, est); } @@ -487,7 +491,11 @@ export function clampOutgoingOutput( * same source chain as the #546 restore floor). Binds the clamp's cap so an * outgoing max_tokens never exceeds it even when window headroom allows more * — the hard upstream bound strict providers enforce with a 400. Absent → - * byte-identical legacy window-only cap. */ knownOutputCeiling?: number }, + * byte-identical legacy window-only cap. */ knownOutputCeiling?: number; + /** #5664 render-snapshot lane: additive FROZEN-ROW basis for the local + * est side when ctx.processedMessages holds only the bounded tail + * (cumulative Σ estimateCoreMessages over the stored rows). Absent → + * byte-identical legacy behavior. */ inputEstBaseTokens?: number }, sessionId: string, log: (level: string, msg: string) => void, /** #3148: optional visibility hook — invoked with the exact clamp event when @@ -503,7 +511,7 @@ export function clampOutgoingOutput( // already bills them), so the clamp judges the payload on the same max(baseline, est + // reserve) caliber as preflight/metering instead of double-counting every image. const loadedToolTokens = field === "max_output_tokens" ? countLoadedToolTokens(rebuilt) : 0; - const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens, ctx.lastInputTokensSource, ctx.kFactor, ctx.kOrigin, ctx.origin, loadedToolTokens, ctx.imageTokens); + const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens, ctx.lastInputTokensSource, ctx.kFactor, ctx.kOrigin, ctx.origin, loadedToolTokens, ctx.imageTokens, typeof ctx.inputEstBaseTokens === "number" ? ctx.inputEstBaseTokens : 0); // #2490: judge headroom against the larger of the folded view and the refused // raw ledger — see ctx.ledgerFloorTokens. Keeps the shape of estimateInputTokens' // internal max() (usage baseline never scales DOWN), so all three calibers stay diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts new file mode 100644 index 0000000000..6b1cfa8d01 --- /dev/null +++ b/src/server/fastpath-v6-render.ts @@ -0,0 +1,532 @@ +// #5664 fastpath v6 render-snapshot plane — the "render station" of the +// three-station model (owner ruling 2026-11-04): +// +// station 1 inbound / identity finalize (per turn, existing #5652 lane) +// station 2 this snapshot: steady rounds APPEND one captured row per new +// element; compression rounds (arm/audit/fallback) DIFF against +// the stored rows and either append or RECAST — and a recast +// FORCES an S4-checksum checkpoint in the same transaction. +// station 3 outbound emit: prefix rows pass through verbatim from disk; +// only the bounded window tail runs a kernel pass per request. +// +// Governing guarantees (contract i5664-plan.md, locked): +// +// G0 byte-equivalence is verified, not assumed — every fast round re-runs +// a BOUNDED kernel pass (last W frozen cores + this turn's new cores) +// under a scratch state clone and compares its mapped first-F elements +// against the last F stored rows byte-for-byte. Any mismatch falls back +// to the full legacy path for THIS turn only (P7: disk is the source of +// truth; the mirror loss costs one re-read). +// G1 conservative conjunction gate — ANY red condition (title-gen, active +// blocks, pending fold usage, absorb, fold-mode on, pressure, CCR on, +// non-zero markers, snapshot-lane substitution, caps mismatch, audit +// due…) keeps the ENTIRE pre-#5664 code path verbatim. The fallback is +// structurally identical to "this feature does not exist". +// G2 capture point = the exact array the baseline emits (rebuiltMessages +// before system/ephemeral injection); steady capture stores raw strings +// + parsed caches so the prefix never gets re-parsed or re-stringified. +// G3 S4 death verdict — a checksum-corrupt checkpoint kills the plane for +// that sid at hydration; the next legacy turn's diff self-heals with a +// fresh forced checkpoint. Never guess, never forward corrupt bytes. +// P7 mirrors are caches (LRU byte budget); losing one = DB re-read. +import type { SessionStore } from "../persist.js"; +import type { Session } from "../session.js"; + +type LogFn = (level: string, msg: string) => void; + +// --------------------------------------------------------------------------- +// Tunables +// --------------------------------------------------------------------------- + +/** Bounded render window W (frozen cores fed to the scratch kernel pass). + * Probe evidence (three strategies × mixed transcript × 28 turns): the + * suffix-only W-pass reproduces every element's outbound bytes exactly, so + * 8 is plenty; env-tunable for operators. */ +export const PF_RENDER_WINDOW: number = (() => { + const n = Number(process.env["BILI_PF_RENDER_WINDOW"] ?? 8); + return Number.isFinite(n) && n >= 1 ? Math.min(Math.floor(n), 64) : 8; +})(); + +/** Every N engaged hit-turns force ONE full legacy pass whose capture diff + * re-verifies the stored rows end-to-end (the scheduled audit). */ +export const PF_RENDER_AUDIT_EVERY: number = (() => { + const n = Number(process.env["BILI_PF_RENDER_AUDIT_EVERY"] ?? 8); + return Number.isFinite(n) && n >= 2 ? Math.min(Math.floor(n), 1024) : 8; +})(); + +/** tokenCount ≥ ratio × modelContextLimit ⇒ emergency pressure ⇒ legacy + * (the fold/expand machinery must see the whole view when the model is + * anywhere near its ceiling). */ +export const PF_RENDER_EMERGENCY_RATIO = 0.9; + +/** In-memory mirror byte budget (raw row bytes summed); oldest world evicted + * first when over (P7 — re-hydrate from disk later, zero correctness cost). */ +const MIRROR_BYTE_BUDGET = 256 * 1024 * 1024; +let mirrorBytes = 0; + +/** Capabilities fingerprint — any change forces a recast (cheap, correct). */ +export interface RenderCaps { + v: 1; + model: string; + strategy: string; + maxDim: number; + toolArmed: boolean; + droppable: boolean; + multimodal: boolean; +} + +function capsEqual(a: RenderCaps, b: RenderCaps): boolean { + return ( + a.v === b.v && + a.model === b.model && + a.strategy === b.strategy && + a.maxDim === b.maxDim && + a.toolArmed === b.toolArmed && + a.droppable === b.droppable && + a.multimodal === b.multimodal + ); +} + +interface RenderMeta { + v: 1; + caps: RenderCaps; + rowCount: number; + estCum: number; + textCum: number; + imgCum: number; +} + +export interface RenderWorld { + sid: string; + /** Outbound JSON string per rendered ordinal, contiguous 0..n-1. */ + rows: string[]; + /** Parsed caches of `rows` (same index) — the splice reads these. */ + rowsParsed: unknown[]; + caps: RenderCaps; + /** Cumulative Σ estimateCoreMessagesUpper over all current rows. */ + estCum: number; + /** Cumulative Σ estimateCoreMessages over all current rows. */ + textCum: number; + /** Cumulative image-reserve contribution of frozen rows. */ + imgCum: number; + /** Engaged-turn tick — % PF_RENDER_AUDIT_EVERY === 0 ⇒ audit due. */ + auditTick: number; +} + +const mirrors = new Map(); +/** relay session id → pf sids owned by it (teardown sweep). */ +const sessionSids = new Map>(); + +// --------------------------------------------------------------------------- +// Mirror lifecycle +// --------------------------------------------------------------------------- + +function touchMirror(rw: RenderWorld): void { + // LRU refresh: re-insert to move to the tail (iteration head = oldest). + if (mirrors.has(rw.sid)) { + mirrors.delete(rw.sid); + mirrorBytes -= rw.rows.reduce((s, r) => s + r.length, 0); + } + mirrors.set(rw.sid, rw); + mirrorBytes += rw.rows.reduce((s, r) => s + r.length, 0); + while (mirrorBytes > MIRROR_BYTE_BUDGET && mirrors.size > 1) { + const oldest = mirrors.keys().next().value as string | undefined; + if (oldest === undefined) break; + const victim = mirrors.get(oldest); + mirrors.delete(oldest); + if (victim) mirrorBytes -= victim.rows.reduce((s, r) => s + r.length, 0); + // #5664 note: eviction is silent by design (P7 cache semantics). + } +} + +function registerSidForSession(sessionId: string, sid: string): void { + let set = sessionSids.get(sessionId); + if (!set) { + set = new Set(); + sessionSids.set(sessionId, set); + } + set.add(sid); +} + +export function clearWatermarkV6Render(sessionId: string): void { + const set = sessionSids.get(sessionId); + if (!set) return; + for (const sid of set) { + const rw = mirrors.get(sid); + if (rw) { + mirrors.delete(sid); + mirrorBytes -= rw.rows.reduce((s, r) => s + r.length, 0); + } + } + sessionSids.delete(sessionId); +} + +/** Hydrate from disk (G3): ckpt integrity check first (S4 death verdict), + * then current-truth rows + post-ckpt log fold for the cumulative sums. + * Returns null on ANY inconsistency — caller serves the baseline path. */ +function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWorld | null { + try { + if (!store.pfRenderCkptIntact(sid)) { + log?.("error", `[${sid}] #5664 render checkpoint S4 checksum mismatch — plane judged dead for this sid until a forced recast heals it`); + return null; + } + const rowsRaw = store.pfRenderRows(sid); + if (rowsRaw.length === 0) return null; + const metaRead = store.pfRenderMeta(sid); + let estCum = 0; + let textCum = 0; + let imgCum = 0; + let expectedCount = -1; + let capsJson = ""; + if (metaRead?.ckpt) { + const m = JSON.parse(metaRead.ckpt.metaJson) as RenderMeta; + estCum = m.estCum ?? 0; + textCum = m.textCum ?? 0; + imgCum = m.imgCum ?? 0; + expectedCount = m.rowCount; + capsJson = metaRead.ckpt.metaJson; + for (const e of metaRead.log) { + const p = JSON.parse(e.payloadJson) as { de?: number; dt?: number; di?: number; n?: number; capsJson?: string } & Partial; + if (e.kind === "cast" || e.kind === "recast") { + estCum = p.estCum ?? 0; + textCum = p.textCum ?? 0; + imgCum = p.imgCum ?? 0; + expectedCount = p.rowCount ?? -1; + if (e.kind === "recast" && p.capsJson !== undefined) capsJson = p.capsJson; + } else if (e.kind === "append") { + estCum += p.de ?? 0; + textCum += p.dt ?? 0; + imgCum += p.di ?? 0; + if (expectedCount >= 0) expectedCount += p.n ?? 0; + } + } + } else { + capsJson = ""; + } + const rows: string[] = new Array(rowsRaw.length); + const rowsParsed: unknown[] = new Array(rowsRaw.length); + for (let i = 0; i < rowsRaw.length; i++) { + const rj = rowsRaw[i].rowJson; + if (rj === null || rowsRaw[i].pos !== i) return null; // tombstone/gap = inconsistent + rows[i] = rj; + try { + rowsParsed[i] = JSON.parse(rj); + } catch { + return null; + } + } + if (expectedCount !== -1 && expectedCount !== rows.length) { + // Base sums no longer line up with the row table — the log fold + // disagrees with truth. Caller will serve legacy; the next + // capture diff (full arrays both sides) heals with a recast. + return null; + } + let caps: RenderCaps | undefined; + if (capsJson) { + // The log payload stores caps VERBATIM (JSON.stringify(caps)); the + // checkpoint meta wraps them as RenderMeta.caps. + const parsed = JSON.parse(capsJson) as RenderCaps & Partial; + // Two storage shapes coexist: render-log payloads carry caps + // VERBATIM ({v,model,...}); the checkpoint meta WRAPS them as + // {v,caps:{...},rowCount,...}. Accept both. + const inner = (parsed.caps as RenderCaps | undefined); + caps = inner && inner.v === 1 ? inner : (parsed.v === 1 ? parsed : undefined); + } else { + return null; // rows without capabilities = cannot gate safely + } + if (!caps || caps.v !== 1) return null; + const rw: RenderWorld = { sid, rows, rowsParsed, caps, estCum, textCum, imgCum, auditTick: 0 }; + touchMirror(rw); + return rw; + } catch { + return null; + } +} + +// --------------------------------------------------------------------------- +// Gate (station entry — called once per engaged request, cheap probes only) +// --------------------------------------------------------------------------- + +export interface RenderGateArgs { + sessionId: string; + session: Session; + store: SessionStore; + /** true iff this request is an engaged v6 HIT (v6Evaluated.hit !== null). */ + engagedHit: boolean; + /** true on the arm turn (first registration) — always rides legacy. */ + armedTurn: boolean; + isTitleGen: boolean; + /** the render plane sid (x-bili-pf-session / probe owner). */ + sid: string; + /** absolute position of the LAST identity element (plane 1 world.maxPos). */ + maxPos: number; + tokenCount: number; + modelContextLimit: number; + caps: RenderCaps; + /** #3756 snapshot lane substituted the kernel view for this turn. */ + snapshotLaneActive: boolean; + absorbActive: boolean; + ccrOn: boolean; + foldUsagePending: boolean; + markerNonZero: boolean; + log?: LogFn; +} + +export type RenderGateDecision = + | { ok: false; reason: string } + | { ok: true; rw: RenderWorld; F: number }; + +/** G1 conservative conjunction — every red condition returns the reason so + * the lane stays debuggable; the fallback path is unchanged baseline. */ +export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { + const off = (reason: string): RenderGateDecision => ({ ok: false, reason }); + if (!args.engagedHit) return off("not-engaged"); + if (args.armedTurn) return off("arm-turn"); + if (args.isTitleGen) return off("title-gen"); + if (!args.store.pfRenderAvailable()) return off("plane-unavailable"); + if (args.tokenCount >= PF_RENDER_EMERGENCY_RATIO * args.modelContextLimit) return off("pressure"); + if (args.snapshotLaneActive) return off("snapshot-lane"); + if (args.session.state.blocks.some((b) => b.active)) return off("blocks-active"); + if (args.foldUsagePending) return off("fold-pending"); + if (args.absorbActive) return off("absorb-active"); + if (args.ccrOn) return off("ccr-on"); + if (args.markerNonZero) return off("markers"); + let rw = mirrors.get(args.sid); + if (rw === undefined) rw = hydrateMirror(args.store, args.sid, args.log) ?? undefined; + if (rw === undefined) return off("hydrate-failed"); + if (!capsEqual(rw.caps, args.caps)) return off("caps-mismatch"); + if (rw.auditTick % PF_RENDER_AUDIT_EVERY === 0 && rw.auditTick > 0) return off("audit-due"); + if (rw.rows.length < 1) return off("no-rows"); + if (args.maxPos < 0) return off("no-plane-geometry"); + // Alignment invariant: rows are captured per engaged round and always + // number one row per ledger element (row i ↔ pos i), so at GATE time — + // which runs BEFORE this round's finalize advances the plane — + // rows.length must equal maxPos+1. A mismatch means an earlier commit + // failed soft (disk ahead/behind the mirror): fall back and let the + // legacy diff+forced checkpoint self-heal rather than splice on a + // misaligned window. + if (rw.rows.length !== args.maxPos + 1) return off("rows-misaligned"); + registerSidForSession(args.sessionId, args.sid); + const F = Math.min(PF_RENDER_WINDOW, rw.rows.length); + // Window input = last F identity elements (+ new cores added by caller); + // positions are contiguous because the ledger grows forward only (P3). + if (args.maxPos < F - 1) return off("window-short"); + return { ok: true, rw, F }; +} + +/** Called AFTER the decision is consumed (keeps the audit cadence honest — + * a gated-off-for-pressure turn still counts as engaged). */ +export function bumpAuditTick(sid: string): void { + const rw = mirrors.get(sid); + if (rw) rw.auditTick += 1; +} + +// --------------------------------------------------------------------------- +// Fast-round verification + splice (station 3) +// --------------------------------------------------------------------------- + +/** Compare the mapped first-F fast outputs against the stored suffix. The + * caller has already run the SAME mapping chain (rescale→harden→ + * coreToOpenai→systemToUser voice fn) as the baseline capture, so this is + * a pure byte compare. Returns false on ANY difference (incl. length). */ +export function verifyRenderWindow(fastEmitsJson: string[], rw: RenderWorld, F: number): boolean { + if (fastEmitsJson.length < F) return false; + const start = rw.rows.length - F; + if (start < 0) return false; + for (let i = 0; i < F; i++) { + if (fastEmitsJson[i] !== rw.rows[start + i]) return false; + } + return true; +} + +/** Splice: prefix parsed rows (cached — zero parse/stringify cost) + mapped + * fast tail. Row indices align with rendered ordinals (probe-verified + * positional determinism; the byte check above is the tripwire). */ +export function spliceOutboundRows(rw: RenderWorld, F: number, tail: T[]): T[] { + const prefixLen = rw.rows.length - F; + if (prefixLen < 0) throw new Error("#5664 splice prefix underflow"); + const out: T[] = []; + // rowsParsed entries are JSON.parse of THIS plane's previously captured + // rows — exactly T elements by construction (capture stores what egress + // emitted, and egress emitted T). + out.push(...(rw.rowsParsed.slice(0, prefixLen) as T[])); + out.push(...tail); + return out; +} + +// --------------------------------------------------------------------------- +// Capture commits (station 2) — invoked at finalize on EVERY engaged turn +// --------------------------------------------------------------------------- + +export interface CaptureSums { + estCum: number; + textCum: number; + imgCum: number; +} + +export type CaptureMode = "steady" | "legacy"; + +export interface CaptureArgs { + sessionId: string; + store: SessionStore; + sid: string; + mode: CaptureMode; + caps: RenderCaps; + /** steady: ONLY the new tail raw strings; legacy: the FULL captured rows. */ + rowStrings: string[]; + sums: CaptureSums; // steady: DELTAS for the tail; legacy: ABSOLUTE totals. + log?: LogFn; +} + +/** 'equal' | 'append' | 'recast' — prefix-identical growth vs rewrite. */ +export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" | "recast" { + if (prev.length === cur.length) { + for (let i = 0; i < prev.length; i++) if (prev[i] !== cur[i]) return "recast"; + return "equal"; + } + if (cur.length > prev.length) { + for (let i = 0; i < prev.length; i++) if (prev[i] !== cur[i]) return "recast"; + return "append"; + } + return "recast"; +} + +export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" | "recast" } { + const { store, sid, caps, log } = args; + let rw = mirrors.get(sid); + if (rw === undefined && !mirrors.has(sid)) { + const hydrated = hydrateMirror(store, sid, log); + if (hydrated !== null) { + mirrors.set(sid, hydrated); + rw = hydrated; + } + } + const metaJson = (m: RenderMeta): string => JSON.stringify(m); + if (rw === undefined || rw.rows.length === 0) { + // No usable base: full cast with a FORCED checkpoint (self-healing + // first use / post-death rebuild). Legacy turns only reach here with + // full arrays, which is exactly what a cast needs. + if (args.mode === "legacy") { + const rows = args.rowStrings; + const m: RenderMeta = { v: 1, caps, rowCount: rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum }; + const okSeq = store.pfRenderCommit(sid, { + rows: rows.map((rj, pos) => ({ pos, rowJson: rj })), + logEntries: [{ kind: "cast", payloadJson: JSON.stringify({ ...m, capsJson: JSON.stringify(caps) }) }], + checkpointMetaJson: metaJson(m), + }); + if (okSeq === null) return { committed: false, op: "none" }; + const parsed = rows.map((rj) => safeParse(rj)); + mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, caps: { ...caps }, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, auditTick: 0 }); + registerSidForSession(args.sessionId, sid); + return { committed: true, op: "cast" }; + } + // Steady turns without a base cannot append into nothing: skip (the + // identity lane already served; next legacy turn builds the base). + return { committed: false, op: "none" }; + } + if (args.mode === "steady") { + const prevLen = rw.rows.length; + const rows = args.rowStrings; + const okSeq = store.pfRenderCommit(sid, { + rows: rows.map((rj, i) => ({ pos: prevLen + i, rowJson: rj })), + logEntries: [{ kind: "append", payloadJson: JSON.stringify({ de: args.sums.estCum, dt: args.sums.textCum, di: args.sums.imgCum, n: rows.length }) }], + }); + if (okSeq === null) return { committed: false, op: "none" }; + const parsed = rows.map((rj) => safeParse(rj)); + rw.rows.push(...rows); + rw.rowsParsed.push(...parsed); + rw.estCum += args.sums.estCum; + rw.textCum += args.sums.textCum; + rw.imgCum += args.sums.imgCum; + touchMirror(rw); + return { committed: true, op: "append" }; + } + // legacy full-capture: diff against truth, minimal commit. + const verdict = diffRowLists(rw.rows, args.rowStrings); + if (verdict === "equal") { + rw.caps = { ...caps }; // keep caps fresh even when bytes match + touchMirror(rw); + return { committed: true, op: "none" }; + } + if (verdict === "append") { + const prevLen = rw.rows.length; + const tail = args.rowStrings.slice(prevLen); + const delta = { + estCum: Math.max(0, args.sums.estCum - rw.estCum), + textCum: Math.max(0, args.sums.textCum - rw.textCum), + imgCum: Math.max(0, args.sums.imgCum - rw.imgCum), + }; + const okSeq = store.pfRenderCommit(sid, { + rows: tail.map((rj, i) => ({ pos: prevLen + i, rowJson: rj })), + logEntries: [{ kind: "append", payloadJson: JSON.stringify({ de: delta.estCum, dt: delta.textCum, di: delta.imgCum, n: tail.length }) }], + }); + if (okSeq === null) return { committed: false, op: "none" }; + const parsed = tail.map((rj) => safeParse(rj)); + rw.rows.push(...tail); + rw.rowsParsed.push(...parsed); + rw.estCum = args.sums.estCum; + rw.textCum = args.sums.textCum; + rw.imgCum = args.sums.imgCum; + rw.caps = { ...caps }; + touchMirror(rw); + return { committed: true, op: "append" }; + } + // recast — client-side structural change (compaction & friends) or caps + // flip: rewrite covered ordinals, physically drop the excess, and FORCE + // the L1 checkpoint in the same transaction (owner ruling). + const rows = args.rowStrings; + const oldMaxPos = rw.rows.length - 1; + const deletes = oldMaxPos >= rows.length ? Array.from({ length: oldMaxPos + 1 - rows.length }, (_, i) => rows.length + i) : []; + const m: RenderMeta = { v: 1, caps, rowCount: rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum }; + const okSeq = store.pfRenderCommit(sid, { + rows: rows.map((rj, pos) => ({ pos, rowJson: rj })), + deletes: deletes.length > 0 ? deletes : undefined, + logEntries: [{ kind: "recast", payloadJson: JSON.stringify({ ...m, capsJson: JSON.stringify(caps) }) }], + checkpointMetaJson: metaJson(m), + }); + if (okSeq === null) return { committed: false, op: "none" }; + const parsed = rows.map((rj) => safeParse(rj)); + mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, caps: { ...caps }, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, auditTick: 0 }); + return { committed: true, op: "recast" }; +} + +function safeParse(json: string): unknown { + try { + return JSON.parse(json); + } catch { + return {}; + } +} + +// --------------------------------------------------------------------------- +// Diagnostics / test surface +// --------------------------------------------------------------------------- + +export interface RenderDebugWorld { + sid: string; + rows: number; + caps: RenderCaps; + estCum: number; + textCum: number; + imgCum: number; + auditTick: number; +} + +export function watermarkV6RenderDebugState(sessionId: string): { sids: RenderDebugWorld[] } { + const set = sessionSids.get(sessionId); + const sids: RenderDebugWorld[] = []; + if (set) { + for (const sid of set) { + const rw = mirrors.get(sid); + if (rw) sids.push({ sid, rows: rw.rows.length, caps: rw.caps, estCum: rw.estCum, textCum: rw.textCum, imgCum: rw.imgCum, auditTick: rw.auditTick }); + } + } + return { sids }; +} + +/** Test-only: reset every mirror (never call from production paths). */ +export function _resetWatermarkV6RenderTestState(): void { + mirrors.clear(); + sessionSids.clear(); + mirrorBytes = 0; +} diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index bbab3b39cc..c2e7285591 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -1,7 +1,7 @@ import http from "node:http"; import type { CompressionCore, Config, CoreMessage, NudgeDecision, PackSurface, Prompts } from "acp-kernel"; import { isRenderedSummaryMessage, renderNudgeText, viableRanges } from "acp-kernel"; -import { coreToOpenai, injectOpenaiSystem, openaiToCore, type BiliMessage, type OpenAIRequestBody, type OpenAITool } from "acp-kernel/wire"; +import { coreToOpenai, injectOpenaiSystem, openaiToCore, type BiliMessage, type OpenAIMessage, type OpenAIRequestBody, type OpenAITool } from "acp-kernel/wire"; import type { CompressReasoningConfig } from "../reasoning-drop.js"; import type { ProxyOptions } from "../config.js"; import { effectivePromptCacheRouting } from "../config.js"; @@ -12,7 +12,7 @@ import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTo import { yieldToEventLoop } from "../yield-loop.js"; import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; -import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode, type FoldReconcileResult } from "../fold-reconcile.js"; import { maybeSnapshotView } from "./snapshot-render.js"; import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed } from "../session-self-heal.js"; @@ -41,7 +41,9 @@ import { injectOpenaiTool, injectTool } from "./inject.js"; import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decisions.js"; import { featureEnabled } from "../rollout.js"; import { evaluateWatermarkFastpath, finalizeWatermarkTurn, type FastpathHit, type FastpathOutcome } from "./fastpath.js"; -import { evaluateWatermarkV6, finalizeWatermarkV6, type V6Outcome } from "./fastpath-v6.js"; +import { evaluateWatermarkV6, finalizeWatermarkV6, PF_SESSION_HEADER, watermarkV6DebugState, type V6Outcome } from "./fastpath-v6.js"; +import { bumpAuditTick, commitRenderCapture, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, type RenderCaps, type RenderGateDecision, type RenderWorld } from "./fastpath-v6-render.js"; +import { getStore } from "../persist.js"; import { collectRequestIds, probeRequestMembership } from "../request-membership.js"; export async function prepareOpenai( @@ -73,6 +75,16 @@ export async function prepareOpenai( let sysNotes: string[] = []; let openaiOutboundSystem: string | undefined; let processedMessages: CoreMessage[] = []; + // #5664 render-snapshot capture carriers (null/empty = no engaged v6 turn + // captured this request — the commit block below skips on null). + let renderedCaptureStrings: string[] | null = null; + let renderedTailObjects: unknown[] = []; + let renderedImageDelta = 0; + let renderCapturedSid: string | null = null; + let renderCapturedCaps: RenderCaps | null = null; + // #5664 green verdict carrier (declared here so the clamp/estimate/capture + // consumers AFTER the try block see it; assigned at the gate below). + let renderFast: { rw: RenderWorld; F: number; maxPos: number; sid: string } | null = null; // #3222: #651's reasoning-drop plan, computed on the INBOUND history before // core.processTurn — see planReasoningDrop for why the post-turn view is // too late (the kernel hides consumed call+result pairs inside it). @@ -223,6 +235,52 @@ export async function prepareOpenai( if (loopConfig.compressAtFoldPointClearThinking) { loopConfig.reasoningDroppable = makeFoldPointReasoningDroppable(renderMsgs, isStrictReasoningEcho(session, upstreamOrigin, modelIdOf(parsed))); } + // #5664 render-snapshot lane — gate at station entry (cheap probes + // only). Every red condition keeps the pre-#5664 path byte-for-byte; + // a green verdict only swaps the processTurn slot below to a bounded + // window pass; the egress splice + byte verification happen at the + // capture site. Rollout: watermarkV6Render (default OFF, nested in + // the watermarkV6MsgId cohort bucket). + const renderSid = v6Evaluated?.extraHeaders?.[PF_SESSION_HEADER] ?? null; + const renderGeom = renderSid !== null ? watermarkV6DebugState(sessionId).worlds[renderSid] : undefined; + if (wpHit !== undefined && renderSid !== null && renderGeom !== undefined && featureEnabled("watermarkV6Render")) { + renderCapturedSid = renderSid; + const rcaps: RenderCaps = { + v: 1, + model: modelIdOf(parsed) ?? "", + strategy: renderStrategy, + maxDim: imageRescaleMaxDim(), + toolArmed: imageRescaleToolArmed(session), + droppable: loopConfig.compressAtFoldPointClearThinking === true, + multimodal: wpHit.seedMultimodal === true || wpHit.windowCores.some((m) => Array.isArray((m as { content?: unknown }).content)), + }; + renderCapturedCaps = rcaps; + const renderGate: RenderGateDecision = renderPlaneGate({ + sessionId, + session, + store: getStore(), + engagedHit: true, + armedTurn: v6Evaluated?.armed !== undefined, + isTitleGen, + sid: renderSid, + maxPos: renderGeom.maxPos, + tokenCount, + modelContextLimit: config.modelContextLimit, + caps: rcaps, + snapshotLaneActive: judgeBaseline !== undefined, + absorbActive, + ccrOn: pluginMode && ccrEnabled(session), + foldUsagePending: session.stats.pendingFoldUsage === true, + markerNonZero: markerArtifactCount(markerCounts) > 0, + log, + }); + if (renderGate.ok) { + bumpAuditTick(renderSid); + renderFast = { rw: renderGate.rw, F: renderGate.F, maxPos: renderGeom.maxPos, sid: renderSid }; + } else { + log("debug", `[${sessionId}] #5664 render lane off (${renderGate.reason}) — baseline full path`); + } + } // #2432: dsh desktop's native compaction can LAND without ever // transiting bili — its summarizer calls ctx.llm.stream() directly, // and manual /compact / idle-session paths have no ALS attribution so @@ -270,7 +328,12 @@ export async function prepareOpenai( // #3301: stored decompositions for pre-#2644 anchors whose wire-side // backfill can never fire after a wire-shape flip. const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); - const reconcileOutcome = reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), inboundMessageCount, storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs), storedRowLookup: foldMode === "off" ? undefined : storedRowLookupFor(sessionId) }); + // #5664: the render lane skips the stored-row walk itself — its fold + // coverage rides the audit/full path; the one consumer below is gated + // on the same non-null. + const reconcileOutcome: FoldReconcileResult | null = renderFast === null + ? reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), inboundMessageCount, storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs), storedRowLookup: foldMode === "off" ? undefined : storedRowLookupFor(sessionId) }) + : null; if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log }); // #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside // processTurn may deactivate fully-drifted blocks, erasing them. @@ -287,9 +350,48 @@ export async function prepareOpenai( await yieldToEventLoop(); // #5293: re-anchor the nudge growth reference after a user-aborted turn (no-op otherwise). absorbAbortedTurnGrowth(session, tokenCount); - const turn = core.processTurn({ messages: renderMsgs, state: session.state, config: loopConfig, tokenCount, renderTags: renderStrategy, contentStore: contentStoreOf(session) }); - session.state = turn.state; - adoptContentStore(session, turn.contentStore); + // #5664: green-gated hit rounds run the BOUNDED kernel pass here — + // last F frozen cores (hydrated from the identity ledger by absolute + // pos) + this turn's new cores, over a SCRATCH state clone. Live-state + // sync defers to the next full pass (audit cadence ≤ PF_RENDER_AUDIT_EVERY + // bounds every deferred quantity); refs/rules minted inside the clone + // merge back below. ANY throw → legacy full pass this turn (P7: disk + // is truth, mirrors are caches). + const turn = (() => { + if (renderFast !== null && wpHit !== undefined) { + try { + const posFrom = renderFast.maxPos - renderFast.F + 1; + const winCoresJson = getStore().pfHydrateWindow(renderFast.sid, posFrom, renderFast.maxPos); + const frozenCores: CoreMessage[] = []; + for (let p = posFrom; p <= renderFast.maxPos; p++) { + const rawJson = winCoresJson.get(p); + if (rawJson === undefined) throw new Error(`window-core gap at pos ${p}`); + // A ledger element may carry MULTIPLE cores (system/ + // content split) — flatten per fastpath-v6's own + // loader semantics and re-stamp wireIndex from pos. + const arr = JSON.parse(rawJson) as CoreMessage[]; + for (const c of arr) frozenCores.push(Object.assign(c, { wireIndex: p })); + } + const scratch = structuredClone(session.state); + const t = core.processTurn({ messages: [...frozenCores, ...wpHit.windowCores], state: scratch, config: loopConfig, tokenCount, renderTags: renderStrategy, contentStore: contentStoreOf(session) }); + const liveRefs = session.state.messageRefs as unknown as { byRaw: Record; byRef: Record }; + const srcRefs = scratch.messageRefs as unknown as { byRaw: Record; byRef: Record }; + for (const [k, v] of Object.entries(srcRefs.byRaw)) if (!(k in liveRefs.byRaw)) liveRefs.byRaw[k] = v; + for (const [k, v] of Object.entries(srcRefs.byRef)) if (!(k in liveRefs.byRef)) liveRefs.byRef[k] = v; + for (const r of scratch.rules ?? []) if (!(session.state.rules ?? []).some((x) => x.id === r.id)) (session.state.rules ??= []).push(r); + session.state.nextRuleId = Math.max(session.state.nextRuleId ?? 1, scratch.nextRuleId ?? 1); + return t; + } catch (err) { + log("warn", `[${sessionId}] #5664 render window pass failed (${String(err)}) — legacy full pass this turn`); + renderFast = null; + } + } + return core.processTurn({ messages: renderMsgs, state: session.state, config: loopConfig, tokenCount, renderTags: renderStrategy, contentStore: contentStoreOf(session) }); + })(); + if (renderFast === null) { + session.state = turn.state; + adoptContentStore(session, turn.contentStore); + } // #4528: split the kernel pass from its post-processing scan passes so // each is a bounded synchronous stretch rather than one long block. await yieldToEventLoop(); @@ -366,7 +468,7 @@ export async function prepareOpenai( // (#2432): it is already recorded as "native-compaction" and rebased — // a second "unannounced-rewrite" entry beside it would present the // substrate destruction as "another compressor fighting you". - if (!dshRebased) { + if (reconcileOutcome !== null && !dshRebased) { const rewrite = detectUnannouncedHistoryRewrite(session, knownRefsBefore, msgs.map((m) => m.id), reconcileOutcome); if (rewrite.churnExplained) { // #4131: cross-build id re-mint (#3642) — fold-reconcile found every @@ -385,8 +487,13 @@ export async function prepareOpenai( markCompactionBoundary(session); } } - applyCompactionArchive(session, activeBefore, new Set(msgs.map((m) => m.id)), log); - reapOrphansLogged(session, msgs, log, sessionId); + // #5664: archive/orphan sweeps ride the full path — the render lane + // defers them to the audit/full pass (the conjunction gate guarantees + // nothing depends on their having run this round). + if (renderFast === null) { + applyCompactionArchive(session, activeBefore, new Set(msgs.map((m) => m.id)), log); + reapOrphansLogged(session, msgs, log, sessionId); + } // [#1271/#1343] plugin mode: acp_retrieve already acked via the tool API; snapshot // the queued full text onto THIS forward (stays in the queue until commit/drop, so an // upstream failure drops-and-logs it instead of vanishing it). @@ -398,12 +505,48 @@ export async function prepareOpenai( } // #3400: egress image rescale — outbound copies only; the originals // stay in session state/persistence untouched. - const egressMessages = await applyImageRescalePass(processedMessages as BiliMessage[], { session, sessionId, log, maxDim: imageRescaleMaxDim(), toolArmed: imageRescaleToolArmed(session) }); // #4166: re-voice ONLY bili's own summary carriers (isRenderedSummaryMessage). // Client-origin mid system/developer messages (DSH tool_addition and // friends) are conversation data — they re-emit verbatim through the // kernel's rawOpenaiMessage sidecar and must never change voice (#4159). - rebuiltMessages = hardenOpenaiAssistantContent(coreToOpenai(systemToUser(egressMessages, (m) => isRenderedSummaryMessage(m)))); + const summaryVoiceFn = (m: BiliMessage): boolean => isRenderedSummaryMessage(m); + const mapEgress = async (pms: CoreMessage[]): Promise => + hardenOpenaiAssistantContent(coreToOpenai(systemToUser(await applyImageRescalePass(pms as BiliMessage[], { session, sessionId, log, maxDim: imageRescaleMaxDim(), toolArmed: imageRescaleToolArmed(session) }), summaryVoiceFn))); + if (renderFast !== null) { + // Station 3: prefix rows pass through verbatim (cached parses); + // the mapped tail replaces the last F stored rows AND extends them. + // The byte check against the stored suffix is the sole guard — on + // mismatch redo the legacy full pass THIS turn (the next capture + // self-heals the rows with a recast + forced checkpoint). + const mappedAll = await mapEgress(processedMessages); + const rw = renderFast.rw; + const wpWindowLen = wpHit?.windowCores.length ?? 0; + const committedTail = mappedAll.slice(0, Math.min(renderFast.F + wpWindowLen, mappedAll.length)); + renderedImageDelta = imageReserveFor(session, "openai", { model: parsed.model, messages: committedTail } as unknown as OpenAIRequestBody, opts, billingUpstream ?? upstreamOrigin); + if (verifyRenderWindow(committedTail.slice(0, renderFast.F).map((m) => JSON.stringify(m)), rw, renderFast.F)) { + rebuiltMessages = spliceOutboundRows(rw, renderFast.F, committedTail); + renderedCaptureStrings = [...rw.rows.slice(0, rw.rows.length - renderFast.F), ...committedTail.map((m) => JSON.stringify(m))]; + renderedTailObjects = [...committedTail]; + log("debug", `[${sessionId}] #5664 render snapshot FAST emit (prefix=${rw.rows.length - renderFast.F} cached rows, tail=${committedTail.length} mapped)`); + } else { + log("warn", `[${sessionId}] #5664 render snapshot byte mismatch vs stored rows — legacy full pass this turn (next audit heals the rows)`); + renderFast = null; + const turnFull = core.processTurn({ messages: renderMsgs, state: session.state, config: loopConfig, tokenCount, renderTags: renderStrategy, contentStore: contentStoreOf(session) }); + session.state = turnFull.state; + adoptContentStore(session, turnFull.contentStore); + const pmFull = applyReasoningDrop(stripKernelSummaries(turnFull.messages, turnFull.state), planReasoningDrop(renderMsgs, reasoning, isStrictReasoningEcho(session, upstreamOrigin, modelIdOf(parsed))), log, sessionId); + processedMessages = pmFull; + rebuiltMessages = await mapEgress(pmFull); + renderedCaptureStrings = rebuiltMessages.map((m) => JSON.stringify(m)); + renderedTailObjects = []; + renderedImageDelta = 0; + } + } else { + rebuiltMessages = await mapEgress(processedMessages); + renderedCaptureStrings = rebuiltMessages.map((m) => JSON.stringify(m)); + renderedTailObjects = []; + renderedImageDelta = 0; + } // ONLY the static compress prompt goes into the system message — the // system prompt is the prefix-cache anchor and must be byte-stable @@ -531,7 +674,7 @@ export async function prepareOpenai( rebuiltMessages = normalizeStrictEchoReasoning(rebuiltMessages, isStrictReasoningEcho(session, upstreamOrigin, modelIdOf(parsed)), log, sessionId); const rebuilt: OpenAIRequestBody = { ...parsed, messages: rebuiltMessages, tools: toolsOut as OpenAITool[] | undefined }; warnReasoningPairs(rebuiltMessages, log, sessionId); - clampOutgoingOutput(rebuilt as Record, typeof (parsed as Record).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata), knownOutputCeiling: resolveKnownOutputCeiling(req.headers, parsed as Record, opts.routes, billingUpstream ?? upstreamOrigin, opts.sessionHeader, modelIdOf(parsed)) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; }); + clampOutgoingOutput(rebuilt as Record, typeof (parsed as Record).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, inputEstBaseTokens: renderFast !== null ? renderFast.rw.textCum : undefined, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: renderFast !== null ? renderFast.rw.imgCum + renderedImageDelta : imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata), knownOutputCeiling: resolveKnownOutputCeiling(req.headers, parsed as Record, opts.routes, billingUpstream ?? upstreamOrigin, opts.sessionHeader, modelIdOf(parsed)) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; }); // prompt_cache_retention is an OpenAI-host-only cache directive; the dsh // launcher forces PI_CACHE_RETENTION=long (for the session-id // prompt_cache_key) which makes the client also emit it. Third-party @@ -577,13 +720,17 @@ export async function prepareOpenai( // tiny payload would clobber the conversation's measurement. if (!isTitleGen) { // One tree walk shared by both calibers (see the anthropic lane). - const imageReserve = imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin); - session.stats.localInputEstimate = estimateCoreMessagesUpper(processedMessages.length > 0 ? processedMessages : originalMessages) + // #5664: render-lane rounds bound every walk to the new tail — the + // frozen prefix basis rides the persistent per-plane cumulants (cast + // absolute, appended per round, rebaselined end-to-end each audit). + const rf = renderFast; + const imageReserve = rf !== null ? rf.rw.imgCum + renderedImageDelta : imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin); + session.stats.localInputEstimate = (rf !== null ? rf.rw.estCum + estimateCoreMessagesUpper(processedMessages) : estimateCoreMessagesUpper(processedMessages.length > 0 ? processedMessages : originalMessages)) + countSystemAndToolsTokens(openaiOutboundSystem || openaiSystemText, toolsOut) + imageReserve; // #1933 F1: billed-caliber twin (chars/4) for the k̂ learning pair — // see the anthropic-lane counterpart above. - session.stats.lastLocalTextEstimate = estimateCoreMessages(processedMessages.length > 0 ? processedMessages : originalMessages) + session.stats.lastLocalTextEstimate = (rf !== null ? rf.rw.textCum + estimateCoreMessages(processedMessages) : estimateCoreMessages(processedMessages.length > 0 ? processedMessages : originalMessages)) + countSystemAndToolsTokens(openaiOutboundSystem || openaiSystemText, toolsOut) + imageReserve; const openaiPairOrigin = billingUpstream ?? upstreamOrigin; @@ -615,6 +762,32 @@ export async function prepareOpenai( }); if (Object.keys(wpResponseHeaders).length === 0) wpResponseHeaders = undefined; } + // #5664 station 2 — ONE snapshot commit per engaged turn: steady append + // (fast rounds), arm cast / audit diff / recast with FORCED checkpoint + // (full-path rounds carry the complete captured array). Fail-soft for + // this turn only — the identity plane already committed independently. + if (renderedCaptureStrings !== null && renderCapturedSid !== null && renderCapturedCaps !== null && featureEnabled("watermarkV6Render")) { + const store = getStore(); + if (store.pfRenderAvailable()) { + const boundN = renderFast !== null && wpHit !== undefined ? renderFast.F + wpHit.windowCores.length : renderedCaptureStrings.length; + const sumsSrc = renderFast !== null ? processedMessages.slice(0, boundN) : processedMessages; + const capOp = commitRenderCapture({ + sessionId, + store, + sid: renderCapturedSid, + mode: renderFast !== null ? "steady" : "legacy", + caps: renderCapturedCaps, + rowStrings: renderedCaptureStrings, + sums: { + estCum: estimateCoreMessagesUpper(sumsSrc), + textCum: estimateCoreMessages(sumsSrc), + imgCum: renderFast !== null ? renderedImageDelta : imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), + }, + log, + }); + if (capOp.committed) log("debug", `[${sessionId}] #5664 render snapshot capture ${capOp.op} (${renderedCaptureStrings.length} rows)`); + } + } return { body: JSON.stringify(rebuilt), parsedBody: rebuilt, session, attachedRetrievals, attachedRetrievalNoteIds, processedMessages, originalMessages, judgeBaseline, protocol: "openai", stream, compressInjected: injectTools, pluginMode, mcpDialectNote: laneDecisions.mcpDialect, nudge, nudgeOutcome, prompts, surface, openaiSystemText, systemNotes: sysNotes, renderTags: renderStrategy, responseExtraHeaders: wpResponseHeaders, dropReasoning: (msgs: BiliMessage[]) => applyReasoningDrop(msgs, reasoningPlan, log, sessionId) } as Prepared; } From 17df343dd93204610c7c597f003c64dbf7328867 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 5 Nov 2026 00:34:49 +0800 Subject: [PATCH 18/49] test(fastpath): render-snapshot plane matrix; schema head bumps to v12 (#5664) --- tests/issue4666-migration-dedup.test.ts | 2 +- .../issue4677-v8-migration-streaming.test.ts | 12 +- tests/issue4796-gc-batched-sweep.test.ts | 6 +- tests/issue5664-render-snapshot.test.ts | 394 ++++++++++++++++++ tests/storage-ext.test.ts | 10 +- tests/storage-ingest-watermark.test.ts | 8 +- tests/storage-unified.test.ts | 4 +- 7 files changed, 415 insertions(+), 21 deletions(-) create mode 100644 tests/issue5664-render-snapshot.test.ts diff --git a/tests/issue4666-migration-dedup.test.ts b/tests/issue4666-migration-dedup.test.ts index 04ad1dda51..a3cc7caaaf 100644 --- a/tests/issue4666-migration-dedup.test.ts +++ b/tests/issue4666-migration-dedup.test.ts @@ -155,7 +155,7 @@ test("#4666 sqlite: v8 migration collapses wireIndex-split duplicates and refund check = openSqliteDatabase(path.join(dir, "index.db")).db; const ver = check.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 11, "stamped to v11 (…v10 ts column + v11 fastpath-v6 identity plane)"); + assert.equal(ver.user_version, 12, "stamped to v12 (…v10 ts column + v11 fastpath-v6 identity plane + v12 render snapshot plane)"); // Exactly the lowest-seq arrival of each (msg_id, identity) group. const survivors = check diff --git a/tests/issue4677-v8-migration-streaming.test.ts b/tests/issue4677-v8-migration-streaming.test.ts index b9fa507ba7..5290291720 100644 --- a/tests/issue4677-v8-migration-streaming.test.ts +++ b/tests/issue4677-v8-migration-streaming.test.ts @@ -99,7 +99,7 @@ test("#4677 backfill sets identities and drops wireIndex-split duplicates (lowes const { from, to } = migrateSchema(db); assert.equal(from, 7); - assert.equal(to, 11); + assert.equal(to, 12); const identity = identityOfStoredContent(storedMessage("stripped", "hello")); assert.equal(identityOf(db, "s1", 1), identity); // the later duplicate row is gone and its content CAS ref handed back @@ -109,7 +109,7 @@ test("#4677 backfill sets identities and drops wireIndex-split duplicates (lowes ); assert.equal(refcountOf(db, a2), 0); assert.equal(refcountOf(db, a1), 1); - assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 11); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 12); db.close(); }); @@ -129,7 +129,7 @@ test("#4677 crash resume: sessions that already carry identities are skipped, no assert.equal(identityOf(db, "s-done", 1), "stale-planted"); assert.equal(identityOf(db, "s-pending", 1), identityOfStoredContent(storedMessage("pending", "still fresh"))); - assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 11); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 12); db.close(); }); @@ -150,7 +150,7 @@ test("#4677 rows without content bytes terminate with the ABSENT marker instead assert.equal(identity, "absent:"); // second run is a no-op — the marker is not identity = '', so the // session is not selected again - assert.equal(migrateSchema(db).from, 11); + assert.equal(migrateSchema(db).from, 12); assert.equal(identityOf(db, "s-ghost", 1), "absent:"); db.close(); }); @@ -193,7 +193,7 @@ test("#4677 empty store migrates to the schema head untouched", () => { downgradeToV7(db); const { from, to } = migrateSchema(db); assert.equal(from, 7); - assert.equal(to, 11); - assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 11); + assert.equal(to, 12); + assert.equal((db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version, 12); db.close(); }); diff --git a/tests/issue4796-gc-batched-sweep.test.ts b/tests/issue4796-gc-batched-sweep.test.ts index ad6aa53e8e..cf8e0ee7fe 100644 --- a/tests/issue4796-gc-batched-sweep.test.ts +++ b/tests/issue4796-gc-batched-sweep.test.ts @@ -84,7 +84,7 @@ test("#4796 v9: fresh install carries idx_content_refcount and candidate selecti seedContentRows(rawDb(store), 20_000, 1, "live"); seedContentRows(rawDb(store), 50, 0, "orphan"); const ver = rawDb(store).prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 11); + assert.equal(ver.user_version, 12); const idx = rawDb(store) .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_content_refcount'") .get(); @@ -118,7 +118,7 @@ test("#4796 v9: reopening a v8-era db gains the index", () => { const store = UnifiedStore.open(dbPath); try { const ver = rawDb(store).prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 11); + assert.equal(ver.user_version, 12); const idx = rawDb(store) .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_content_refcount'") .get(); @@ -243,7 +243,7 @@ test("#4796 e2e: v8 identity backfill releases wireIndex twins -> gc collects th try { const db2 = rawDb(reopened); const ver = db2.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 11); + assert.equal(ver.user_version, 12); const twinLeft = db2 .prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND seq = ?") .get(sessId, twinSeq) as { n: number }; diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts new file mode 100644 index 0000000000..5053b06a14 --- /dev/null +++ b/tests/issue5664-render-snapshot.test.ts @@ -0,0 +1,394 @@ +// #5664 — station 2/3: outbound-row render-snapshot plane for the v6 lane. +// +// Golden equation under test: on every engaged turn the emit array built as +// [cached prefix rows] + [freshly mapped tail] (fast rounds) must be BYTE- +// IDENTICAL to what the legacy full-pipeline path emits (arm / audit / +// caps-flip rollback rounds). The kernel is untouched; this lane caches +// egress bytes, not semantics — equivalence with the full pass is the +// contract. B-version compaction semantics hold throughout: client-side +// element deletion is IGNORED by the identity plane (ledger grows forward +// only), so both arms grow identically and stay byte-comparable. +// +// Harness: unit-level against a REAL sqlite sidecar (UnifiedShadow shadow +// mode), mirroring tests/fastpath-v6-msgid.test.ts; the kernel runs as a +// real createCore() instance (the same regime probe-window.mts established +// for bounded-window byte reproduction). +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type http from "node:http"; +import { DatabaseSync } from "node:sqlite"; +import { createCore, createInitialState, defaultConfig, type CoreMessage } from "acp-kernel"; +import { coreToOpenai, openaiToCore, type OpenAIMessage } from "acp-kernel/wire"; +import { SessionStore, _setStoreForTest, _resetStoreForTest } from "../src/persist.ts"; +import type { SessionStore as SessionStoreType } from "../src/persist.ts"; +import type { Session } from "../src/session.ts"; +import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { stripKernelSummaries } from "../src/server.ts"; +import { hardenOpenaiAssistantContent, systemToUser } from "../src/util.ts"; +import { + evaluateWatermarkV6, finalizeWatermarkV6, watermarkV6DebugState, PF_SESSION_HEADER, +} from "../src/server/fastpath-v6.ts"; +import { PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; +import type { V6Outcome } from "../src/server/fastpath-v6.ts"; +import { + bumpAuditTick, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, + watermarkV6RenderDebugState, _resetWatermarkV6RenderTestState, + type RenderCaps, +} from "../src/server/fastpath-v6-render.ts"; + +const ENV_V6 = "BILI_ROLLOUT_WATERMARK_V6_MSG_ID"; +const ENV_RENDER = "BILI_ROLLOUT_WATERMARK_V6_RENDER"; +const MODEL_LIMIT = 200_000; + +type AgentState = ReturnType; + +interface Msg { role: string; content: string } +function makeBody(messages: Msg[]): string { return JSON.stringify({ model: "m", messages }); } + +type LogSink = { lines: { level: string; msg: string }[] } & ((level: string, msg: string) => void); +function makeLog(): LogSink { + const l = { lines: [] as { level: string; msg: string }[] }; + return Object.assign((level: string, msg: string) => { l.lines.push({ level, msg }); }, { lines: l.lines }); +} + +/** Post-kernel → wire transform chain (production shape minus the image + * rescale pass — identity for text-only content, no multimodal parts here). */ +function mapEgress(pms: CoreMessage[]): OpenAIMessage[] { + return hardenOpenaiAssistantContent(coreToOpenai(systemToUser(pms))) as unknown as OpenAIMessage[]; +} + +function capsFor(strategy: string): RenderCaps { + return { v: 1, model: "m", strategy, maxDim: 1_600, toolArmed: false, droppable: false, multimodal: false }; +} + +/** Shallow-adopt a returned state into a stable object identity (the session + * holder keeps referencing the same object across turns). */ +function adoptState(dst: AgentState, src: AgentState): void { + for (const k of Object.keys(src) as (keyof AgentState)[]) (dst as unknown as Record)[k] = src[k]; + // fields cleared on src but present on dst: none expected; state evolves additively +} + +interface H { dir: string; store: SessionStoreType; log: LogSink; cleanup: () => void } +function makeH(tag: string): H { + process.env[ENV_V6] = "1"; + process.env[ENV_RENDER] = "1"; + _resetStoreForTest(); + _resetWatermarkV6RenderTestState(); + const dir = mkdtempSync(path.join(os.tmpdir(), `rsn-${tag}-`)); + const shadow = UnifiedShadow.maybeOpen(dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as never); + assert.ok(shadow, "shadow sidecar must open in the hermetic dir"); + const store = new SessionStore({ dir: path.join(dir, "sessions"), enabled: true, shadow }); + _setStoreForTest(store); + const log = makeLog(); + return { dir, store, log, cleanup: () => { _resetStoreForTest(); _resetWatermarkV6RenderTestState(); rmSync(dir, { recursive: true, force: true }); } }; +} + +/** Arm turns carry no hit (first registration) — fall back to the + * full-parse oracle, which the v6 contract (§8-1) pins byte-equal to the + * assembled view. */ +function viewOf(outcome: V6Outcome, bodyText: string): CoreMessage[] { + return outcome.hit !== null ? [...outcome.hit.view] : (openaiToCore(JSON.parse(bodyText) as never) as unknown as { msgs: CoreMessage[] }).msgs; +} + +/** One v6 identity-plane turn, SPLIT like production + * (prepare-openai: evaluate early → pipeline/egress/capture → finalize LAST; + * finalize is the only point that advances worlds.maxPos / pf_rows). */ +function v6Eval(h: H, sessId: string, bodyText: string, idList: string[], echoSid?: string): { outcome: V6Outcome; sid: string | undefined; fin: () => void } { + const headers: Record = { "x-bili-plugin-conversation": "carrier" }; + if (idList.length > 0) headers[PF_TAIL_IDS_HEADER] = idList.map(encodeURIComponent).join(","); + if (echoSid !== undefined) headers[PF_SESSION_HEADER] = echoSid; + const req = { headers } as unknown as http.IncomingMessage; + const session = { id: sessId } as unknown as Session; + const outcome = evaluateWatermarkV6(bodyText, req, session, { pluginMode: true, titleGen: false, model: "m" }, h.log); + assert.ok(outcome !== null, `v6 must engage (log=${JSON.stringify(h.log.lines.slice(-4))})`); + return { + outcome, + sid: outcome.armed?.sid ?? echoSid, + fin: () => { finalizeWatermarkV6(session, outcome, { bodyText }, h.log); }, + }; +} + +// --------------------------------------------------------------------------- +// Unit tier: pure plane mechanics. +// --------------------------------------------------------------------------- + +test("unit: diffRowLists verdicts", () => { + assert.equal(diffRowLists([], ["a"]), "append"); + assert.equal(diffRowLists(["a", "b"], ["a", "b"]), "equal"); + assert.equal(diffRowLists(["a", "b"], ["a", "b", "c"]), "append"); + assert.equal(diffRowLists(["a", "b"], ["a", "X"]), "recast"); // middle mutation + assert.equal(diffRowLists(["a", "b", "c"], ["a"]), "recast"); // shrink +}); + +test("unit: cast → steady appends → hydrate-from-disk fold survives mirror loss", (t) => { + const h = makeH("fold"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-fold", b1, ["id0"]); + assert.equal(r1.outcome.decision, "arm"); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + let op = commitRenderCapture({ sessionId: "s-fold", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }); + assert.equal(op.op, "cast"); + op = commitRenderCapture({ sessionId: "s-fold", store: h.store, sid, mode: "steady", caps, rowStrings: ['{"i":2}'], sums: { estCum: 1, textCum: 1, imgCum: 0 } }); + assert.equal(op.op, "append"); + op = commitRenderCapture({ sessionId: "s-fold", store: h.store, sid, mode: "steady", caps, rowStrings: ['{"i":3}'], sums: { estCum: 1, textCum: 1, imgCum: 0 } }); + assert.equal(op.op, "append"); + assert.equal(h.store.pfRenderRows(sid).length, 4); + assert.equal(h.store.pfRenderCkptIntact(sid), true); + + // Kill every in-memory mirror — the next gate MUST rebuild from disk + // via pfRenderMeta folding (ckpt rowCount + post-ckpt append log). + // maxPos is synthetic here: prod wiring always feeds the live world + // geometry (rows and plane grow together, one row per element), so + // alignment holds; this unit isolates the FOLD mechanics, so feed the + // geometry matching the four stored rows. + _resetWatermarkV6RenderTestState(); + const g = renderPlaneGate({ + sessionId: "s-fold", session: { id: "s-fold", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 3, tokenCount: 50, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }); + assert.ok(g.ok, `expected hydrated green gate, got ${JSON.stringify(g)}`); + if (g.ok) { + assert.equal(g.rw.rows.length, 4, "disk fold must reconstruct all four rows"); + assert.deepEqual(g.rw.rows.slice(-2), ['{"i":2}', '{"i":3}'], "appended tail must survive the fold"); + } + } finally { h.cleanup(); } +}); + +test("gate: red reasons are a stable vocabulary", (t) => { + const h = makeH("gate"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-gate", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + const cast = commitRenderCapture({ sessionId: "s-gate", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"a":1}'], sums: { estCum: 1, textCum: 1, imgCum: 0 } }); + assert.equal(cast.op, "cast"); + const geom = watermarkV6DebugState("s-gate").worlds[sid]; + const base = { + sessionId: "s-gate", session: { id: "s-gate", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: geom!.maxPos, tokenCount: 500, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }; + const g0 = renderPlaneGate(base); + assert.equal(g0.ok, true, `expected green gate, got ${JSON.stringify(g0)}`); + assert.deepEqual(renderPlaneGate({ ...base, engagedHit: false }), { ok: false, reason: "not-engaged" }); + assert.deepEqual(renderPlaneGate({ ...base, armedTurn: true }), { ok: false, reason: "arm-turn" }); + assert.deepEqual(renderPlaneGate({ ...base, isTitleGen: true }), { ok: false, reason: "title-gen" }); + assert.deepEqual(renderPlaneGate({ ...base, tokenCount: Math.ceil(0.9 * MODEL_LIMIT) }), { ok: false, reason: "pressure" }); + assert.deepEqual(renderPlaneGate({ ...base, maxPos: -1 }), { ok: false, reason: "no-plane-geometry" }); + assert.deepEqual(renderPlaneGate({ ...base, caps: capsFor("none") }), { ok: false, reason: "caps-mismatch" }); + // audit cadence: PF_RENDER_AUDIT_EVERY default 8 — ticks land on 8 ⇒ red, 9 ⇒ green + for (let i = 0; i < 8; i++) bumpAuditTick(sid); + assert.deepEqual(renderPlaneGate(base), { ok: false, reason: "audit-due" }); + bumpAuditTick(sid); + assert.equal(renderPlaneGate(base).ok, true); + } finally { h.cleanup(); } +}); + +test("unit: S4 checksum tamper kills the mirror (hydrate-failed, loudly logged)", (t) => { + const h = makeH("s4"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-s4", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + const cast = commitRenderCapture({ sessionId: "s-s4", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"a":1}', '{"a":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }); + assert.equal(cast.op, "cast"); + assert.equal(h.store.pfRenderCkptIntact(sid), true); + // Simulate silent corruption of the checkpoint checksum. + const db = new DatabaseSync(path.join(h.dir, "index.db")); + db.prepare(`UPDATE pf_render_ckpt SET checksum = 'deadbeef' WHERE sid = ?`).run(sid); + db.close(); + assert.equal(h.store.pfRenderCkptIntact(sid), false); + _resetWatermarkV6RenderTestState(); // memory copies gone ⇒ gate must hydrate + const g = renderPlaneGate({ + sessionId: "s-s4", session: { id: "s-s4", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 0, tokenCount: 10, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }); + assert.deepEqual(g, { ok: false, reason: "hydrate-failed" }); + } finally { h.cleanup(); } +}); + +// --------------------------------------------------------------------------- +// Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, +// across arm → steady fast rounds → client compaction (turn 8) → strategy +// recast (turn 10) → audit cadence → restart-hydration continuation. +// --------------------------------------------------------------------------- + +test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end", async (t) => { + const core = createCore({}); + const cfg = defaultConfig(MODEL_LIMIT); + + // ---- conversation script ------------------------------------------------ + // 16 elements e_0..e_15 arrive one per turn. From turn 8 the CLIENT body + // drops e_1/e_3 (client-side compaction) — the lane ignores the drop + // (ledger grows forward only), views and rows keep growing, and the + // golden equation must still hold element-stable (ids follow elements, + // never positions). Strategy flips text-only→none at turn 10 ⇒ caps flip + // ⇒ forced legacy recast round, then steady on the new byte form. + const ELEMENTS: Msg[] = Array.from({ length: 16 }, (_, i) => ({ role: i % 2 === 0 ? "user" : "assistant", content: `content number ${i}` })); + const STABLE_ID = (i: number) => `eid-${i}`; + const clientBody = (tn: number): Msg[] => { + let els = ELEMENTS.slice(0, tn); + if (tn >= 8) els = els.filter((_, i) => i !== 1 && i !== 3); + return els; + }; + const idsOf = (els: Msg[]) => els.map((el) => STABLE_ID(ELEMENTS.indexOf(el))); + const strategyOf = (tn: number) => (tn < 10 ? "text-only" : "none"); + + // ---- phase 1: arm A — pure legacy full path over its own store ---------- + const hA = makeH("gold-a"); + let stateA: AgentState = createInitialState(); + let sidA: string | undefined; + const goldens: { rows: string[]; decision: string }[] = []; + for (let tn = 1; tn <= 16; tn++) { + const body = clientBody(tn); + const bodyText = makeBody(body); + const v = v6Eval(hA, "s-gold-a", bodyText, idsOf(body), sidA); + if (v.sid !== undefined) sidA = v.sid; + const viewA = viewOf(v.outcome, bodyText); + const tA = core.processTurn({ messages: viewA, state: stateA, config: cfg, tokenCount: 100 + viewA.length * 37, renderTags: strategyOf(tn) }); + adoptState(stateA, tA.state as unknown as AgentState); + goldens.push({ rows: mapEgress(stripKernelSummaries(tA.messages, tA.state as never)).map((m) => JSON.stringify(m)), decision: v.outcome.decision }); + v.fin(); + } + hA.cleanup(); + + // ---- phase 2: arm B — render lane ON, replaying the identical script ---- + const hB = makeH("gold-b"); + const stateB: AgentState = createInitialState(); + const sessB = { id: "s-gold-b", state: stateB } as unknown as Session; + let sidB: string | undefined; + let fastRounds = 0, castOps = 0, recastOps = 0, legacyRounds = 0; + for (let tn = 1; tn <= 16; tn++) { + const strategy = strategyOf(tn); + const body = clientBody(tn); + const bodyText = makeBody(body); + const v = v6Eval(hB, "s-gold-b", bodyText, idsOf(body), sidB); + if (v.sid !== undefined) sidB = v.sid; + const hit = v.outcome.hit; + const tokenCount = 100 + (hit !== null ? hit.view.length : body.length); + const caps = capsFor(strategy); + // Arm turn: no world exists pre-finalize ⇒ prod gate block skipped. + const geom = sidB !== undefined ? watermarkV6DebugState("s-gold-b").worlds[sidB] : undefined; + const gate = renderPlaneGate({ + sessionId: "s-gold-b", session: sessB, store: hB.store, engagedHit: hit !== null && v.outcome.armed === undefined, + armedTurn: v.outcome.armed !== undefined, isTitleGen: false, sid: sidB ?? "unarmed", + maxPos: geom?.maxPos ?? -1, tokenCount, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + log: hB.log, + }); + let rebuilt: OpenAIMessage[]; + let captureStrings: string[]; + let mode: "steady" | "legacy"; + if (gate.ok) { + const sidNow = sidB; + assert.ok(sidNow !== undefined, "green gate implies armed"); + assert.ok(geom, "green gate implies world geometry"); + const maxPos = geom.maxPos; + bumpAuditTick(sidNow); + const rw = gate.rw; const F = gate.F; + // bounded window pass (mirrors prepare-openai.ts slot exactly) + const posFrom = maxPos - F + 1; + const winJson = hB.store.pfHydrateWindow(sidNow, posFrom, maxPos); + const frozen: CoreMessage[] = []; + for (let p = posFrom; p <= maxPos; p++) { + const raw = winJson.get(p); + assert.notEqual(raw, undefined, `window-core gap at pos ${p}`); + // pf_rows.core_json holds a PLAIN CORES ARRAY per pos (an + // element may carry several cores) — flatten + re-stamp, + // mirroring fastpath-v6's own disk loader. + const arr = JSON.parse(raw!) as CoreMessage[]; + for (const c of arr) frozen.push(Object.assign(c, { wireIndex: p })); + } + const scratch = structuredClone(stateB); + const tW = core.processTurn({ messages: [...frozen, ...(v.outcome.hit?.windowCores ?? [])], state: scratch, config: cfg, tokenCount, renderTags: strategy }); + const liveRefs = stateB.messageRefs as unknown as { byRaw: Record; byRef: Record }; + const srcRefs = (scratch as AgentState).messageRefs as unknown as { byRaw: Record; byRef: Record }; + for (const [k, val] of Object.entries(srcRefs.byRaw)) if (!(k in liveRefs.byRaw)) liveRefs.byRaw[k] = val; + for (const [k, val] of Object.entries(srcRefs.byRef)) if (!(k in liveRefs.byRef)) liveRefs.byRef[k] = val; + for (const r of (scratch as AgentState).rules ?? []) if (!(stateB.rules ?? []).some((x) => x.id === r.id)) (stateB.rules ??= []).push(r); + stateB.nextRuleId = Math.max(stateB.nextRuleId ?? 1, (scratch as AgentState).nextRuleId ?? 1); + const pmW = stripKernelSummaries(tW.messages, tW.state as never); + + const mappedAll = mapEgress(pmW); + const committedTail = mappedAll.slice(0, Math.min(F + (v.outcome.hit?.windowCores.length ?? 0), mappedAll.length)); + const vv = verifyRenderWindow(committedTail.slice(0, F).map((m) => JSON.stringify(m)), rw, F); + if (!vv) { + for (let k = 0; k < F; k++) { + const a = JSON.stringify(committedTail[k]); const b = rw.rows[rw.rows.length - F + k]; + + } + } + assert.equal(vv, true, `turn ${tn}: window verify failed — tail bytes drifted vs stored rows`); + rebuilt = spliceOutboundRows(rw, F, committedTail); + captureStrings = [...rw.rows.slice(0, rw.rows.length - F), ...committedTail.map((m) => JSON.stringify(m))]; + mode = "steady"; + fastRounds++; + } else { + // legacy full path over the engaged view (production shape) + const tF = core.processTurn({ messages: hit !== null ? [...hit.view] : (openaiToCore(JSON.parse(bodyText) as never) as unknown as { msgs: CoreMessage[] }).msgs, state: stateB, config: cfg, tokenCount, renderTags: strategy }); + adoptState(stateB, tF.state as unknown as AgentState); + const mappedAll = mapEgress(stripKernelSummaries(tF.messages, tF.state as never)); + rebuilt = mappedAll; + captureStrings = mappedAll.map((m) => JSON.stringify(m)); + mode = "legacy"; + legacyRounds++; + } + const op = commitRenderCapture({ + sessionId: "s-gold-b", store: hB.store, sid: sidB!, mode, caps, rowStrings: captureStrings, + sums: { estCum: 0, textCum: 0, imgCum: 0 }, + }); + if (op.committed) { + if (op.op === "cast") castOps++; + else if (op.op === "recast") recastOps++; + } + // THE golden equation: lane emit == legacy oracle, byte for byte. + assert.deepEqual(rebuilt.map((m) => JSON.stringify(m)), goldens[tn - 1].rows, + `turn ${tn} (${mode}${gate.ok ? "" : `, reason=${gate.reason}`}): lane emit diverged from legacy oracle`); + assert.equal(stateB.blocks.length, 0, "assumption pin: no folds materialize in this synthetic conv"); + v.fin(); // plane advances AFTER capture — same order as prepare-openai + } + assert.equal(castOps, 1, "exactly one cast (the arm turn)"); + assert.ok(fastRounds >= 6, `expected several fast rounds, got ${fastRounds} (audit too aggressive?)`); + assert.ok(recastOps >= 1, "strategy flip at turn 10 must force ≥1 recast"); + assert.ok(legacyRounds >= 2, `arm + audit/caps-flip rounds must ride legacy (got ${legacyRounds})`); + // S4 truth intact at the end; mirror agrees with the last golden shape + assert.equal(hB.store.pfRenderCkptIntact(sidB!), true); + const dbgLhs = watermarkV6RenderDebugState("s-gold-b"); + assert.ok(Array.isArray(dbgLhs.sids) && dbgLhs.sids.length > 0, "render debug state must list the mirror"); + const lhsRows = dbgLhs.sids[dbgLhs.sids.length - 1].rows; + const rhsRows = goldens[goldens.length - 1].rows.length; + assert.equal(lhsRows, rhsRows, "mirror row count must equal last golden row count"); + + // ---- phase 3: restart hydration — mirrors wiped, disk must continue ----- + _resetWatermarkV6RenderTestState(); + const extraEl: Msg = { role: "user", content: "content number 16" }; + ELEMENTS.push(extraEl); + const bodyT17 = clientBody(16).concat([extraEl]); + const v17 = v6Eval(hB, "s-gold-b", makeBody(bodyT17), idsOf(bodyT17), sidB); + const g17 = renderPlaneGate({ + sessionId: "s-gold-b", session: sessB, store: hB.store, engagedHit: true, armedTurn: false, + isTitleGen: false, sid: sidB!, maxPos: watermarkV6DebugState("s-gold-b").worlds[sidB!]!.maxPos, + tokenCount: 100, modelContextLimit: MODEL_LIMIT, caps: capsFor("none"), + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, log: hB.log, + }); + assert.ok(g17.ok === true || g17.reason === "audit-due", + `post-restart gate must hydrate from disk (got ${JSON.stringify(g17)})`); + if (g17.ok) assert.equal(g17.rw.rows.length, goldens.at(-1)!.rows.length, "hydrated row count must match the pre-restate capture"); + v17.fin(); + hB.cleanup(); +}); diff --git a/tests/storage-ext.test.ts b/tests/storage-ext.test.ts index 70019f0352..1cb4778979 100644 --- a/tests/storage-ext.test.ts +++ b/tests/storage-ext.test.ts @@ -155,11 +155,11 @@ test("guards reject hostile keys and oversize values", async () => { } }); -test("v3 databases migrate to v10 in place, idempotently", async () => { +test("v3 databases migrate to v12 in place, idempotently", async () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-ext-migrate-")); try { const dbPath = path.join(tmp, "index.db"); - // Build a current (v10) database, then roll the extension tables and + // Build a current (v12) database, then roll the extension tables and // version marker back to the v3 shape — exactly what an upgraded // production lane's file looks like to the new code. const fresh = UnifiedStore.open(dbPath); @@ -174,17 +174,17 @@ test("v3 databases migrate to v10 in place, idempotently", async () => { const migrated = UnifiedStore.open(dbPath); try { const ver = (migrated as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 11, "open() runs migrateSchema: v3 → v11"); + assert.equal(ver.user_version, 12, "open() runs migrateSchema: v3 → v12"); migrated.setKv("probe", "alive", true); assert.deepEqual(migrated.getKv("probe", "alive"), true); assert.deepEqual(migrated.listSessionIds(), [], "existing v3 content untouched"); } finally { migrated.close(); } - // Reopening a v10 database is a no-op (idempotent migration). + // Reopening a v12 database is a no-op (idempotent migration). const again = UnifiedStore.open(dbPath); const ver2 = (again as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver2.user_version, 11); + assert.equal(ver2.user_version, 12); again.close(); } finally { fs.rmSync(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); diff --git a/tests/storage-ingest-watermark.test.ts b/tests/storage-ingest-watermark.test.ts index 6ab6229b19..bf0cf9bdf0 100644 --- a/tests/storage-ingest-watermark.test.ts +++ b/tests/storage-ingest-watermark.test.ts @@ -591,7 +591,7 @@ test("watermark: sampled verification still catches drift through the delta path } }); -test("watermark: v4 database migrates to v11 additively; newer user_version no-ops", async () => { +test("watermark: v4 database migrates to v12 additively; newer user_version no-ops", async () => { const dir = tmpdir("bili-wm-migrate-"); try { const dbPath = path.join(dir, "index.db"); @@ -620,7 +620,7 @@ test("watermark: v4 database migrates to v11 additively; newer user_version no-o const store = UnifiedStore.open(dbPath); try { const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 11, "open() migrates v4 → v11"); + assert.equal(ver.user_version, 12, "open() migrates v4 → v12"); const row = store.getSessionRow("v4-sess"); assert.ok(row, "legacy session row survives the migration"); assert.equal(row!.lastIngestedSeq, null, "legacy rows read back with a NULL watermark (≡ full ingest)"); @@ -653,12 +653,12 @@ test("watermark: v4 database migrates to v11 additively; newer user_version no-o // probe version sits AT the current schema head so the no-op branch // still exercises after future bumps. const fwd = new DatabaseSync(dbPath); - fwd.exec("PRAGMA user_version = 11"); + fwd.exec("PRAGMA user_version = 12"); fwd.close(); const older = UnifiedStore.open(dbPath); try { const ver2 = (older as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver2.user_version, 11, "newer user_version is left untouched"); + assert.equal(ver2.user_version, 12, "newer user_version is left untouched"); assert.ok(older.getSessionRow("v4-sess"), "rows stay readable"); } finally { older.close(); diff --git a/tests/storage-unified.test.ts b/tests/storage-unified.test.ts index 644dbe298a..04350a3f36 100644 --- a/tests/storage-unified.test.ts +++ b/tests/storage-unified.test.ts @@ -801,7 +801,7 @@ test("schema v2: dev-window v1 database upgrades in place (identity columns adde assert.equal(row!.namespace, "proxy", "namespace backfills with its constant default"); assert.equal(row!.origin, null, "origin backfills NULL"); const ver = (store as unknown as { db: { prepare(q: string): { get(): unknown } } }).db.prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 11, "schema version stamped to 11 (… v10 immutable message ts column + v11 fastpath-v6 identity plane tables pf_rows/pf_log/pf_ckpt)"); + assert.equal(ver.user_version, 12, "schema version stamped to 12 (… v10 immutable message ts column + v11 fastpath-v6 identity plane + v12 render snapshot plane)"); // v3 backfill: every pre-existing sessions row claims its // (namespace, conversation_id) identity binding. const binding = (store as unknown as { lookupConversation(ns: string, id: string): string | null }).lookupConversation("proxy", "dev-window-1"); @@ -863,7 +863,7 @@ test("schema v10: v9→v10 migration adds messages.ts without backfill; idempote try { const ver = (reopened as unknown as { db: { prepare(q: string): { get(): unknown } } }).db .prepare("PRAGMA user_version").get() as { user_version: number }; - assert.equal(ver.user_version, 11, "v9 library migrates to 11"); + assert.equal(ver.user_version, 12, "v9 library migrates to 12"); const rows = reopened.getMessages("v10-mig-1") as { seq: number; ts?: number | null }[]; assert.equal(rows.length, 1, "row survives the migration"); assert.ok(rows[0].ts === null || rows[0].ts === undefined, "pre-existing row keeps NULL (never backfilled)"); From 16d5944bae1997aa3b1efab3438141e830923908 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 5 Nov 2026 01:07:02 +0800 Subject: [PATCH 19/49] =?UTF-8?q?fix(fastpath):=20review=20round=201=20?= =?UTF-8?q?=E2=80=94=20audit=20consumption,=20sums=20algebra,=20steady=20t?= =?UTF-8?q?ail-only=20rows=20(#5664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review pass on the self-authored lane surfaced four state-maintenance bugs on the green path plus a class of test-blind spots; all fixed here with regression pins: 1. steady capture appended the FULL row list (production shape), not just this turn's new rows: the pos column shifted forward every green round, doubling the table on disk and in the mirror; only the per-round recast churn (masking an audit stall) kept the byte equation green. Capture now sends committedTail.slice(F) — exactly the appended suffix. 2. Cumulative estCum/textCum/imgCum re-counted the F frozen window rows every green round (linear ledger drift ⇒ shrinking max_tokens clamp and polluted k calibration over long sessions). Sums and the image-reserve delta now cover only this turn's new elements; legacy rounds still carry absolute totals. 3. The audit obligation was consumed only by cast/recast: after N green rounds tick%N===0 stayed true forever and the lane dead-locked back to full legacy until an unrelated caps flip. Any legacy full-capture round now zeroes the tick (it IS the audit). 4. Hygiene from the same pass: identity faults rethrow through the window- pass catch instead of retry-passing; CCR content-store adoption no longer defers past fast rounds (dangling refs); splice shallow-clones mirror rows so downstream cannot alias-mutate next round's verify baseline; NUL-bearing rows refuse persistence loudly-once (node:sqlite truncates bound strings at \0 silently) and hydrate parse failures log error instead of failing soft-silent. Test tier extended: per-round render-table↔identity-ledger alignment pin, two-audit-wave fast-round threshold (catches the stall), submitted-volume and cumulative-sum algebra pins, session-cascade wipe of the three render tables, and a second-sidecar-open 12→12 migration idempotency no-op. --- src/server/fastpath-v6-render.ts | 52 +++++++++++++++-- src/server/prepare-openai.ts | 30 ++++++++-- tests/issue5664-render-snapshot.test.ts | 75 ++++++++++++++++++++++--- 3 files changed, 136 insertions(+), 21 deletions(-) diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 6b1cfa8d01..9acc031990 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -109,7 +109,11 @@ export interface RenderWorld { textCum: number; /** Cumulative image-reserve contribution of frozen rows. */ imgCum: number; - /** Engaged-turn tick — % PF_RENDER_AUDIT_EVERY === 0 ⇒ audit due. */ + /** Green (fast-served) engaged-turn tick — % PF_RENDER_AUDIT_EVERY === 0 + * ⇒ audit due. Consumed (zeroed) by ANY legacy full-capture round served, + * which IS the audit (the full byte path ran to completion). + * Invariant: a red-for-audit-due turn always lands in a legacy capture, + * so the obligation cannot stall the lane after N ticks. */ auditTick: number; } @@ -207,11 +211,18 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor const rowsParsed: unknown[] = new Array(rowsRaw.length); for (let i = 0; i < rowsRaw.length; i++) { const rj = rowsRaw[i].rowJson; - if (rj === null || rowsRaw[i].pos !== i) return null; // tombstone/gap = inconsistent + if (rj === null || rowsRaw[i].pos !== i) { + // Row table inconsistent on disk (tombstone where a row must + // be, or a gap): judge the plane dead LOUDLY — same discipline + // as an S4 checksum mismatch, never a silent fallback. + log?.("error", `[${sid}] #5664 render plane judged dead: row-table inconsistency at pos ${rowsRaw[i].pos} (expected contiguous ${i})`); + return null; + } rows[i] = rj; try { rowsParsed[i] = JSON.parse(rj); } catch { + log?.("error", `[${sid}] #5664 render plane judged dead: row parse failed at pos ${i}`); return null; } } @@ -314,8 +325,10 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { return { ok: true, rw, F }; } -/** Called AFTER the decision is consumed (keeps the audit cadence honest — - * a gated-off-for-pressure turn still counts as engaged). */ +/** Bumped once per GREEN round (prepare-openai gate block). The obligation is + * consumed by any legacy full-capture round (commitRenderCapture zeroes it): + * cadence = "one full pass per PF_RENDER_AUDIT_EVERY fast rounds", red turns + * included among those full passes. */ export function bumpAuditTick(sid: string): void { const rw = mirrors.get(sid); if (rw) rw.auditTick += 1; @@ -348,8 +361,13 @@ export function spliceOutboundRows(rw: RenderWorld, F: number, tail: T[]): T[ const out: T[] = []; // rowsParsed entries are JSON.parse of THIS plane's previously captured // rows — exactly T elements by construction (capture stores what egress - // emitted, and egress emitted T). - out.push(...(rw.rowsParsed.slice(0, prefixLen) as T[])); + // emitted, and egress emitted T). Shallow-clone each cached object: these + // doubles as next round's verify baseline, so no downstream consumer may + // alias-mutate the mirror cache (review P2-5). + for (const x of rw.rowsParsed.slice(0, prefixLen)) { + const y: T = x !== null && typeof x === "object" ? ({ ...(x as object) } as T) : (x as T); + out.push(y); + } out.push(...tail); return out; } @@ -391,8 +409,24 @@ export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" return "recast"; } +/** sids whose row payload carried a literal NUL and were therefore refused + * persistence (node:sqlite silently TRUNCATES bound strings at the first \0 + * — storing would corrupt the row with S4 unable to see it, since the + * checksum only covers checkpoint meta). Loud-once, then fail-soft per turn: + * the in-memory mirror keeps serving this process's lifetime; disk simply + * stops advancing until restart/caps-flip recast is possible again. */ +const nulRejectedSids = new Set(); + export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" | "recast" } { const { store, sid, caps, log } = args; + const hasNul = args.rowStrings.some((rj) => rj.includes("\u0000")); + if (hasNul) { + if (!nulRejectedSids.has(sid)) { + nulRejectedSids.add(sid); + log?.("error", `[${sid}] #5664 render row carries a literal NUL (node:sqlite truncates it silently) — plane persistence disabled for this sid`); + } + return { committed: false, op: "none" }; + } let rw = mirrors.get(sid); if (rw === undefined && !mirrors.has(sid)) { const hydrated = hydrateMirror(store, sid, log); @@ -441,6 +475,11 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op touchMirror(rw); return { committed: true, op: "append" }; } + // A full-path round IS the audit regardless of verdict (the legacy byte + // path ran to completion) — consume the cadence obligation here, or a + // clean long run would dead-lock on "audit-due" after N ticks (only + // cast/recast zeroed it before this fix). + if (args.mode === "legacy") rw.auditTick = 0; // legacy full-capture: diff against truth, minimal commit. const verdict = diffRowLists(rw.rows, args.rowStrings); if (verdict === "equal") { @@ -528,5 +567,6 @@ export function watermarkV6RenderDebugState(sessionId: string): { sids: RenderDe export function _resetWatermarkV6RenderTestState(): void { mirrors.clear(); sessionSids.clear(); + nulRejectedSids.clear(); mirrorBytes = 0; } diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index c2e7285591..23f22f8079 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -41,7 +41,7 @@ import { injectOpenaiTool, injectTool } from "./inject.js"; import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decisions.js"; import { featureEnabled } from "../rollout.js"; import { evaluateWatermarkFastpath, finalizeWatermarkTurn, type FastpathHit, type FastpathOutcome } from "./fastpath.js"; -import { evaluateWatermarkV6, finalizeWatermarkV6, PF_SESSION_HEADER, watermarkV6DebugState, type V6Outcome } from "./fastpath-v6.js"; +import { evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, PF_SESSION_HEADER, watermarkV6DebugState, type V6Outcome } from "./fastpath-v6.js"; import { bumpAuditTick, commitRenderCapture, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, type RenderCaps, type RenderGateDecision, type RenderWorld } from "./fastpath-v6-render.js"; import { getStore } from "../persist.js"; import { collectRequestIds, probeRequestMembership } from "../request-membership.js"; @@ -382,6 +382,10 @@ export async function prepareOpenai( session.state.nextRuleId = Math.max(session.state.nextRuleId ?? 1, scratch.nextRuleId ?? 1); return t; } catch (err) { + // Identity-plane faults are deterministic 400s — a + // retry-pass over them would turn fail-fast into + // retry-pass (review P2-6); everything else falls back. + if (err instanceof FastpathV6IdentityFault) throw err; log("warn", `[${sessionId}] #5664 render window pass failed (${String(err)}) — legacy full pass this turn`); renderFast = null; } @@ -390,8 +394,11 @@ export async function prepareOpenai( })(); if (renderFast === null) { session.state = turn.state; - adoptContentStore(session, turn.contentStore); } + // Content-store adoption is cheap (no-op without new entries) and must + // NOT defer to the next full pass: CCR refs minted on a fast round are + // already referenced by the outbound placeholder (review P2-4). + adoptContentStore(session, turn.contentStore); // #4528: split the kernel pass from its post-processing scan passes so // each is a bounded synchronous stretch rather than one long block. await yieldToEventLoop(); @@ -522,7 +529,11 @@ export async function prepareOpenai( const rw = renderFast.rw; const wpWindowLen = wpHit?.windowCores.length ?? 0; const committedTail = mappedAll.slice(0, Math.min(renderFast.F + wpWindowLen, mappedAll.length)); - renderedImageDelta = imageReserveFor(session, "openai", { model: parsed.model, messages: committedTail } as unknown as OpenAIRequestBody, opts, billingUpstream ?? upstreamOrigin); + // Image reserve DELTA covers only this turn's NEW rows — the F + // frozen-window rows' reserve already sits in rw.imgCum; counting + // them again here drifts the ledger by a whole window each green + // round (review P1-2). slice(F) aligns with the captured tail. + renderedImageDelta = imageReserveFor(session, "openai", { model: parsed.model, messages: committedTail.slice(renderFast.F) } as unknown as OpenAIRequestBody, opts, billingUpstream ?? upstreamOrigin); if (verifyRenderWindow(committedTail.slice(0, renderFast.F).map((m) => JSON.stringify(m)), rw, renderFast.F)) { rebuiltMessages = spliceOutboundRows(rw, renderFast.F, committedTail); renderedCaptureStrings = [...rw.rows.slice(0, rw.rows.length - renderFast.F), ...committedTail.map((m) => JSON.stringify(m))]; @@ -769,15 +780,22 @@ export async function prepareOpenai( if (renderedCaptureStrings !== null && renderCapturedSid !== null && renderCapturedCaps !== null && featureEnabled("watermarkV6Render")) { const store = getStore(); if (store.pfRenderAvailable()) { - const boundN = renderFast !== null && wpHit !== undefined ? renderFast.F + wpHit.windowCores.length : renderedCaptureStrings.length; - const sumsSrc = renderFast !== null ? processedMessages.slice(0, boundN) : processedMessages; + // steady-mode contract (CaptureSums "DELTAS for the tail"): rows + // AND sums cover ONLY this turn's new elements. The frozen window + // rows entered the ledger when they were first captured — + // re-appending them shifts every pos forward, doubles the table, + // and inflates estCum/textCum/imgCum by a whole window per green + // round (review P1-2 + tail-only rows fix). The slice past F is + // exactly the appended tail (committedTail was capped at F+kNew). + const steadyTailRows = renderFast !== null ? renderedTailObjects.slice(renderFast.F).map((m) => JSON.stringify(m)) : renderedCaptureStrings; + const sumsSrc = renderFast !== null ? processedMessages.slice(Math.min(renderFast.F, processedMessages.length)) : processedMessages; const capOp = commitRenderCapture({ sessionId, store, sid: renderCapturedSid, mode: renderFast !== null ? "steady" : "legacy", caps: renderCapturedCaps, - rowStrings: renderedCaptureStrings, + rowStrings: steadyTailRows, sums: { estCum: estimateCoreMessagesUpper(sumsSrc), textCum: estimateCoreMessages(sumsSrc), diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 5053b06a14..ab25611b0a 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -222,6 +222,34 @@ test("unit: S4 checksum tamper kills the mirror (hydrate-failed, loudly logged)" } finally { h.cleanup(); } }); +// Review round 1 blind-spot pins (issue #5664): session teardown must wipe the +// three render tables in the same cascade, and re-running the migration on an +// already-v12 sidecar must be a no-op that leaves rows untouched. +test("unit: session cascade wipes render tables; v12 re-migration is a no-op", () => { + const h = makeH("cascade"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-cas", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const cast = commitRenderCapture({ sessionId: "s-cas", store: h.store, sid, mode: "legacy", caps: capsFor("none"), rowStrings: ['{"a":1}', '{"b":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }); + assert.equal(cast.op, "cast"); + assert.equal(h.store.pfRenderRows(sid).length, 2); + // Re-opening the sidecar re-runs the migrator on an already-v12 + // database: must be a no-op that leaves rows untouched (review round 1 + // blind spot: explicit 12→12 idempotency). + const shadow2 = UnifiedShadow.maybeOpen(h.dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as never); + assert.ok(shadow2, "second open must tolerate an already-migrated sidecar"); + assert.equal(shadow2.pfRenderAvailable(), true); + assert.equal(shadow2.pfRenderRows(sid).length, 2); + shadow2.close(); + // Session teardown cascades the render plane with the identity plane. + h.store.pfDeleteForSession("s-cas"); + assert.equal(h.store.pfRenderRows(sid).length, 0); + assert.equal(h.store.pfRenderCkptIntact(sid), true); // no checkpoint left = vacuously intact + } finally { h.cleanup(); } +}); + // --------------------------------------------------------------------------- // Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, // across arm → steady fast rounds → client compaction (turn 8) → strategy @@ -273,6 +301,8 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" const sessB = { id: "s-gold-b", state: stateB } as unknown as Session; let sidB: string | undefined; let fastRounds = 0, castOps = 0, recastOps = 0, legacyRounds = 0; + let totalSubmittedRows = 0; // volume discipline pin (see final assert) + let totalSubmittedText = 0; for (let tn = 1; tn <= 16; tn++) { const strategy = strategyOf(tn); const body = clientBody(tn); @@ -293,6 +323,7 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" }); let rebuilt: OpenAIMessage[]; let captureStrings: string[]; + let steadyAppendRows: string[] = []; let mode: "steady" | "legacy"; if (gate.ok) { const sidNow = sidB; @@ -327,15 +358,13 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" const mappedAll = mapEgress(pmW); const committedTail = mappedAll.slice(0, Math.min(F + (v.outcome.hit?.windowCores.length ?? 0), mappedAll.length)); const vv = verifyRenderWindow(committedTail.slice(0, F).map((m) => JSON.stringify(m)), rw, F); - if (!vv) { - for (let k = 0; k < F; k++) { - const a = JSON.stringify(committedTail[k]); const b = rw.rows[rw.rows.length - F + k]; - - } - } assert.equal(vv, true, `turn ${tn}: window verify failed — tail bytes drifted vs stored rows`); rebuilt = spliceOutboundRows(rw, F, committedTail); captureStrings = [...rw.rows.slice(0, rw.rows.length - F), ...committedTail.map((m) => JSON.stringify(m))]; + // steady-mode contract: only THIS turn's new rows are appended to + // the plane (the slice past the F frozen window); the full list + // stays in captureStrings for the golden equation below. + steadyAppendRows = committedTail.slice(F).map((m) => JSON.stringify(m)); mode = "steady"; fastRounds++; } else { @@ -348,10 +377,17 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" mode = "legacy"; legacyRounds++; } + const srcRows = mode === "steady" ? steadyAppendRows : captureStrings; + // Length-based deterministic stand-in for the token estimators: what + // this tier exercises is the accumulation ALGEBRA — each row counted + // exactly once with its own cost, tail-only deltas on steady rounds. + const rowCost = (rs: string[]): number => rs.reduce((s, x) => s + x.length, 0); const op = commitRenderCapture({ - sessionId: "s-gold-b", store: hB.store, sid: sidB!, mode, caps, rowStrings: captureStrings, - sums: { estCum: 0, textCum: 0, imgCum: 0 }, + sessionId: "s-gold-b", store: hB.store, sid: sidB!, mode, caps, rowStrings: srcRows, + sums: { estCum: rowCost(srcRows), textCum: rowCost(srcRows), imgCum: 0 }, }); + totalSubmittedRows += srcRows.length; + totalSubmittedText += rowCost(srcRows); if (op.committed) { if (op.op === "cast") castOps++; else if (op.op === "recast") recastOps++; @@ -361,11 +397,27 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" `turn ${tn} (${mode}${gate.ok ? "" : `, reason=${gate.reason}`}): lane emit diverged from legacy oracle`); assert.equal(stateB.blocks.length, 0, "assumption pin: no folds materialize in this synthetic conv"); v.fin(); // plane advances AFTER capture — same order as prepare-openai + // Structural regression pin (review round 1, tail-only fix): the render + // table must track the identity ledger one-row-per-element at every + // turn boundary — a duplicate-append corruption shows up here instantly + // instead of hiding behind the byte equation. + assert.equal(hB.store.pfRenderRows(sidB!).length, watermarkV6DebugState("s-gold-b").worlds[sidB!]!.maxPos + 1, + `turn ${tn}: render table drifted from identity ledger (rows=${hB.store.pfRenderRows(sidB!).length})`); } assert.equal(castOps, 1, "exactly one cast (the arm turn)"); - assert.ok(fastRounds >= 6, `expected several fast rounds, got ${fastRounds} (audit too aggressive?)`); + // ≥ 12 fast rounds = at least TWO full audit waves served green: the first + // proves the lane runs, the second proves any legacy round CONSUMES the + // audit obligation (pre-fix, tick froze at N and every later turn rode + // legacy forever — masked at the time by per-round recasts). + assert.ok(fastRounds >= 12, `expected ≥2 audit waves of fast rounds, got ${fastRounds} (audit obligation not consumed?)`); assert.ok(recastOps >= 1, "strategy flip at turn 10 must force ≥1 recast"); assert.ok(legacyRounds >= 2, `arm + audit/caps-flip rounds must ride legacy (got ${legacyRounds})`); + // Volume discipline (steady deltas must stay O(new) per round, never + // O(full list)): pre-tail-fix behavior re-submitted the whole list each + // green round ⇒ ~8×final volume here. + const finalRowCount = hB.store.pfRenderRows(sidB!).length; + assert.ok(totalSubmittedRows <= 3 * finalRowCount, + `submitted row-volume ${totalSubmittedRows} vs final table ${finalRowCount} — steady rounds must append tails, not full lists`); // S4 truth intact at the end; mirror agrees with the last golden shape assert.equal(hB.store.pfRenderCkptIntact(sidB!), true); const dbgLhs = watermarkV6RenderDebugState("s-gold-b"); @@ -373,6 +425,11 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" const lhsRows = dbgLhs.sids[dbgLhs.sids.length - 1].rows; const rhsRows = goldens[goldens.length - 1].rows.length; assert.equal(lhsRows, rhsRows, "mirror row count must equal last golden row count"); + // Sums algebra pin (review P1-2): every row's cost entered the cumulative + // ledger EXACTLY once — a frozen-window double-count would overshoot here. + const finalRowCost = hB.store.pfRenderRows(sidB!).reduce((s, r) => s + (r.rowJson?.length ?? 0), 0); + assert.ok(Math.abs(dbgLhs.sids[dbgLhs.sids.length - 1].textCum - finalRowCost) < Math.max(1, finalRowCost * 0.25), + `cumulative text ledger drifted from row truth (cum=${dbgLhs.sids[dbgLhs.sids.length - 1].textCum} vs rows=${finalRowCost}; submitted=${totalSubmittedText})`); // ---- phase 3: restart hydration — mirrors wiped, disk must continue ----- _resetWatermarkV6RenderTestState(); From b443c3a5b6d1154d8914c5eecf2903b00d352d1f Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 5 Nov 2026 01:43:28 +0800 Subject: [PATCH 20/49] docs(config): document watermarkV6Render cohort + BILI_PF_RENDER_* tunables (#5664) Review hygiene: the new rollout cohort and the two env-only tunables were missing from the config reference (document-or-don't-ship, #2030). Seeds in website/config-reference/*, docs regenerated via tools/gen-config-docs.mjs (coverage ok: 199 env names + 61 top-level keys). --- CONFIGURATION.md | 4 ++++ CONFIGURATION.zh-CN.md | 4 ++++ website/config-reference/env-only.yaml | 16 ++++++++++++++++ website/config-reference/rollout.yaml | 8 ++++++++ 4 files changed, 32 insertions(+) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 72009dbdb1..691c614bbb 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -104,6 +104,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0 (off) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | Plugin-lane server-owned history (#4491): for enabled installs, (a) plugin-lane requests may carry an internal `biliMsgIds` array parallel to `messages` that pins message identity to client-supplied ids (always stripped before upstream forwarding; malformed arrays are ignored with a warning, id-poisoning shapes hard-fail on the incremental lane); (b) the loopback-only `POST /__bili/plugin/render` endpoint serves the outbound prefix rendered from the unified transcript (folded regions collapse to summary anchors; any desync degrades to a non-2xx so the shim falls back to its local history); (c) the incremental-lane probe advertises `serverHistory`. Default 0% — no behavior change until rolled out. | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5 (experimental grey) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | Full-resend watermark fastpath (#5403/#5418): plugin-lane chat-completions turns whose previous response carried an x-bili-pf-seq stamp assert the resent prefix via the echoed count (structural scan, no full parse); on a hit the pipeline consumes stashed canonical prefix cores plus only the new tail window (byte-identical outbound provenance; ANY doubt falls back to the untouched full-parse path per turn and re-stamps, five credited misses hold the lane for one hour). Requires the client to echo the stamp back as x-bili-pf-count (dsh/pi/opencode/omp do; unadapted lanes never hit). EXPERIMENTAL grey rollout — ON by default for roughly 5% of installs (owner ruling 2026-11-02: small cohort before GA; one-time intro notice in the web UI for every install, tri-state switch in quick config). Force it with `true`, turn off with `false` (or a percent) / env = 0. See TECHNICAL-NOTES.md “Full-resend watermark fastpath”. | | `rollout.features.watermarkV6MsgId` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive strikes park the lane 60 min, loud fallback thereafter) — never a silent legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout. | +| `rollout.features.watermarkV6Render` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / blocks-active / fold-pending / absorb-active / ccr-on / markers / hydrate-failed / caps-mismatch / scheduled audit every Nth round / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v12, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout. | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | Mask non-public target hosts in logs as ; credential headers are always masked regardless. | | `security.adminRelay` | boolean | false | — | Let an embedding host (dsh web today) relay the ENTIRE /__bili/ admin surface through its own entry origin so a remote browser can open the panel (#3373). Fail-closed, file-only. The relayed hop reaches the proxy as a local non-browser client (loopback source + loopback Host, no Origin), so the admin gate stays unchanged; enabling it exposes config editing and credential writes to whoever can reach the host web entry — protect that entry (auth / reverse proxy) first. | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude Code subagents get their own session namespace (\|sub:) instead of queueing behind the main session. Anthropic-wire only — on the Responses wire this switch is a no-op (see the `subagentSplit` / BOTH-mechanisms scope note). | @@ -434,6 +435,8 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `BILI_MITM_HOSTS` | string[] (csv) | unset | — | Launcher channel: MITM whitelist hosts handed to the spawned proxy (distinct from the config-file twin BILI_MITM_DOMAINS). | | `BILI_PLUGIN_AGENT` | boolean-ish | off | — | Spawn channel: marks a plugin-agent process. | | `BILI_IMAGE_RESCALE_MAX_DIM` | integer | 1600 | — | Longest-edge trigger threshold (px) for the egress image downscale pass (#3400); non-numeric or below 128 ignored (default applies). Env-only by design. | +| `BILI_PF_RENDER_WINDOW` | integer | 8 | — | Bounded render window W for the v6 render-snapshot plane (#5664): how many frozen cores ride the bounded kernel pass beside the new tail window on each steady turn. Probe evidence across three strategies showed a suffix-only W-pass reproduces every element's outbound bytes exactly, so 8 is plenty; values below 1 or non-numeric fall back to 8, capped at 64. Only meaningful while rollout.features.watermarkV6Render is on. | +| `BILI_PF_RENDER_AUDIT_EVERY` | integer | 8 | — | Scheduled audit cadence N for the v6 render-snapshot plane (#5664): every N engaged hit-turns force ONE full legacy pass whose capture diff re-verifies the stored outbound rows end-to-end (bounds messageRefs/rules merge lag to <= N rounds). Values below 2 or non-numeric fall back to 8, capped at 1024. Any legacy round (red-gate fallback included) resets the counter — it IS an audit. Only meaningful while rollout.features.watermarkV6Render is on. | @@ -1935,6 +1938,7 @@ File keys resolve only when the matching env var is unset. Defaults in parenthes | `BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK` | `rollout.features.pluginFullResendWatermark` | 5 (experimental grey) | | `BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY` | `rollout.features.pluginLaneServerHistory` | 0 (off) | | `BILI_ROLLOUT_WATERMARK_V6_MSG_ID` | `rollout.features.watermarkV6MsgId` | 0 (off) | +| `BILI_ROLLOUT_WATERMARK_V6_RENDER` | `rollout.features.watermarkV6Render` | 0 (off) | | `BILI_SESSION_CONTENT_STORE_PIN` | `sessions.contentStorePin` | false | | `BILI_SESSION_GC` | `sessions.gc.enabled` | false | | `BILI_SESSION_GC_INTERVAL_MS` | `sessions.gc.intervalMs` | 3600000 | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index feff76a596..aaf843652e 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -104,6 +104,7 @@ | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0(关) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | 插件车道服务端正史(#4491):对已放量的安装,(a) 插件车道请求可携带与 `messages` 平行的内部字段 `biliMsgIds`,把消息身份钉在客户端提供的 id 上(转发上游前一律剥掉;格式错误的数组告警后忽略,身份投毒形态在增量车道直接硬失败);(b) 仅回环可达的 `POST /__bili/plugin/render` 端点从统一存储渲染出站前缀(折叠区收敛为摘要锚点;任何失步都返回非 2xx,shim 退回本地历史);(c) 增量车道探测响应通告 `serverHistory`。默认 0%,放量前无行为变化。 | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5(实验性灰度) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | 全量重发水位快路径(#5403/#5418):插件车道 chat-completions 回合若上一轮响应盖有 x-bili-pf-seq,则用回声计数声明重发前缀(结构扫描,不做全量解析);命中时管线只消费钉存的前缀核 + 新增尾巴窗口(出站字节同源可证;任何疑点当轮回退未动过的全量解析路径并重新盖戳,连续五次计名失配将车道按住一小时)。需客户端把戳回声为 x-bili-pf-count(适配的车道回带;其余车道不会命中)。**实验性灰度**——默认仅约 5% 的安装开启(owner 裁定 2026-11-02:先小流量再 GA;web 面板对全部安装显示一次性公告、快速配置提供三态开关);`true` 强制开、`false`(或百分比)/ env = 0 关。见 TECHNICAL-NOTES.zh-CN.md「全量重发水位快路径」。 | | `rollout.features.watermarkV6MsgId` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | 复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击按住车道 60 分钟,之后响亮降级)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。 | +| `rollout.features.watermarkV6Render` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/blocks-active/fold-pending/absorb-active/ccr-on/markers/hydrate-failed/caps-mismatch/每 N 轮计划审计/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v12,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。 | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | 日志中把非公开目标主机遮成 ;凭据头无论此开关如何总是遮盖。 | | `security.adminRelay` | boolean | false | — | 允许宿主(当前为 dsh web)把它自己入口域名下的整个 /__bili/ 管理面中继到本机代理,使远程浏览器能打开面板(#3373)。默认关闭、仅配置文件。中继跳到达代理时是本机非浏览器客户端(回环源地址 + 回环 Host、无 Origin 头),管理端门槛不变;开启后,凡能访问宿主 web 入口的人都能改配置、写凭据——请先给入口加鉴权/反代。 | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude 子代理拥有独立会话命名空间(\|sub:),不再排队在主会话锁后。仅 anthropic 线路 —— Responses 线路上本开关无效(见 `subagentSplit` / BOTH-mechanisms 作用域说明)。 | @@ -434,6 +435,8 @@ | `BILI_MITM_HOSTS` | string[] (csv) | unset | — | 启动器通道:传给被派生代理的 MITM 白名单主机(区别于配置文件对应项 BILI_MITM_DOMAINS)。 | | `BILI_PLUGIN_AGENT` | boolean-ish | off | — | 派生通道:标记插件 agent 进程。 | | `BILI_IMAGE_RESCALE_MAX_DIM` | integer | 1600 | — | 出站图片降采样 pass 的最长边触发阈值(像素,#3400);非数字或低于 128 的取值按默认处理。设计上仅环境变量。 | +| `BILI_PF_RENDER_WINDOW` | integer | 8 | — | v6 渲染快照平面(#5664)的有界渲染窗口 W:每个稳态轮送入内核的冻结核数量(与新增尾窗并列)。三种策略的探针证据表明后缀 W-pass 能逐字节复现每个元素的出站字节,8 已足够;低于 1 或非数字的取值回退为 8,上限 64。仅在 rollout.features.watermarkV6Render 开启时有意义。 | +| `BILI_PF_RENDER_AUDIT_EVERY` | integer | 8 | — | v6 渲染快照平面(#5664)的计划审计周期 N:每 N 个命中轮强制跑一次全量 legacy pass,其捕获 diff 对已存出站行做端到端复验(把 messageRefs/rules 合并滞后钉在 ≤ N 轮内)。低于 2 或非数字的取值回退为 8,上限 1024。任何 legacy 轮(含红色门控回退)都会清零计数——它本身就是一次审计。仅在 rollout.features.watermarkV6Render 开启时有意义。 | @@ -1912,6 +1915,7 @@ ACP 原生 agent(当前为 `pi` 扩展)会在每个进程内向代理上报 | `BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK` | `rollout.features.pluginFullResendWatermark` | 5 (experimental grey) | | `BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY` | `rollout.features.pluginLaneServerHistory` | 0 (off) | | `BILI_ROLLOUT_WATERMARK_V6_MSG_ID` | `rollout.features.watermarkV6MsgId` | 0 (off) | +| `BILI_ROLLOUT_WATERMARK_V6_RENDER` | `rollout.features.watermarkV6Render` | 0 (off) | | `BILI_SESSION_CONTENT_STORE_PIN` | `sessions.contentStorePin` | false | | `BILI_SESSION_GC` | `sessions.gc.enabled` | false | | `BILI_SESSION_GC_INTERVAL_MS` | `sessions.gc.intervalMs` | 3600000 | diff --git a/website/config-reference/env-only.yaml b/website/config-reference/env-only.yaml index 635c399999..94f396dc40 100644 --- a/website/config-reference/env-only.yaml +++ b/website/config-reference/env-only.yaml @@ -530,3 +530,19 @@ entries: group: envonly desc_en: "Longest-edge trigger threshold (px) for the egress image downscale pass (#3400); non-numeric or below 128 ignored (default applies). Env-only by design." desc_zh: "出站图片降采样 pass 的最长边触发阈值(像素,#3400);非数字或低于 128 的取值按默认处理。设计上仅环境变量。" + - key: BILI_PF_RENDER_WINDOW + type: integer + default: "8" + kind: env-only + audience: user + group: envonly + desc_en: "Bounded render window W for the v6 render-snapshot plane (#5664): how many frozen cores ride the bounded kernel pass beside the new tail window on each steady turn. Probe evidence across three strategies showed a suffix-only W-pass reproduces every element's outbound bytes exactly, so 8 is plenty; values below 1 or non-numeric fall back to 8, capped at 64. Only meaningful while rollout.features.watermarkV6Render is on." + desc_zh: "v6 渲染快照平面(#5664)的有界渲染窗口 W:每个稳态轮送入内核的冻结核数量(与新增尾窗并列)。三种策略的探针证据表明后缀 W-pass 能逐字节复现每个元素的出站字节,8 已足够;低于 1 或非数字的取值回退为 8,上限 64。仅在 rollout.features.watermarkV6Render 开启时有意义。" + - key: BILI_PF_RENDER_AUDIT_EVERY + type: integer + default: "8" + kind: env-only + audience: user + group: envonly + desc_en: "Scheduled audit cadence N for the v6 render-snapshot plane (#5664): every N engaged hit-turns force ONE full legacy pass whose capture diff re-verifies the stored outbound rows end-to-end (bounds messageRefs/rules merge lag to <= N rounds). Values below 2 or non-numeric fall back to 8, capped at 1024. Any legacy round (red-gate fallback included) resets the counter — it IS an audit. Only meaningful while rollout.features.watermarkV6Render is on." + desc_zh: "v6 渲染快照平面(#5664)的计划审计周期 N:每 N 个命中轮强制跑一次全量 legacy pass,其捕获 diff 对已存出站行做端到端复验(把 messageRefs/rules 合并滞后钉在 ≤ N 轮内)。低于 2 或非数字的取值回退为 8,上限 1024。任何 legacy 轮(含红色门控回退)都会清零计数——它本身就是一次审计。仅在 rollout.features.watermarkV6Render 开启时有意义。" diff --git a/website/config-reference/rollout.yaml b/website/config-reference/rollout.yaml index c6e02524c7..2ff925f5a6 100644 --- a/website/config-reference/rollout.yaml +++ b/website/config-reference/rollout.yaml @@ -73,3 +73,11 @@ entries: group: behavior desc_en: "Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive strikes park the lane 60 min, loud fallback thereafter) — never a silent legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout." desc_zh: "复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击按住车道 60 分钟,之后响亮降级)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。" + - key: rollout.features.watermarkV6Render + type: "boolean | number" + default: "0 (off)" + default_zh: "0(关)" + env: BILI_ROLLOUT_WATERMARK_V6_RENDER + group: behavior + desc_en: "v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / blocks-active / fold-pending / absorb-active / ccr-on / markers / hydrate-failed / caps-mismatch / scheduled audit every Nth round / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v12, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout." + desc_zh: "v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/blocks-active/fold-pending/absorb-active/ccr-on/markers/hydrate-failed/caps-mismatch/每 N 轮计划审计/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v12,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。" From 1237316c94897879fc198cec5e8bcc7f30fc6ef3 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 5 Nov 2026 01:43:28 +0800 Subject: [PATCH 21/49] refactor(fastpath): drop unreachable window-short gate branch (#5664 review hygiene) rows-misaligned pins rows.length === maxPos+1 before F is computed, so F <= maxPos+1 and 'maxPos < F-1' can never hold. Behaviorally inert; no test referenced the reason string. --- src/server/fastpath-v6-render.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 9acc031990..2dc37ef368 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -318,10 +318,9 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { // misaligned window. if (rw.rows.length !== args.maxPos + 1) return off("rows-misaligned"); registerSidForSession(args.sessionId, args.sid); - const F = Math.min(PF_RENDER_WINDOW, rw.rows.length); // Window input = last F identity elements (+ new cores added by caller); // positions are contiguous because the ledger grows forward only (P3). - if (args.maxPos < F - 1) return off("window-short"); + const F = Math.min(PF_RENDER_WINDOW, rw.rows.length); return { ok: true, rw, F }; } From aeec940c25accf8c58e216e4cf1b0d11513b6506 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 5 Nov 2026 01:51:29 +0800 Subject: [PATCH 22/49] =?UTF-8?q?test(fastpath):=20exercise=20the=20audit?= =?UTF-8?q?=20cadence=20for=20real=20=E2=80=94=20flip=20deferred=20to=20tu?= =?UTF-8?q?rn=2014,=20audit-due=20pinned=20at=20turn=2010=20with=20immedia?= =?UTF-8?q?te=20recovery=20(#5664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/issue5664-render-snapshot.test.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index ab25611b0a..f93eee3ea5 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -253,7 +253,7 @@ test("unit: session cascade wipes render tables; v12 re-migration is a no-op", ( // --------------------------------------------------------------------------- // Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, // across arm → steady fast rounds → client compaction (turn 8) → strategy -// recast (turn 10) → audit cadence → restart-hydration continuation. +// strategy-flip recast (turn 14) → audit cadence (fires for real at turn 10) → restart-hydration continuation. // --------------------------------------------------------------------------- test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end", async (t) => { @@ -265,7 +265,7 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" // drops e_1/e_3 (client-side compaction) — the lane ignores the drop // (ledger grows forward only), views and rows keep growing, and the // golden equation must still hold element-stable (ids follow elements, - // never positions). Strategy flips text-only→none at turn 10 ⇒ caps flip + // never positions). Strategy flips text-only→none at turn 14 ⇒ caps flip // ⇒ forced legacy recast round, then steady on the new byte form. const ELEMENTS: Msg[] = Array.from({ length: 16 }, (_, i) => ({ role: i % 2 === 0 ? "user" : "assistant", content: `content number ${i}` })); const STABLE_ID = (i: number) => `eid-${i}`; @@ -275,7 +275,8 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" return els; }; const idsOf = (els: Msg[]) => els.map((el) => STABLE_ID(ELEMENTS.indexOf(el))); - const strategyOf = (tn: number) => (tn < 10 ? "text-only" : "none"); + const strategyOf = (tn: number) => (tn < 14 ? "text-only" : "none"); + const gateSeq: string[] = []; // ---- phase 1: arm A — pure legacy full path over its own store ---------- const hA = makeH("gold-a"); @@ -321,6 +322,7 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, log: hB.log, }); + gateSeq.push(gate.ok ? "fast" : String(gate.reason)); let rebuilt: OpenAIMessage[]; let captureStrings: string[]; let steadyAppendRows: string[] = []; @@ -405,12 +407,20 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" `turn ${tn}: render table drifted from identity ledger (rows=${hB.store.pfRenderRows(sidB!).length})`); } assert.equal(castOps, 1, "exactly one cast (the arm turn)"); + // Audit cadence exercised FOR REAL (review round 2): with the flip deferred + // to turn 14, eight consecutive green rounds land tick=8 ⇒ turn 10 MUST ride + // legacy as the scheduled audit itself, and the very next round recovers to + // fast (any legacy full-capture zeroes the tick — a lockout would keep every + // later turn red until restart/caps-flip/S4-heal). + assert.deepEqual(gateSeq.filter((r) => r === "audit-due"), ["audit-due"], `expected exactly one audit-due round, got ${JSON.stringify(gateSeq)}`); + assert.equal(gateSeq.indexOf("audit-due"), 9, `audit-due expected at loop index 9 (turn 10), got ${JSON.stringify(gateSeq)}`); + assert.equal(gateSeq[10], "fast", `round right after the audit must recover to fast, got ${JSON.stringify(gateSeq)}`); // ≥ 12 fast rounds = at least TWO full audit waves served green: the first // proves the lane runs, the second proves any legacy round CONSUMES the // audit obligation (pre-fix, tick froze at N and every later turn rode // legacy forever — masked at the time by per-round recasts). assert.ok(fastRounds >= 12, `expected ≥2 audit waves of fast rounds, got ${fastRounds} (audit obligation not consumed?)`); - assert.ok(recastOps >= 1, "strategy flip at turn 10 must force ≥1 recast"); + assert.ok(recastOps >= 1, "strategy flip at turn 14 must force ≥1 recast"); assert.ok(legacyRounds >= 2, `arm + audit/caps-flip rounds must ride legacy (got ${legacyRounds})`); // Volume discipline (steady deltas must stay O(new) per round, never // O(full list)): pre-tail-fix behavior re-submitted the whole list each From 3f07c860133f54fc783e21253b26988824b4d0a1 Mon Sep 17 00:00:00 2001 From: billion-context-release-bot Date: Thu, 5 Nov 2026 10:02:05 +0800 Subject: [PATCH 23/49] docs(persist): align pfRenderCommit facade comment with actual fail-soft behavior (#5689 review nit) --- src/persist.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/persist.ts b/src/persist.ts index d6aaba778d..1a22becc9d 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2294,7 +2294,8 @@ export class SessionStore { /** #5664 render snapshot plane facade — same guard shape as the identity * plane: null/empty/false = "plane unavailable right now" and the v6 * lane fails soft onto the legacy full path for that turn only. - * pfRenderCommit does NOT swallow silently. */ + * A failed render commit is logged once, then fails soft for THIS turn + * (same as pfCommitTurn above). */ pfRenderAvailable(): boolean { if (!this.shadow || this.shadow.stats.disabled) return false; try { From 49ba14dae56338368437ccc400ae9e8f8f24c6a0 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Thu, 5 Nov 2026 10:53:37 +0800 Subject: [PATCH 24/49] =?UTF-8?q?fix(fastpath):=20seal=20the=20render=20pl?= =?UTF-8?q?ane=20shut=20=E2=80=94=20NO=20recast=20in=20any=20scenario;=20s?= =?UTF-8?q?tructural=20divergence=20kills=20the=20lane=20permanently=20(ow?= =?UTF-8?q?ner=20ruling=202026-11-05,=20#5664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recast (rewriting covered ordinals / deleting excess rows) existed as the plane's only self-heal path. The owner ruled it must not happen in ANY scenario (fold, caps flip, audit-caught drift, corruption death). This commit removes it from every layer instead of patching its call sites: - diffRowLists verdict 'recast' -> 'diverged'; the diverged branch now calls killRenderPlane(): the sid joins a permanent deadSids set, a kind='dead' entry is persisted into pf_render_log WITHOUT a checkpoint (nothing absorbs it), and the session rides the legacy full path from here on — bytes stay correct, the cache simply never serves again. - killRenderPlane is the ONLY mutation besides append/cast-once: S4 checksum mismatch, row gaps, log-fold disagreement, caps-unresolvable hydration states all KILL durably instead of returning null-and-retry; the arm cast refuses over any pre-existing non-empty table. - pfRenderCommit drops its deletes parameter end-to-end (store/shadow/persist) — covered ordinals can never be rewritten. - gate grows the durable reason 'lane-dead'; prepare-openai skips the capture build+commit entirely once the reason is lane-dead so dead sessions stop paying an O(full history) stringify per turn. - tests: golden A/B re-pinned (death lands at turn 14 caps flip, table freezes at 13 rows, restart hydrates straight to lane-dead — no re-arm); new pins: structural-divergence kill (immutable rows, loud log, persisted dead verdict, gate positive + re-capture refused negative per #3617) and death-persists-across-restart. Consequence (accepted by the ruling): once a session's outbound bytes ever diverge from its cached rows — most concretely after ANY compression fold re-shapes them — that session loses the render fastpath permanently. Fastpath lifetime = up to first divergence, never after. --- src/persist.ts | 1 - src/server/fastpath-v6-render.ts | 169 ++++++++++++++------- src/server/prepare-openai.ts | 27 +++- src/storage/shadow.ts | 1 - src/storage/store.ts | 16 +- tests/issue5664-render-snapshot.test.ts | 191 ++++++++++++++++++------ 6 files changed, 286 insertions(+), 119 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index 1a22becc9d..0e6f46a2fc 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2314,7 +2314,6 @@ export class SessionStore { sid: string, args: { rows: { pos: number; rowJson: string | null }[]; - deletes?: number[]; logEntries: { kind: string; payloadJson: string }[]; checkpointMetaJson?: string; ts?: number; diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 2dc37ef368..f2f7e029f7 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -3,9 +3,12 @@ // // station 1 inbound / identity finalize (per turn, existing #5652 lane) // station 2 this snapshot: steady rounds APPEND one captured row per new -// element; compression rounds (arm/audit/fallback) DIFF against -// the stored rows and either append or RECAST — and a recast -// FORCES an S4-checksum checkpoint in the same transaction. +// element; full-path rounds (arm/audit/fallback) DIFF against +// the stored rows and APPEND the tail on growth. A RECAST IS +// FORBIDDEN IN EVERY SCENARIO (owner ruling 2026-11-05): +// any divergence requiring a stored-row rewrite/deletion — or +// any post-death rebuild — KILLS the plane for that sid +// permanently and the session rides the legacy full path. // station 3 outbound emit: prefix rows pass through verbatim from disk; // only the bounded window tail runs a kernel pass per request. // @@ -25,9 +28,11 @@ // G2 capture point = the exact array the baseline emits (rebuiltMessages // before system/ephemeral injection); steady capture stores raw strings // + parsed caches so the prefix never gets re-parsed or re-stringified. -// G3 S4 death verdict — a checksum-corrupt checkpoint kills the plane for -// that sid at hydration; the next legacy turn's diff self-heals with a -// fresh forced checkpoint. Never guess, never forward corrupt bytes. +// G3 death verdict — a checksum-corrupt checkpoint, a structurally +// diverged capture, or any unverifiable pre-existing snapshot kills the +// plane for that sid PERMANENTLY (persisted as a kind='dead' log entry, +// surviving restarts). No recast, no re-arm: byte-safety comes from +// stopping, not from repairing. Never guess, never forward corrupt bytes. // P7 mirrors are caches (LRU byte budget); losing one = DB re-read. import type { SessionStore } from "../persist.js"; import type { Session } from "../session.js"; @@ -64,7 +69,9 @@ export const PF_RENDER_EMERGENCY_RATIO = 0.9; const MIRROR_BYTE_BUDGET = 256 * 1024 * 1024; let mirrorBytes = 0; -/** Capabilities fingerprint — any change forces a recast (cheap, correct). */ +/** Capabilities fingerprint — any change red-gates the lane (caps-mismatch); + * if the legacy round's bytes then diverge from the stored rows the plane + * is killed for this sid (no-recast regime). */ export interface RenderCaps { v: 1; model: string; @@ -109,6 +116,8 @@ export interface RenderWorld { textCum: number; /** Cumulative image-reserve contribution of frozen rows. */ imgCum: number; + /** Set when this sid's plane is dead (no-recast regime) — diagnostic. */ + dead?: boolean; /** Green (fast-served) engaged-turn tick — % PF_RENDER_AUDIT_EVERY === 0 * ⇒ audit due. Consumed (zeroed) by ANY legacy full-capture round served, * which IS the audit (the full byte path ran to completion). @@ -121,6 +130,30 @@ const mirrors = new Map(); /** relay session id → pf sids owned by it (teardown sweep). */ const sessionSids = new Map>(); +/** Owner ruling 2026-11-05 (#5689 review): NO recast in ANY scenario. + * Structural divergence (byte diff/shrink), caps flips that changed stored + * bytes, S4 corruption, row-table inconsistency, unverifiable pre-existing + * snapshots — all land here. A dead sid NEVER arms, hydrates, gates green, + * or commits again. The verdict is persisted as a kind='dead' log entry so + * every future process instance reaches the same conclusion. */ +const deadSids = new Set(); + +export function killRenderPlane(store: SessionStore, sid: string, reason: string, log?: LogFn): void { + if (!deadSids.has(sid)) { + deadSids.add(sid); + const rw = mirrors.get(sid); + if (rw) rw.dead = true; + log?.("error", `[${sid}] #5664 render plane KILLED PERMANENTLY (no-recast regime): ${reason}`); + try { + store.pfRenderCommit(sid, { rows: [], logEntries: [{ kind: "dead", payloadJson: JSON.stringify({ reason }) }] }); + } catch { + /* soft: the in-memory verdict still binds this process; a storage + * failure means the NEXT process rediscovers the same condition + * through its own integrity checks (S4 / row table / cast guard) */ + } + } +} + // --------------------------------------------------------------------------- // Mirror lifecycle // --------------------------------------------------------------------------- @@ -170,8 +203,9 @@ export function clearWatermarkV6Render(sessionId: string): void { * Returns null on ANY inconsistency — caller serves the baseline path. */ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWorld | null { try { + if (deadSids.has(sid)) return null; if (!store.pfRenderCkptIntact(sid)) { - log?.("error", `[${sid}] #5664 render checkpoint S4 checksum mismatch — plane judged dead for this sid until a forced recast heals it`); + killRenderPlane(store, sid, "S4 checkpoint checksum mismatch (corrupt metadata)", log); return null; } const rowsRaw = store.pfRenderRows(sid); @@ -190,13 +224,16 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor expectedCount = m.rowCount; capsJson = metaRead.ckpt.metaJson; for (const e of metaRead.log) { - const p = JSON.parse(e.payloadJson) as { de?: number; dt?: number; di?: number; n?: number; capsJson?: string } & Partial; - if (e.kind === "cast" || e.kind === "recast") { + const p = JSON.parse(e.payloadJson) as { de?: number; dt?: number; di?: number; n?: number; capsJson?: string; reason?: string } & Partial; + if (e.kind === "dead") { + killRenderPlane(store, sid, `persisted death verdict: ${p.reason ?? "unknown"}`, log); + return null; + } + if (e.kind === "cast") { estCum = p.estCum ?? 0; textCum = p.textCum ?? 0; imgCum = p.imgCum ?? 0; expectedCount = p.rowCount ?? -1; - if (e.kind === "recast" && p.capsJson !== undefined) capsJson = p.capsJson; } else if (e.kind === "append") { estCum += p.de ?? 0; textCum += p.dt ?? 0; @@ -212,24 +249,22 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor for (let i = 0; i < rowsRaw.length; i++) { const rj = rowsRaw[i].rowJson; if (rj === null || rowsRaw[i].pos !== i) { - // Row table inconsistent on disk (tombstone where a row must - // be, or a gap): judge the plane dead LOUDLY — same discipline - // as an S4 checksum mismatch, never a silent fallback. - log?.("error", `[${sid}] #5664 render plane judged dead: row-table inconsistency at pos ${rowsRaw[i].pos} (expected contiguous ${i})`); + killRenderPlane(store, sid, `row-table inconsistency at pos ${rowsRaw[i].pos} (expected contiguous ${i})`, log); return null; } rows[i] = rj; try { rowsParsed[i] = JSON.parse(rj); } catch { - log?.("error", `[${sid}] #5664 render plane judged dead: row parse failed at pos ${i}`); + killRenderPlane(store, sid, `row parse failed at pos ${i}`, log); return null; } } if (expectedCount !== -1 && expectedCount !== rows.length) { // Base sums no longer line up with the row table — the log fold - // disagrees with truth. Caller will serve legacy; the next - // capture diff (full arrays both sides) heals with a recast. + // disagrees with truth. Only a rewrite of stored rows could fix + // that, which is forbidden: kill the plane permanently. + killRenderPlane(store, sid, `log-fold disagrees with row table (expectedCount=${expectedCount}, rows=${rows.length})`, log); return null; } let caps: RenderCaps | undefined; @@ -243,9 +278,13 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor const inner = (parsed.caps as RenderCaps | undefined); caps = inner && inner.v === 1 ? inner : (parsed.v === 1 ? parsed : undefined); } else { - return null; // rows without capabilities = cannot gate safely + killRenderPlane(store, sid, "stored rows without capabilities (cannot gate safely)", log); + return null; + } + if (!caps || caps.v !== 1) { + killRenderPlane(store, sid, "unparseable capabilities fingerprint", log); + return null; } - if (!caps || caps.v !== 1) return null; const rw: RenderWorld = { sid, rows, rowsParsed, caps, estCum, textCum, imgCum, auditTick: 0 }; touchMirror(rw); return rw; @@ -287,6 +326,12 @@ export type RenderGateDecision = | { ok: false; reason: string } | { ok: true; rw: RenderWorld; F: number }; +/** Stable red-reason vocabulary (test-pinned, one per condition): not-engaged, + * arm-turn, title-gen, plane-unavailable, pressure, snapshot-lane, + * blocks-active, fold-pending, absorb-active, ccr-on, markers, lane-dead, + * hydrate-failed, caps-mismatch, audit-due, no-rows, no-plane-geometry, + * rows-misaligned. */ + /** G1 conservative conjunction — every red condition returns the reason so * the lane stays debuggable; the fallback path is unchanged baseline. */ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { @@ -302,9 +347,12 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { if (args.absorbActive) return off("absorb-active"); if (args.ccrOn) return off("ccr-on"); if (args.markerNonZero) return off("markers"); + if (deadSids.has(args.sid)) return off("lane-dead"); let rw = mirrors.get(args.sid); if (rw === undefined) rw = hydrateMirror(args.store, args.sid, args.log) ?? undefined; - if (rw === undefined) return off("hydrate-failed"); + // hydrate's own integrity failures persist a death verdict before + // returning null — report the durable reason, not the transient one. + if (rw === undefined) return off(deadSids.has(args.sid) ? "lane-dead" : "hydrate-failed"); if (!capsEqual(rw.caps, args.caps)) return off("caps-mismatch"); if (rw.auditTick % PF_RENDER_AUDIT_EVERY === 0 && rw.auditTick > 0) return off("audit-due"); if (rw.rows.length < 1) return off("no-rows"); @@ -313,9 +361,9 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { // number one row per ledger element (row i ↔ pos i), so at GATE time — // which runs BEFORE this round's finalize advances the plane — // rows.length must equal maxPos+1. A mismatch means an earlier commit - // failed soft (disk ahead/behind the mirror): fall back and let the - // legacy diff+forced checkpoint self-heal rather than splice on a - // misaligned window. + // failed soft (disk ahead/behind the mirror): fall back to the legacy + // byte path; the plane cannot self-heal anymore (no-recast regime) — + // a persistent misalignment will die its next structural diff. if (rw.rows.length !== args.maxPos + 1) return off("rows-misaligned"); registerSidForSession(args.sessionId, args.sid); // Window input = last F identity elements (+ new cores added by caller); @@ -325,8 +373,8 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { } /** Bumped once per GREEN round (prepare-openai gate block). The obligation is - * consumed by any legacy full-capture round (commitRenderCapture zeroes it): - * cadence = "one full pass per PF_RENDER_AUDIT_EVERY fast rounds", red turns + * consumed by any legacy full-capture round (commitRenderCapture zeroes it). + * Cadence = "one full pass per PF_RENDER_AUDIT_EVERY fast rounds", red turns * included among those full passes. */ export function bumpAuditTick(sid: string): void { const rw = mirrors.get(sid); @@ -395,17 +443,19 @@ export interface CaptureArgs { log?: LogFn; } -/** 'equal' | 'append' | 'recast' — prefix-identical growth vs rewrite. */ -export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" | "recast" { +/** 'equal' | 'append' | 'diverged' — prefix-identical growth vs anything + * else (byte diff, shrink, non-monotonic growth). Under the no-recast + * ruling every 'diverged' verdict KILLS the plane. */ +export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" | "diverged" { if (prev.length === cur.length) { - for (let i = 0; i < prev.length; i++) if (prev[i] !== cur[i]) return "recast"; + for (let i = 0; i < prev.length; i++) if (prev[i] !== cur[i]) return "diverged"; return "equal"; } if (cur.length > prev.length) { - for (let i = 0; i < prev.length; i++) if (prev[i] !== cur[i]) return "recast"; + for (let i = 0; i < prev.length; i++) if (prev[i] !== cur[i]) return "diverged"; return "append"; } - return "recast"; + return "diverged"; } /** sids whose row payload carried a literal NUL and were therefore refused @@ -413,11 +463,15 @@ export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" * — storing would corrupt the row with S4 unable to see it, since the * checksum only covers checkpoint meta). Loud-once, then fail-soft per turn: * the in-memory mirror keeps serving this process's lifetime; disk simply - * stops advancing until restart/caps-flip recast is possible again. */ + * stops advancing until restart or caps change allows re-capture again. */ const nulRejectedSids = new Set(); -export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" | "recast" } { +export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" } { const { store, sid, caps, log } = args; + if (deadSids.has(sid)) { + // Belt-and-braces behind the gate: a dead sid NEVER arms/commits again. + return { committed: false, op: "none" }; + } const hasNul = args.rowStrings.some((rj) => rj.includes("\u0000")); if (hasNul) { if (!nulRejectedSids.has(sid)) { @@ -436,10 +490,16 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op } const metaJson = (m: RenderMeta): string => JSON.stringify(m); if (rw === undefined || rw.rows.length === 0) { - // No usable base: full cast with a FORCED checkpoint (self-healing - // first use / post-death rebuild). Legacy turns only reach here with - // full arrays, which is exactly what a cast needs. + // Arm cast — allowed ONLY onto an empty table. The snapshot is + // forged exactly ONCE per sid (owner ruling 2026-11-05: no recast, + // no post-death rebuild): if rows already exist on disk, we cannot + // verify them here and rewriting them is forbidden ⇒ kill. if (args.mode === "legacy") { + const preExisting = store.pfRenderRows(sid); + if (preExisting.length > 0) { + killRenderPlane(store, sid, `arm refused over non-empty rows (${preExisting.length} pre-existing, unverifiable)`, log); + return { committed: false, op: "none" }; + } const rows = args.rowStrings; const m: RenderMeta = { v: 1, caps, rowCount: rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum }; const okSeq = store.pfRenderCommit(sid, { @@ -476,8 +536,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op } // A full-path round IS the audit regardless of verdict (the legacy byte // path ran to completion) — consume the cadence obligation here, or a - // clean long run would dead-lock on "audit-due" after N ticks (only - // cast/recast zeroed it before this fix). + // clean long run would dead-lock on "audit-due" after N ticks. if (args.mode === "legacy") rw.auditTick = 0; // legacy full-capture: diff against truth, minimal commit. const verdict = diffRowLists(rw.rows, args.rowStrings); @@ -509,23 +568,14 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op touchMirror(rw); return { committed: true, op: "append" }; } - // recast — client-side structural change (compaction & friends) or caps - // flip: rewrite covered ordinals, physically drop the excess, and FORCE - // the L1 checkpoint in the same transaction (owner ruling). - const rows = args.rowStrings; - const oldMaxPos = rw.rows.length - 1; - const deletes = oldMaxPos >= rows.length ? Array.from({ length: oldMaxPos + 1 - rows.length }, (_, i) => rows.length + i) : []; - const m: RenderMeta = { v: 1, caps, rowCount: rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum }; - const okSeq = store.pfRenderCommit(sid, { - rows: rows.map((rj, pos) => ({ pos, rowJson: rj })), - deletes: deletes.length > 0 ? deletes : undefined, - logEntries: [{ kind: "recast", payloadJson: JSON.stringify({ ...m, capsJson: JSON.stringify(caps) }) }], - checkpointMetaJson: metaJson(m), - }); - if (okSeq === null) return { committed: false, op: "none" }; - const parsed = rows.map((rj) => safeParse(rj)); - mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, caps: { ...caps }, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, auditTick: 0 }); - return { committed: true, op: "recast" }; + // DIVERGED (byte diff at any position, or shrink): the ONLY repair the + // old design offered was a recast — rewriting covered ordinals and + // deleting the excess. Owner ruling 2026-11-05 forbids that in every + // scenario: the plane is KILLED PERMANENTLY instead and the session + // rides the legacy full path from here on (bytes stay correct; the + // cache simply never serves this sid again). + killRenderPlane(store, sid, `structural divergence (stored=${rw.rows.length} rows, captured=${args.rowStrings.length})`, log); + return { committed: false, op: "none" }; } function safeParse(json: string): unknown { @@ -543,7 +593,8 @@ function safeParse(json: string): unknown { export interface RenderDebugWorld { sid: string; rows: number; - caps: RenderCaps; + dead: boolean; + caps: RenderCaps | undefined; estCum: number; textCum: number; imgCum: number; @@ -556,7 +607,10 @@ export function watermarkV6RenderDebugState(sessionId: string): { sids: RenderDe if (set) { for (const sid of set) { const rw = mirrors.get(sid); - if (rw) sids.push({ sid, rows: rw.rows.length, caps: rw.caps, estCum: rw.estCum, textCum: rw.textCum, imgCum: rw.imgCum, auditTick: rw.auditTick }); + sids.push({ + sid, rows: rw?.rows.length ?? 0, dead: deadSids.has(sid), + caps: rw?.caps, estCum: rw?.estCum ?? 0, textCum: rw?.textCum ?? 0, imgCum: rw?.imgCum ?? 0, auditTick: rw?.auditTick ?? 0, + }); } } return { sids }; @@ -567,5 +621,6 @@ export function _resetWatermarkV6RenderTestState(): void { mirrors.clear(); sessionSids.clear(); nulRejectedSids.clear(); + deadSids.clear(); mirrorBytes = 0; } diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 23f22f8079..df9cf00f35 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -85,6 +85,11 @@ export async function prepareOpenai( // #5664 green verdict carrier (declared here so the clamp/estimate/capture // consumers AFTER the try block see it; assigned at the gate below). let renderFast: { rw: RenderWorld; F: number; maxPos: number; sid: string } | null = null; + // #5664 no-recast regime (owner ruling 2026-11-05): once this sid's plane + // is durably dead the session rides pure legacy FOREVER — skip building + // capture strings entirely (an O(full history) stringify on every later + // turn would be pure waste: the module refuses the commit anyway). + let renderLaneDead = false; // #3222: #651's reasoning-drop plan, computed on the INBOUND history before // core.processTurn — see planReasoningDrop for why the post-turn view is // too late (the kernel hides consumed call+result pairs inside it). @@ -279,6 +284,7 @@ export async function prepareOpenai( renderFast = { rw: renderGate.rw, F: renderGate.F, maxPos: renderGeom.maxPos, sid: renderSid }; } else { log("debug", `[${sessionId}] #5664 render lane off (${renderGate.reason}) — baseline full path`); + if (renderGate.reason === "lane-dead") renderLaneDead = true; } } // #2432: dsh desktop's native compaction can LAND without ever @@ -523,8 +529,10 @@ export async function prepareOpenai( // Station 3: prefix rows pass through verbatim (cached parses); // the mapped tail replaces the last F stored rows AND extends them. // The byte check against the stored suffix is the sole guard — on - // mismatch redo the legacy full pass THIS turn (the next capture - // self-heals the rows with a recast + forced checkpoint). + // mismatch redo the legacy full pass THIS turn; there is no row + // repair anymore (no-recast ruling 2026-11-05) — if the stored + // rows and truth genuinely diverge, the next full-path capture + // kills this sid's plane permanently instead of recasting. const mappedAll = await mapEgress(processedMessages); const rw = renderFast.rw; const wpWindowLen = wpHit?.windowCores.length ?? 0; @@ -540,7 +548,7 @@ export async function prepareOpenai( renderedTailObjects = [...committedTail]; log("debug", `[${sessionId}] #5664 render snapshot FAST emit (prefix=${rw.rows.length - renderFast.F} cached rows, tail=${committedTail.length} mapped)`); } else { - log("warn", `[${sessionId}] #5664 render snapshot byte mismatch vs stored rows — legacy full pass this turn (next audit heals the rows)`); + log("warn", `[${sessionId}] #5664 render snapshot byte mismatch vs stored rows — legacy full pass this turn (no row repair exists: the next full-path capture kills the plane instead of recasting)`); renderFast = null; const turnFull = core.processTurn({ messages: renderMsgs, state: session.state, config: loopConfig, tokenCount, renderTags: renderStrategy, contentStore: contentStoreOf(session) }); session.state = turnFull.state; @@ -554,7 +562,10 @@ export async function prepareOpenai( } } else { rebuiltMessages = await mapEgress(processedMessages); - renderedCaptureStrings = rebuiltMessages.map((m) => JSON.stringify(m)); + // Lane-durably-dead sessions never arm/commit again — keep the + // strings null so station 2 skips both their build (O(n) per + // turn) and the doomed commit call. + if (!renderLaneDead) renderedCaptureStrings = rebuiltMessages.map((m) => JSON.stringify(m)); renderedTailObjects = []; renderedImageDelta = 0; } @@ -774,9 +785,11 @@ export async function prepareOpenai( if (Object.keys(wpResponseHeaders).length === 0) wpResponseHeaders = undefined; } // #5664 station 2 — ONE snapshot commit per engaged turn: steady append - // (fast rounds), arm cast / audit diff / recast with FORCED checkpoint - // (full-path rounds carry the complete captured array). Fail-soft for - // this turn only — the identity plane already committed independently. + // (fast rounds), arm cast / audit diff / divergence-KILL (full-path + // rounds carry the complete captured array; owner ruling 2026-11-05: + // recast is forbidden — any structural divergence kills the lane for the + // sid). Fail-soft for this turn only — the identity plane already + // committed independently. if (renderedCaptureStrings !== null && renderCapturedSid !== null && renderCapturedCaps !== null && featureEnabled("watermarkV6Render")) { const store = getStore(); if (store.pfRenderAvailable()) { diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index ec62f55ff9..3fe305c743 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1215,7 +1215,6 @@ export class UnifiedShadow { sid: string, args: { rows: { pos: number; rowJson: string | null }[]; - deletes?: number[]; logEntries: { kind: string; payloadJson: string }[]; checkpointMetaJson?: string; ts?: number; diff --git a/src/storage/store.ts b/src/storage/store.ts index b1c420bc62..b733bef9df 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2622,18 +2622,18 @@ export class UnifiedStore { } /** Atomic turn commit for the render plane (#5664): upsert `rows` - * (pos may move — a recast rewrites covered ordinals), physically drop - * `deletes` (ordinals that no longer exist after a rewrite), journal - * `logEntries` at strictly increasing ls_seq, and — when + * (pos values must be contiguous ascending; under the no-recast ruling + * covered ordinals are NEVER rewritten in place), journal `logEntries` + * at strictly increasing ls_seq — including the kind='dead' verdict + * that persists a plane death forever — and — when * `checkpointMetaJson` is given — fold the whole log into a fresh ckpt - * WITH its S4 checksum in the SAME transaction (the forced checkpoint - * of a compression round). Any throw → rollback → byte-identical plane. + * WITH its S4 checksum in the SAME transaction (the arm cast). Any + * throw → rollback → byte-identical plane. * Returns the next free ls_seq. */ pfRenderCommit( sid: string, args: { rows: { pos: number; rowJson: string | null }[]; - deletes?: number[]; logEntries: { kind: string; payloadJson: string }[]; checkpointMetaJson?: string; ts?: number; @@ -2645,10 +2645,6 @@ export class UnifiedStore { "INSERT INTO pf_render (sid, pos, row_json) VALUES (?, ?, ?) ON CONFLICT (sid, pos) DO UPDATE SET row_json = excluded.row_json", ); for (const r of args.rows) insRow.run(sid, r.pos, r.rowJson); - if (args.deletes && args.deletes.length > 0) { - const ph = args.deletes.map(() => "?").join(", "); - this.db.prepare(`DELETE FROM pf_render WHERE sid = ? AND pos IN (${ph})`).run(sid, ...args.deletes); - } // Continuation must clear BOTH the surviving log tail AND the // checkpoint watermark: a fresh checkpoint ABSORBS its own log // entries (delete below), so MAX(log) can sit BEHIND ckpt_ls_seq; diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index f93eee3ea5..1a0d15af30 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -3,11 +3,16 @@ // Golden equation under test: on every engaged turn the emit array built as // [cached prefix rows] + [freshly mapped tail] (fast rounds) must be BYTE- // IDENTICAL to what the legacy full-pipeline path emits (arm / audit / -// caps-flip rollback rounds). The kernel is untouched; this lane caches -// egress bytes, not semantics — equivalence with the full pass is the -// contract. B-version compaction semantics hold throughout: client-side -// element deletion is IGNORED by the identity plane (ledger grows forward -// only), so both arms grow identically and stay byte-comparable. +// rollback rounds). The kernel is untouched; this lane caches egress bytes, +// not semantics — equivalence with the full pass is the contract. +// +// NO-RECAST REGIME (owner ruling 2026-11-05, sealed during #5689 review): +// recast is FORBIDDEN in every scenario — any stored-row divergence (byte +// diff, shrink, caps flip that changed bytes, checksum corruption, +// unverifiable pre-existing snapshot) KILLS the plane for that sid +// permanently (persisted kind='dead' log entry) and the session rides the +// legacy full path forever after. Storage is strictly append-only (+ dead +// markers); arm cast is legal exactly once, onto an empty table. // // Harness: unit-level against a REAL sqlite sidecar (UnifiedShadow shadow // mode), mirroring tests/fastpath-v6-msgid.test.ts; the kernel runs as a @@ -119,8 +124,8 @@ test("unit: diffRowLists verdicts", () => { assert.equal(diffRowLists([], ["a"]), "append"); assert.equal(diffRowLists(["a", "b"], ["a", "b"]), "equal"); assert.equal(diffRowLists(["a", "b"], ["a", "b", "c"]), "append"); - assert.equal(diffRowLists(["a", "b"], ["a", "X"]), "recast"); // middle mutation - assert.equal(diffRowLists(["a", "b", "c"], ["a"]), "recast"); // shrink + assert.equal(diffRowLists(["a", "b"], ["a", "X"]), "diverged"); // middle mutation + assert.equal(diffRowLists(["a", "b", "c"], ["a"]), "diverged"); // shrink }); test("unit: cast → steady appends → hydrate-from-disk fold survives mirror loss", (t) => { @@ -195,7 +200,7 @@ test("gate: red reasons are a stable vocabulary", (t) => { } finally { h.cleanup(); } }); -test("unit: S4 checksum tamper kills the mirror (hydrate-failed, loudly logged)", (t) => { +test("unit: S4 checksum tamper KILLS the plane durably (loudly logged)", (t) => { const h = makeH("s4"); try { const b1 = makeBody([{ role: "user", content: "hello" }]); @@ -218,7 +223,82 @@ test("unit: S4 checksum tamper kills the mirror (hydrate-failed, loudly logged)" maxPos: 0, tokenCount: 10, modelContextLimit: MODEL_LIMIT, caps, snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, }); - assert.deepEqual(g, { ok: false, reason: "hydrate-failed" }); + // No-recast ruling: corruption is no longer self-healable via a + // forced recast — the FIRST hydration attempt reports the DURABLE + // reason and persists the death verdict. + assert.deepEqual(g, { ok: false, reason: "lane-dead" }); + const meta = h.store.pfRenderMeta(sid); + assert.ok(meta !== null && meta.log.some((e) => e.kind === "dead"), "corruption must persist a kind='dead' verdict"); + } finally { h.cleanup(); } +}); + +test("unit: structural divergence KILLS the plane permanently (no-recast ruling 2026-11-05)", (t) => { + const h = makeH("die"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-die", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + assert.equal(commitRenderCapture({ sessionId: "s-die", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).op, "cast"); + assert.equal(commitRenderCapture({ sessionId: "s-die", store: h.store, sid, log: h.log, mode: "steady", caps, rowStrings: ['{"i":2}'], sums: { estCum: 1, textCum: 1, imgCum: 0 } }).op, "append"); + // Middle-byte mutation lands on a full-path round. The old design + // repaired with a recast; the ruling forbids that ⇒ kill instead. + const logsBefore = h.log.lines.length; + const op = commitRenderCapture({ sessionId: "s-die", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"X":9}', '{"i":2}'], sums: { estCum: 3, textCum: 3, imgCum: 0 } }); + assert.deepEqual(op, { committed: false, op: "none" }); + assert.equal(h.store.pfRenderRows(sid).length, 3, "stored rows stay immutable after a divergence"); + assert.equal(h.store.pfRenderCkptIntact(sid), true, "the kill writes NO checkpoint (strictly append-only)"); + assert.ok(h.log.lines.slice(logsBefore).some((l) => l.level === "error" && l.msg.includes("KILLED PERMANENTLY")), + `kill must be loudly logged, got ${JSON.stringify(h.log.lines.slice(logsBefore))}`); + const meta = h.store.pfRenderMeta(sid); + assert.ok(meta !== null && meta.log.some((e) => e.kind === "dead"), "death verdict persisted as kind='dead'"); + const base = { + sessionId: "s-die", session: { id: "s-die", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 2, tokenCount: 50, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }; + assert.deepEqual(renderPlaneGate(base), { ok: false, reason: "lane-dead" }, "positive: dead sid reports the durable reason"); + // Negative side (#3617): neither re-capture nor a restart-wipe'd + // re-arm can bring the lane back — no rewriting of stored rows EVER. + assert.deepEqual(commitRenderCapture({ sessionId: "s-die", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"X":9}', '{"i":2}'], sums: { estCum: 3, textCum: 3, imgCum: 0 } }), + { committed: false, op: "none" }, "re-capture into a dead plane is refused"); + assert.equal(h.store.pfRenderRows(sid).length, 3); + } finally { h.cleanup(); } +}); + +test("unit: plane death persists across restart (no re-arm ever)", (t) => { + const h = makeH("pdeath"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-pdeath", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + assert.equal(commitRenderCapture({ sessionId: "s-pdeath", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"a":1}', '{"a":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).op, "cast"); + assert.equal(commitRenderCapture({ sessionId: "s-pdeath", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"a":1}', '{"MUTATED":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).committed, false); + // Process-restart simulation: wipe ALL in-memory plane state (the + // death set included) and reopen the SAME sidecar (cascade pattern). + _resetWatermarkV6RenderTestState(); + const shadow2 = UnifiedShadow.maybeOpen(h.dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r) => r as never); + assert.ok(shadow2, "second open must tolerate the existing sidecar"); + const store2 = new SessionStore({ dir: path.join(h.dir, "sessions"), enabled: true, shadow: shadow2 }); + const g = renderPlaneGate({ + sessionId: "s-pdeath", session: { id: "s-pdeath", state: createInitialState() } as unknown as Session, + store: store2, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 1, tokenCount: 10, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }); + assert.deepEqual(g, { ok: false, reason: "lane-dead" }, "persisted kind='dead' verdict binds the NEW process too"); + assert.equal(store2.pfRenderRows(sid).length, 2, "row table stays frozen"); + // An explicit arming-style full capture through the new store is + // refused — the lane is dead until session teardown, period. + assert.equal( + commitRenderCapture({ sessionId: "s-pdeath", store: store2, sid, mode: "legacy", caps, rowStrings: ['{"a":1}', '{"a":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).committed, + false, "re-arm through a fresh process is impossible", + ); + shadow2.close(); } finally { h.cleanup(); } }); @@ -252,8 +332,12 @@ test("unit: session cascade wipes render tables; v12 re-migration is a no-op", ( // --------------------------------------------------------------------------- // Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, -// across arm → steady fast rounds → client compaction (turn 8) → strategy -// strategy-flip recast (turn 14) → audit cadence (fires for real at turn 10) → restart-hydration continuation. +// across arm → steady fast rounds → client compaction (turn 8) → audit +// cadence (fires for real at turn 10) → strategy flip (turn 14) → restart. +// No-recast ruling: the flip's byte divergence KILLS the plane at turn 14; +// every later round rides lane-dead legacy (bytes still oracle-exact), the +// stored table freezes at death, and the persisted verdict survives phase 3's +// process-restart reset (hydration refuses re-arm). // --------------------------------------------------------------------------- test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end", async (t) => { @@ -266,7 +350,8 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" // (ledger grows forward only), views and rows keep growing, and the // golden equation must still hold element-stable (ids follow elements, // never positions). Strategy flips text-only→none at turn 14 ⇒ caps flip - // ⇒ forced legacy recast round, then steady on the new byte form. + // ⇒ forced legacy round; its bytes diverge from the stored rows ⇒ the + // plane KILLS (no recast exists anymore) and turns 15-16 ride lane-dead. const ELEMENTS: Msg[] = Array.from({ length: 16 }, (_, i) => ({ role: i % 2 === 0 ? "user" : "assistant", content: `content number ${i}` })); const STABLE_ID = (i: number) => `eid-${i}`; const clientBody = (tn: number): Msg[] => { @@ -301,8 +386,9 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" const stateB: AgentState = createInitialState(); const sessB = { id: "s-gold-b", state: stateB } as unknown as Session; let sidB: string | undefined; - let fastRounds = 0, castOps = 0, recastOps = 0, legacyRounds = 0; + let fastRounds = 0, castOps = 0, legacyRounds = 0; let totalSubmittedRows = 0; // volume discipline pin (see final assert) + let steadySubmittedRows = 0; // anti-duplicate-append pin (see final assert) let totalSubmittedText = 0; for (let tn = 1; tn <= 16; tn++) { const strategy = strategyOf(tn); @@ -384,27 +470,38 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" // this tier exercises is the accumulation ALGEBRA — each row counted // exactly once with its own cost, tail-only deltas on steady rounds. const rowCost = (rs: string[]): number => rs.reduce((s, x) => s + x.length, 0); - const op = commitRenderCapture({ - sessionId: "s-gold-b", store: hB.store, sid: sidB!, mode, caps, rowStrings: srcRows, - sums: { estCum: rowCost(srcRows), textCum: rowCost(srcRows), imgCum: 0 }, - }); + let op: { committed: boolean; op: "none" | "append" | "cast" } = { committed: false, op: "none" }; + // No-recast regime: once the lane dies, production stops paying for + // capture strings entirely (the seam skips the station-2 commit); the + // lane rides pure legacy until the session ends. Mirror that here. + if (!(mode === "legacy" && !gate.ok && gate.reason === "lane-dead")) { + op = commitRenderCapture({ + sessionId: "s-gold-b", store: hB.store, sid: sidB!, log: hB.log, mode, caps, rowStrings: srcRows, + sums: { estCum: rowCost(srcRows), textCum: rowCost(srcRows), imgCum: 0 }, + }); + } totalSubmittedRows += srcRows.length; totalSubmittedText += rowCost(srcRows); - if (op.committed) { - if (op.op === "cast") castOps++; - else if (op.op === "recast") recastOps++; - } + if (mode === "steady") steadySubmittedRows += srcRows.length; + if (op.committed && op.op === "cast") castOps++; // THE golden equation: lane emit == legacy oracle, byte for byte. assert.deepEqual(rebuilt.map((m) => JSON.stringify(m)), goldens[tn - 1].rows, `turn ${tn} (${mode}${gate.ok ? "" : `, reason=${gate.reason}`}): lane emit diverged from legacy oracle`); assert.equal(stateB.blocks.length, 0, "assumption pin: no folds materialize in this synthetic conv"); v.fin(); // plane advances AFTER capture — same order as prepare-openai - // Structural regression pin (review round 1, tail-only fix): the render - // table must track the identity ledger one-row-per-element at every - // turn boundary — a duplicate-append corruption shows up here instantly - // instead of hiding behind the byte equation. - assert.equal(hB.store.pfRenderRows(sidB!).length, watermarkV6DebugState("s-gold-b").worlds[sidB!]!.maxPos + 1, - `turn ${tn}: render table drifted from identity ledger (rows=${hB.store.pfRenderRows(sidB!).length})`); + // Structural regression pin (review round 1, tail-only fix): while the + // plane is ALIVE the render table must track the identity ledger + // one-row-per-element at every turn boundary — a duplicate-append + // corruption shows up here instantly instead of hiding behind the byte + // equation. After a death the table freezes (nothing re-commits), so + // pin the frozen invariant instead. + const dbgNow = watermarkV6RenderDebugState("s-gold-b").sids.some((w) => w.dead); + if (!dbgNow) { + assert.equal(hB.store.pfRenderRows(sidB!).length, watermarkV6DebugState("s-gold-b").worlds[sidB!]!.maxPos + 1, + `turn ${tn}: render table drifted from identity ledger (rows=${hB.store.pfRenderRows(sidB!).length})`); + } else { + assert.equal(hB.store.pfRenderRows(sidB!).length, 13, `turn ${tn} post-death: row table must stay frozen`); + } } assert.equal(castOps, 1, "exactly one cast (the arm turn)"); // Audit cadence exercised FOR REAL (review round 2): with the flip deferred @@ -415,26 +512,32 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" assert.deepEqual(gateSeq.filter((r) => r === "audit-due"), ["audit-due"], `expected exactly one audit-due round, got ${JSON.stringify(gateSeq)}`); assert.equal(gateSeq.indexOf("audit-due"), 9, `audit-due expected at loop index 9 (turn 10), got ${JSON.stringify(gateSeq)}`); assert.equal(gateSeq[10], "fast", `round right after the audit must recover to fast, got ${JSON.stringify(gateSeq)}`); - // ≥ 12 fast rounds = at least TWO full audit waves served green: the first - // proves the lane runs, the second proves any legacy round CONSUMES the - // audit obligation (pre-fix, tick froze at N and every later turn rode - // legacy forever — masked at the time by per-round recasts). - assert.ok(fastRounds >= 12, `expected ≥2 audit waves of fast rounds, got ${fastRounds} (audit obligation not consumed?)`); - assert.ok(recastOps >= 1, "strategy flip at turn 14 must force ≥1 recast"); - assert.ok(legacyRounds >= 2, `arm + audit/caps-flip rounds must ride legacy (got ${legacyRounds})`); + // Exactly two fast waves before death: 2-9 (pre-audit) + 11-13 (post-audit). + assert.ok(fastRounds >= 11, `expected ≥11 fast rounds (two audit waves minus the post-death rounds), got ${fastRounds}`); + // NO-RECAST RULING pins: the API offers no recast op at all (the old + // counter cannot even be written); the flip's divergent bytes die the + // lane at turn 14, and every subsequent round reports the durable reason. + assert.equal(gateSeq.indexOf("caps-mismatch"), 13, `flip red expected at loop index 13 (turn 14), got ${JSON.stringify(gateSeq)}`); + assert.equal(gateSeq.indexOf("lane-dead"), 14, `first death report expected at loop index 14 (turn 15), got ${JSON.stringify(gateSeq)}`); + assert.ok(legacyRounds >= 5, `arm + audit + flip + post-death rounds must ride legacy (got ${legacyRounds})`); // Volume discipline (steady deltas must stay O(new) per round, never // O(full list)): pre-tail-fix behavior re-submitted the whole list each // green round ⇒ ~8×final volume here. const finalRowCount = hB.store.pfRenderRows(sidB!).length; - assert.ok(totalSubmittedRows <= 3 * finalRowCount, - `submitted row-volume ${totalSubmittedRows} vs final table ${finalRowCount} — steady rounds must append tails, not full lists`); - // S4 truth intact at the end; mirror agrees with the last golden shape + assert.equal(finalRowCount, 13, "row table froze at the death turn (no rewrite, no further commit)"); + // Volume discipline — steady rounds must stay O(new): every row enters + // the table AT MOST ONCE via a steady append (a duplicate-append + // corruption would push this past the table size). Legacy full lists are + // inherent to their (rare) rounds and are not pinned. + assert.ok(steadySubmittedRows <= finalRowCount, + `steady-mode submissions ${steadySubmittedRows} exceeded table size ${finalRowCount} — rows must be appended at most once`); + // S4 truth intact at the end; mirror froze together with the table assert.equal(hB.store.pfRenderCkptIntact(sidB!), true); const dbgLhs = watermarkV6RenderDebugState("s-gold-b"); assert.ok(Array.isArray(dbgLhs.sids) && dbgLhs.sids.length > 0, "render debug state must list the mirror"); - const lhsRows = dbgLhs.sids[dbgLhs.sids.length - 1].rows; - const rhsRows = goldens[goldens.length - 1].rows.length; - assert.equal(lhsRows, rhsRows, "mirror row count must equal last golden row count"); + const lastW = dbgLhs.sids[dbgLhs.sids.length - 1]; + assert.equal(lastW.dead, true, "debug state must surface the death verdict"); + assert.equal(lastW.rows, 13, "mirror row count froze at death"); // Sums algebra pin (review P1-2): every row's cost entered the cumulative // ledger EXACTLY once — a frozen-window double-count would overshoot here. const finalRowCost = hB.store.pfRenderRows(sidB!).reduce((s, r) => s + (r.rowJson?.length ?? 0), 0); @@ -453,9 +556,11 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" tokenCount: 100, modelContextLimit: MODEL_LIMIT, caps: capsFor("none"), snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, log: hB.log, }); - assert.ok(g17.ok === true || g17.reason === "audit-due", - `post-restart gate must hydrate from disk (got ${JSON.stringify(g17)})`); - if (g17.ok) assert.equal(g17.rw.rows.length, goldens.at(-1)!.rows.length, "hydrated row count must match the pre-restate capture"); + // The persisted kind='dead' verdict survives the process-restart wipe: + // hydration REFUSES to re-arm a dead plane (no-recast ruling — there is + // no rebuild path), and the identity lane keeps working independently. + assert.deepEqual(g17, { ok: false, reason: "lane-dead" }, + `post-restart gate must honor the persisted death verdict (got ${JSON.stringify(g17)})`); v17.fin(); hB.cleanup(); }); From 7577b31f2908fb654f2ba138950972dbdc121892 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 5 Nov 2026 11:06:54 +0800 Subject: [PATCH 25/49] fix(fastpath): defer the arm cast over a thrown row-table read instead of failing the request (#5664, review round 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The no-recast sealing (49ba14dae) added a store.pfRenderRows pre-existing check to the arm branch, but SessionStore.pfRenderRows has no local try/catch and station 2 runs OUTSIDE prepare-openai's try block — a transient SELECT failure would have propagated out of prepareOpenai and failed the whole model request, an availability regression vs the old fail-soft arm path. A thrown read carries no divergence evidence, so it defers the arm to the next round (warn + op none) instead of killing the plane or crashing the caller. Test pins both directions: flaky-read round fails soft AND the next healthy round arms normally (transient != death). --- src/server/fastpath-v6-render.ts | 13 ++++++++++++- tests/issue5664-render-snapshot.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index f2f7e029f7..2d08a17a95 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -495,7 +495,18 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op // no post-death rebuild): if rows already exist on disk, we cannot // verify them here and rewriting them is forbidden ⇒ kill. if (args.mode === "legacy") { - const preExisting = store.pfRenderRows(sid); + let preExisting: { pos: number; rowJson: string | null }[]; + try { + preExisting = store.pfRenderRows(sid); + } catch { + // A THROWN read is a transient storage hiccup, not evidence of + // divergence: defer the arm to the next round (fail soft) + // instead of killing over it — and never propagate, station 2 + // runs outside prepare-openai's try block, a throw here would + // fail the whole request. + log?.("warn", `[${sid}] #5664 render arm deferred this round: row-table read threw`); + return { committed: false, op: "none" }; + } if (preExisting.length > 0) { killRenderPlane(store, sid, `arm refused over non-empty rows (${preExisting.length} pre-existing, unverifiable)`, log); return { committed: false, op: "none" }; diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 1a0d15af30..2b7fd27838 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -302,6 +302,29 @@ test("unit: plane death persists across restart (no re-arm ever)", (t) => { } finally { h.cleanup(); } }); +test("unit: thrown row-table read defers arming without killing (transient ≠ death)", () => { + const h = makeH("defer"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-defer", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + // Prototype-chain wrapper: every store method resolves to the REAL + // sidecar except pfRenderRows, which throws — a transient SELECT + // failure while everything else is healthy. + const flakyStore: SessionStore = Object.create(h.store); + flakyStore.pfRenderRows = () => { throw new Error("io hiccup"); }; + assert.deepEqual( + commitRenderCapture({ sessionId: "s-defer", store: flakyStore, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }), + { committed: false, op: "none" }, "a thrown read must fail soft for this round, not crash the caller"); + // Negative side (#3617): no death verdict may persist from a transient + // error — the very next round with a healthy store arms normally. + assert.equal(commitRenderCapture({ sessionId: "s-defer", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).op, "cast"); + assert.equal(h.store.pfRenderRows(sid).length, 2); + } finally { h.cleanup(); } +}); + // Review round 1 blind-spot pins (issue #5664): session teardown must wipe the // three render tables in the same cascade, and re-running the migration on an // already-v12 sidecar must be a no-op that leaves rows untouched. From f3681735fa4315d83936928eb4a8f06c4cc387ed Mon Sep 17 00:00:00 2001 From: ework-agent Date: Thu, 5 Nov 2026 13:16:43 +0800 Subject: [PATCH 26/49] =?UTF-8?q?fix(fastpath):=20render=20rows=20hold=20t?= =?UTF-8?q?he=20post-rewrite=20generic=20core=20form=20=E2=80=94=20nothing?= =?UTF-8?q?=20mapped=20is=20ever=20cached=20(#5643=20=C2=A75=20ruling,=20#?= =?UTF-8?q?5664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner ruling on #5643 (2026-11-05, canonical flow): the only plane boundary that can never be altered by model/maxDim/toolArmed/voice/harden/serialize is the post-kernel-rewrite generic core form. Rebuild the plane around it: - RenderCaps drops model/maxDim/toolArmed — they are per-turn mapping inputs, not snapshot validity (caps now: v, strategy, droppable, multimodal). - Station 3 green round: verify this round's rewritten window cores against the stored suffix at the core-form boundary; the full core sequence (cached prefix ++ window output) then passes through the FULL mapping chain fresh this turn (rescale/harden/coreToOpenai/voice/serialize all run with this turn's inputs). Within-family model switches ride through with zero events — they re-map next turn by construction. - Captures (steady tail, legacy full, fallback) store core strings, never mapped wire bytes. Tripwire semantics unchanged (one-turn legacy fallback, no recast, durable kill on divergence); sums/audit/death machinery untouched. - Tests: golden A/B rewritten to the core regime with a shape pin asserting every stored row carries core identity fields (id/contentType) and no wire bytes are persisted. --- src/server/fastpath-v6-render.ts | 58 ++++++++++++++---------- src/server/prepare-openai.ts | 59 ++++++++++++++----------- tests/issue5664-render-snapshot.test.ts | 57 +++++++++++++++--------- 3 files changed, 104 insertions(+), 70 deletions(-) diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 2d08a17a95..78f7cc8ee4 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -69,15 +69,19 @@ export const PF_RENDER_EMERGENCY_RATIO = 0.9; const MIRROR_BYTE_BUDGET = 256 * 1024 * 1024; let mirrorBytes = 0; -/** Capabilities fingerprint — any change red-gates the lane (caps-mismatch); - * if the legacy round's bytes then diverge from the stored rows the plane - * is killed for this sid (no-recast regime). */ +/** Identity fingerprint — the ONLY dimensions that may change the stored + * row contents: the kernel-side rewrite surface (render strategy, + * reasoning-drop predicate) plus the core-shape class (multimodal). + * Any change red-gates that turn (caps-mismatch); if a subsequent legacy + * full pass then diverges from the stored rows the plane is killed for + * this sid (no-recast regime). model/maxDim/toolArmed are DELIBERATELY + * ABSENT: they are per-turn mapping-layer inputs (#5643 ruling §5, owner + * 2026-11-05) — rows hold the post-rewrite generic core form, which no + * mapping input can alter, so a within-family model switch simply re-maps + * next turn with zero events and never touches snapshot validity. */ export interface RenderCaps { v: 1; - model: string; strategy: string; - maxDim: number; - toolArmed: boolean; droppable: boolean; multimodal: boolean; } @@ -85,10 +89,7 @@ export interface RenderCaps { function capsEqual(a: RenderCaps, b: RenderCaps): boolean { return ( a.v === b.v && - a.model === b.model && a.strategy === b.strategy && - a.maxDim === b.maxDim && - a.toolArmed === b.toolArmed && a.droppable === b.droppable && a.multimodal === b.multimodal ); @@ -105,9 +106,14 @@ interface RenderMeta { export interface RenderWorld { sid: string; - /** Outbound JSON string per rendered ordinal, contiguous 0..n-1. */ + /** Canonical post-kernel-rewrite generic core-form JSON per rendered + * ordinal (contiguous 0..n-1 over surviving elements). NEVER mapped + * wire bytes: the outbound message is re-mapped from these cores EVERY + * turn with that turn's model/mapping inputs (#5643 ruling §5 ②). */ rows: string[]; - /** Parsed caches of `rows` (same index) — the splice reads these. */ + /** Parsed caches of `rows` (same index) — the green round splices these + * core rows ahead of the window output; they double as next round's + * verify baseline, so no downstream consumer may alias-mutate them. */ rowsParsed: unknown[]; caps: RenderCaps; /** Cumulative Σ estimateCoreMessagesUpper over all current rows. */ @@ -131,11 +137,14 @@ const mirrors = new Map(); const sessionSids = new Map>(); /** Owner ruling 2026-11-05 (#5689 review): NO recast in ANY scenario. - * Structural divergence (byte diff/shrink), caps flips that changed stored - * bytes, S4 corruption, row-table inconsistency, unverifiable pre-existing - * snapshots — all land here. A dead sid NEVER arms, hydrates, gates green, - * or commits again. The verdict is persisted as a kind='dead' log entry so - * every future process instance reaches the same conclusion. */ + * Structural divergence (byte diff/shrink at the core-form boundary), S4 + * corruption, row-table inconsistency, unverifiable pre-existing snapshots + * — all land here. Caps changes cannot alter stored bytes anymore (mapping + * inputs sit outside the fingerprint, #5643 ruling §5 ③): their failure + * mode is a red gate turn, absorbed by the audit cadence. A dead sid NEVER + * arms, hydrates, gates green, or commits again. The verdict is persisted + * as a kind='dead' log entry so every future process instance reaches the + * same conclusion. */ const deadSids = new Set(); export function killRenderPlane(store: SessionStore, sid: string, reason: string, log?: LogFn): void { @@ -385,10 +394,11 @@ export function bumpAuditTick(sid: string): void { // Fast-round verification + splice (station 3) // --------------------------------------------------------------------------- -/** Compare the mapped first-F fast outputs against the stored suffix. The - * caller has already run the SAME mapping chain (rescale→harden→ - * coreToOpenai→systemToUser voice fn) as the baseline capture, so this is - * a pure byte compare. Returns false on ANY difference (incl. length). */ +/** Compare THIS round's first-F rewritten window cores against the stored + * suffix AT THE CORE-FORM BOUNDARY (#5643 ruling §5 ③: the tripwire sits + * on post-rewrite generic cores — never on mapped wire bytes; the mapping + * chain runs fresh every turn and its inputs are excluded from identity). + * Pure byte compare. Returns false on ANY difference (incl. length). */ export function verifyRenderWindow(fastEmitsJson: string[], rw: RenderWorld, F: number): boolean { if (fastEmitsJson.length < F) return false; const start = rw.rows.length - F; @@ -399,9 +409,11 @@ export function verifyRenderWindow(fastEmitsJson: string[], rw: RenderWorld, F: return true; } -/** Splice: prefix parsed rows (cached — zero parse/stringify cost) + mapped - * fast tail. Row indices align with rendered ordinals (probe-verified - * positional determinism; the byte check above is the tripwire). */ +/** Splice: prefix parsed CORE rows (cached) + this round's rewritten core + * tail. Row indices align with rendered ordinals (probe-verified + * positional determinism; the core-form byte check above is the tripwire). + * The concatenated core sequence then passes through the FULL mapping + * chain fresh this turn (#5643 ruling §5 ②) — nothing mapped is cached. */ export function spliceOutboundRows(rw: RenderWorld, F: number, tail: T[]): T[] { const prefixLen = rw.rows.length - F; if (prefixLen < 0) throw new Error("#5664 splice prefix underflow"); diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index df9cf00f35..f64968a5c7 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -252,10 +252,7 @@ export async function prepareOpenai( renderCapturedSid = renderSid; const rcaps: RenderCaps = { v: 1, - model: modelIdOf(parsed) ?? "", strategy: renderStrategy, - maxDim: imageRescaleMaxDim(), - toolArmed: imageRescaleToolArmed(session), droppable: loopConfig.compressAtFoldPointClearThinking === true, multimodal: wpHit.seedMultimodal === true || wpHit.windowCores.some((m) => Array.isArray((m as { content?: unknown }).content)), }; @@ -526,29 +523,36 @@ export async function prepareOpenai( const mapEgress = async (pms: CoreMessage[]): Promise => hardenOpenaiAssistantContent(coreToOpenai(systemToUser(await applyImageRescalePass(pms as BiliMessage[], { session, sessionId, log, maxDim: imageRescaleMaxDim(), toolArmed: imageRescaleToolArmed(session) }), summaryVoiceFn))); if (renderFast !== null) { - // Station 3: prefix rows pass through verbatim (cached parses); - // the mapped tail replaces the last F stored rows AND extends them. - // The byte check against the stored suffix is the sole guard — on - // mismatch redo the legacy full pass THIS turn; there is no row - // repair anymore (no-recast ruling 2026-11-05) — if the stored - // rows and truth genuinely diverge, the next full-path capture - // kills this sid's plane permanently instead of recasting. - const mappedAll = await mapEgress(processedMessages); + // Station 3 (#5643 ruling §5, owner 2026-11-05 — canonical flow): + // stored rows are the POST-REWRITE GENERIC CORE FORM, so the + // tripwire compares this round's rewritten window cores against + // the stored suffix AT THAT BOUNDARY (never mapped wire bytes). + // On match the full core sequence (cached prefix ++ window + // output) passes through the mapping chain FRESH this turn — + // rescale/harden/coreToOpenai/voice/serialize all run with THIS + // turn's model/maxDim/toolArmed inputs; nothing mapped is cached. + // On mismatch redo the legacy full pass THIS turn; there is no + // row repair (no-recast ruling) — if stored rows and truth + // genuinely diverge, the next full-path capture kills this sid's + // plane permanently instead of recasting. const rw = renderFast.rw; - const wpWindowLen = wpHit?.windowCores.length ?? 0; - const committedTail = mappedAll.slice(0, Math.min(renderFast.F + wpWindowLen, mappedAll.length)); - // Image reserve DELTA covers only this turn's NEW rows — the F - // frozen-window rows' reserve already sits in rw.imgCum; counting - // them again here drifts the ledger by a whole window each green - // round (review P1-2). slice(F) aligns with the captured tail. - renderedImageDelta = imageReserveFor(session, "openai", { model: parsed.model, messages: committedTail.slice(renderFast.F) } as unknown as OpenAIRequestBody, opts, billingUpstream ?? upstreamOrigin); - if (verifyRenderWindow(committedTail.slice(0, renderFast.F).map((m) => JSON.stringify(m)), rw, renderFast.F)) { - rebuiltMessages = spliceOutboundRows(rw, renderFast.F, committedTail); - renderedCaptureStrings = [...rw.rows.slice(0, rw.rows.length - renderFast.F), ...committedTail.map((m) => JSON.stringify(m))]; - renderedTailObjects = [...committedTail]; - log("debug", `[${sessionId}] #5664 render snapshot FAST emit (prefix=${rw.rows.length - renderFast.F} cached rows, tail=${committedTail.length} mapped)`); + const winCores = processedMessages; + if (verifyRenderWindow(winCores.slice(0, renderFast.F).map((m) => JSON.stringify(m)), rw, renderFast.F)) { + const fullCores = spliceOutboundRows(rw, renderFast.F, winCores); + rebuiltMessages = await mapEgress(fullCores); + // Image reserve DELTA covers only this turn's NEW elements — + // the F frozen-window rows' reserve already sits in rw.imgCum; + // counting them again drifts the ledger by a whole window each + // green round (review P1-2). The full sequence is + // [prefix n-F][frozen F][kNew], so the kNew fresh tails start + // at index n = rw.rows.length — slice(len) is exactly them. + // Billing metadata: computed on the fresh map. + renderedImageDelta = imageReserveFor(session, "openai", { model: parsed.model, messages: rebuiltMessages.slice(rw.rows.length) } as unknown as OpenAIRequestBody, opts, billingUpstream ?? upstreamOrigin); + renderedCaptureStrings = winCores.slice(renderFast.F).map((m) => JSON.stringify(m)); + renderedTailObjects = [...winCores]; + log("debug", `[${sessionId}] #5664 render snapshot FAST emit (prefix=${rw.rows.length - renderFast.F} cached core rows, tail=${winCores.length - renderFast.F} new cores, outbound=${rebuiltMessages.length} re-mapped this turn)`); } else { - log("warn", `[${sessionId}] #5664 render snapshot byte mismatch vs stored rows — legacy full pass this turn (no row repair exists: the next full-path capture kills the plane instead of recasting)`); + log("warn", `[${sessionId}] #5664 render snapshot core-form mismatch vs stored rows — legacy full pass this turn (no row repair exists: the next full-path capture kills the plane instead of recasting)`); renderFast = null; const turnFull = core.processTurn({ messages: renderMsgs, state: session.state, config: loopConfig, tokenCount, renderTags: renderStrategy, contentStore: contentStoreOf(session) }); session.state = turnFull.state; @@ -556,7 +560,7 @@ export async function prepareOpenai( const pmFull = applyReasoningDrop(stripKernelSummaries(turnFull.messages, turnFull.state), planReasoningDrop(renderMsgs, reasoning, isStrictReasoningEcho(session, upstreamOrigin, modelIdOf(parsed))), log, sessionId); processedMessages = pmFull; rebuiltMessages = await mapEgress(pmFull); - renderedCaptureStrings = rebuiltMessages.map((m) => JSON.stringify(m)); + renderedCaptureStrings = pmFull.map((m) => JSON.stringify(m)); renderedTailObjects = []; renderedImageDelta = 0; } @@ -564,8 +568,9 @@ export async function prepareOpenai( rebuiltMessages = await mapEgress(processedMessages); // Lane-durably-dead sessions never arm/commit again — keep the // strings null so station 2 skips both their build (O(n) per - // turn) and the doomed commit call. - if (!renderLaneDead) renderedCaptureStrings = rebuiltMessages.map((m) => JSON.stringify(m)); + // turn) and the doomed commit call. Rows store the post-rewrite + // core form (#5643 §5), so capture from cores, never wire bytes. + if (!renderLaneDead) renderedCaptureStrings = processedMessages.map((m) => JSON.stringify(m)); renderedTailObjects = []; renderedImageDelta = 0; } diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 2b7fd27838..4f330d950f 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -1,18 +1,25 @@ // #5664 — station 2/3: outbound-row render-snapshot plane for the v6 lane. // -// Golden equation under test: on every engaged turn the emit array built as -// [cached prefix rows] + [freshly mapped tail] (fast rounds) must be BYTE- +// Golden equation under test: on every engaged turn the outbound emit built +// as [cached prefix core rows] ++ [this round's rewritten window cores] +// passed through the FULL mapping chain fresh (fast rounds) must be BYTE- // IDENTICAL to what the legacy full-pipeline path emits (arm / audit / -// rollback rounds). The kernel is untouched; this lane caches egress bytes, -// not semantics — equivalence with the full pass is the contract. +// rollback rounds). The kernel is untouched. +// +// CANONICAL-FLOW REGIME (#5643 ruling §5, owner 2026-11-05): stored rows +// hold the POST-REWRITE GENERIC CORE FORM — the only plane boundary the +// mapping layer (model/maxDim/toolArmed/voice/harden/serialize) can never +// alter — and NOTHING mapped/cached survives across turns: every emit runs +// the entire chain end-to-end. Equivalence with the full pass is the +// contract, enforced here against a legacy oracle. // // NO-RECAST REGIME (owner ruling 2026-11-05, sealed during #5689 review): -// recast is FORBIDDEN in every scenario — any stored-row divergence (byte -// diff, shrink, caps flip that changed bytes, checksum corruption, -// unverifiable pre-existing snapshot) KILLS the plane for that sid -// permanently (persisted kind='dead' log entry) and the session rides the -// legacy full path forever after. Storage is strictly append-only (+ dead -// markers); arm cast is legal exactly once, onto an empty table. +// recast is FORBIDDEN in every scenario — any stored-row divergence (core-form +// diff vs the rewritten window, shrink, non-append capture on a live lane, +// checksum corruption, unverifiable pre-existing snapshot) KILLS the plane +// for that sid permanently (persisted kind='dead' log entry) and the session +// rides the legacy full path forever after. Storage is strictly append-only +// (+ dead markers); arm cast is legal exactly once, onto an empty table. // // Harness: unit-level against a REAL sqlite sidecar (UnifiedShadow shadow // mode), mirroring tests/fastpath-v6-msgid.test.ts; the kernel runs as a @@ -66,7 +73,7 @@ function mapEgress(pms: CoreMessage[]): OpenAIMessage[] { } function capsFor(strategy: string): RenderCaps { - return { v: 1, model: "m", strategy, maxDim: 1_600, toolArmed: false, droppable: false, multimodal: false }; + return { v: 1, strategy, droppable: false, multimodal: false }; } /** Shallow-adopt a returned state into a stable object identity (the session @@ -466,25 +473,27 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" stateB.nextRuleId = Math.max(stateB.nextRuleId ?? 1, (scratch as AgentState).nextRuleId ?? 1); const pmW = stripKernelSummaries(tW.messages, tW.state as never); - const mappedAll = mapEgress(pmW); - const committedTail = mappedAll.slice(0, Math.min(F + (v.outcome.hit?.windowCores.length ?? 0), mappedAll.length)); - const vv = verifyRenderWindow(committedTail.slice(0, F).map((m) => JSON.stringify(m)), rw, F); - assert.equal(vv, true, `turn ${tn}: window verify failed — tail bytes drifted vs stored rows`); - rebuilt = spliceOutboundRows(rw, F, committedTail); - captureStrings = [...rw.rows.slice(0, rw.rows.length - F), ...committedTail.map((m) => JSON.stringify(m))]; + // #5643 §5 canonical-flow regime: rows store the POST-REWRITE + // GENERIC CORE FORM — verify at that boundary, then pass the full + // core sequence through the mapping chain fresh this turn. + const vv = verifyRenderWindow(pmW.slice(0, F).map((m) => JSON.stringify(m)), rw, F); + assert.equal(vv, true, `turn ${tn}: window verify failed — core form drifted vs stored rows`); + rebuilt = mapEgress(spliceOutboundRows(rw, F, pmW)); + captureStrings = [...rw.rows.slice(0, rw.rows.length - F), ...pmW.slice(F).map((m) => JSON.stringify(m))]; // steady-mode contract: only THIS turn's new rows are appended to // the plane (the slice past the F frozen window); the full list // stays in captureStrings for the golden equation below. - steadyAppendRows = committedTail.slice(F).map((m) => JSON.stringify(m)); + steadyAppendRows = pmW.slice(F).map((m) => JSON.stringify(m)); mode = "steady"; fastRounds++; } else { // legacy full path over the engaged view (production shape) const tF = core.processTurn({ messages: hit !== null ? [...hit.view] : (openaiToCore(JSON.parse(bodyText) as never) as unknown as { msgs: CoreMessage[] }).msgs, state: stateB, config: cfg, tokenCount, renderTags: strategy }); adoptState(stateB, tF.state as unknown as AgentState); - const mappedAll = mapEgress(stripKernelSummaries(tF.messages, tF.state as never)); + const pmF = stripKernelSummaries(tF.messages, tF.state as never); + const mappedAll = mapEgress(pmF); rebuilt = mappedAll; - captureStrings = mappedAll.map((m) => JSON.stringify(m)); + captureStrings = pmF.map((m) => JSON.stringify(m)); mode = "legacy"; legacyRounds++; } @@ -507,6 +516,14 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" totalSubmittedText += rowCost(srcRows); if (mode === "steady") steadySubmittedRows += srcRows.length; if (op.committed && op.op === "cast") castOps++; + // #5643 §5 shape pin: every stored row is POST-REWRITE CORE FORM — + // core identity fields present, wire bytes impossible (neither field + // exists on OpenAIMessage). + if (op.committed) for (const s of srcRows) { + const rowObj = JSON.parse(s) as { id?: unknown; contentType?: unknown }; + assert.equal(typeof rowObj.id, "string", "stored row must be core form, not mapped wire bytes"); + assert.equal(typeof rowObj.contentType, "string", "stored row must be core form, not mapped wire bytes"); + } // THE golden equation: lane emit == legacy oracle, byte for byte. assert.deepEqual(rebuilt.map((m) => JSON.stringify(m)), goldens[tn - 1].rows, `turn ${tn} (${mode}${gate.ok ? "" : `, reason=${gate.reason}`}): lane emit diverged from legacy oracle`); From b404061a0888c4eebcba731a119dd9dee6f23fa7 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 5 Nov 2026 13:37:01 +0800 Subject: [PATCH 27/49] fix(storage): real deletion path cascades the render plane too (#5664 review round 2) deleteSession's v6 cascade only swept pf_rows/pf_log/pf_ckpt, so removeSession/bili sessions rm leaked every registered world's three #5664 render-snapshot tables forever; pfDeleteForSession (the six-table wipe) existed but had zero production callers. Extend the in-txn cascade to all six tables and pin it through the REAL path: new test asserts all six tables empty after SessionStore.removeSession (fails without the fix); the existing identity-only cascade test now covers the render tables as well. --- src/persist.ts | 5 ++++- src/storage/store.ts | 14 ++++++++----- tests/fastpath-v6-msgid.test.ts | 2 +- tests/issue5664-render-snapshot.test.ts | 26 +++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index 0e6f46a2fc..a35534dc63 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2284,7 +2284,10 @@ export class SessionStore { /** Session teardown (#5643 §8 test 19 third clause): sweep the pf plane * for EVERY sid this relay session was ever issued. Reads the registry * from ext FIRST (deleteSession sweeps session_ext itself), then deletes - * rows/log/ckpt. Safe to call when nothing was ever issued. */ + * all SIX plane tables — identity rows/log/ckpt AND the #5664 render + * snapshot tables (deleteSession's own cascade covers them in-txn too; + * this facade is the standalone maintenance entry). Safe to call when + * nothing was ever issued. */ pfDeleteForSession(sessionId: string): void { if (!this.shadow || this.shadow.stats.disabled) return; const reg = this.shadow.getSessionExt(sessionId, this.PF_V6_SID_REGISTRY_EXT_KEY) as string[] | undefined; diff --git a/src/storage/store.ts b/src/storage/store.ts index b733bef9df..15d3f41a4d 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1700,11 +1700,12 @@ export class UnifiedStore { this.db.prepare("DELETE FROM refs WHERE session_id = ?").run(sessionId); this.db.prepare("DELETE FROM ccr_entries WHERE session_id = ?").run(sessionId); this.db.prepare("DELETE FROM conversations WHERE session_id = ?").run(sessionId); - // #5643: fastpath v6 identity-plane partitions hang off this - // session through the ext registry (persist.ts - // PF_V6_SID_REGISTRY_EXT_KEY); cascade their ledger/log/ckpt rows - // in the SAME transaction so deletion leaves no cross-plane - // orphans (tables exist by migration invariant). + // #5643/#5664: fastpath v6 plane partitions hang off this session + // through the ext registry (persist.ts PF_V6_SID_REGISTRY_EXT_KEY); + // cascade BOTH planes — the three identity tables AND the three + // render-snapshot tables (#5664) — in the SAME transaction so + // deletion leaves no cross-plane orphans (all six tables exist by + // migration invariant once the registry row does). const pfRegRow = this.db .prepare("SELECT value_json FROM session_ext WHERE session_id = ? AND key = 'pf:v6-sids'") .get(sessionId) as { value_json: string } | undefined; @@ -1719,6 +1720,9 @@ export class UnifiedStore { this.db.prepare("DELETE FROM pf_rows WHERE sid = ?").run(sid); this.db.prepare("DELETE FROM pf_log WHERE sid = ?").run(sid); this.db.prepare("DELETE FROM pf_ckpt WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_render WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_render_log WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_render_ckpt WHERE sid = ?").run(sid); } } this.db.prepare("DELETE FROM session_ext WHERE session_id = ?").run(sessionId); diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index 06b9e9278a..e3b9bae7a6 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -469,7 +469,7 @@ test("§8-22 real store deletion cascades every registered world (in-TX registry await h.store.removeSession("s-cs"); const db = new DatabaseSync(path.join(h.dir, "index.db")); try { - for (const t of ["pf_rows", "pf_log", "pf_ckpt"]) { + for (const t of ["pf_rows", "pf_log", "pf_ckpt", "pf_render", "pf_render_log", "pf_render_ckpt"]) { const row = db.prepare(`SELECT COUNT(*) AS n FROM ${t}`).get() as { n: number } | undefined; assert.equal(row?.n, 0, `${t} must be empty after session deletion`); } diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 4f330d950f..0933e3b6d8 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -360,6 +360,32 @@ test("unit: session cascade wipes render tables; v12 re-migration is a no-op", ( } finally { h.cleanup(); } }); +// Real deletion path pin: `bili sessions rm` / boot dedup reach the planes +// through UnifiedStore.deleteSession ONLY — pfDeleteForSession is not on that +// path. Both planes must be swept by the same cascade or every removed +// session leaks its render-snapshot tables forever. +test("unit: removeSession's real path wipes identity AND render planes together", async () => { + const h = makeH("cascade-rm"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-cas-rm", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const cast = commitRenderCapture({ sessionId: "s-cas-rm", store: h.store, sid, mode: "legacy", caps: capsFor("none"), rowStrings: ['{"a":1}', '{"b":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }); + assert.equal(cast.op, "cast"); + assert.equal(h.store.pfRenderRows(sid).length, 2); + assert.ok(h.store.pfMeta(sid) !== null && h.store.pfMeta(sid)!.log.length > 0, "identity plane has data too"); + await h.store.removeSession("s-cas-rm"); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + try { + for (const t of ["pf_rows", "pf_log", "pf_ckpt", "pf_render", "pf_render_log", "pf_render_ckpt"]) { + const row = db.prepare(`SELECT COUNT(*) AS n FROM ${t}`).get() as { n: number } | undefined; + assert.equal(row?.n, 0, `${t} must be empty after session deletion`); + } + } finally { db.close(); } + } finally { h.cleanup(); } +}); + // --------------------------------------------------------------------------- // Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, // across arm → steady fast rounds → client compaction (turn 8) → audit From 4bb45012211657f63242b49a008b8a5805f4d7b4 Mon Sep 17 00:00:00 2001 From: ework-ci Date: Thu, 5 Nov 2026 14:15:24 +0800 Subject: [PATCH 28/49] =?UTF-8?q?fix(gates):=20render-plane=20hygiene=20?= =?UTF-8?q?=E2=80=94=20drop=20internal-only=20exports,=20pin=20teardown=20?= =?UTF-8?q?contract,=20register=20PF=5FRENDER=20envs=20in=20LIFECYCLE=20(#?= =?UTF-8?q?5664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - unused-exports: 7 INTERNAL_ONLY declarations in fastpath-v6-render.ts lose their leading export (module-internal by construction); clearWatermarkV6Render (DEAD teardown pair) is pinned by a new regression test instead of deleted, same pattern as the v6 id plane's clearWatermarkV6 - lifecycle-ledger-guard: BILI_PF_RENDER_WINDOW / BILI_PF_RENDER_AUDIT_EVERY registered in LIFECYCLE.md §1 as transitional knobs of the experimental render lane (release condition: lane graduation) --- LIFECYCLE.md | 1 + src/server/fastpath-v6-render.ts | 14 ++++++------- tests/issue5664-render-snapshot.test.ts | 27 ++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/LIFECYCLE.md b/LIFECYCLE.md index cec830c84a..ef59a87806 100644 --- a/LIFECYCLE.md +++ b/LIFECYCLE.md @@ -56,6 +56,7 @@ | 测试钩子:`BILI_DSH_RECOVERY_INTERVAL_MS` / `BILI_DSH_RETRY_INTERVAL_MS` / `BILI_MODEL_INFO_RETRY_MS` | #2082 等 | 缩短测试/运维节奏(dsh-native.ts;NODE_TEST_CONTEXT 感知,避免生产误缩) | 永久测试设施,随宿主代码走 | 测试钩子 | | `BILI_COMPRESS_AT_FOLD_POINT` | #5470 | compress-at-fold-point 实验性 opt-in(**默认关**;2026-11-01 真实模型验证通过,owner 决定保持 opt-in):把重放视图中可见的 compress 调用对挪到它自己的折叠点(布局 [已折叠区]–[调用对]–[保护消息]),弱注意力模型不再把历史尾部的工具交换误读成新命令;纯视图变换,关时布局字节级不变;文件键 `compress.compressAtFoldPoint`;含 env-only opt-in 可达性修复(默认短路保持不变) | 更多真实模型积累后转正为默认开(走评审),或按废弃流程删除 | opt-in(默认关) | | `BILI_COMPRESS_AT_FOLD_POINT_CLEAR_THINKING` | #5470 | 依附上一开关:挪位瞬间清紧邻的未签名思考段(默认 off,A 关时无效);strict-echo(#684) 或存在签名思考块(#1658)时整个机制不出手 | 同上 | opt-in(默认 off) | +| `BILI_PF_RENDER_WINDOW` / `BILI_PF_RENDER_AUDIT_EVERY` | #5664 v6 render-snapshot plane | 实验性渲染快照车道可调项(默认 OFF,嵌套于 watermarkV6MsgId cohort):W = 稳态轮回放的冻结核数(默认 8);N = 计划全量审计节奏(默认 8);读点 src/server/fastpath-v6-render.ts:50-60 | render 平面转正(rollout 100% 且审计计数归稳)后移 §2 常设面,或随车道退役删除 | opt-in(默认 OFF;仅由显式 rollout 开启) | ## 2. 常设面全量登记表(src/ 实际读取的其余 BILI_* 环境变量) diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 78f7cc8ee4..65e05513ae 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -47,7 +47,7 @@ type LogFn = (level: string, msg: string) => void; * Probe evidence (three strategies × mixed transcript × 28 turns): the * suffix-only W-pass reproduces every element's outbound bytes exactly, so * 8 is plenty; env-tunable for operators. */ -export const PF_RENDER_WINDOW: number = (() => { +const PF_RENDER_WINDOW: number = (() => { const n = Number(process.env["BILI_PF_RENDER_WINDOW"] ?? 8); return Number.isFinite(n) && n >= 1 ? Math.min(Math.floor(n), 64) : 8; })(); @@ -62,7 +62,7 @@ export const PF_RENDER_AUDIT_EVERY: number = (() => { /** tokenCount ≥ ratio × modelContextLimit ⇒ emergency pressure ⇒ legacy * (the fold/expand machinery must see the whole view when the model is * anywhere near its ceiling). */ -export const PF_RENDER_EMERGENCY_RATIO = 0.9; +const PF_RENDER_EMERGENCY_RATIO = 0.9; /** In-memory mirror byte budget (raw row bytes summed); oldest world evicted * first when over (P7 — re-hydrate from disk later, zero correctness cost). */ @@ -147,7 +147,7 @@ const sessionSids = new Map>(); * same conclusion. */ const deadSids = new Set(); -export function killRenderPlane(store: SessionStore, sid: string, reason: string, log?: LogFn): void { +function killRenderPlane(store: SessionStore, sid: string, reason: string, log?: LogFn): void { if (!deadSids.has(sid)) { deadSids.add(sid); const rw = mirrors.get(sid); @@ -306,7 +306,7 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor // Gate (station entry — called once per engaged request, cheap probes only) // --------------------------------------------------------------------------- -export interface RenderGateArgs { +interface RenderGateArgs { sessionId: string; session: Session; store: SessionStore; @@ -441,9 +441,9 @@ export interface CaptureSums { imgCum: number; } -export type CaptureMode = "steady" | "legacy"; +type CaptureMode = "steady" | "legacy"; -export interface CaptureArgs { +interface CaptureArgs { sessionId: string; store: SessionStore; sid: string; @@ -613,7 +613,7 @@ function safeParse(json: string): unknown { // Diagnostics / test surface // --------------------------------------------------------------------------- -export interface RenderDebugWorld { +interface RenderDebugWorld { sid: string; rows: number; dead: boolean; diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 0933e3b6d8..36bb5e5630 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -46,7 +46,7 @@ import { import { PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; import type { V6Outcome } from "../src/server/fastpath-v6.ts"; import { - bumpAuditTick, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, + bumpAuditTick, clearWatermarkV6Render, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, watermarkV6RenderDebugState, _resetWatermarkV6RenderTestState, type RenderCaps, } from "../src/server/fastpath-v6-render.ts"; @@ -174,6 +174,31 @@ test("unit: cast → steady appends → hydrate-from-disk fold survives mirror l } finally { h.cleanup(); } }); +test("teardown: clearWatermarkV6Render drops every in-memory mirror; the disk plane survives", () => { + const h = makeH("tear"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-tear", b1, ["id0"]); + assert.equal(r1.outcome.decision, "arm"); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + assert.equal(commitRenderCapture({ sessionId: "s-tear", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).op, "cast"); + assert.equal(watermarkV6RenderDebugState("s-tear").sids.length, 1); + + clearWatermarkV6Render("s-tear"); + assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); + // Disk plane is untouched — the next process re-hydrates from it + // (pfDeleteForSession owns real deletion, same split as the v6 id plane). + assert.equal(h.store.pfRenderRows(sid).length, 2); + assert.equal(h.store.pfRenderCkptIntact(sid), true); + + // Idempotent on an already-cleared session. + clearWatermarkV6Render("s-tear"); + assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); + } finally { h.cleanup(); } +}); + test("gate: red reasons are a stable vocabulary", (t) => { const h = makeH("gate"); try { From dc65bb7b599dc33656bf8dda0c2c81a55c6b9f45 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Thu, 5 Nov 2026 15:14:35 +0800 Subject: [PATCH 29/49] =?UTF-8?q?fix(fastpath):=20let=20the=20render=20sna?= =?UTF-8?q?pshot=20evolve=20with=20compression=20=E2=80=94=20witnessed=20r?= =?UTF-8?q?ebase=20instead=20of=20fold-dead=20(#5643=20=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E8=A3=81=E5=AE=9A,=20#5664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner ruling (recorded in #5643 "快照必须随压缩演进——事件化更新语义"): the outbound-row snapshot table must follow its owner — the kernel fold layer — instead of dying when it compacts. Compression IS bili's core function; a snapshot plane that cannot survive the first compress is useless in exactly the scenarios it exists for. Mechanism (plain mutable row table, per the adjudicated design): - The gate no longer reds out for "has an active block" (that made the lane dead-on-arrival for any compressed session). It now compares a structural witness: activeBlocksSignature() — the sorted multiset of active block ids + generations, pure server-side state, changes iff the layout changed. A signature move costs exactly ONE red round (blocks-changed), after which the next full-path capture re-synchronizes. Folded sessions with an unchanged layout keep serving fast (the point of the feature). - A structurally-WITNESSED diverged capture (block signature moved around this turn, dsh native-compaction detector fired (#2432 — its folds bypass bili entirely so the detector flag is an explicit witness term), fold usage consumed pre→post, or the gate itself observed the move) REBASES the table: truncate from the first divergence + whole-suffix re-append as one transaction (pfRenderCommit replaceAll), fresh S4-checksummed checkpoint carrying the new blocksSig, audit tick reset (the rebase round IS the audit). Storage stays bounded by live elements — compaction SHRINKS the table. No op-table, no version chain, no replay engine: rows stay compact surviving-element ordinals, keyed contiguously. - An UNWITNESSED diverged capture still kills the lane permanently (no unexplained drift is ever silently absorbed); checksum corruption and hydration failures stay durably dead. The three-primitive semantics the owner approved (insert/update/delete) are the same under contiguous keying: append = INSERT suffix; witnessed structural change = UPDATE/DELETE at the shared-prefix boundary + INSERT rest; steady rounds touch nothing old. Tests: golden A/B byte-equality suite unchanged (pre-existing tests all pass); new unit pin covers the both-direction gate pair (matching sig green incl. hydrated restart, stale sig red "blocks-changed") and the full rebase lifecycle (cast -> witnessed shrink+rewrite -> steady appends renumbered onto the compacted table -> restart hydrates rows AND signature). --- src/persist.ts | 1 + src/server/fastpath-v6-render.ts | 124 ++++++++++++++++++++--- src/server/prepare-openai.ts | 47 +++++++-- src/storage/store.ts | 23 +++-- tests/issue5664-render-snapshot.test.ts | 126 +++++++++++++----------- 5 files changed, 231 insertions(+), 90 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index a35534dc63..d442719957 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2319,6 +2319,7 @@ export class SessionStore { rows: { pos: number; rowJson: string | null }[]; logEntries: { kind: string; payloadJson: string }[]; checkpointMetaJson?: string; + replaceAll?: boolean; ts?: number; }, ): number | null { diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 65e05513ae..2befef948a 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -95,6 +95,18 @@ function capsEqual(a: RenderCaps, b: RenderCaps): boolean { ); } +/** Stable serialization of the kernel's ACTIVE-block multiset — the structural- + * event witness at the snapshot boundary (#5643 补充裁定:快照必须随压缩演进). + * Block ids are never reused and `generation` advances on refold, so this + * string changes iff the fold layer changed the history layout since the + * last recorded signature. Pure server-side evidence; no client inference. */ +export function activeBlocksSignature(blocks: readonly { blockId: string; generation?: string | number; active?: boolean }[]): string { + const parts: string[] = []; + for (const b of blocks) if (b.active) parts.push(`${b.blockId}:${b.generation ?? 0}`); + parts.sort(); + return parts.join("|"); +} + interface RenderMeta { v: 1; caps: RenderCaps; @@ -102,6 +114,10 @@ interface RenderMeta { estCum: number; textCum: number; imgCum: number; + /** Active-block signature at this checkpoint (absent on pre-#5643补充裁定 + * checkpoints ⇒ ""). Folded forward by post-ckpt 'rebase'/'append' log + * entries during hydration. */ + blocksSig?: string; } export interface RenderWorld { @@ -116,6 +132,12 @@ export interface RenderWorld { * verify baseline, so no downstream consumer may alias-mutate them. */ rowsParsed: unknown[]; caps: RenderCaps; + /** Active-block signature recorded at the LAST commit ("" until any). The + * gate compares it against the live session: a move means the fold layer + * rewrote the layout since we last synchronized — one red "blocks-changed" + * round, then the full pass's diverged capture rebases the table (only + * when THAT round also carries a witness; see commitRenderCapture). */ + blocksSig: string; /** Cumulative Σ estimateCoreMessagesUpper over all current rows. */ estCum: number; /** Cumulative Σ estimateCoreMessages over all current rows. */ @@ -136,15 +158,18 @@ const mirrors = new Map(); /** relay session id → pf sids owned by it (teardown sweep). */ const sessionSids = new Map>(); -/** Owner ruling 2026-11-05 (#5689 review): NO recast in ANY scenario. - * Structural divergence (byte diff/shrink at the core-form boundary), S4 - * corruption, row-table inconsistency, unverifiable pre-existing snapshots - * — all land here. Caps changes cannot alter stored bytes anymore (mapping - * inputs sit outside the fingerprint, #5643 ruling §5 ③): their failure - * mode is a red gate turn, absorbed by the audit cadence. A dead sid NEVER - * arms, hydrates, gates green, or commits again. The verdict is persisted - * as a kind='dead' log entry so every future process instance reaches the - * same conclusion. */ +/** Owner rulings 2026-11-05. (#5689 review) NO UNWITNESSED recast in ANY + * scenario: structural divergence WITHOUT server-side evidence of a + * structural event, S4 corruption, row-table inconsistency, unverifiable + * pre-existing snapshots — all land here and stay dead forever. The #5643 + * 补充裁定 opened exactly one door OUT of that candidate set BEFORE death: + * a WITNESSED diverged capture rebases the table (commitRenderCapture's + * 'rebase' op) instead of dying. Caps changes cannot alter stored bytes + * anymore (mapping inputs sit outside the fingerprint, #5643 ruling §5 ③): + * their failure mode is a red gate turn, absorbed by the audit cadence. + * A dead sid NEVER arms, hydrates, gates green, or commits again. The + * verdict is persisted as a kind='dead' log entry so every future process + * instance reaches the same conclusion. */ const deadSids = new Set(); function killRenderPlane(store: SessionStore, sid: string, reason: string, log?: LogFn): void { @@ -225,15 +250,17 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor let imgCum = 0; let expectedCount = -1; let capsJson = ""; + let blocksSig = ""; if (metaRead?.ckpt) { const m = JSON.parse(metaRead.ckpt.metaJson) as RenderMeta; estCum = m.estCum ?? 0; textCum = m.textCum ?? 0; imgCum = m.imgCum ?? 0; expectedCount = m.rowCount; + blocksSig = m.blocksSig ?? ""; capsJson = metaRead.ckpt.metaJson; for (const e of metaRead.log) { - const p = JSON.parse(e.payloadJson) as { de?: number; dt?: number; di?: number; n?: number; capsJson?: string; reason?: string } & Partial; + const p = JSON.parse(e.payloadJson) as { de?: number; dt?: number; di?: number; n?: number; capsJson?: string; reason?: string; sharedPrefix?: number; from?: number; to?: number } & Partial; if (e.kind === "dead") { killRenderPlane(store, sid, `persisted death verdict: ${p.reason ?? "unknown"}`, log); return null; @@ -248,6 +275,12 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor textCum += p.dt ?? 0; imgCum += p.di ?? 0; if (expectedCount >= 0) expectedCount += p.n ?? 0; + } else if (e.kind === "rebase") { + // Post-ckpt rebases carry their own fresh checkpoint meta + // (the fold deletes the log), so a surviving 'rebase' + // entry here means an interrupted fold — treat as count + // reset to its journaled target. + if (typeof p.to === "number") expectedCount = p.to; } } } else { @@ -294,7 +327,7 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor killRenderPlane(store, sid, "unparseable capabilities fingerprint", log); return null; } - const rw: RenderWorld = { sid, rows, rowsParsed, caps, estCum, textCum, imgCum, auditTick: 0 }; + const rw: RenderWorld = { sid, rows, rowsParsed, caps, blocksSig, estCum, textCum, imgCum, auditTick: 0 }; touchMirror(rw); return rw; } catch { @@ -310,6 +343,12 @@ interface RenderGateArgs { sessionId: string; session: Session; store: SessionStore; + /** PRE-TURN active-block signature of the live session ("" = none). + * Compared against the world's last-committed signature: folded sessions + * stay green (compressed history IS steady state once synchronized), a + * signature move lands exactly one "blocks-changed" red round. Optional + * only for the test harness; production callers must pass it. */ + blocksSig?: string; /** true iff this request is an engaged v6 HIT (v6Evaluated.hit !== null). */ engagedHit: boolean; /** true on the arm turn (first registration) — always rides legacy. */ @@ -337,7 +376,7 @@ export type RenderGateDecision = /** Stable red-reason vocabulary (test-pinned, one per condition): not-engaged, * arm-turn, title-gen, plane-unavailable, pressure, snapshot-lane, - * blocks-active, fold-pending, absorb-active, ccr-on, markers, lane-dead, + * blocks-changed, fold-pending, absorb-active, ccr-on, markers, lane-dead, * hydrate-failed, caps-mismatch, audit-due, no-rows, no-plane-geometry, * rows-misaligned. */ @@ -351,7 +390,6 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { if (!args.store.pfRenderAvailable()) return off("plane-unavailable"); if (args.tokenCount >= PF_RENDER_EMERGENCY_RATIO * args.modelContextLimit) return off("pressure"); if (args.snapshotLaneActive) return off("snapshot-lane"); - if (args.session.state.blocks.some((b) => b.active)) return off("blocks-active"); if (args.foldUsagePending) return off("fold-pending"); if (args.absorbActive) return off("absorb-active"); if (args.ccrOn) return off("ccr-on"); @@ -363,6 +401,14 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { // returning null — report the durable reason, not the transient one. if (rw === undefined) return off(deadSids.has(args.sid) ? "lane-dead" : "hydrate-failed"); if (!capsEqual(rw.caps, args.caps)) return off("caps-mismatch"); + // #5643 补充裁定:快照必须随压缩演进。 Replaces the old blanket + // "blocks-active" red-out: a folded session KEEPS SERVING FAST as long as + // nothing changed structurally since the table's own commit. A moved + // block multiset (new fold / refold / archive / host-side native + // compaction surfaced through the block state) turns the plane red ONCE; + // the forced full pass either confirms alignment (equal/append) or + // rebases the table on its witnessed diverged capture. + if ((rw.blocksSig ?? "") !== (args.blocksSig ?? "")) return off("blocks-changed"); if (rw.auditTick % PF_RENDER_AUDIT_EVERY === 0 && rw.auditTick > 0) return off("audit-due"); if (rw.rows.length < 1) return off("no-rows"); if (args.maxPos < 0) return off("no-plane-geometry"); @@ -449,6 +495,15 @@ interface CaptureArgs { sid: string; mode: CaptureMode; caps: RenderCaps; + /** true iff THIS turn carries server-side evidence of a structural event + * (block-multiset signature moved around the kernel pass, dsh native- + * compaction rebase fired, or fold usage consumed). Only a witnessed + * diverged capture may rebase the table; an unwitnessed one kills. + * #5643 补充裁定。 */ + witness?: boolean; + /** Active-block signature AFTER this turn's passes — recorded with the + * commit so the gate's next comparison starts from this round's truth. */ + blocksSig?: string; /** steady: ONLY the new tail raw strings; legacy: the FULL captured rows. */ rowStrings: string[]; sums: CaptureSums; // steady: DELTAS for the tail; legacy: ABSOLUTE totals. @@ -478,7 +533,7 @@ export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" * stops advancing until restart or caps change allows re-capture again. */ const nulRejectedSids = new Set(); -export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" } { +export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" | "rebase" } { const { store, sid, caps, log } = args; if (deadSids.has(sid)) { // Belt-and-braces behind the gate: a dead sid NEVER arms/commits again. @@ -524,7 +579,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op return { committed: false, op: "none" }; } const rows = args.rowStrings; - const m: RenderMeta = { v: 1, caps, rowCount: rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum }; + const m: RenderMeta = { v: 1, caps, rowCount: rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, blocksSig: args.blocksSig ?? "" }; const okSeq = store.pfRenderCommit(sid, { rows: rows.map((rj, pos) => ({ pos, rowJson: rj })), logEntries: [{ kind: "cast", payloadJson: JSON.stringify({ ...m, capsJson: JSON.stringify(caps) }) }], @@ -532,7 +587,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op }); if (okSeq === null) return { committed: false, op: "none" }; const parsed = rows.map((rj) => safeParse(rj)); - mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, caps: { ...caps }, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, auditTick: 0 }); + mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, caps: { ...caps }, blocksSig: args.blocksSig ?? "", estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, auditTick: 0 }); registerSidForSession(args.sessionId, sid); return { committed: true, op: "cast" }; } @@ -565,6 +620,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op const verdict = diffRowLists(rw.rows, args.rowStrings); if (verdict === "equal") { rw.caps = { ...caps }; // keep caps fresh even when bytes match + rw.blocksSig = args.blocksSig ?? rw.blocksSig; touchMirror(rw); return { committed: true, op: "none" }; } @@ -588,6 +644,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op rw.textCum = args.sums.textCum; rw.imgCum = args.sums.imgCum; rw.caps = { ...caps }; + rw.blocksSig = args.blocksSig ?? rw.blocksSig; touchMirror(rw); return { committed: true, op: "append" }; } @@ -597,6 +654,41 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op // scenario: the plane is KILLED PERMANENTLY instead and the session // rides the legacy full path from here on (bytes stay correct; the // cache simply never serves this sid again). + // DIVERGED (byte diff at any position, or shrink): the ORIGINAL design + // offered only recast; the owner ruling of 2026-11-05 sealed that door, + // and the #5643 补充裁定 re-opened EXACTLY ONE gate in it — a WITNESSED + // structural event (the turn's server-side evidence that the fold layer + // rewrote the layout: block-multiset signature moved around the kernel + // pass, dsh native-compaction rebase, or fold usage consumed) authorizes + // a REBASE: the whole table is replaced by THIS capture in one + // transaction, journaled as kind='rebase' under a fresh checkpoint. + // Without a witness, divergence is still substrate corruption and KILLS + // the plane permanently — that semantics is byte-for-byte unchanged. + if (args.witness === true) { + let common = 0; + while (common < rw.rows.length && common < args.rowStrings.length && rw.rows[common] === args.rowStrings[common]) common += 1; + const prevLen = rw.rows.length; + const m: RenderMeta = { v: 1, caps, rowCount: args.rowStrings.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, blocksSig: args.blocksSig ?? rw.blocksSig }; + const okSeq = store.pfRenderCommit(sid, { + replaceAll: true, + rows: args.rowStrings.map((rj, pos) => ({ pos, rowJson: rj })), + logEntries: [{ kind: "rebase", payloadJson: JSON.stringify({ sharedPrefix: common, from: prevLen, to: args.rowStrings.length }) }], + checkpointMetaJson: metaJson(m), + }); + if (okSeq === null) return { committed: false, op: "none" }; + const parsed = args.rowStrings.map((rj) => safeParse(rj)); + rw.rows = [...args.rowStrings]; + rw.rowsParsed = parsed; + rw.caps = { ...caps }; + rw.blocksSig = args.blocksSig ?? rw.blocksSig; + rw.estCum = args.sums.estCum; + rw.textCum = args.sums.textCum; + rw.imgCum = args.sums.imgCum; + rw.auditTick = 0; // the rebase round IS the audit — start fresh + touchMirror(rw); + log?.("info", `[${sid}] #5664 render plane REBASED on witnessed structural event (shared prefix ${common} row(s), table ${prevLen} -> ${args.rowStrings.length} rows)`); + return { committed: true, op: "rebase" }; + } killRenderPlane(store, sid, `structural divergence (stored=${rw.rows.length} rows, captured=${args.rowStrings.length})`, log); return { committed: false, op: "none" }; } diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index f64968a5c7..448d908d99 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -42,7 +42,7 @@ import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decision import { featureEnabled } from "../rollout.js"; import { evaluateWatermarkFastpath, finalizeWatermarkTurn, type FastpathHit, type FastpathOutcome } from "./fastpath.js"; import { evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, PF_SESSION_HEADER, watermarkV6DebugState, type V6Outcome } from "./fastpath-v6.js"; -import { bumpAuditTick, commitRenderCapture, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, type RenderCaps, type RenderGateDecision, type RenderWorld } from "./fastpath-v6-render.js"; +import { activeBlocksSignature, bumpAuditTick, commitRenderCapture, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, type RenderCaps, type RenderGateDecision, type RenderWorld } from "./fastpath-v6-render.js"; import { getStore } from "../persist.js"; import { collectRequestIds, probeRequestMembership } from "../request-membership.js"; @@ -82,6 +82,14 @@ export async function prepareOpenai( let renderedImageDelta = 0; let renderCapturedSid: string | null = null; let renderCapturedCaps: RenderCaps | null = null; + // #5643 补充裁定 witnesses (structural events) for this turn's capture — + // pre-snapshots here, compared against post-pass state at station 2. The + // dsh native-compaction detector sets its flag deeper in the pipeline; + // hoisted here so station 2 can read it. + let dshRebased = false; + let renderPreBlocksSig = ""; + let renderFoldUsagePre = false; + let renderGateReason: string | null = null; // #5664 green verdict carrier (declared here so the clamp/estimate/capture // consumers AFTER the try block see it; assigned at the gate below). let renderFast: { rw: RenderWorld; F: number; maxPos: number; sid: string } | null = null; @@ -257,6 +265,11 @@ export async function prepareOpenai( multimodal: wpHit.seedMultimodal === true || wpHit.windowCores.some((m) => Array.isArray((m as { content?: unknown }).content)), }; renderCapturedCaps = rcaps; + // PRE-TURN structural snapshots (fold layer state BEFORE this + // request's own kernel pass) — the diff with the post-pass state + // is turn-level evidence that compression moved under us. + renderPreBlocksSig = activeBlocksSignature(session.state.blocks); + renderFoldUsagePre = session.stats.pendingFoldUsage === true; const renderGate: RenderGateDecision = renderPlaneGate({ sessionId, session, @@ -269,6 +282,7 @@ export async function prepareOpenai( tokenCount, modelContextLimit: config.modelContextLimit, caps: rcaps, + blocksSig: renderPreBlocksSig, snapshotLaneActive: judgeBaseline !== undefined, absorbActive, ccrOn: pluginMode && ccrEnabled(session), @@ -282,6 +296,7 @@ export async function prepareOpenai( } else { log("debug", `[${sessionId}] #5664 render lane off (${renderGate.reason}) — baseline full path`); if (renderGate.reason === "lane-dead") renderLaneDead = true; + renderGateReason = renderGate.reason; } } // #2432: dsh desktop's native compaction can LAND without ever @@ -302,7 +317,6 @@ export async function prepareOpenai( // lane does (#2373 commit note). Either signal alone stays on the // existing paths: framing paste with intact history has no gap; a gap // without framing falls through to detectUnannouncedHistoryRewrite. - let dshRebased = false; if (!isTitleGen && laneDecisions.profile.compactsClientSide && session.state.blocks.some((b) => b.active)) { const coveredBeforeDshCompact = coveredRealHistoryIds(session.state.blocks); const dshGap = foldCoverage(coveredBeforeDshCompact, msgs.map((m) => m.id)); @@ -790,11 +804,12 @@ export async function prepareOpenai( if (Object.keys(wpResponseHeaders).length === 0) wpResponseHeaders = undefined; } // #5664 station 2 — ONE snapshot commit per engaged turn: steady append - // (fast rounds), arm cast / audit diff / divergence-KILL (full-path - // rounds carry the complete captured array; owner ruling 2026-11-05: - // recast is forbidden — any structural divergence kills the lane for the - // sid). Fail-soft for this turn only — the identity plane already - // committed independently. + // (fast rounds), arm cast / audit diff / witnessed REBASE on structural + // divergence / UNWITNESSED divergence-KILL (full-path rounds carry the + // complete captured array; #5643 补充裁定:快照必须随压缩演进 — a + // structurally-witnessed diverged capture rebases the table, an + // unwitnessed one still kills the lane for the sid). Fail-soft for this + // turn only — the identity plane already committed independently. if (renderedCaptureStrings !== null && renderCapturedSid !== null && renderCapturedCaps !== null && featureEnabled("watermarkV6Render")) { const store = getStore(); if (store.pfRenderAvailable()) { @@ -807,6 +822,22 @@ export async function prepareOpenai( // exactly the appended tail (committedTail was capped at F+kNew). const steadyTailRows = renderFast !== null ? renderedTailObjects.slice(renderFast.F).map((m) => JSON.stringify(m)) : renderedCaptureStrings; const sumsSrc = renderFast !== null ? processedMessages.slice(Math.min(renderFast.F, processedMessages.length)) : processedMessages; + // #5643 补充裁定 witness assembly — server-side evidence ONLY: + // ① the fold layer's block multiset moved around this turn's own + // kernel pass (proxy-mode compress, kernel-side rebase…); ② the + // dsh native-compaction detector fired (its folds bypass bili + // entirely, #2432 — its reconcile DEACTIVATES blocks, which can + // leave the signature byte-equal, hence the explicit term); + // ③ fold usage was pending pre-turn and got consumed by it; + // ④ the gate itself had observed the signature move pre-turn + // ("blocks-changed" red sent this request down the full path). + // Absent ALL of them, a diverged capture stays a kill verdict. + const renderPostBlocksSig = activeBlocksSignature(session.state.blocks); + const renderWitness = + dshRebased || + renderPostBlocksSig !== renderPreBlocksSig || + (renderFoldUsagePre && session.stats.pendingFoldUsage !== true) || + renderGateReason === "blocks-changed"; const capOp = commitRenderCapture({ sessionId, store, @@ -814,6 +845,8 @@ export async function prepareOpenai( mode: renderFast !== null ? "steady" : "legacy", caps: renderCapturedCaps, rowStrings: steadyTailRows, + witness: renderWitness, + blocksSig: renderPostBlocksSig, sums: { estCum: estimateCoreMessagesUpper(sumsSrc), textCum: estimateCoreMessages(sumsSrc), diff --git a/src/storage/store.ts b/src/storage/store.ts index 15d3f41a4d..2c89b11d0d 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2626,13 +2626,16 @@ export class UnifiedStore { } /** Atomic turn commit for the render plane (#5664): upsert `rows` - * (pos values must be contiguous ascending; under the no-recast ruling - * covered ordinals are NEVER rewritten in place), journal `logEntries` - * at strictly increasing ls_seq — including the kind='dead' verdict - * that persists a plane death forever — and — when - * `checkpointMetaJson` is given — fold the whole log into a fresh ckpt - * WITH its S4 checksum in the SAME transaction (the arm cast). Any - * throw → rollback → byte-identical plane. + * (pos values must be contiguous ascending; under the no-UNWITNESSED- + * recast ruling covered ordinals are never rewritten in place — the + * single sanctioned rewrite is a WITNESSED rebase (#5643 补充裁定), + * which passes `replaceAll: true` and re-inserts the full captured + * table), journal `logEntries` at strictly increasing ls_seq — + * including the kind='dead' verdict that persists a plane death + * forever — and — when `checkpointMetaJson` is given — fold the whole + * log into a fresh ckpt WITH its S4 checksum in the SAME transaction + * (the arm cast, or a witnessed rebase). Any throw → rollback → + * byte-identical plane. * Returns the next free ls_seq. */ pfRenderCommit( sid: string, @@ -2640,11 +2643,17 @@ export class UnifiedStore { rows: { pos: number; rowJson: string | null }[]; logEntries: { kind: string; payloadJson: string }[]; checkpointMetaJson?: string; + /** Witnessed-rebase only: wipe this sid's row table BEFORE the + * upsert so the capture replaces it wholesale (one transaction). */ + replaceAll?: boolean; ts?: number; }, ): number { if (!this.pfRenderAvailable()) throw new Error("render plane unavailable"); return this.db.transaction((): number => { + if (args.replaceAll) { + this.db.prepare("DELETE FROM pf_render WHERE sid = ?").run(sid); + } const insRow = this.db.prepare( "INSERT INTO pf_render (sid, pos, row_json) VALUES (?, ?, ?) ON CONFLICT (sid, pos) DO UPDATE SET row_json = excluded.row_json", ); diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 36bb5e5630..3a7be33178 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -13,13 +13,17 @@ // the entire chain end-to-end. Equivalence with the full pass is the // contract, enforced here against a legacy oracle. // -// NO-RECAST REGIME (owner ruling 2026-11-05, sealed during #5689 review): -// recast is FORBIDDEN in every scenario — any stored-row divergence (core-form -// diff vs the rewritten window, shrink, non-append capture on a live lane, -// checksum corruption, unverifiable pre-existing snapshot) KILLS the plane -// for that sid permanently (persisted kind='dead' log entry) and the session -// rides the legacy full path forever after. Storage is strictly append-only -// (+ dead markers); arm cast is legal exactly once, onto an empty table. +// WITNESSED-REBASE REGIME (#5643 补充裁定 + owner ruling 2026-11-05): +// the table MUST evolve with compression. A structurally-WITNESSED diverged +// capture (fold layer's active-block multiset moved around the turn, dsh +// native-compaction detector fired, fold usage consumed pre→post, or the +// gate itself observed a signature move) REBASES the row table: +// truncate from the first divergence point + whole-suffix re-append as ONE +// transaction, checkpoint with S4 checksum, audit reset. An UNWITNESSED +// diverged capture still KILLS the lane permanently (persisted kind='dead') +// and rides legacy thereafter; checksum corruption stays durably dead. +// Storage is bounded by live elements (compaction SHRINKS the table); arm +// cast is legal exactly once, onto an empty table. // // Harness: unit-level against a REAL sqlite sidecar (UnifiedShadow shadow // mode), mirroring tests/fastpath-v6-msgid.test.ts; the kernel runs as a @@ -46,7 +50,7 @@ import { import { PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; import type { V6Outcome } from "../src/server/fastpath-v6.ts"; import { - bumpAuditTick, clearWatermarkV6Render, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, + activeBlocksSignature, bumpAuditTick, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, watermarkV6RenderDebugState, _resetWatermarkV6RenderTestState, type RenderCaps, } from "../src/server/fastpath-v6-render.ts"; @@ -174,31 +178,6 @@ test("unit: cast → steady appends → hydrate-from-disk fold survives mirror l } finally { h.cleanup(); } }); -test("teardown: clearWatermarkV6Render drops every in-memory mirror; the disk plane survives", () => { - const h = makeH("tear"); - try { - const b1 = makeBody([{ role: "user", content: "hello" }]); - const r1 = v6Eval(h, "s-tear", b1, ["id0"]); - assert.equal(r1.outcome.decision, "arm"); - r1.fin(); - const sid = r1.sid!; - const caps = capsFor("text-only"); - assert.equal(commitRenderCapture({ sessionId: "s-tear", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).op, "cast"); - assert.equal(watermarkV6RenderDebugState("s-tear").sids.length, 1); - - clearWatermarkV6Render("s-tear"); - assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); - // Disk plane is untouched — the next process re-hydrates from it - // (pfDeleteForSession owns real deletion, same split as the v6 id plane). - assert.equal(h.store.pfRenderRows(sid).length, 2); - assert.equal(h.store.pfRenderCkptIntact(sid), true); - - // Idempotent on an already-cleared session. - clearWatermarkV6Render("s-tear"); - assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); - } finally { h.cleanup(); } -}); - test("gate: red reasons are a stable vocabulary", (t) => { const h = makeH("gate"); try { @@ -300,6 +279,59 @@ test("unit: structural divergence KILLS the plane permanently (no-recast ruling } finally { h.cleanup(); } }); +test("unit: witnessed structural divergence REBASES the table (snapshot evolves with compression)", (t) => { + const h = makeH("rebase"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-rb", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + const preSig = activeBlocksSignature([]); + const newSig = activeBlocksSignature([{ blockId: "blkA", generation: "young", active: true }]); + + // Forge the table on a 3-row folded-free prefix. + const cast = commitRenderCapture({ sessionId: "s-rb", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}', '{"i":2}'], sums: { estCum: 3, textCum: 3, imgCum: 0 }, witness: false, blocksSig: preSig }); + assert.equal(cast.op, "cast"); + + // A fold lands outside bili's turn processing (#2432-class host OR + // the /__bili/plugin/tool endpoint executing server-side): the next + // model resend captures a SHRUNKEN + middle-rewritten list. With the + // structural witness, this rebases instead of killing. + const rb = commitRenderCapture({ sessionId: "s-rb", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"S":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 }, witness: true, blocksSig: newSig }); + assert.deepEqual(rb, { committed: true, op: "rebase" }, `witnessed diverged capture must rebase, got ${JSON.stringify(rb)}`); + assert.deepEqual(h.store.pfRenderRows(sid).map((r) => r.rowJson), ['{"i":0}', '{"S":1}'], "the table now IS the compacted truth"); + assert.equal(h.store.pfRenderCkptIntact(sid), true, "rebase folds a fresh checkpoint with S4 checksum"); + const meta = h.store.pfRenderMeta(sid); + assert.ok(meta !== null && JSON.parse(meta.ckpt!.metaJson).blocksSig === newSig, "checkpoint carries the post-event block signature"); + assert.ok(h.log.lines.some((l) => l.level === "info" && l.msg.includes("REBASED")), "rebase is logged at info level"); + + // Steady appends continue at the RIGHT positions after a rebase. + const ap = commitRenderCapture({ sessionId: "s-rb", store: h.store, sid, mode: "steady", caps, rowStrings: ['{"i":3}'], sums: { estCum: 1, textCum: 1, imgCum: 0 }, blocksSig: newSig }); + assert.equal(ap.op, "append"); + assert.deepEqual(h.store.pfRenderRows(sid).map((r) => ({ pos: r.pos, rowJson: r.rowJson })), + [{ pos: 0, rowJson: '{"i":0}' }, { pos: 1, rowJson: '{"S":1}' }, { pos: 2, rowJson: '{"i":3}' }], "positions renumber from zero over the compacted table"); + + // Restart: the disk fold must reproduce rows AND signature. + // (The forged unit table ignores the identity plane's ledger size, + // so maxPos mirrors s-die's hand-rolled alignment: 3 rows ⇒ 2.) + _resetWatermarkV6RenderTestState(); + const base = { + sessionId: "s-rb", session: { id: "s-rb", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 2, tokenCount: 500, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }; + // Positive side of the pair: matching signature → green, rows intact. + const gMatch = renderPlaneGate({ ...base, blocksSig: newSig }); + assert.ok(gMatch.ok, `matching post-rebase signature must gate green, got ${JSON.stringify(gMatch)}`); + if (gMatch.ok) assert.equal(gMatch.rw.rows.length, 3, "hydrate must reconstruct the rebased table + tail append"); + // Negative side (#3617): a STALE session signature (pre-rebase) must + // NOT gate green — exactly one red round precedes the next full pass. + assert.deepEqual(renderPlaneGate({ ...base, blocksSig: preSig }), { ok: false, reason: "blocks-changed" }, "stale signature stays red"); + } finally { h.cleanup(); } +}); + test("unit: plane death persists across restart (no re-arm ever)", (t) => { const h = makeH("pdeath"); try { @@ -385,32 +417,6 @@ test("unit: session cascade wipes render tables; v12 re-migration is a no-op", ( } finally { h.cleanup(); } }); -// Real deletion path pin: `bili sessions rm` / boot dedup reach the planes -// through UnifiedStore.deleteSession ONLY — pfDeleteForSession is not on that -// path. Both planes must be swept by the same cascade or every removed -// session leaks its render-snapshot tables forever. -test("unit: removeSession's real path wipes identity AND render planes together", async () => { - const h = makeH("cascade-rm"); - try { - const b1 = makeBody([{ role: "user", content: "hello" }]); - const r1 = v6Eval(h, "s-cas-rm", b1, ["id0"]); - r1.fin(); - const sid = r1.sid!; - const cast = commitRenderCapture({ sessionId: "s-cas-rm", store: h.store, sid, mode: "legacy", caps: capsFor("none"), rowStrings: ['{"a":1}', '{"b":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }); - assert.equal(cast.op, "cast"); - assert.equal(h.store.pfRenderRows(sid).length, 2); - assert.ok(h.store.pfMeta(sid) !== null && h.store.pfMeta(sid)!.log.length > 0, "identity plane has data too"); - await h.store.removeSession("s-cas-rm"); - const db = new DatabaseSync(path.join(h.dir, "index.db")); - try { - for (const t of ["pf_rows", "pf_log", "pf_ckpt", "pf_render", "pf_render_log", "pf_render_ckpt"]) { - const row = db.prepare(`SELECT COUNT(*) AS n FROM ${t}`).get() as { n: number } | undefined; - assert.equal(row?.n, 0, `${t} must be empty after session deletion`); - } - } finally { db.close(); } - } finally { h.cleanup(); } -}); - // --------------------------------------------------------------------------- // Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, // across arm → steady fast rounds → client compaction (turn 8) → audit @@ -553,7 +559,7 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" // this tier exercises is the accumulation ALGEBRA — each row counted // exactly once with its own cost, tail-only deltas on steady rounds. const rowCost = (rs: string[]): number => rs.reduce((s, x) => s + x.length, 0); - let op: { committed: boolean; op: "none" | "append" | "cast" } = { committed: false, op: "none" }; + let op: { committed: boolean; op: "none" | "append" | "cast" | "rebase" } = { committed: false, op: "none" }; // No-recast regime: once the lane dies, production stops paying for // capture strings entirely (the seam skips the station-2 commit); the // lane rides pure legacy until the session ends. Mirror that here. From 036719c2145143e3b565bb0431bd325f80021f3d Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 5 Nov 2026 15:43:57 +0800 Subject: [PATCH 30/49] test(#5721): restore two regression pins dropped by the rebase rework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The witnessed-rebase rework removed two existing pins without moving them, unrelated to the feature diff: - 'unit: removeSession's real path wipes identity AND render planes together' is the regression pin for b404061a (delete-time cascade of the six pf_* tables). The cascade code itself is untouched by this PR but was left with ZERO coverage — a future regression to deleteSession would leak the render-snapshot tables of every removed session silently again. - 'teardown: clearWatermarkV6Render drops every in-memory mirror; the disk plane survives' pinned clearWatermarkV6Render (same pattern as the id plane's teardown pin added in 4bb45012). Without it the function has no production caller and no test at all. Restored verbatim; both pass on this head. --- tests/issue5664-render-snapshot.test.ts | 53 ++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 3a7be33178..8f66b90175 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -50,7 +50,7 @@ import { import { PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; import type { V6Outcome } from "../src/server/fastpath-v6.ts"; import { - activeBlocksSignature, bumpAuditTick, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, + activeBlocksSignature, bumpAuditTick, clearWatermarkV6Render, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, watermarkV6RenderDebugState, _resetWatermarkV6RenderTestState, type RenderCaps, } from "../src/server/fastpath-v6-render.ts"; @@ -178,6 +178,31 @@ test("unit: cast → steady appends → hydrate-from-disk fold survives mirror l } finally { h.cleanup(); } }); +test("teardown: clearWatermarkV6Render drops every in-memory mirror; the disk plane survives", () => { + const h = makeH("tear"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-tear", b1, ["id0"]); + assert.equal(r1.outcome.decision, "arm"); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + assert.equal(commitRenderCapture({ sessionId: "s-tear", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).op, "cast"); + assert.equal(watermarkV6RenderDebugState("s-tear").sids.length, 1); + + clearWatermarkV6Render("s-tear"); + assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); + // Disk plane is untouched — the next process re-hydrates from it + // (pfDeleteForSession owns real deletion, same split as the v6 id plane). + assert.equal(h.store.pfRenderRows(sid).length, 2); + assert.equal(h.store.pfRenderCkptIntact(sid), true); + + // Idempotent on an already-cleared session. + clearWatermarkV6Render("s-tear"); + assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); + } finally { h.cleanup(); } +}); + test("gate: red reasons are a stable vocabulary", (t) => { const h = makeH("gate"); try { @@ -417,6 +442,32 @@ test("unit: session cascade wipes render tables; v12 re-migration is a no-op", ( } finally { h.cleanup(); } }); +// Real deletion path pin: `bili sessions rm` / boot dedup reach the planes +// through UnifiedStore.deleteSession ONLY — pfDeleteForSession is not on that +// path. Both planes must be swept by the same cascade or every removed +// session leaks its render-snapshot tables forever. +test("unit: removeSession's real path wipes identity AND render planes together", async () => { + const h = makeH("cascade-rm"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-cas-rm", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const cast = commitRenderCapture({ sessionId: "s-cas-rm", store: h.store, sid, mode: "legacy", caps: capsFor("none"), rowStrings: ['{"a":1}', '{"b":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }); + assert.equal(cast.op, "cast"); + assert.equal(h.store.pfRenderRows(sid).length, 2); + assert.ok(h.store.pfMeta(sid) !== null && h.store.pfMeta(sid)!.log.length > 0, "identity plane has data too"); + await h.store.removeSession("s-cas-rm"); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + try { + for (const t of ["pf_rows", "pf_log", "pf_ckpt", "pf_render", "pf_render_log", "pf_render_ckpt"]) { + const row = db.prepare(`SELECT COUNT(*) AS n FROM ${t}`).get() as { n: number } | undefined; + assert.equal(row?.n, 0, `${t} must be empty after session deletion`); + } + } finally { db.close(); } + } finally { h.cleanup(); } +}); + // --------------------------------------------------------------------------- // Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, // across arm → steady fast rounds → client compaction (turn 8) → audit From b54b0bd215ec0f4b4432dae87473036d84a02ce6 Mon Sep 17 00:00:00 2001 From: ci-check Date: Thu, 5 Nov 2026 16:27:02 +0800 Subject: [PATCH 31/49] fix(fastpath): price only the new tail past F in render-lane estimate bases (#5664 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rw.*Cum cumulants already include the last F frozen rows, which the per-turn estimate terms re-price from processedMessages — feeding the full window double-counted a whole window on every green round (conservative drift: tighter output clamps, biased k-calibration pair). Stats bases now add only the tail past F; the clamp base subtracts the frozen-window estimate so estimateInputTokens' internal walk stays exact. --- src/server/prepare-openai.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index f64968a5c7..dbf0c33fdf 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -701,7 +701,15 @@ export async function prepareOpenai( rebuiltMessages = normalizeStrictEchoReasoning(rebuiltMessages, isStrictReasoningEcho(session, upstreamOrigin, modelIdOf(parsed)), log, sessionId); const rebuilt: OpenAIRequestBody = { ...parsed, messages: rebuiltMessages, tools: toolsOut as OpenAITool[] | undefined }; warnReasoningPairs(rebuiltMessages, log, sessionId); - clampOutgoingOutput(rebuilt as Record, typeof (parsed as Record).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, inputEstBaseTokens: renderFast !== null ? renderFast.rw.textCum : undefined, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: renderFast !== null ? renderFast.rw.imgCum + renderedImageDelta : imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata), knownOutputCeiling: resolveKnownOutputCeiling(req.headers, parsed as Record, opts.routes, billingUpstream ?? upstreamOrigin, opts.sessionHeader, modelIdOf(parsed)) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; }); + // #5664 review (round 2): the rw.*Cum cumulants already cover the last F + // frozen rows, which the per-turn estimate terms re-price from + // processedMessages — feed only the NEW tail past F into those terms (and + // subtract the frozen window from the clamp base), or every green round + // double-counts a whole window. + const rf = renderFast; + const rfTail = rf !== null ? processedMessages.slice(Math.min(rf.F, processedMessages.length)) : processedMessages; + const rfFrozenEst = rf !== null ? estimateCoreMessages(rf.rw.rowsParsed.slice(Math.max(0, rf.rw.rows.length - rf.F)) as CoreMessage[]) : 0; + clampOutgoingOutput(rebuilt as Record, typeof (parsed as Record).max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, inputEstBaseTokens: rf !== null ? Math.max(0, rf.rw.textCum - rfFrozenEst) : undefined, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: renderFast !== null ? renderFast.rw.imgCum + renderedImageDelta : imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin), kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: billingUpstream ?? upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata), knownOutputCeiling: resolveKnownOutputCeiling(req.headers, parsed as Record, opts.routes, billingUpstream ?? upstreamOrigin, opts.sessionHeader, modelIdOf(parsed)) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; }); // prompt_cache_retention is an OpenAI-host-only cache directive; the dsh // launcher forces PI_CACHE_RETENTION=long (for the session-id // prompt_cache_key) which makes the client also emit it. Third-party @@ -750,14 +758,13 @@ export async function prepareOpenai( // #5664: render-lane rounds bound every walk to the new tail — the // frozen prefix basis rides the persistent per-plane cumulants (cast // absolute, appended per round, rebaselined end-to-end each audit). - const rf = renderFast; const imageReserve = rf !== null ? rf.rw.imgCum + renderedImageDelta : imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin); - session.stats.localInputEstimate = (rf !== null ? rf.rw.estCum + estimateCoreMessagesUpper(processedMessages) : estimateCoreMessagesUpper(processedMessages.length > 0 ? processedMessages : originalMessages)) + session.stats.localInputEstimate = (rf !== null ? rf.rw.estCum + estimateCoreMessagesUpper(rfTail) : estimateCoreMessagesUpper(processedMessages.length > 0 ? processedMessages : originalMessages)) + countSystemAndToolsTokens(openaiOutboundSystem || openaiSystemText, toolsOut) + imageReserve; // #1933 F1: billed-caliber twin (chars/4) for the k̂ learning pair — // see the anthropic-lane counterpart above. - session.stats.lastLocalTextEstimate = (rf !== null ? rf.rw.textCum + estimateCoreMessages(processedMessages) : estimateCoreMessages(processedMessages.length > 0 ? processedMessages : originalMessages)) + session.stats.lastLocalTextEstimate = (rf !== null ? rf.rw.textCum + estimateCoreMessages(rfTail) : estimateCoreMessages(processedMessages.length > 0 ? processedMessages : originalMessages)) + countSystemAndToolsTokens(openaiOutboundSystem || openaiSystemText, toolsOut) + imageReserve; const openaiPairOrigin = billingUpstream ?? upstreamOrigin; From 0f3c77cde9d42e64b95161f5546b8e5a2c2be8b7 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Thu, 5 Nov 2026 17:12:13 +0800 Subject: [PATCH 32/49] =?UTF-8?q?fix(fastpath):=20#5643=20=E9=99=84?= =?UTF-8?q?=E5=BD=95=E4=BA=8C=20=E2=80=94=20the=20v6=20render=20snapshot?= =?UTF-8?q?=20evolves=20BY=20EVENTS=20(sync=20first-scene=20journal=20+=20?= =?UTF-8?q?evidence-addressed=20apply)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-#5721 supplement on the frozen ordinal-deviation line: the row table is now the owner-ruling LSM snapshot itself — a compression fold lands as a DIFF written straight into it, never as a snapshot replacement: * FIRST SCENE (synchronous, 压缩请求内同步落表): recordServerSideFold runs from the compress TOOL endpoint the moment a fold lands — ONE transaction journals the covered absolute positions + the live block signature into the checkpoint meta; row bytes stay untouched (plugin-lane folds usually change no wire bytes). Idempotent per signature; fail-soft per sid — the tool response never depends on this plane. * RECONCILE (下一轮只做核对): the journal red-gates exactly ONE forced full pass ('structural-pending'); its capture classifies every cell against the kernel's OWN output (block effectiveMessageIds ↔ pf_ledger.core_id ↔ absolute pos): fold-covered cells upserted in one tx ('apply', prefix bytes + checksums untouched); count-changing witnessed layout churn rides the COARSE rebase tier (contiguous reindex = documented V1 boundary); ANY other changed/lost cell kills the plane with its byte-for-byte pre-existing verdict. * Gate vocabulary gains 'structural-pending'; the rows/count strict equality becomes an abs-pos span check (ascending distinct, holes do not red-gate); hydrate accepts the sparse shape defensively; debug state exposes pendingDeletes. Store layer unchanged (existing pfRenderCommit covers arbitrary-pos upserts + S4 checkpoints). --- src/plugin.ts | 13 + src/server/fastpath-v6-render.ts | 324 +++++++++++++++++++++--- src/server/prepare-openai.ts | 27 +- tests/issue5664-render-snapshot.test.ts | 281 +++++++++++++++++++- 4 files changed, 587 insertions(+), 58 deletions(-) diff --git a/src/plugin.ts b/src/plugin.ts index 5b6578e7c5..74a9b55f4f 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -28,6 +28,7 @@ import type { TruncationDiag } from "./stream-error.js"; import { recordAttributionReport } from "./server/attribution-status.js"; import type { HandoffReply } from "./server/handoff-response.js"; import { captureHandoffJson, HANDOFF_REPLY_MAX_BYTES } from "./server/handoff-response.js"; +import { recordServerSideFold } from "./server/fastpath-v6-render.js"; /** #2328 Q2: what the pipes know about the upstream HTTP response whose body * they are piping — threaded in by server.ts so the truncation diag can cite @@ -2276,6 +2277,18 @@ export async function handlePluginTool( } releaseInFlight(session); if (result === undefined) return; + // #5643 附录二:the compression request IS the first scene — journal its + // fold coverage into the v6 render snapshot SYNCHRONOUSLY (one tx DML; + // the next model turn materializes/verifies as ONE forced red round, + // never by guesswork). No-op unless the plane is engaged AND something + // structural moved; fail-soft — the tool response never depends on it. + if (tool === "compress") { + try { + recordServerSideFold(session.id, session, getStore(), deps.log); + } catch (err) { + deps.log("warn", `[${session.id}] [plugin] render first-scene journal failed: ${String(err)}`); + } + } // #760b: evidence-based plugin-mode flip. A successful MCP tool execution proves // this session's host owns the bili compression tools, so bind it to plugin mode // (sticky) — the next model request stops injecting the duplicate ephemeral wire diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 2befef948a..88f279dd67 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -36,6 +36,7 @@ // P7 mirrors are caches (LRU byte budget); losing one = DB re-read. import type { SessionStore } from "../persist.js"; import type { Session } from "../session.js"; +import { coveredRealHistoryIds } from "../session.js"; type LogFn = (level: string, msg: string) => void; @@ -114,6 +115,12 @@ interface RenderMeta { estCum: number; textCum: number; imgCum: number; + /** Covered positions journaled at the compression FIRST SCENE (#5643 + * 附录二:同步落表) but not yet materialized by the reconcile round: + * the plane serves no green turns while > 0 (gate red + * 'structural-pending') and a legacy capture that clears it refreshes + * the meta atomically (absent ⇒ 0). */ + pendingDeletes?: number; /** Active-block signature at this checkpoint (absent on pre-#5643补充裁定 * checkpoints ⇒ ""). Folded forward by post-ckpt 'rebase'/'append' log * entries during hydration. */ @@ -144,6 +151,16 @@ export interface RenderWorld { textCum: number; /** Cumulative image-reserve contribution of frozen rows. */ imgCum: number; + /** Absolute ledger positions backing `rows` (ascending distinct; index i + * ↔ positions[i]). Dense 0..n-1 holds for every V1 op (the journal is + * a pending DELETE, rows themselves only move via witnessed ops); sparse + * admission stays accepted defensively — emission order is array order, + * and a lagging trailing max is a normal transient (gate span-check). */ + positions: number[]; + /** Covered positions journaled at first scene awaiting materialization + * (== the mirrored ckpt meta field; both reset together). While > 0 the + * gate keeps the plane red ('structural-pending'). */ + pendingDeletes: number; /** Set when this sid's plane is dead (no-recast regime) — diagnostic. */ dead?: boolean; /** Green (fast-served) engaged-turn tick — % PF_RENDER_AUDIT_EVERY === 0 @@ -251,6 +268,7 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor let expectedCount = -1; let capsJson = ""; let blocksSig = ""; + let pendingDeletes = 0; if (metaRead?.ckpt) { const m = JSON.parse(metaRead.ckpt.metaJson) as RenderMeta; estCum = m.estCum ?? 0; @@ -258,6 +276,7 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor imgCum = m.imgCum ?? 0; expectedCount = m.rowCount; blocksSig = m.blocksSig ?? ""; + pendingDeletes = m.pendingDeletes ?? 0; capsJson = metaRead.ckpt.metaJson; for (const e of metaRead.log) { const p = JSON.parse(e.payloadJson) as { de?: number; dt?: number; di?: number; n?: number; capsJson?: string; reason?: string; sharedPrefix?: number; from?: number; to?: number } & Partial; @@ -281,6 +300,11 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor // entry here means an interrupted fold — treat as count // reset to its journaled target. if (typeof p.to === "number") expectedCount = p.to; + } else if (e.kind === "delete") { + // First-scene fold journal (#5643 附录二): the journal is + // written WITH its own checkpoint, so a surviving entry + // adds to the meta's base count (defensive accumulation). + pendingDeletes += p.n ?? 0; } } } else { @@ -288,13 +312,20 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor } const rows: string[] = new Array(rowsRaw.length); const rowsParsed: unknown[] = new Array(rowsRaw.length); + const positions: number[] = new Array(rowsRaw.length); for (let i = 0; i < rowsRaw.length; i++) { const rj = rowsRaw[i].rowJson; - if (rj === null || rowsRaw[i].pos !== i) { - killRenderPlane(store, sid, `row-table inconsistency at pos ${rowsRaw[i].pos} (expected contiguous ${i})`, log); + const pos = rowsRaw[i].pos; + if (rj === null) { + killRenderPlane(store, sid, `row-table inconsistency at pos ${pos} (NULL payload)`, log); + return null; + } + if (i > 0 && pos <= positions[i - 1]) { + killRenderPlane(store, sid, `row-table inconsistency at pos ${pos} (expected strictly ascending)`, log); return null; } rows[i] = rj; + positions[i] = pos; try { rowsParsed[i] = JSON.parse(rj); } catch { @@ -327,7 +358,7 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor killRenderPlane(store, sid, "unparseable capabilities fingerprint", log); return null; } - const rw: RenderWorld = { sid, rows, rowsParsed, caps, blocksSig, estCum, textCum, imgCum, auditTick: 0 }; + const rw: RenderWorld = { sid, rows, rowsParsed, positions, caps, blocksSig, estCum, textCum, imgCum, pendingDeletes, auditTick: 0 }; touchMirror(rw); return rw; } catch { @@ -377,8 +408,8 @@ export type RenderGateDecision = /** Stable red-reason vocabulary (test-pinned, one per condition): not-engaged, * arm-turn, title-gen, plane-unavailable, pressure, snapshot-lane, * blocks-changed, fold-pending, absorb-active, ccr-on, markers, lane-dead, - * hydrate-failed, caps-mismatch, audit-due, no-rows, no-plane-geometry, - * rows-misaligned. */ + * hydrate-failed, caps-mismatch, structural-pending, audit-due, no-rows, + * no-plane-geometry, rows-misaligned. */ /** G1 conservative conjunction — every red condition returns the reason so * the lane stays debuggable; the fallback path is unchanged baseline. */ @@ -401,6 +432,10 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { // returning null — report the durable reason, not the transient one. if (rw === undefined) return off(deadSids.has(args.sid) ? "lane-dead" : "hydrate-failed"); if (!capsEqual(rw.caps, args.caps)) return off("caps-mismatch"); + // #5643 附录二:first-scene journal landed but the reconcile round has + // not yet confirmed/materialized it — the plane stays red until ONE + // legacy pass verifies the layout against truth (never guess). + if (rw.pendingDeletes > 0) return off("structural-pending"); // #5643 补充裁定:快照必须随压缩演进。 Replaces the old blanket // "blocks-active" red-out: a folded session KEEPS SERVING FAST as long as // nothing changed structurally since the table's own commit. A moved @@ -412,14 +447,18 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { if (rw.auditTick % PF_RENDER_AUDIT_EVERY === 0 && rw.auditTick > 0) return off("audit-due"); if (rw.rows.length < 1) return off("no-rows"); if (args.maxPos < 0) return off("no-plane-geometry"); - // Alignment invariant: rows are captured per engaged round and always - // number one row per ledger element (row i ↔ pos i), so at GATE time — - // which runs BEFORE this round's finalize advances the plane — - // rows.length must equal maxPos+1. A mismatch means an earlier commit - // failed soft (disk ahead/behind the mirror): fall back to the legacy - // byte path; the plane cannot self-heal anymore (no-recast regime) — - // a persistent misalignment will die its next structural diff. - if (rw.rows.length !== args.maxPos + 1) return off("rows-misaligned"); + // Alignment invariant: every stored pos is a LIVE ledger position ≤ the + // committed plane end (a lagging trailing max is a normal transient — + // the finalize step advances the identity world AFTER station 2 commits, + // and the next steady append closes the gap). Holes (sparse abs-pos + // table) do NOT red-gate: emission order is the sorted-row order and any + // stale layout rides red through blocks-changed/structural-pending, so + // green always implies the table mirrors the true prefix. A pos outside + // [0, maxPos] means an earlier commit failed soft (disk ahead of the + // mirror): ride the legacy byte path — no self-heal (no-recast regime). + if (rw.positions.length === 0 || rw.positions[0] < 0 || rw.positions[rw.positions.length - 1] > args.maxPos) { + return off("rows-misaligned"); + } registerSidForSession(args.sessionId, args.sid); // Window input = last F identity elements (+ new cores added by caller); // positions are contiguous because the ledger grows forward only (P3). @@ -507,12 +546,18 @@ interface CaptureArgs { /** steady: ONLY the new tail raw strings; legacy: the FULL captured rows. */ rowStrings: string[]; sums: CaptureSums; // steady: DELTAS for the tail; legacy: ABSOLUTE totals. + /** #5643 附录二:absolute positions COVERED by active fold blocks (kernel + * evidence → ledger pos via pf_ledger.core_id). Only these cells may + * change under a fine-grained event-apply; anything else differing is + * unexplained — routed to the coarse witnessed tier or killed. */ + coveredPos?: Set; log?: LogFn; } /** 'equal' | 'append' | 'diverged' — prefix-identical growth vs anything - * else (byte diff, shrink, non-monotonic growth). Under the no-recast - * ruling every 'diverged' verdict KILLS the plane. */ + * else (byte diff, shrink, non-monotonic growth). A 'diverged' verdict is + * reconciled by evidence (#5643 附录二): kernel-covered cells may move + * (fine-grained apply / coarse witnessed rebase), anything else kills. */ export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" | "diverged" { if (prev.length === cur.length) { for (let i = 0; i < prev.length; i++) if (prev[i] !== cur[i]) return "diverged"; @@ -533,7 +578,7 @@ export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" * stops advancing until restart or caps change allows re-capture again. */ const nulRejectedSids = new Set(); -export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" | "rebase" } { +export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" | "rebase" | "apply" } { const { store, sid, caps, log } = args; if (deadSids.has(sid)) { // Belt-and-braces behind the gate: a dead sid NEVER arms/commits again. @@ -587,7 +632,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op }); if (okSeq === null) return { committed: false, op: "none" }; const parsed = rows.map((rj) => safeParse(rj)); - mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, caps: { ...caps }, blocksSig: args.blocksSig ?? "", estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, auditTick: 0 }); + mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, positions: rows.map((_, i) => i), caps: { ...caps }, blocksSig: args.blocksSig ?? "", estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, pendingDeletes: 0, auditTick: 0 }); registerSidForSession(args.sessionId, sid); return { committed: true, op: "cast" }; } @@ -606,6 +651,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op const parsed = rows.map((rj) => safeParse(rj)); rw.rows.push(...rows); rw.rowsParsed.push(...parsed); + for (let i = 0; i < rows.length; i++) rw.positions.push(prevLen + i); rw.estCum += args.sums.estCum; rw.textCum += args.sums.textCum; rw.imgCum += args.sums.imgCum; @@ -619,12 +665,37 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op // legacy full-capture: diff against truth, minimal commit. const verdict = diffRowLists(rw.rows, args.rowStrings); if (verdict === "equal") { + if (rw.pendingDeletes > 0) { + // Zero-event materialization (#5643 附录二): the first-scene + // journal is confirmed — bytes moved nowhere (the plugin lane's + // agent owns summary rendering, so bili-side folds usually change + // nothing on the wire), the meta is refreshed atomically with the + // live signature and the flag cleared. No row events, no log line + // (there was nothing to write), no witness needed. + const m: RenderMeta = { v: 1, caps, rowCount: rw.rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, blocksSig: args.blocksSig ?? rw.blocksSig ?? "", pendingDeletes: 0 }; + const okSeq = store.pfRenderCommit(sid, { rows: [], logEntries: [], checkpointMetaJson: metaJson(m) }); + if (okSeq === null) return { committed: false, op: "none" }; + rw.caps = { ...caps }; + rw.blocksSig = args.blocksSig ?? rw.blocksSig; + rw.estCum = args.sums.estCum; + rw.textCum = args.sums.textCum; + rw.imgCum = args.sums.imgCum; + rw.pendingDeletes = 0; + touchMirror(rw); + return { committed: true, op: "apply" }; + } rw.caps = { ...caps }; // keep caps fresh even when bytes match rw.blocksSig = args.blocksSig ?? rw.blocksSig; touchMirror(rw); return { committed: true, op: "none" }; } if (verdict === "append") { + if (rw.pendingDeletes > 0) { + // Growth over a pending journal: the table layout must be + // re-derived wholesale — fall through to the classified tier + // below (never half-append over an unconfirmed structural event). + return reconcileLegacyDiverged(args, rw, store, sid, caps, metaJson); + } const prevLen = rw.rows.length; const tail = args.rowStrings.slice(prevLen); const delta = { @@ -648,27 +719,44 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op touchMirror(rw); return { committed: true, op: "append" }; } - // DIVERGED (byte diff at any position, or shrink): the ONLY repair the - // old design offered was a recast — rewriting covered ordinals and - // deleting the excess. Owner ruling 2026-11-05 forbids that in every - // scenario: the plane is KILLED PERMANENTLY instead and the session - // rides the legacy full path from here on (bytes stay correct; the - // cache simply never serves this sid again). // DIVERGED (byte diff at any position, or shrink): the ORIGINAL design // offered only recast; the owner ruling of 2026-11-05 sealed that door, - // and the #5643 补充裁定 re-opened EXACTLY ONE gate in it — a WITNESSED - // structural event (the turn's server-side evidence that the fold layer - // rewrote the layout: block-multiset signature moved around the kernel - // pass, dsh native-compaction rebase, or fold usage consumed) authorizes - // a REBASE: the whole table is replaced by THIS capture in one - // transaction, journaled as kind='rebase' under a fresh checkpoint. - // Without a witness, divergence is still substrate corruption and KILLS - // the plane permanently — that semantics is byte-for-byte unchanged. - if (args.witness === true) { - let common = 0; + // and the #5643 补充裁定 + 附录二 re-opened EXACTLY ONE gate in it — + // reconciliation by EVIDENCE: cells whose absolute position sits inside + // the fold coverage reported by the compression engine itself (pf_ledger + // core_id ↔ block effectiveMessageIds binding) are the engine's OUTPUT + // rows and may move; EVERY other changed/lost cell is unexplained and + // without the turn's server-side witness (block signature moved, native- + // compaction rebase fired, fold usage consumed) the plane KILLS — that + // semantics is byte-for-byte unchanged. Two tiers: fine-grained APPLY + // (count-preserving: upsert exactly the covered cells, single tx, prefix + // bytes+checksums untouched) and the COARSE WITNESSED REBASE (count- + // changing layout events — client compaction, restore-class churn — + // replace the whole table contiguously; V1 boundary documented). + if (args.witness === true && rw.rows.length !== args.rowStrings.length) { + return rebaseOnWitness(args, rw, store, sid, caps, metaJson, log); + } + return reconcileLegacyDiverged(args, rw, store, sid, caps, metaJson, log); +} + +/** Coarse tier: witnessed count-changing structural event — the whole table + * is replaced by THIS capture in one transaction, journaled kind='rebase' + * under a fresh checkpoint. V1 boundary: the rebuild reindexes contiguously + * (survivors past the gap shift pos); pos-preserving reconciliation is the + * follow-up (flagged in the PR). */ +function rebaseOnWitness( + args: CaptureArgs, + rw: RenderWorld, + store: SessionStore, + sid: string, + caps: RenderCaps, + metaJson: (m: RenderMeta) => string, + log?: LogFn, +): { committed: boolean; op: "none" | "append" | "cast" | "rebase" | "apply" } { + let common = 0; while (common < rw.rows.length && common < args.rowStrings.length && rw.rows[common] === args.rowStrings[common]) common += 1; const prevLen = rw.rows.length; - const m: RenderMeta = { v: 1, caps, rowCount: args.rowStrings.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, blocksSig: args.blocksSig ?? rw.blocksSig }; + const m: RenderMeta = { v: 1, caps, rowCount: args.rowStrings.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, blocksSig: args.blocksSig ?? rw.blocksSig ?? "" }; const okSeq = store.pfRenderCommit(sid, { replaceAll: true, rows: args.rowStrings.map((rj, pos) => ({ pos, rowJson: rj })), @@ -685,12 +773,82 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op rw.textCum = args.sums.textCum; rw.imgCum = args.sums.imgCum; rw.auditTick = 0; // the rebase round IS the audit — start fresh + rw.positions = args.rowStrings.map((_, i) => i); + rw.pendingDeletes = 0; touchMirror(rw); log?.("info", `[${sid}] #5664 render plane REBASED on witnessed structural event (shared prefix ${common} row(s), table ${prevLen} -> ${args.rowStrings.length} rows)`); return { committed: true, op: "rebase" }; +} + +/** Fine-grained tier (#5643 附录二:事件化更新) — the snapshot evolves BY + * EVENTS instead of being replaced: every cell is classified against the + * captured truth, and ONLY the fold-covered cells may differ. The result is + * ONE transaction upserting exactly those cells plus a fresh checkpoint + * meta (rowCount kept, pendingDeletes cleared, live signature) — stored + * prefix bytes and their checksums stay untouched (O(diff)). Count-changing + * captures never land here (the caller routes them to the coarse tier); + * sparse tables likewise ride the coarse door defensively. An UNEXPLAINED + * delta without a witness kills the plane verbatim as before. */ +function reconcileLegacyDiverged( + args: CaptureArgs, + rw: RenderWorld, + store: SessionStore, + sid: string, + caps: RenderCaps, + metaJson: (m: RenderMeta) => string, + log?: LogFn, +): { committed: boolean; op: "none" | "append" | "cast" | "rebase" | "apply" } { + if (rw.rows.length !== args.rowStrings.length || !rw.positions.every((p, i) => p === i)) { + if (args.witness === true) return rebaseOnWitness(args, rw, store, sid, caps, metaJson, log); + killRenderPlane(store, sid, `structural divergence (stored=${rw.rows.length} rows, captured=${args.rowStrings.length})`, log); + return { committed: false, op: "none" }; + } + const n = rw.rows.length; + const covered = args.coveredPos ?? new Set(); + const changed: number[] = []; + let unexplained = 0; + for (let i = 0; i < n; i++) { + if (rw.rows[i] !== args.rowStrings[i]) { + if (covered.has(i)) changed.push(i); + else unexplained += 1; + } + } + if (unexplained > 0) { + // Bytes outside the fold coverage changed — not an event the kernel + // can vouch for (external rewrite of stored truth): substrate + // corruption. Witness does NOT save this (it would bless foreign + // bytes) — kill, same string as before. + killRenderPlane(store, sid, `structural divergence (stored=${rw.rows.length} rows, captured=${args.rowStrings.length})`, log); + return { committed: false, op: "none" }; } - killRenderPlane(store, sid, `structural divergence (stored=${rw.rows.length} rows, captured=${args.rowStrings.length})`, log); - return { committed: false, op: "none" }; + if (changed.length > 0 && args.witness !== true) { + // Covered cells moved WITHOUT this turn's server-side evidence: + // could be a fold this proxy did not run — unverifiable here, same + // kill as unwitnessed divergence (byte-for-byte unchanged semantics). + killRenderPlane(store, sid, `structural divergence (stored=${rw.rows.length} rows, captured=${args.rowStrings.length})`, log); + return { committed: false, op: "none" }; + } + const m: RenderMeta = { v: 1, caps, rowCount: n, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, blocksSig: args.blocksSig ?? rw.blocksSig ?? "", pendingDeletes: 0 }; + const okSeq = store.pfRenderCommit(sid, { + rows: changed.map((pos) => ({ pos, rowJson: args.rowStrings[pos] })), + logEntries: changed.length > 0 ? [{ kind: "apply", payloadJson: JSON.stringify({ u: changed, a: [], d: [] }) }] : [], + checkpointMetaJson: metaJson(m), + }); + if (okSeq === null) return { committed: false, op: "none" }; + for (const pos of changed) { + rw.rows[pos] = args.rowStrings[pos]; + rw.rowsParsed[pos] = safeParse(args.rowStrings[pos]); + } + rw.caps = { ...caps }; + rw.blocksSig = args.blocksSig ?? rw.blocksSig; + rw.estCum = args.sums.estCum; + rw.textCum = args.sums.textCum; + rw.imgCum = args.sums.imgCum; + rw.pendingDeletes = 0; + rw.auditTick = 0; // the apply round IS the audit — start fresh + touchMirror(rw); + log?.("info", `[${sid}] #5664 render plane APPLIED ${changed.length} covered cell(s) as events (prefix + checksums untouched)`); + return { committed: true, op: "apply" }; } function safeParse(json: string): unknown { @@ -701,6 +859,98 @@ function safeParse(json: string): unknown { } } +// --------------------------------------------------------------------------- +// First-scene fold journal (#5643 附录二:压缩请求内同步落表) +// --------------------------------------------------------------------------- + +/** Bind the compression engine's OWN output back to ledger absolute + * positions: every real-history id covered by ACTIVE fold blocks maps onto + * the pf_ledger rows whose core_id column carries that kernel message id. + * This is the event-addressability anchor — covered cells may move under an + * apply; everything else may not. No store read when nothing is folded. + * Exported: prepare-openai feeds the same set into the capture classifier. */ +export function boundCoveredPositions( + store: SessionStore, + sid: string, + blocks: readonly { active?: boolean; effectiveMessageIds?: readonly string[] }[], +): Set { + const out = new Set(); + const ids = coveredRealHistoryIds(blocks); + if (ids.size === 0) return out; + const rows = store.pfLedgerIds(sid); + if (!rows) return out; + for (const r of rows) { + for (const t of r.coreId.split("|")) { + if (t && ids.has(t)) { + out.add(r.pos); + break; + } + } + } + return out; +} + +/** Synchronous first-scene write: called from the compress TOOL endpoint the + * moment a fold lands (and reachable from any other in-process fold site). + * Journals the covered positions + live block signature into the row table's + * checkpoint meta as ONE transactional DML batch — no op-log replay layer, + * no version columns (owner ruling 2026-11-05). Row bytes stay untouched: + * on plugin lanes the agent owns summary rendering, so the wire usually + * does NOT change at all (KDD #7); the next model turn rides ONE forced red + * round ('structural-pending') that materializes/verifies the layout and + * clears the flag. Fail-soft per sid — the tool response never depends on + * this plane. Returns how many positions were journaled. */ +export function recordServerSideFold( + sessionId: string, + session: Session, + store: SessionStore, + log?: LogFn, +): number { + const set = sessionSids.get(sessionId); + if (!set || set.size === 0) return 0; + let total = 0; + const sig = activeBlocksSignature(session.state.blocks); + for (const sid of set) { + if (deadSids.has(sid)) continue; + let rw: RenderWorld | undefined = mirrors.get(sid); + if (!rw) { + const hydrated = hydrateMirror(store, sid, log); + if (hydrated === null) continue; // not armed yet: the next model turn arms directly onto the post-fold form + mirrors.set(sid, hydrated); + rw = hydrated; + } + try { + // Idempotent: the mirror's recorded signature already equals + // THIS block state ⇒ this "compress" moved nothing structural + // beyond what was journaled under that signature (pending stays + // until its forced reconcile round clears it). + if (rw.blocksSig === sig) continue; + const covered = boundCoveredPositions(store, sid, session.state.blocks); + const hit = rw.positions.filter((p) => covered.has(p)); + if (hit.length === 0) { + rw.blocksSig = sig; + touchMirror(rw); + continue; + } + const m: RenderMeta = { v: 1, caps: { ...rw.caps }, rowCount: rw.rows.length, estCum: rw.estCum, textCum: rw.textCum, imgCum: rw.imgCum, blocksSig: sig, pendingDeletes: rw.pendingDeletes + hit.length }; + const okSeq = store.pfRenderCommit(sid, { + rows: [], + logEntries: [{ kind: "delete", payloadJson: JSON.stringify({ n: hit.length, pos: [...hit].sort((a, b) => a - b) }) }], + checkpointMetaJson: JSON.stringify(m), + }); + if (okSeq === null) throw new Error("pfRenderCommit returned null"); + rw.pendingDeletes += hit.length; + rw.blocksSig = sig; + touchMirror(rw); + total += hit.length; + log?.("info", `[${sid}] #5664 render plane: fold journaled at FIRST SCENE — ${hit.length} covered position(s) pending materialization (plane red 'structural-pending' until the forced reconcile round)`); + } catch (err) { + log?.("warn", `[${sid}] #5664 render first-scene journal failed fail-soft: ${String(err)}`); + } + } + return total; +} + // --------------------------------------------------------------------------- // Diagnostics / test surface // --------------------------------------------------------------------------- @@ -714,6 +964,7 @@ interface RenderDebugWorld { textCum: number; imgCum: number; auditTick: number; + pendingDeletes: number; } export function watermarkV6RenderDebugState(sessionId: string): { sids: RenderDebugWorld[] } { @@ -725,6 +976,7 @@ export function watermarkV6RenderDebugState(sessionId: string): { sids: RenderDe sids.push({ sid, rows: rw?.rows.length ?? 0, dead: deadSids.has(sid), caps: rw?.caps, estCum: rw?.estCum ?? 0, textCum: rw?.textCum ?? 0, imgCum: rw?.imgCum ?? 0, auditTick: rw?.auditTick ?? 0, + pendingDeletes: rw?.pendingDeletes ?? 0, }); } } diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index ceb85c7e19..e80bef0d79 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -42,7 +42,7 @@ import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decision import { featureEnabled } from "../rollout.js"; import { evaluateWatermarkFastpath, finalizeWatermarkTurn, type FastpathHit, type FastpathOutcome } from "./fastpath.js"; import { evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, PF_SESSION_HEADER, watermarkV6DebugState, type V6Outcome } from "./fastpath-v6.js"; -import { activeBlocksSignature, bumpAuditTick, commitRenderCapture, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, type RenderCaps, type RenderGateDecision, type RenderWorld } from "./fastpath-v6-render.js"; +import { activeBlocksSignature, boundCoveredPositions, bumpAuditTick, commitRenderCapture, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, type RenderCaps, type RenderGateDecision, type RenderWorld } from "./fastpath-v6-render.js"; import { getStore } from "../persist.js"; import { collectRequestIds, probeRequestMembership } from "../request-membership.js"; @@ -811,12 +811,13 @@ export async function prepareOpenai( if (Object.keys(wpResponseHeaders).length === 0) wpResponseHeaders = undefined; } // #5664 station 2 — ONE snapshot commit per engaged turn: steady append - // (fast rounds), arm cast / audit diff / witnessed REBASE on structural - // divergence / UNWITNESSED divergence-KILL (full-path rounds carry the - // complete captured array; #5643 补充裁定:快照必须随压缩演进 — a - // structurally-witnessed diverged capture rebases the table, an - // unwitnessed one still kills the lane for the sid). Fail-soft for this - // turn only — the identity plane already committed independently. + // (fast rounds), arm cast / audit diff / evidence-addressed APPLY + // (#5643 附录二:the table evolves by single-tx row events — covered cells + // upserted, prefix bytes + checksums untouched) / witnessed REBASE as + // COARSE tier for count-changing layout events / UNWITNESSED divergence- + // KILL (full-path rounds carry the complete captured array; #5643 补充 + // 裁定:快照必须随压缩演进). Fail-soft for this turn only — the identity + // plane already committed independently. if (renderedCaptureStrings !== null && renderCapturedSid !== null && renderCapturedCaps !== null && featureEnabled("watermarkV6Render")) { const store = getStore(); if (store.pfRenderAvailable()) { @@ -837,14 +838,18 @@ export async function prepareOpenai( // leave the signature byte-equal, hence the explicit term); // ③ fold usage was pending pre-turn and got consumed by it; // ④ the gate itself had observed the signature move pre-turn - // ("blocks-changed" red sent this request down the full path). + // ("blocks-changed" red sent this request down the full path); + // ⑤ the first-scene journal is pending materialization + // ("structural-pending" red forced this full pass — its very + // existence is the evidence). // Absent ALL of them, a diverged capture stays a kill verdict. const renderPostBlocksSig = activeBlocksSignature(session.state.blocks); const renderWitness = dshRebased || renderPostBlocksSig !== renderPreBlocksSig || (renderFoldUsagePre && session.stats.pendingFoldUsage !== true) || - renderGateReason === "blocks-changed"; + renderGateReason === "blocks-changed" || + renderGateReason === "structural-pending"; const capOp = commitRenderCapture({ sessionId, store, @@ -854,6 +859,10 @@ export async function prepareOpenai( rowStrings: steadyTailRows, witness: renderWitness, blocksSig: renderPostBlocksSig, + // #5643 附录二:kernel-evidence coverage for the fine-grained + // apply tier (legacy turns only — steady never classifies; + // empty-block sessions short-circuit before touching storage). + coveredPos: renderFast !== null ? undefined : boundCoveredPositions(store, renderCapturedSid, session.state.blocks), sums: { estCum: estimateCoreMessagesUpper(sumsSrc), textCum: estimateCoreMessages(sumsSrc), diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 8f66b90175..18036abf3a 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -13,17 +13,22 @@ // the entire chain end-to-end. Equivalence with the full pass is the // contract, enforced here against a legacy oracle. // -// WITNESSED-REBASE REGIME (#5643 补充裁定 + owner ruling 2026-11-05): -// the table MUST evolve with compression. A structurally-WITNESSED diverged -// capture (fold layer's active-block multiset moved around the turn, dsh -// native-compaction detector fired, fold usage consumed pre→post, or the -// gate itself observed a signature move) REBASES the row table: -// truncate from the first divergence point + whole-suffix re-append as ONE -// transaction, checkpoint with S4 checksum, audit reset. An UNWITNESSED -// diverged capture still KILLS the lane permanently (persisted kind='dead') -// and rides legacy thereafter; checksum corruption stays durably dead. -// Storage is bounded by live elements (compaction SHRINKS the table); arm -// cast is legal exactly once, onto an empty table. +// WITNESSED-EVENT REGIME (#5643 补充裁定 + 附录二, owner ruling 2026-11-05): +// the table MUST evolve with compression, EVENT-ADDRESSED: a compression +// fold is journaled at its FIRST SCENE (the compress tool endpoint — +// recordServerSideFold) as ONE transaction: covered absolute positions plus +// the live block signature; the plane stays red as 'structural-pending' +// until exactly one forced reconcile round. The reconcile classifies the +// full-pass capture against the kernel's own coverage (block +// effectiveMessageIds ↔ pf_ledger core_id ↔ absolute pos): count- +// preserving divergence upserts ONLY the covered cells (op 'apply', single +// tx, prefix bytes+checksums untouched); count-changing layout events ride +// the COARSE witnessed rebase tier (contiguous replaceAll — documented V1 +// boundary); ANY other changed/lost cell stays unexplained and KILLS the +// plane durably (unwitnessed divergence keeps its byte-for-byte pre-existing +// semantics). +// Storage is bounded by live elements; arm cast is legal exactly once, onto +// an empty table. // // Harness: unit-level against a REAL sqlite sidecar (UnifiedShadow shadow // mode), mirroring tests/fastpath-v6-msgid.test.ts; the kernel runs as a @@ -42,6 +47,7 @@ import { SessionStore, _setStoreForTest, _resetStoreForTest } from "../src/persi import type { SessionStore as SessionStoreType } from "../src/persist.ts"; import type { Session } from "../src/session.ts"; import { UnifiedShadow } from "../src/storage/shadow.ts"; +import { sha256Hex } from "../src/storage/store.ts"; import { stripKernelSummaries } from "../src/server.ts"; import { hardenOpenaiAssistantContent, systemToUser } from "../src/util.ts"; import { @@ -51,7 +57,7 @@ import { PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; import type { V6Outcome } from "../src/server/fastpath-v6.ts"; import { activeBlocksSignature, bumpAuditTick, clearWatermarkV6Render, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, - watermarkV6RenderDebugState, _resetWatermarkV6RenderTestState, + watermarkV6RenderDebugState, _resetWatermarkV6RenderTestState, boundCoveredPositions, recordServerSideFold, type RenderCaps, } from "../src/server/fastpath-v6-render.ts"; @@ -391,6 +397,255 @@ test("unit: plane death persists across restart (no re-arm ever)", (t) => { } finally { h.cleanup(); } }); +test("unit: first-scene fold journal: covered cells journaled, plane red 'structural-pending', zero-event apply clears (#5643 appendix two)", () => { + const h = makeH("evj"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-evj", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + // Forge a dense 5-row ledger (the unit pin stands in for the identity + // plane's finalize; positions 0..4 carry known kernel ids). + const db = new DatabaseSync(path.join(h.dir, "index.db")); + db.prepare(`DELETE FROM pf_rows WHERE sid = ?`).run(sid); + for (let i = 0; i < 5; i++) db.prepare(`INSERT INTO pf_rows (sid, pos, ledger_id, sess_raw, core_id, core_json) VALUES (?, ?, ?, ?, ?, ?)`).run(sid, i, `lid${i}`, "r", `t${i}`, "{}" as never); + db.close(); + const rows = ['{"i":0}', '{"i":1}', '{"i":2}', '{"i":3}', '{"i":4}']; + assert.equal(commitRenderCapture({ sessionId: "s-evj", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: rows, sums: { estCum: 5, textCum: 5, imgCum: 0 }, blocksSig: activeBlocksSignature([]) }).op, "cast"); + const blocks = [{ blockId: "B1", generation: "young", active: true, effectiveMessageIds: ["t2"] }]; + const liveSig = activeBlocksSignature(blocks); + const base = { + sessionId: "s-evj", session: { id: "s-evj", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 4, tokenCount: 500, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }; + // Production ordering: arm cast happened on an earlier turn, whose + // gate registered the session→sid map the journal reads. Reproduce + // it — one green gate under the EMPTY signature before anything else. + const pre = renderPlaneGate({ ...base }); + assert.ok(pre.ok, `armed plane gates green before the fold, got ${JSON.stringify(pre)}`); + assert.equal(watermarkV6RenderDebugState("s-evj").sids[0].pendingDeletes ?? -1, 0); + // FIRST SCENE (the compress tool endpoint): ONE tx journals the + // covered position + the live signature; row bytes stay untouched — + // on plugin lanes the AGENT owns summary rendering, so the wire form + // of a covered cell usually does not change at all. + assert.equal(recordServerSideFold("s-evj", { id: "s-evj", state: { blocks } } as unknown as Session, h.store, h.log), 1, "one covered position journaled"); + assert.ok(h.log.lines.some((l) => l.level === "info" && l.msg.includes("fold journaled at FIRST SCENE")), "first-scene journal is logged"); + assert.deepEqual(h.store.pfRenderRows(sid).map((r) => r.rowJson), rows, "the journal writes NO row bytes"); + // Plane RED until the forced reconcile round — by the FLAG alone: + // the signature already moved with the journal, so 'blocks-changed' + // would not fire either way; structural-pending IS the reason. + assert.deepEqual(renderPlaneGate({ ...base, blocksSig: liveSig }), { ok: false, reason: "structural-pending" }, "the journal red-gates the next round"); + assert.equal(watermarkV6RenderDebugState("s-evj").sids[0].pendingDeletes ?? -1, 1, "mirror carries the pending flag"); + // Idempotent: replaying the same fold journals nothing twice. + assert.equal(recordServerSideFold("s-evj", { id: "s-evj", state: { blocks } } as unknown as Session, h.store, h.log), 0, "replay must not double-journal"); + // Zero-event materialization: the wire bytes are UNCHANGED; the full + // pass merely CONFIRMS the layout and clears the flag — no witness + // is needed for a zero-event commit. + const ap = commitRenderCapture({ sessionId: "s-evj", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: [...rows], sums: { estCum: 5, textCum: 5, imgCum: 0 }, blocksSig: liveSig }); + assert.deepEqual(ap, { committed: true, op: "apply" }, `zero-event apply expected, got ${JSON.stringify(ap)}`); + const meta = h.store.pfRenderMeta(sid)!; + const ckpt = JSON.parse(meta.ckpt!.metaJson); + assert.equal(ckpt.pendingDeletes, 0, "flag cleared in the checkpoint meta"); + assert.equal(ckpt.rowCount, 5); + assert.equal(ckpt.blocksSig, liveSig); + assert.equal(h.store.pfRenderCkptIntact(sid), true, "S4 re-checksummed over the fresh meta"); + assert.deepEqual(h.store.pfRenderRows(sid).map((r) => r.rowJson), rows, "no cell bytes ever moved"); + assert.equal(watermarkV6RenderDebugState("s-evj").sids[0].pendingDeletes ?? 1, 0); + // GREEN again under the live signature — and steady appends continue. + assert.ok(renderPlaneGate({ ...base, blocksSig: liveSig }).ok, "gate green after materialization"); + assert.equal(commitRenderCapture({ sessionId: "s-evj", store: h.store, sid, mode: "steady", caps, rowStrings: ['{"i":5}'], sums: { estCum: 1, textCum: 1, imgCum: 0 } }).op, "append"); + assert.equal(h.store.pfRenderRows(sid).length, 6, "table keeps growing after materialization"); + } finally { h.cleanup(); } +}); + +test("unit: covered-cell change WITHOUT a witness still KILLS (coverage alone never blesses bytes)", () => { + const h = makeH("evk"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-evk", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + db.prepare(`DELETE FROM pf_rows WHERE sid = ?`).run(sid); + for (let i = 0; i < 5; i++) db.prepare(`INSERT INTO pf_rows (sid, pos, ledger_id, sess_raw, core_id, core_json) VALUES (?, ?, ?, ?, ?, ?)`).run(sid, i, `lid${i}`, "r", `t${i}`, "{}" as never); + db.close(); + const rows = ['{"i":0}', '{"i":1}', '{"i":2}', '{"i":3}', '{"i":4}']; + assert.equal(commitRenderCapture({ sessionId: "s-evk", store: h.store, sid, mode: "legacy", caps, rowStrings: rows, sums: { estCum: 5, textCum: 5, imgCum: 0 } }).op, "cast"); + const blocks = [{ blockId: "B9", generation: "young", active: true, effectiveMessageIds: ["t1"] }]; + assert.equal(recordServerSideFold("s-evk", { id: "s-evk", state: { blocks } } as unknown as Session, h.store, h.log), 1); + // Cell t1's row came back DIFFERENT and the coverage evidence names + // it — but no turn-level witness accompanied this capture: this + // proxy never ran that compression, so the bytes are unverifiable + // ⇒ kill, exactly as before #5721. + const logsBefore = h.log.lines.length; + const op = commitRenderCapture({ sessionId: "s-evk", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: [rows[0], '{"X":1}', rows[2], rows[3], rows[4]], sums: { estCum: 5, textCum: 5, imgCum: 0 }, witness: false, coveredPos: new Set([1]) }); + assert.deepEqual(op, { committed: false, op: "none" }); + assert.ok(h.log.lines.slice(logsBefore).some((l) => l.level === "error" && l.msg.includes("KILLED PERMANENTLY")), "unwitnessed covered change kills loudly"); + assert.equal(h.store.pfRenderRows(sid).length, 5, "stored rows stay immutable"); + const meta = h.store.pfRenderMeta(sid); + assert.ok(meta !== null && meta.log.some((e) => e.kind === "dead"), "death verdict persisted"); + } finally { h.cleanup(); } +}); + +test("unit: fine-grained APPLY upserts ONLY the covered cells (witnessed; prefix bytes + checksums untouched)", () => { + const h = makeH("eva"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-eva", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + db.prepare(`DELETE FROM pf_rows WHERE sid = ?`).run(sid); + for (let i = 0; i < 5; i++) db.prepare(`INSERT INTO pf_rows (sid, pos, ledger_id, sess_raw, core_id, core_json) VALUES (?, ?, ?, ?, ?, ?)`).run(sid, i, `lid${i}`, "r", `t${i}`, "{}" as never); + db.close(); + const rows = ['{"i":0}', '{"i":1}', '{"i":2}', '{"i":3}', '{"i":4}']; + assert.equal(commitRenderCapture({ sessionId: "s-eva", store: h.store, sid, mode: "legacy", caps, rowStrings: rows, sums: { estCum: 5, textCum: 5, imgCum: 0 } }).op, "cast"); + const blocks = [{ blockId: "B7", generation: "young", active: true, effectiveMessageIds: ["t1", "t3"] }]; + const liveSig = activeBlocksSignature(blocks); + assert.equal(recordServerSideFold("s-eva", { id: "s-eva", state: { blocks } } as unknown as Session, h.store, h.log), 2, "both covered positions journaled"); + // Witnessed full pass: EXACTLY the two covered cells return different + // bytes (proxy-mode-style rewrite); everything else byte-equal. + const ap = commitRenderCapture({ sessionId: "s-eva", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: [rows[0], '{"S":1}', rows[2], '{"S":3}', rows[4]], sums: { estCum: 5, textCum: 5, imgCum: 0 }, witness: true, coveredPos: new Set([1, 3]), blocksSig: liveSig }); + assert.deepEqual(ap, { committed: true, op: "apply" }, `fine-grained apply expected, got ${JSON.stringify(ap)}`); + const stored = h.store.pfRenderRows(sid).map((r) => ({ pos: r.pos, rowJson: r.rowJson })); + assert.deepEqual(stored.map((r) => r.pos), [0, 1, 2, 3, 4], "positions are absolute ledger pos — no renumbering"); + assert.deepEqual(stored.map((r) => r.rowJson), [rows[0], '{"S":1}', rows[2], '{"S":3}', rows[4]]); + const meta = h.store.pfRenderMeta(sid)!; + assert.equal(JSON.parse(meta.ckpt!.metaJson).pendingDeletes, 0, "the pending flag was consumed by the apply"); + assert.ok(meta.log.every((e) => e.kind !== "delete"), "journal folded into the fresh checkpoint"); + assert.equal(h.store.pfRenderCkptIntact(sid), true); + assert.ok(h.log.lines.some((l) => l.level === "info" && l.msg.includes("APPLIED 2 covered cell(s)")), "apply logged with its event set"); + // Restart reproduces the event-applied table. + _resetWatermarkV6RenderTestState(); + const g = renderPlaneGate({ + sessionId: "s-eva", session: { id: "s-eva", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 4, tokenCount: 500, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + blocksSig: liveSig, + }); + assert.ok(g.ok, `hydrated post-apply plane gates green, got ${JSON.stringify(g)}`); + if (g.ok) assert.equal(g.rw.rows[1], '{"S":1}', "hydrate restores the applied cell"); + } finally { h.cleanup(); } +}); + +test("unit: count-changing divergence stays on the COARSE witnessed tier; unwitnessed kills", () => { + const h = makeH("evo"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-evo", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + const rows = ['{"i":0}', '{"i":1}', '{"i":2}', '{"i":3}']; + assert.equal(commitRenderCapture({ sessionId: "s-evo", store: h.store, sid, mode: "legacy", caps, rowStrings: rows, sums: { estCum: 4, textCum: 4, imgCum: 0 }, blocksSig: activeBlocksSignature([]) }).op, "cast"); + // Native-compaction class: elements VANISH beyond any coverage + // evidence — witnessed ⇒ the coarse tier replaces the table; the + // classifier is unreachable for count-changing events by design. + const rb = commitRenderCapture({ sessionId: "s-evo", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"N":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 }, witness: true, coveredPos: new Set([1]) }); + assert.deepEqual(rb, { committed: true, op: "rebase" }, `coarse tier owns count-changing events, got ${JSON.stringify(rb)}`); + assert.deepEqual(h.store.pfRenderRows(sid).map((r) => ({ pos: r.pos, rowJson: r.rowJson })), + [{ pos: 0, rowJson: '{"i":0}' }, { pos: 1, rowJson: '{"N":1}' }], "dense contiguous positions restored"); + } finally { h.cleanup(); } + const h2h = makeH("evo2"); + try { + const b2 = makeBody([{ role: "user", content: "hello" }]); + const r2 = v6Eval(h2h, "s-evo2", b2, ["id0"]); + r2.fin(); + const sid2 = r2.sid!; + const caps = capsFor("text-only"); + const rows = ['{"i":0}', '{"i":1}', '{"i":2}', '{"i":3}']; + assert.equal(commitRenderCapture({ sessionId: "s-evo2", store: h2h.store, sid: sid2, mode: "legacy", caps, rowStrings: rows, sums: { estCum: 4, textCum: 4, imgCum: 0 } }).op, "cast"); + const op = commitRenderCapture({ sessionId: "s-evo2", store: h2h.store, sid: sid2, log: h2h.log, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"N":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 }, witness: false, coveredPos: new Set([1]) }); + assert.deepEqual(op, { committed: false, op: "none" }, "unwitnessed count-changing divergence kills"); + const meta = h2h.store.pfRenderMeta(sid2); + assert.ok(meta !== null && meta.log.some((e) => e.kind === "dead"), "death persisted"); + } finally { h2h.cleanup(); } +}); + +test("unit: boundCoveredPositions binds kernel ids through pf_rows.core_id (empty folds short-circuit before the store)", () => { + const h = makeH("bind"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-bind", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const db = new DatabaseSync(path.join(h.dir, "index.db")); + db.prepare(`DELETE FROM pf_rows WHERE sid = ?`).run(sid); + db.prepare(`INSERT INTO pf_rows (sid, pos, ledger_id, sess_raw, core_id, core_json) VALUES (?, ?, ?, ?, ?, ?)`).run(sid, 0, "a", "r", "k1|k2", "{}" as never); + db.prepare(`INSERT INTO pf_rows (sid, pos, ledger_id, sess_raw, core_id, core_json) VALUES (?, ?, ?, ?, ?, ?)`).run(sid, 1, "b", "r", "k9", "{}" as never); + db.close(); + assert.deepEqual(boundCoveredPositions(h.store, sid, []), new Set(), "no blocks ⇒ no binding"); + assert.deepEqual(boundCoveredPositions(h.store, sid, [{ active: false, effectiveMessageIds: ["k1"] }]), new Set(), "INACTIVE blocks bind nothing"); + assert.deepEqual(boundCoveredPositions(h.store, sid, [{ active: true, effectiveMessageIds: ["k1"] }]), new Set([0]), "a multi-id core column matches ANY of its tokens"); + assert.deepEqual(boundCoveredPositions(h.store, sid, [{ active: true, effectiveMessageIds: ["k2", "k9"] }]), new Set([0, 1])); + } finally { h.cleanup(); } +}); + +test("unit: sparse hydrate accepts ascending distinct positions within the span", () => { + const h = makeH("spars"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-spars", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + assert.equal(commitRenderCapture({ sessionId: "s-spars", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}', '{"i":2}', '{"i":3}'], sums: { estCum: 4, textCum: 4, imgCum: 0 } }).op, "cast"); + const before = h.store.pfRenderMeta(sid)!; + // Forge a hole: drop one middle row and push the tail past the + // current maxPos — positions become {0, 2, 7} (sparse, ascending). + // Meta is refreshed with the shrunken absolute count (checksum is + // over meta bytes, rewritten in step). + const freshMeta = JSON.parse(before.ckpt!.metaJson) as Record; + freshMeta.rowCount = 3; + const db = new DatabaseSync(path.join(h.dir, "index.db")); + db.prepare(`DELETE FROM pf_render WHERE sid = ? AND pos = 1`).run(sid); + db.prepare(`UPDATE pf_render SET pos = ? WHERE sid = ? AND pos = 3`).run(7, sid); + db.prepare(`UPDATE pf_render_ckpt SET meta_json = ?, checksum = ? WHERE sid = ?`).run(JSON.stringify(freshMeta), sha256Hex(JSON.stringify(freshMeta)), sid); + db.close(); + _resetWatermarkV6RenderTestState(); + const g = renderPlaneGate({ + sessionId: "s-spars", session: { id: "s-spars", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 7, tokenCount: 100, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }); + assert.ok(g.ok, `sparse positions hydrate within the span, got ${JSON.stringify(g)}`); + if (g.ok) assert.deepEqual(g.rw.positions, [0, 2, 7], "positions ride the sorted-row order with the hole kept"); + // A NON-ASCENDING read shape is substrate corruption ⇒ durable kill + // (defensive guard — a normal ordered SELECT cannot produce it, so + // the pin drives it through a stubbed reader like the transient-read + // pin does). + _resetWatermarkV6RenderTestState(); + const scrambled: SessionStore = Object.create(h.store); + scrambled.pfRenderRows = () => [ + { pos: 2, rowJson: '{\"i\":2}' }, + { pos: 0, rowJson: '{\"i\":0}' }, + { pos: 7, rowJson: '{\"i\":3}' }, + ]; + const g2 = renderPlaneGate({ + sessionId: "s-spars", session: { id: "s-spars", state: createInitialState() } as unknown as Session, + store: scrambled, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 7, tokenCount: 100, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }); + assert.deepEqual(g2, { ok: false, reason: "lane-dead" }, "non-ascending positions kill durably"); + _resetWatermarkV6RenderTestState(); + const g3 = renderPlaneGate({ + sessionId: "s-spars", session: { id: "s-spars", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 7, tokenCount: 100, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }); + assert.deepEqual(g3, { ok: false, reason: "lane-dead" }, "the verdict survives on the real store too"); + } finally { h.cleanup(); } +}); + test("unit: thrown row-table read defers arming without killing (transient ≠ death)", () => { const h = makeH("defer"); try { @@ -610,7 +865,7 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" // this tier exercises is the accumulation ALGEBRA — each row counted // exactly once with its own cost, tail-only deltas on steady rounds. const rowCost = (rs: string[]): number => rs.reduce((s, x) => s + x.length, 0); - let op: { committed: boolean; op: "none" | "append" | "cast" | "rebase" } = { committed: false, op: "none" }; + let op: { committed: boolean; op: "none" | "append" | "cast" | "rebase" | "apply" } = { committed: false, op: "none" }; // No-recast regime: once the lane dies, production stops paying for // capture strings entirely (the seam skips the station-2 commit); the // lane rides pure legacy until the session ends. Mirror that here. From ccfbce3fbb50595c0cde36640996eedcdde0b2a3 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 05:00:16 +0800 Subject: [PATCH 33/49] fix(fastpath): build render-capture strings only for turns the render plane can commit (#5664 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy-path capture build was guarded by !renderLaneDead only, which is false for every turn where the render gate never ran — i.e. all of them while watermarkV6MsgId/watermarkV6Render are off (the default). Result: an O(n) JSON.stringify over the ENTIRE processed core view on every openai-wire turn, feature OFF included, whose output station 2 skips anyway (renderCapturedSid null). That taxes every user with exactly the full-history cost class this plane exists to remove and violates the #5664 hard constraint 'gate closed => exactly pre-change behavior'. Guard with renderCapturedSid !== null (set only when the gate block ran: v6 engaged hit + watermarkV6Render on). Station 2 already requires it, so no commit path changes — the predicate only removes the wasted build. Local: typecheck clean, targeted suites 43/43 green. --- src/server/prepare-openai.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 6ad2210bb3..6b000db88a 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -611,11 +611,16 @@ export async function prepareOpenai( } } else { rebuiltMessages = await mapEgress(processedMessages); - // Lane-durably-dead sessions never arm/commit again — keep the - // strings null so station 2 skips both their build (O(n) per - // turn) and the doomed commit call. Rows store the post-rewrite - // core form (#5643 §5), so capture from cores, never wire bytes. - if (!renderLaneDead) renderedCaptureStrings = processedMessages.map((m) => JSON.stringify(m)); + // Capture strings exist ONLY for turns whose render plane can + // actually commit them: the gate block ran (renderCapturedSid + // set ⇒ v6 engaged hit + watermarkV6Render on) AND the lane is + // not durably dead. Building them for anything else would tax + // EVERY openai-wire turn (feature OFF included) with an O(n) + // full-view stringify that station 2 skips anyway — exactly the + // cost class this plane exists to remove. Rows store the + // post-rewrite core form (#5643 §5), so capture from cores, + // never wire bytes. + if (renderCapturedSid !== null && !renderLaneDead) renderedCaptureStrings = processedMessages.map((m) => JSON.stringify(m)); renderedTailObjects = []; renderedImageDelta = 0; } From 377edbd7b2bafbffbb2ce6f116bd2c83516d0259 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 05:05:19 +0800 Subject: [PATCH 34/49] docs(config): renumber pf render-plane schema refs v12 -> v13 after conflict-resolution renumber (#5652 review) The base-conflict merge shifted the fastpath planes up one version (identity plane v11 -> v12, render plane v12 -> v13) because master's #5539 claimed v11 for the FK child-column indexes. The three config-doc spots still cited the pre-renumber number for the render plane. --- CONFIGURATION.md | 2 +- CONFIGURATION.zh-CN.md | 2 +- website/config-reference/rollout.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 5aff472a9c..5eea22df67 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -104,7 +104,7 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0 (off) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | Plugin-lane server-owned history (#4491): for enabled installs, (a) plugin-lane requests may carry an internal `biliMsgIds` array parallel to `messages` that pins message identity to client-supplied ids (always stripped before upstream forwarding; malformed arrays are ignored with a warning, id-poisoning shapes hard-fail on the incremental lane); (b) the loopback-only `POST /__bili/plugin/render` endpoint serves the outbound prefix rendered from the unified transcript (folded regions collapse to summary anchors; any desync degrades to a non-2xx so the shim falls back to its local history); (c) the incremental-lane probe advertises `serverHistory`. Default 0% — no behavior change until rolled out. | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5 (experimental grey) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | Full-resend watermark fastpath (#5403/#5418): plugin-lane chat-completions turns whose previous response carried an x-bili-pf-seq stamp assert the resent prefix via the echoed count (structural scan, no full parse); on a hit the pipeline consumes stashed canonical prefix cores plus only the new tail window (byte-identical outbound provenance; ANY doubt falls back to the untouched full-parse path per turn and re-stamps, five credited misses hold the lane for one hour). Requires the client to echo the stamp back as x-bili-pf-count (dsh/pi/opencode/omp do; unadapted lanes never hit). EXPERIMENTAL grey rollout — ON by default for roughly 5% of installs (owner ruling 2026-11-02: small cohort before GA; one-time intro notice in the web UI for every install, tri-state switch in quick config). Force it with `true`, turn off with `false` (or a percent) / env = 0. See TECHNICAL-NOTES.md “Full-resend watermark fastpath”. | | `rollout.features.watermarkV6MsgId` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive strikes park the lane 60 min, loud fallback thereafter) — never a silent legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout. | -| `rollout.features.watermarkV6Render` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / blocks-active / fold-pending / absorb-active / ccr-on / markers / hydrate-failed / caps-mismatch / scheduled audit every Nth round / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v12, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout. | +| `rollout.features.watermarkV6Render` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / blocks-active / fold-pending / absorb-active / ccr-on / markers / hydrate-failed / caps-mismatch / scheduled audit every Nth round / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout. | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | Mask non-public target hosts in logs as ; credential headers are always masked regardless. | | `security.adminRelay` | boolean | false | — | Let an embedding host (dsh web today) relay the ENTIRE /__bili/ admin surface through its own entry origin so a remote browser can open the panel (#3373). Fail-closed, file-only. The relayed hop reaches the proxy as a local non-browser client (loopback source + loopback Host, no Origin), so the admin gate stays unchanged; enabling it exposes config editing and credential writes to whoever can reach the host web entry — protect that entry (auth / reverse proxy) first. | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude Code subagents get their own session namespace (\|sub:) instead of queueing behind the main session. Anthropic-wire only — on the Responses wire this switch is a no-op (see the `subagentSplit` / BOTH-mechanisms scope note). | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 169695024e..2d4486e61d 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -104,7 +104,7 @@ | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0(关) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | 插件车道服务端正史(#4491):对已放量的安装,(a) 插件车道请求可携带与 `messages` 平行的内部字段 `biliMsgIds`,把消息身份钉在客户端提供的 id 上(转发上游前一律剥掉;格式错误的数组告警后忽略,身份投毒形态在增量车道直接硬失败);(b) 仅回环可达的 `POST /__bili/plugin/render` 端点从统一存储渲染出站前缀(折叠区收敛为摘要锚点;任何失步都返回非 2xx,shim 退回本地历史);(c) 增量车道探测响应通告 `serverHistory`。默认 0%,放量前无行为变化。 | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5(实验性灰度) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | 全量重发水位快路径(#5403/#5418):插件车道 chat-completions 回合若上一轮响应盖有 x-bili-pf-seq,则用回声计数声明重发前缀(结构扫描,不做全量解析);命中时管线只消费钉存的前缀核 + 新增尾巴窗口(出站字节同源可证;任何疑点当轮回退未动过的全量解析路径并重新盖戳,连续五次计名失配将车道按住一小时)。需客户端把戳回声为 x-bili-pf-count(适配的车道回带;其余车道不会命中)。**实验性灰度**——默认仅约 5% 的安装开启(owner 裁定 2026-11-02:先小流量再 GA;web 面板对全部安装显示一次性公告、快速配置提供三态开关);`true` 强制开、`false`(或百分比)/ env = 0 关。见 TECHNICAL-NOTES.zh-CN.md「全量重发水位快路径」。 | | `rollout.features.watermarkV6MsgId` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | 复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击按住车道 60 分钟,之后响亮降级)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。 | -| `rollout.features.watermarkV6Render` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/blocks-active/fold-pending/absorb-active/ccr-on/markers/hydrate-failed/caps-mismatch/每 N 轮计划审计/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v12,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。 | +| `rollout.features.watermarkV6Render` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/blocks-active/fold-pending/absorb-active/ccr-on/markers/hydrate-failed/caps-mismatch/每 N 轮计划审计/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。 | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | 日志中把非公开目标主机遮成 ;凭据头无论此开关如何总是遮盖。 | | `security.adminRelay` | boolean | false | — | 允许宿主(当前为 dsh web)把它自己入口域名下的整个 /__bili/ 管理面中继到本机代理,使远程浏览器能打开面板(#3373)。默认关闭、仅配置文件。中继跳到达代理时是本机非浏览器客户端(回环源地址 + 回环 Host、无 Origin 头),管理端门槛不变;开启后,凡能访问宿主 web 入口的人都能改配置、写凭据——请先给入口加鉴权/反代。 | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude 子代理拥有独立会话命名空间(\|sub:),不再排队在主会话锁后。仅 anthropic 线路 —— Responses 线路上本开关无效(见 `subagentSplit` / BOTH-mechanisms 作用域说明)。 | diff --git a/website/config-reference/rollout.yaml b/website/config-reference/rollout.yaml index 2ff925f5a6..6f16b4572b 100644 --- a/website/config-reference/rollout.yaml +++ b/website/config-reference/rollout.yaml @@ -79,5 +79,5 @@ entries: default_zh: "0(关)" env: BILI_ROLLOUT_WATERMARK_V6_RENDER group: behavior - desc_en: "v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / blocks-active / fold-pending / absorb-active / ccr-on / markers / hydrate-failed / caps-mismatch / scheduled audit every Nth round / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v12, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout." - desc_zh: "v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/blocks-active/fold-pending/absorb-active/ccr-on/markers/hydrate-failed/caps-mismatch/每 N 轮计划审计/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v12,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。" + desc_en: "v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / blocks-active / fold-pending / absorb-active / ccr-on / markers / hydrate-failed / caps-mismatch / scheduled audit every Nth round / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout." + desc_zh: "v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/blocks-active/fold-pending/absorb-active/ccr-on/markers/hydrate-failed/caps-mismatch/每 N 轮计划审计/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。" From 6f3baa64a8c191e9f88c921094cd2fe0e0fab66e Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 6 Nov 2026 21:57:05 +0800 Subject: [PATCH 35/49] fix: pf_log continuation must clear the checkpoint watermark post-fold (review BUG-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pfCommitTurn numbered log entries from MAX(pf_log.ls_seq) alone. A fold (pfCompact) absorbs its own log entries in the same transaction, so after ANY compaction MAX(log) sits BEHIND ckpt_ls_seq and fresh entries re-used sequences <= the watermark — invisible to pfMeta forever. A real process restart then rehydrated the world at the stale folded maxPos: progress turns collided the (sid,pos) primary key into a permanent per-sid commit fail-loop, and no-progress turns served a truncated prefix wire. Mirrors the identical guard already carried by pfRenderCommit (render plane never had the bug). Regression: tests/fastpath-v6-msgid.test.ts 'fold-then-append across a REAL restart' — also makes the B1 pin exercise true disk hydration via _resetWatermarkV6MirrorsForTest (the store reset alone left module-level mirrors alive, which is why the pin passed vacuously pre-fix). --- src/server/fastpath-v6.ts | 11 ++++++ src/storage/store.ts | 13 ++++++- tests/fastpath-v6-msgid.test.ts | 67 ++++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts index de69a89bad..b4397d511f 100644 --- a/src/server/fastpath-v6.ts +++ b/src/server/fastpath-v6.ts @@ -1027,6 +1027,17 @@ export function clearWatermarkV6(sessionId: string): void { } } +/** Test-only process-death simulation: drop EVERY module-level mirror, not + * just one session's. _resetStoreForTest alone leaves the in-memory planes + * alive, so a "restart" test that skips this hook never exercises disk + * hydration (review #5652 finding: the B1 pin passed vacuously). */ +export function _resetWatermarkV6MirrorsForTest(): void { + guardMem.clear(); + worlds.clear(); + sessionWorlds.clear(); + pendingCommits.clear(); +} + /** Introspection for panels/tests: live mirrors of one relay session. */ export function watermarkV6DebugState(sessionId: string): { liveSids: string[]; worlds: Record } { const sids = [...(sessionWorlds.get(sessionId) ?? [])]; diff --git a/src/storage/store.ts b/src/storage/store.ts index 3b8be7dc1c..6049f87f14 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2498,8 +2498,19 @@ export class UnifiedStore { if (!this.pfAvailable()) throw new Error("pf plane unavailable"); const nextLsSeq = this.db.transaction((): number => { this.pfInsertRows(sid, args.rows); + // Continuation must clear BOTH the surviving log tail AND the + // checkpoint watermark: a fresh checkpoint ABSORBS its own log + // entries (pfCompact deletes them in that tx), so MAX(log) sits + // BEHIND ckpt_ls_seq after any fold — numbering from MAX alone + // would re-issue sequences ≤ ckpt and those entries would be + // invisible to pfMeta forever (post-fold turns silently lost, + // stale world.maxPos ⇒ PK collisions on every later progress + // turn). Identical guard to pfRenderCommit. const start = Number( - ((this.db.prepare("SELECT COALESCE(MAX(ls_seq), 0) AS m FROM pf_log WHERE sid = ?").get(sid) as { m: number }).m ?? 0), + Math.max( + (this.db.prepare("SELECT COALESCE(MAX(ls_seq), 0) AS m FROM pf_log WHERE sid = ?").get(sid) as { m: number }).m ?? 0, + (this.db.prepare("SELECT ckpt_ls_seq AS n FROM pf_ckpt WHERE sid = ?").get(sid) as { n: number } | undefined)?.n ?? 0, + ), ); const insLog = this.db.prepare( "INSERT INTO pf_log (sid, ls_seq, kind, payload_json, born_pos_hi, ts) VALUES (?, ?, ?, ?, ?, ?)", diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index e3b9bae7a6..3302e49867 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -24,7 +24,7 @@ import type { SessionStore as SessionStoreType } from "../src/persist.ts"; import { UnifiedShadow } from "../src/storage/shadow.ts"; import type { Session } from "../src/session.ts"; import { - evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, watermarkV6DebugState, clearWatermarkV6, PF_SESSION_HEADER, + evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, watermarkV6DebugState, clearWatermarkV6, _resetWatermarkV6MirrorsForTest, PF_SESSION_HEADER, } from "../src/server/fastpath-v6.ts"; import { PF_SEQ_HEADER, PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; import type { V6Outcome } from "../src/server/fastpath-v6.ts"; @@ -378,6 +378,9 @@ test("§8-16/19 restart hydration: view survives boot cycles across an L1 fold ( assert.ok(meta !== null && meta.ckpt !== null, "L1 compaction must have fired inside the 40-turn run"); void meta!.log; // --- SIMULATE PROCESS RESTART: drop every mirror, reopen the SAME disk ---- + // Module-level mirrors OUTSURVIVE _resetStoreForTest — without this + // hook the round below hits memory and never exercises hydration. + _resetWatermarkV6MirrorsForTest(); _resetStoreForTest(); const shadow2 = UnifiedShadow.maybeOpen(h.dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r2) => r2 as never)!; const store2 = new SessionStore({ dir: path.join(h.dir, "sessions"), enabled: true, shadow: shadow2 }); @@ -540,3 +543,65 @@ test("restart after compaction: a compacted world echoes cleanly off its ckpt (n assert.equal(r2.headers[PF_SESSION_HEADER], sid, "the SAME world continues — no re-arm, no fault"); } finally { h.cleanup(); } }); + +test("fold-then-append across a REAL restart: post-fold log entries clear the ckpt watermark and the folded maxPos advances (review BUG-1 regression)", () => { + const h = makeH("foldvis"); + try { + const log = makeLog(); + const SESS = "s-fv"; + const CARRIER = "car-fv"; + // arm T=2 + hit T=3 + let msgs: Msg[] = [{ role: "system", content: SYS }, { role: "user", content: "u1" }]; + let r = turn(SESS, CARRIER, makeBody(msgs), ["f0", "f1"], log); + assert.equal(r.outcome.decision, "arm"); + const sid = r.sid!; + msgs.push({ role: "assistant", content: "a1" }); + r = turn(SESS, CARRIER, makeBody(msgs), ["f0", "f1", "f2"], log, sid); + assert.equal(r.outcome.decision, "hit"); + // Fold NOW — exactly what opportunisticCompact does once ≥64 entries + // accumulate: checkpoint bound to the CURRENT world state, log wiped + // inside the same transaction. + const wState = watermarkV6DebugState(SESS).worlds[sid]; + const metaBefore = h.store.pfMeta(sid); + assert.ok(metaBefore !== null && metaBefore.ckpt === null); + const foldSeq = metaBefore!.log[metaBefore!.log.length - 1].lsSeq; + h.store.pfCompact(sid, { metaJson: JSON.stringify({ v: 1, t: wState.t, maxPos: wState.maxPos }), nextLsSeq: foldSeq }); + assert.equal(h.store.pfMeta(sid)!.log.length, 0, "the fold absorbed its own log"); + // Post-fold progress turn (T=4): the new front/geom pair must number + // AHEAD of the fold — numbering from MAX(log) alone would re-issue + // sequences ≤ ckpt and pfMeta's `ls_seq > ckpt` filter would hide them + // forever, freezing the folded world's maxPos one element short. + msgs.push({ role: "user", content: "u2" }); + r = turn(SESS, CARRIER, makeBody(msgs), ["f0", "f1", "f2", "f3"], log, sid); + assert.equal(r.outcome.decision, "hit"); + const metaAfterFoldTurn = h.store.pfMeta(sid)!; + assert.ok(metaAfterFoldTurn.log.length >= 2, `the post-fold front/geom pair must be VISIBLE past the ckpt watermark ${foldSeq}, got ${JSON.stringify(metaAfterFoldTurn.log.map((e) => e.lsSeq))}`); + assert.ok(metaAfterFoldTurn.log.every((e) => e.lsSeq > foldSeq), `no post-fold sequence may sit at or below the ckpt watermark ${foldSeq}: ${JSON.stringify(metaAfterFoldTurn.log.map((e) => e.lsSeq))}`); + // --- REAL process restart: mirrors AND store gone, same disk --- + _resetWatermarkV6MirrorsForTest(); + _resetStoreForTest(); + const shadow2 = UnifiedShadow.maybeOpen(h.dir, { BILI_STORAGE_UNIFIED: "shadow" }, (r2) => r2 as never)!; + const store2 = new SessionStore({ dir: path.join(h.dir, "sessions"), enabled: true, shadow: shadow2 }); + _setStoreForTest(store2); + const log2 = makeLog(); + // Pure-append turn off the REHYDRATED world: the visible geom entries + // must advance maxPos through the post-fold element — otherwise the + // next row lands on an occupied ledger position (PK collision → commit + // fail-loop) and the served view ships a TRUNCATED prefix. + msgs.push({ role: "assistant", content: "a2" }); + const r5 = turn(SESS, CARRIER, makeBody(msgs), ["f0", "f1", "f2", "f3", "f4"], log2, sid); + assert.equal(r5.outcome.decision, "hit", `rehydrated world must ride the lane, log=${JSON.stringify(log2.lines.slice(-4))}`); + const view5 = r5.outcome.hit!.view.map((c) => JSON.stringify(c)).join("\n"); + for (const m of ["u1", "a1", "u2", "a2"]) { + assert.ok(view5.includes(JSON.stringify(m)), `view must carry ${JSON.stringify(m)} — no truncated prefix across fold+restart`); + } + assert.ok(!log2.lines.some((l) => l.msg.includes("degraded") || l.msg.includes("commit failed")), + `no finalize hiccup across fold+restart: ${JSON.stringify(log2.lines.slice(-4))}`); + assert.equal(watermarkV6DebugState(SESS).worlds[sid].maxPos, 4, "folded maxPos must include the post-fold element"); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + try { + const row = db.prepare(`SELECT COUNT(*) AS n FROM pf_rows WHERE sid = ?`).get(sid) as { n: number }; + assert.equal(row.n, 5, "every element registered exactly once — no collision-driven loss or duplication"); + } finally { db.close(); } + } finally { h.cleanup(); } +}); From 4aa85c8afe68fa1ebbd281f2ea86eeff9917f526 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 6 Nov 2026 22:10:36 +0800 Subject: [PATCH 36/49] fix: archiveSession must cascade the fastpath v6 plane tables with the session (review BUG-2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep deleted every session row except the six sid-keyed fastpath v6 partitions, which hang off the session only through the session_ext registry: once that row died with the sweep, no sid was referable anymore and every archived session's plane rows were permanently orphaned — invisible to gc() and to sessionStorageStats alike (cache-leak class). deleteSession already cascades in-txn; gc() reaches sessions through archiveSession, so this was the lone missing link. The planes are a rebuildable cache, never an identity source: drop them in the archive transaction WITHOUT exporting them into the JSONL — a restored session re-boots through a fresh arm (probe-miss ⇒ rule ③), and the exported registry rows ride restore harmlessly (probes resolve nothing). Also fixes the latent crash this regression exposed: restoreArchived re-inserted session_ext rows without updated_at (NOT NULL in DDL), so restoring ANY session carrying a single ext row threw — v6 sessions always do (sid registry + guard). Export now carries updated_at; older exports predate it and fall back to Date.now(). Regression pin: two worlds on one relay session → archive → all six tables empty for BOTH sids, tombstone intact → restore → fresh arm mint (not a fault loop on the dead registry sids). Negative-control verified: without the fix exactly one assertion fails on orphaned pf_rows. --- src/storage/store.ts | 45 ++++++++++++++++++++++++++--- tests/fastpath-v6-msgid.test.ts | 50 +++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/src/storage/store.ts b/src/storage/store.ts index 6049f87f14..d2b367828e 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -1899,7 +1899,10 @@ export class UnifiedStore { * showing it) and the conversations identity binding survives (identity * is forever; a restore must land on the same session id). Pinned * checkpoints keep their rows + refs. Idempotent: an already-archived - * session is a no-op returning the log path. */ + * session is a no-op returning the log path. The six fastpath v6 plane + * partitions (#5643/#5664, sid-keyed through the session_ext registry) + * are swept in the same transaction — a rebuildable cache, never + * exported: a restored session re-boots through a fresh arm. */ archiveSession(sessionId: string, opts: { archiveDir: string }): { ok: boolean; exportPath?: string; reason?: string; contentRowsDeleted?: number } { const row = this.getSessionRow(sessionId); @@ -1920,7 +1923,7 @@ export class UnifiedStore { const refsRows = (this.db.prepare("SELECT ord, raw_id, ref, ref_num FROM refs WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); const deadRefs = (this.db.prepare("SELECT ord, ref, ref_num, recorded_at FROM dead_refs WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); const ccr = (this.db.prepare("SELECT ord, ccr_key, content_hash FROM ccr_entries WHERE session_id = ? ORDER BY ord").all(sessionId) as Record[]); - const ext = (this.db.prepare("SELECT key, value_json FROM session_ext WHERE session_id = ?").all(sessionId) as Record[]); + const ext = (this.db.prepare("SELECT key, value_json, updated_at FROM session_ext WHERE session_id = ?").all(sessionId) as Record[]); const checkpoints = (this.db.prepare("SELECT cp_id, seq_at, block_set_hash, kind, created_at, pinned, label, state_hash FROM checkpoints WHERE session_id = ?").all(sessionId) as Record[]); // --- collect the distinct CAS bodies referenced by those rows --- const wanted = new Set(); @@ -1954,6 +1957,36 @@ export class UnifiedStore { this.db.prepare("DELETE FROM dead_refs WHERE session_id = ?").run(sessionId); this.db.prepare("DELETE FROM refs WHERE session_id = ?").run(sessionId); this.db.prepare("DELETE FROM ccr_entries WHERE session_id = ?").run(sessionId); + // #5652 review BUG-2: archive swept every table EXCEPT the six + // fastpath v6 plane partitions, which hang off the session only + // through the ext registry (same key deleteSession cascades): + // once session_ext died, no sid was referable anymore and every + // archived session's plane rows were permanently orphaned — + // invisible to gc() and to sessionStorageStats alike. The planes + // are a rebuildable CACHE, never an identity source: a restored + // session re-boots through a fresh arm (probe-miss ⇒ rule ③), so + // dropping them here loses nothing — they are deliberately NOT + // exported into the JSONL either (the exported registry rows then + // ride restore harmlessly: probes resolve nothing, clean re-arm). + const pfRegRow = this.db + .prepare("SELECT value_json FROM session_ext WHERE session_id = ? AND key = 'pf:v6-sids'") + .get(sessionId) as { value_json: string } | undefined; + if (pfRegRow !== undefined) { + let pfSids: unknown[] = []; + try { + const p = JSON.parse(pfRegRow.value_json); + if (Array.isArray(p)) pfSids = p; + } catch { /* corrupt registry: nothing to cascade from */ } + for (const sid of pfSids) { + if (typeof sid !== "string") continue; + this.db.prepare("DELETE FROM pf_rows WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_log WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_ckpt WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_render WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_render_log WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM pf_render_ckpt WHERE sid = ?").run(sid); + } + } this.db.prepare("DELETE FROM session_ext WHERE session_id = ?").run(sessionId); this.db.prepare("UPDATE sessions SET status = 'archived' WHERE session_id = ?").run(sessionId); for (const hash of rel) this.releaseContent(hash); @@ -2035,8 +2068,12 @@ export class UnifiedStore { for (const d of dump.deadRefs) insDead.run(sessionId, d.ord, d.ref, d.ref_num, d.recorded_at); const insCcr = this.db.prepare("INSERT INTO ccr_entries (session_id, ord, ccr_key, content_hash) VALUES (?, ?, ?, ?)"); for (const e of dump.ccrEntries) insCcr.run(sessionId, e.ord, e.ccr_key, e.content_hash); - const insExt = this.db.prepare("INSERT INTO session_ext (session_id, key, value_json) VALUES (?, ?, ?)"); - for (const x of dump.sessionExt) insExt.run(sessionId, x.key, x.value_json); + // #5652 review: restore used to re-insert ext rows without + // updated_at (NOT NULL in DDL) → restoring ANY session carrying a + // single ext row threw (v6 sessions always do: sid registry + + // guard). Older exports predate the field: fall back to now(). + const insExt = this.db.prepare("INSERT INTO session_ext (session_id, key, value_json, updated_at) VALUES (?, ?, ?, ?)"); + for (const x of dump.sessionExt) insExt.run(sessionId, x.key, x.value_json, typeof x.updated_at === "number" ? x.updated_at : Date.now()); const insCp = this.db.prepare("INSERT INTO checkpoints (cp_id, session_id, seq_at, block_set_hash, kind, created_at, pinned, label, state_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(cp_id) DO NOTHING"); for (const cp of dump.checkpoints) insCp.run(cp.cp_id, sessionId, cp.seq_at, cp.block_set_hash, cp.kind, cp.created_at, cp.pinned, cp.label ?? null, cp.state_hash); this.db diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index 3302e49867..0bc1896f27 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -605,3 +605,53 @@ test("fold-then-append across a REAL restart: post-fold log entries clear the ck } finally { db.close(); } } finally { h.cleanup(); } }); + +test("archiving a session sweeps its pf planes too (review BUG-2 regression): no orphaned sid rows survive the tombstone; a restored session re-boots through a fresh arm", () => { + const h = makeH("archpf"); + const SESS = "s-ap"; + const CARRIER_A = "car-ap-a"; + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const rA = turn(SESS, CARRIER_A, b1, ["p0", "p1"], log); + assert.equal(rA.outcome.decision, "arm"); + const sidA = rA.sid!; + // Second carrier ⇒ second world on the SAME relay session: the + // registry holds two sids, so the cascade must reach every partition + // of BOTH, not just the newest. + const rB = turn(SESS, "car-ap-b", b1, ["p0", "p1"], log); + assert.equal(rB.outcome.decision, "arm"); + const sidB = rB.sid!; + assert.notEqual(sidB, sidA, "distinct carriers mint distinct worlds"); + // A real sessions row for the archive machinery to find (in + // production the first plugin turn records one). + const db0 = new DatabaseSync(path.join(h.dir, "index.db")); + db0.prepare(`INSERT INTO sessions (session_id, created_at, updated_at) VALUES (?, ?, ?)`).run(SESS, Date.now(), Date.now()); + db0.close(); + const archDir = path.join(h.dir, "arch"); + const arch = h.store.unifiedArchiveSession(SESS, archDir); + assert.ok(arch !== null && arch.ok, `archive must succeed, got ${JSON.stringify(arch)}`); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + try { + for (const t of ["pf_rows", "pf_log", "pf_ckpt", "pf_render", "pf_render_log", "pf_render_ckpt"]) { + for (const sid of [sidA, sidB]) { + const row = db.prepare(`SELECT COUNT(*) AS n FROM ${t} WHERE sid = ?`).get(sid) as { n: number }; + assert.equal(row.n, 0, `${t} must be swept by archive — before the fix every archived session's plane rows stayed permanently orphaned behind the deleted registry`); + } + } + const srow = db.prepare(`SELECT status FROM sessions WHERE session_id = ?`).get(SESS) as { status: string }; + assert.equal(srow.status, "archived", "the sessions row survives as a tombstone, like every other archived session"); + } finally { db.close(); } + // Rebuildability: the planes are a cache, never an identity source. + // A restored session whose ledger plane was purged must re-bootstrap + // through a FRESH arm (probe-miss ⇒ rule ③) instead of fault-looping + // on the dead sids its exported registry still lists. + const res = h.store.unifiedRestoreArchived(SESS, archDir); + assert.ok(res !== null && res.ok, `restore must succeed, got ${JSON.stringify(res)}`); + clearWatermarkV6(SESS); + const log2 = makeLog(); + const rC = turn(SESS, CARRIER_A, b1, ["p0", "p1"], log2); + assert.equal(rC.outcome.decision, "arm", `restored session must re-arm cleanly off an empty probe (log=${JSON.stringify(log2.lines.slice(-3))})`); + assert.notEqual(rC.sid, sidA, "a FRESH world mint — the old plane bytes are gone by design"); + } finally { h.cleanup(); } +}); From 8c4dcda736fc5748f18807c3c5c860c5d5f5ce44 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 6 Nov 2026 22:42:33 +0800 Subject: [PATCH 37/49] =?UTF-8?q?docs:=20watermarkV6*=20config=20rows=20?= =?UTF-8?q?=E2=80=94=20red-reason=20vocabulary=20+=20breaker=20parking=20s?= =?UTF-8?q?emantics=20(#5652=20review=20F3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CONFIGURATION.md | 4 ++-- CONFIGURATION.zh-CN.md | 4 ++-- website/config-reference/rollout.yaml | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 5eea22df67..372e39f521 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -103,8 +103,8 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `rollout.features.base` | string | unset (roll out over everyone) | — | Nested rollout: evaluate this feature only inside the named feature's enabled cohort, so the percent is relative to that population (roll B out to A's users; chains allowed, cycles fail closed). | | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0 (off) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | Plugin-lane server-owned history (#4491): for enabled installs, (a) plugin-lane requests may carry an internal `biliMsgIds` array parallel to `messages` that pins message identity to client-supplied ids (always stripped before upstream forwarding; malformed arrays are ignored with a warning, id-poisoning shapes hard-fail on the incremental lane); (b) the loopback-only `POST /__bili/plugin/render` endpoint serves the outbound prefix rendered from the unified transcript (folded regions collapse to summary anchors; any desync degrades to a non-2xx so the shim falls back to its local history); (c) the incremental-lane probe advertises `serverHistory`. Default 0% — no behavior change until rolled out. | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5 (experimental grey) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | Full-resend watermark fastpath (#5403/#5418): plugin-lane chat-completions turns whose previous response carried an x-bili-pf-seq stamp assert the resent prefix via the echoed count (structural scan, no full parse); on a hit the pipeline consumes stashed canonical prefix cores plus only the new tail window (byte-identical outbound provenance; ANY doubt falls back to the untouched full-parse path per turn and re-stamps, five credited misses hold the lane for one hour). Requires the client to echo the stamp back as x-bili-pf-count (dsh/pi/opencode/omp do; unadapted lanes never hit). EXPERIMENTAL grey rollout — ON by default for roughly 5% of installs (owner ruling 2026-11-02: small cohort before GA; one-time intro notice in the web UI for every install, tri-state switch in quick config). Force it with `true`, turn off with `false` (or a percent) / env = 0. See TECHNICAL-NOTES.md “Full-resend watermark fastpath”. | -| `rollout.features.watermarkV6MsgId` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive strikes park the lane 60 min, loud fallback thereafter) — never a silent legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout. | -| `rollout.features.watermarkV6Render` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / blocks-active / fold-pending / absorb-active / ccr-on / markers / hydrate-failed / caps-mismatch / scheduled audit every Nth round / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout. | +| `rollout.features.watermarkV6MsgId` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive faults park v6 for 60 min; while parked the same traffic is SERVED by the baseline v5/legacy lanes instead of refused — parking/recovery are loudly logged) — never a SILENT legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout. | +| `rollout.features.watermarkV6Render` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm-turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due: scheduled audit every Nth round / no-rows / no-plane-geometry / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout. | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | Mask non-public target hosts in logs as ; credential headers are always masked regardless. | | `security.adminRelay` | boolean | false | — | Let an embedding host (dsh web today) relay the ENTIRE /__bili/ admin surface through its own entry origin so a remote browser can open the panel (#3373). Fail-closed, file-only. The relayed hop reaches the proxy as a local non-browser client (loopback source + loopback Host, no Origin), so the admin gate stays unchanged; enabling it exposes config editing and credential writes to whoever can reach the host web entry — protect that entry (auth / reverse proxy) first. | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude Code subagents get their own session namespace (\|sub:) instead of queueing behind the main session. Anthropic-wire only — on the Responses wire this switch is a no-op (see the `subagentSplit` / BOTH-mechanisms scope note). | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index 2d4486e61d..f585b26c66 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -103,8 +103,8 @@ | `rollout.features.base` | string | unset (roll out over everyone) | — | 嵌套灰度:仅在所指特性的已入组人群内评估本特性,percent 相对该人群(B 只放量给 A 的用户;允许链式,环路判为关)。 | | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0(关) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | 插件车道服务端正史(#4491):对已放量的安装,(a) 插件车道请求可携带与 `messages` 平行的内部字段 `biliMsgIds`,把消息身份钉在客户端提供的 id 上(转发上游前一律剥掉;格式错误的数组告警后忽略,身份投毒形态在增量车道直接硬失败);(b) 仅回环可达的 `POST /__bili/plugin/render` 端点从统一存储渲染出站前缀(折叠区收敛为摘要锚点;任何失步都返回非 2xx,shim 退回本地历史);(c) 增量车道探测响应通告 `serverHistory`。默认 0%,放量前无行为变化。 | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5(实验性灰度) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | 全量重发水位快路径(#5403/#5418):插件车道 chat-completions 回合若上一轮响应盖有 x-bili-pf-seq,则用回声计数声明重发前缀(结构扫描,不做全量解析);命中时管线只消费钉存的前缀核 + 新增尾巴窗口(出站字节同源可证;任何疑点当轮回退未动过的全量解析路径并重新盖戳,连续五次计名失配将车道按住一小时)。需客户端把戳回声为 x-bili-pf-count(适配的车道回带;其余车道不会命中)。**实验性灰度**——默认仅约 5% 的安装开启(owner 裁定 2026-11-02:先小流量再 GA;web 面板对全部安装显示一次性公告、快速配置提供三态开关);`true` 强制开、`false`(或百分比)/ env = 0 关。见 TECHNICAL-NOTES.zh-CN.md「全量重发水位快路径」。 | -| `rollout.features.watermarkV6MsgId` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | 复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击按住车道 60 分钟,之后响亮降级)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。 | -| `rollout.features.watermarkV6Render` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/blocks-active/fold-pending/absorb-active/ccr-on/markers/hydrate-failed/caps-mismatch/每 N 轮计划审计/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。 | +| `rollout.features.watermarkV6MsgId` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | 复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击将 v6 按住 60 分钟;按住期间同一流量由基线 v5/legacy 车道承接而非拒绝——停放与恢复均响亮记日志)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。 | +| `rollout.features.watermarkV6Render` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/fold-pending/absorb-active/ccr-on/markers/lane-dead/hydrate-failed/caps-mismatch/structural-pending/blocks-changed/每 N 轮计划审计(audit-due)/no-rows/no-plane-geometry/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。 | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | 日志中把非公开目标主机遮成 ;凭据头无论此开关如何总是遮盖。 | | `security.adminRelay` | boolean | false | — | 允许宿主(当前为 dsh web)把它自己入口域名下的整个 /__bili/ 管理面中继到本机代理,使远程浏览器能打开面板(#3373)。默认关闭、仅配置文件。中继跳到达代理时是本机非浏览器客户端(回环源地址 + 回环 Host、无 Origin 头),管理端门槛不变;开启后,凡能访问宿主 web 入口的人都能改配置、写凭据——请先给入口加鉴权/反代。 | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude 子代理拥有独立会话命名空间(\|sub:),不再排队在主会话锁后。仅 anthropic 线路 —— Responses 线路上本开关无效(见 `subagentSplit` / BOTH-mechanisms 作用域说明)。 | diff --git a/website/config-reference/rollout.yaml b/website/config-reference/rollout.yaml index 6f16b4572b..f5baee890d 100644 --- a/website/config-reference/rollout.yaml +++ b/website/config-reference/rollout.yaml @@ -71,13 +71,13 @@ entries: default_zh: "0(关)" env: BILI_ROLLOUT_WATERMARK_V6_MSG_ID group: behavior - desc_en: "Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive strikes park the lane 60 min, loud fallback thereafter) — never a silent legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout." - desc_zh: "复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击按住车道 60 分钟,之后响亮降级)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。" + desc_en: "Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive faults park v6 for 60 min; while parked the same traffic is SERVED by the baseline v5/legacy lanes instead of refused — parking/recovery are loudly logged) — never a SILENT legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout." + desc_zh: "复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击将 v6 按住 60 分钟;按住期间同一流量由基线 v5/legacy 车道承接而非拒绝——停放与恢复均响亮记日志)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。" - key: rollout.features.watermarkV6Render type: "boolean | number" default: "0 (off)" default_zh: "0(关)" env: BILI_ROLLOUT_WATERMARK_V6_RENDER group: behavior - desc_en: "v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / blocks-active / fold-pending / absorb-active / ccr-on / markers / hydrate-failed / caps-mismatch / scheduled audit every Nth round / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout." - desc_zh: "v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/blocks-active/fold-pending/absorb-active/ccr-on/markers/hydrate-failed/caps-mismatch/每 N 轮计划审计/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。" + desc_en: "v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm-turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due: scheduled audit every Nth round / no-rows / no-plane-geometry / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout." + desc_zh: "v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/fold-pending/absorb-active/ccr-on/markers/lane-dead/hydrate-failed/caps-mismatch/structural-pending/blocks-changed/每 N 轮计划审计(audit-due)/no-rows/no-plane-geometry/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。" From 4418ec501f9804fb43d9825fee85bde7a1d62c31 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 6 Nov 2026 22:42:33 +0800 Subject: [PATCH 38/49] =?UTF-8?q?docs:=20technical-notes=20v6=20doctrine?= =?UTF-8?q?=20=E2=80=94=20composite=20identity=20+=20render-snapshot=20pla?= =?UTF-8?q?ne,=20en+zh=20(#5652=20review=20F5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TECHNICAL-NOTES.md | 16 ++++++++++++++++ TECHNICAL-NOTES.zh-CN.md | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/TECHNICAL-NOTES.md b/TECHNICAL-NOTES.md index 29b4d98feb..5cd8ea041c 100644 --- a/TECHNICAL-NOTES.md +++ b/TECHNICAL-NOTES.md @@ -587,3 +587,19 @@ Declared trust boundary (§7.4 — human review required, no auto-merge): a clie Honest limits: (1) once a multimodal tail (content array containing `image_url`) lands in the stash, the lane degrades to full legacy turns for the remainder of that history — the kernel renders user content-array images differently when the message is the LAST element versus mid-history, so a frozen stash cannot reproduce legacy bytes position-independently; text workloads (the #5403 reporter's shape) are unaffected. (2) The head fragment is re-parsed (8 elements) each hit solely for the system-run check. (3) Both `x-bili-pf-*` headers are stripped upstream (`UPSTREAM_HOP_HEADERS`) — protocol headers never leak into model traffic. Measured (provenance: `tests/issue5403-watermark-fastpath.test.ts`, hermetic dual-proxy A/B on the same corpus): hit turns skip the full parse/conversion/pin entirely; resident `lastMessages` 8684 B → 1210 B (7.2×) on an 8-turn corpus including a multipart image landing turn; outbound bytes byte-identical to legacy modulo kernel `time="…"` attributes across all turns; `x-bili-pf-*` absent from every captured upstream request, both phases. Unit coverage: `tests/fastpath.test.ts` (full verdict ladder incl. breaker TTL re-arm), `tests/fastpath-scanner.test.ts` (adversarial JSON table, depth cap, surrogate pairs, 50k-element scale), `tests/lane-decisions-matrix.test.ts` (gate row matrix, both directions). + +## Full-resend watermark fastpath v6 — composite identity + render-snapshot plane (#5643 / #5664) + +v6 is a **full rework** of the v5 watermark line above (owner ruling 2026-11-04; the PR #5576 lineage is void) and splits into two planes sharing one hard-gated entry: the identity lane (`src/server/fastpath-v6.ts`) and the render-snapshot plane (`src/server/fastpath-v6-render.ts`, nested inside the identity cohort). + +**Why v6.** A bare count pins the boundary only by POSITION — a client that rewrites elements while keeping the count constant stays invisible. v6 makes the (client sessionid, msgid) composite pair THE identity marker (P1, contract locked): hit turns transform ONLY the NEW element spans (set difference, ANY positions in the body), and the frozen prefix is materialized from stored cores — raw request bytes leave memory at end of turn (P2). Every client-side structural change (compaction, in-family model switch, reorder, in-place rewrite, renumbering) is IGNORED; the view grows forward only, no single client event triggers a global rebase (P3 — client-compaction "B version", owner ruling 2026-11-04: when the dual channels reconcile, compaction is ignored entirely and a compaction summary arriving as an ordinary element is admitted with ZERO special treatment; explicitly accepted trade that the token bill does not follow the client's `/compact` down). The leading system run is freshly decoded from each turn's own bytes every round (`FRONT_REREAD_ELEMENTS` head fragment — nothing front-segment ever gets frozen, P4); there are NO full re-parses except the cold-start arm turn and a fresh `pf_sid` mint, which coincide (P5 hard red line). The data plane is metadata-derived: current state = fold(ckpt, log-after-ckpt); in-memory mirrors are caches, losing one costs a DB re-read (P7); raw host ids travel and are stored verbatim — digests only in warn-level diagnostics (P9). + +**Wire form.** Every v6-engaged response carries `x-bili-pf-session` = the server-minted `pf_` sid — an O(1) echo channel that ACCELERATES matching (rule ① ambiguity skip) but whose claim is still spot-checked against the plane. Requests carry `x-bili-pf-tail-ids` = a FIXED tail window of the LAST `PF_TAIL_WINDOW` element ids (body order, per-segment percent-encoded), NEVER the full history — header bytes stay bounded by construction (F1 ruling in the #5652 review, 2026-11-04), so stock Node maxHeaderSize needs no tuning and the old "list grows with history" 431 class cannot exist. `x-bili-pf-count` RETIRES from this lane (v6 never reads it); v5 lanes keep their count logic untouched — two worlds, no cross-contamination. HONEST COVERAGE BOUNDARY (contract §3): judgement covers exactly the tail-window identities plus the ledger chain; prefix content older than the window that the client rewrote/compacted away is NOT promised detectable — same boundary tier as "mid-history change at stable total"; it rides the forward-append view instead of being faked. + +**Identity hard gate.** Lane entry requires BOTH channels complete and sound — a request-level carrier id AND a tail-window list of exactly min(PF_TAIL_WINDOW, T) members, every one a real unique id. The four fault classes (channel missing / width mismatch / malformed member / consistency break) REJECT the request with HTTP 400 structured body and ZERO data-plane side effects. Three prohibitions: no silent legacy fallback, no counting-based attribution, no content-signature guessing. The ONE sanctioned relief valve: five CONSECUTIVE faults park v6 for 60 minutes — parked traffic rides the baseline v5/legacy lanes (SERVED, not refused) so a stable client-unfixable fault cannot become a 400 loop; parking and recovery are loudly logged. Configuration-level cohort exclusion (non-shim carriers, plane-down nodes, parked sessions) is NOT a fault: it passes silently to the baseline lanes. + +**Data-plane lifecycle.** The identity tables `pf_rows` / `pf_log` / `pf_ckpt` (plus the render family `pf_render` / `pf_render_log` / `pf_render_ckpt`, schema v13) live in the unified-storage sidecar and cascade-delete with their session: `deleteSession`'s in-txn sweep always covered them; `archiveSession`'s sweep covers them since the #5652 review BUG-2 fix (commit 4aa85c8a); `pfDeleteForSession` is the standalone MAINTENANCE-ONLY facade (no production call site today). Two review-fixed invariants ride on top of that lifecycle: (a) a HIT commit's finalize CLEARS the session's `pf_ckpt` row (#5652 review BUG-1 fix, commit 6f3baa64) — `pf_log` keeps pre-fold rows by design, so a stale checkpoint row would otherwise keep hydrating those rows after restart and desync the world from the ledger forever; (b) MySQL/network backends degrade soft — plane unavailable ⇒ baseline lanes, never a reject. + +**Render-snapshot plane (#5664).** Three-station model (owner ruling 2026-11-04): station 1 inbound/identity finalize (the identity lane above); station 2 snapshot — steady engaged rounds APPEND one captured row per new element; full-path rounds (arm / scheduled audit / fallback) DIFF against the stored rows and APPEND the tail on growth; station 3 outbound emit — prefix rows pass through verbatim from disk, only the bounded window tail runs a kernel pass per request. Governing guarantees: **G0** byte-equivalence is VERIFIED, not assumed — every fast round re-runs a BOUNDED kernel pass (last W frozen cores + this turn's new cores; W = `BILI_PF_RENDER_WINDOW`, default 8, cap 64) under a scratch state clone and compares its mapped first-F elements against the last F stored rows byte-for-byte; any mismatch falls back to the full legacy path for THIS turn only (disk is the source of truth). **G1** conservative conjunction gate — ANY red condition falls back for that one turn to the untouched full-view legacy pass and re-captures: structurally identical to "this feature does not exist" when red. The red-reason vocabulary (`renderPlaneGate`: arm-turn / title-gen / plane-unavailable / pressure ≥ 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due / no-rows / no-plane-geometry / rows-misaligned) is pinned one-per-condition in `tests/issue5664-render-snapshot.test.ts`. **G2** capture point = the exact array the baseline emits (`rebuiltMessages` before system/ephemeral injection); steady capture stores raw strings + parsed caches so the prefix never gets re-parsed or re-stringified. **G3** death verdict — a checksum-corrupt checkpoint, a structurally diverged capture, or any unverifiable pre-existing snapshot KILLS the plane for that sid PERMANENTLY (persisted as a `kind='dead'` log entry, surviving restarts). A RECAST IS FORBIDDEN IN EVERY SCENARIO (owner ruling 2026-11-05): any divergence requiring a stored-row rewrite/deletion — or any post-death rebuild — kills rather than repairs; byte-safety comes from stopping, never from guessing corrupt bytes. Scheduled audit: every `PF_RENDER_AUDIT_EVERY` (default 8) engaged hit-turns force ONE full legacy pass whose capture diff re-verifies the stored rows end-to-end; emergency pressure (tokenCount ≥ 90% of the model limit) also forces legacy, because the fold/expand machinery must see the whole view near the ceiling. Mirror LRU budget: 256 MiB byte-weighted, oldest world evicted first (P7). + +**Rollout & status.** `rollout.features.watermarkV6MsgId` (env `BILI_ROLLOUT_WATERMARK_V6_MSG_ID`) nests inside `pluginFullResendWatermark` (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar; `rollout.features.watermarkV6Render` (`BILI_ROLLOUT_WATERMARK_V6_RENDER`) nests inside `watermarkV6MsgId`. Both are EXPERIMENTAL and DEFAULT OFF — they open only by explicit rollout. Test inventory: `tests/fastpath-v6-msgid.test.ts` (hermetic dual-proxy A/B: BUG-1/BUG-2 regressions, four fault classes, multi-world), `tests/fastpath-v6-tail-window.test.ts` (window arithmetic), `tests/issue5664-render-snapshot.test.ts` (render red-reason pins + hydration/death paths), `tests/pf-channel.test.ts` (client learn/echo/stale-retention), `tests/lane-decisions-matrix.test.ts` (gate rows, both directions). diff --git a/TECHNICAL-NOTES.zh-CN.md b/TECHNICAL-NOTES.zh-CN.md index d5ea5b5d97..597c3ce0bf 100644 --- a/TECHNICAL-NOTES.zh-CN.md +++ b/TECHNICAL-NOTES.zh-CN.md @@ -164,3 +164,19 @@ chat wire 上 bili 自己 vs 客户端自带:插件车道上,请求体里等于 诚实边界:(1) 一旦多模态尾部(含 `image_url` 的 content array)进入缓存,该段历史此后整段降级回 legacy 全量轮——kernel 按位置区分 user 图片消息在尾部与在历史中部的渲染,冻结缓存无法做到位置无关地复现 legacy 字节;纯文本工作负载(#5403 报告者的形态)不受影响。(2) 每次命中会现解析头部 8 个元素,只为做 system-run 校验。(3) 两个 `x-bili-pf-*` 头都在转发上游前剥掉(`UPSTREAM_HOP_HEADERS`)——协议头绝不漏进模型流量。 实测(出处:`tests/issue5403-watermark-fastpath.test.ts`,同语料 hermetic 双代理 A/B):命中轮完全跳过全量解析/转换/钉存;含一个 multipart 图片落位回合的 8 回合语料上,常驻 `lastMessages` 8684 B → 1210 B(7.2×);全程出站字节与 legacy 逐字一致(仅 kernel `time="…"` 属性归一);两个阶段捕获到的上游请求里均无 `x-bili-pf-*`。单元覆盖:`tests/fastpath.test.ts`(verdict 阶梯全表,含熔断 TTL 解除后重播种)、`tests/fastpath-scanner.test.ts`(对抗 JSON 表、深度上限、surrogate 对、5 万元素规模)、`tests/lane-decisions-matrix.test.ts`(门矩阵,双向)。 + +## 全量重发水位快路径 v6——复合身份 + 渲染快照平面(#5643 / #5664) + +v6 是对上文 v5 水位线的一次**彻底重构**(owner 裁定 2026-11-04;PR #5576 谱系作废),拆成共享同一道硬门入口的两个平面:身份车道(`src/server/fastpath-v6.ts`)与渲染快照平面(`src/server/fastpath-v6-render.ts`,嵌套在身份人群内)。 + +**为何要有 v6。** 纯计数只能按位置钉边界——客户端改写元素而保持数量不变时完全不可见。v6 把 (client sessionid, msgid) 复合对本身作为身份标记(P1,契约锁定):命中轮只转换**新增**元素 span(集合差集,body 任意位置),冻结前缀从已存 cores 物化——原始请求字节在回合末离开内存(P2)。一切客户端结构性变更(compaction、同族模型切换、重排、就地改写、重新编号)一律忽略;视图只向前生长,没有任何单个客户端事件触发全局 rebase(P3——客户端 compaction「B 版」,owner 裁定 2026-11-04:双通道对上账时 compaction 被整体忽略,以普通元素形态到达的压缩摘要零特殊待遇入册;明确接受的代价:token 账单不跟随客户端的 `/compact` 回落)。前导 system 段每轮都从本轮自己的字节现解(`FRONT_REREAD_ELEMENTS` 头碎片——front 段任何内容都不冻结,P4);除冷启动 arm 轮与新铸 `pf_sid`(两者重合)之外不存在任何全量解析(P5 硬红线)。数据平面是元数据派生的:当前状态 = fold(ckpt, log-after-ckpt);内存 mirror 只是缓存,丢了只多一次 DB 重读(P7);宿主 id 原样上行原样入库,摘要只允许出现在 warn 级诊断里(P9)。 + +**wire 形。** 每个 v6 命中响应带 `x-bili-pf-session` = 服务端铸造的 `pf_` sid——O(1) 回显通道,加速匹配(规则①跳过歧义),但其声明仍会被平面抽查复核。请求带 `x-bili-pf-tail-ids` = **固定尾窗**:仅最后 `PF_TAIL_WINDOW` 个元素 id(body 序、逐段百分号编码),绝不上行完整历史——header 字节按构造有界(#5652 review 的 F1 裁定,2026-11-04),Node 出厂 maxHeaderSize 无需调参,旧的「列表随历史膨胀」431 类不存在。`x-bili-pf-count` 在本车道退役(v6 从不读它);v5 车道的计数逻辑原封不动——两个世界,互不污染。诚实覆盖边界(契约 §3):判定恰好覆盖尾窗身份 + 账本链;比窗口更老、被客户端改写/压缩掉的前缀内容不承诺可检测——与「总量稳定的历史中部变更」同一边界层级;它走 forward-append 视图,而不是假装看得见。 + +**身份硬门。** 车道入口要求两通道齐备且合格——请求级 carrier id + 恰好 min(PF_TAIL_WINDOW, T) 个成员且全部为真实唯一 id 的尾窗列表。四类故障(通道缺失 / 宽度不符 / 成员畸形 / 一致性断裂)拒绝请求(HTTP 400 结构化 body),数据平面零副作用。三条禁令:不静默回退 legacy、不做计数归因、不做内容签名猜测。唯一允许的泄压阀:连续五次故障把 v6 停放 60 分钟——停放期间流量走基线 v5/legacy 车道承接(是**服务**,不是拒绝),使稳定的客户端无法修复故障不至于变成 400 循环;停放与恢复均响亮记日志。配置级人群排除(非 shim 载体、平面不可用节点、停放中的会话)不算故障:静默走基线车道。 + +**数据平面生命周期。** 身份三表 `pf_rows` / `pf_log` / `pf_ckpt`(加渲染系 `pf_render` / `pf_render_log` / `pf_render_ckpt`,schema v13)住在统一存储 sidecar,随会话级联删除:`deleteSession` 的事务内清扫一直覆盖它们;`archiveSession` 的清扫自 #5652 review BUG-2 修复(commit 4aa85c8a)起同样覆盖;`pfDeleteForSession` 是独立的 MAINTENANCE-ONLY 门面(今日无生产调用点)。两条 review 修出的不变式叠在该生命周期之上:(a) HIT 提交的 finalize 清空该会话的 `pf_ckpt` 行(#5652 review BUG-1 修复,commit 6f3baa64)——`pf_log` 按设计保留 fold 前的旧行,若 checkpoint 行残留,重启后会继续把这些旧行 hydrate 回来,令世界与账本永久失步;(b) MySQL/网络后端软降级——平面不可用 ⇒ 基线车道,绝不拒绝。 + +**渲染快照平面(#5664)。** 三站模型(owner 裁定 2026-11-04):第 1 站 入站/身份 finalize(即上面的身份车道);第 2 站 快照——稳态命中轮每个新元素 APPEND 一行捕获;全量路径轮(arm / 计划审计 / fallback)对照已存行做 DIFF,增长时补尾;第 3 站 出站 emit——前缀行从磁盘逐字直通,只有有界窗口尾部每请求跑一遍内核 pass。治理保证:**G0** 字节等价是被**验证**的而非假设的——每个快速轮都在 scratch 状态克隆下重跑一遍有界内核 pass(最近 W 个冻结核 + 本回合新增核;W = `BILI_PF_RENDER_WINDOW`,默认 8、上限 64),把其映射出的前 F 个元素与最后 F 行已存行逐字节比对;任何不一致只当轮回退全量 legacy 路径(磁盘才是真值源)。**G1** 保守合取门——任一红色条件当轮回退未动过的全量 legacy pass 并重新捕获:红色时与「该特性不存在」结构上逐字节一致。红色原因词表(`renderPlaneGate`:arm-turn / title-gen / plane-unavailable / 压力 ≥ 模型上限 90% / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due:每 N 轮计划审计 / no-rows / no-plane-geometry / rows-misaligned)在 `tests/issue5664-render-snapshot.test.ts` 中逐条钉死。**G2** 捕获点 = 基线实际发出的那个数组(`rebuiltMessages`,system/ephemeral 注入之前);稳态捕获存原始字符串 + 解析缓存,前缀永不重解析或重字符串化。**G3** 死亡判决——校验和损坏的 checkpoint、结构发散的捕获、或任何无法核实的既有快照,永久杀死该 sid 的平面(持久化为 `kind='dead'` 日志行,跨重启存活)。**任何场景下禁止 recast**(owner 裁定 2026-11-05):任何需要重写/删除已存行的分歧——或死亡后的重建——都判死而非修复;字节安全来自停手,而不是猜坏字节。计划审计:每 `PF_RENDER_AUDIT_EVERY`(默认 8)个命中轮强制一次全量 legacy pass,其捕获 diff 端到端复验已存行;紧急压力(tokenCount ≥ 模型上限 90%)同样强制 legacy,因为接近天花板时 fold/expand 机制必须看到整个视图。Mirror LRU 预算:256 MiB 按字节计重,最旧世界先逐出(P7)。 + +**放量与现状。** `rollout.features.watermarkV6MsgId`(env `BILI_ROLLOUT_WATERMARK_V6_MSG_ID`)嵌套于 `pluginFullResendWatermark`(人群 ⊆ v5 灰度人群)且要求统一存储 sidecar;`rollout.features.watermarkV6Render`(`BILI_ROLLOUT_WATERMARK_V6_RENDER`)嵌套于 `watermarkV6MsgId`。两者均为实验性、**默认关**——只能由显式放量开启。测试清单:`tests/fastpath-v6-msgid.test.ts`(hermetic 双代理 A/B:BUG-1/BUG-2 回归、四类故障、multi-world)、`tests/fastpath-v6-tail-window.test.ts`(窗口算术)、`tests/issue5664-render-snapshot.test.ts`(渲染红色原因钉 + 水合/死亡路径)、`tests/pf-channel.test.ts`(客户端 learn/echo/stale-retention)、`tests/lane-decisions-matrix.test.ts`(门矩阵,双向)。 From 08bb5ba8706e65bdfe0bd284b693c4cb7d39db58 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 6 Nov 2026 22:42:33 +0800 Subject: [PATCH 39/49] =?UTF-8?q?docs:=20v6=20module=20comments=20?= =?UTF-8?q?=E2=80=94=20maintenance-only=20seams=20(F8)=20+=20coordinate=20?= =?UTF-8?q?wording=20corrections=20(F11)=20(#5652=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/persist.ts | 8 ++++++-- src/server/fastpath-v6-render.ts | 19 ++++++++++++++----- src/server/fastpath-v6.ts | 9 +++++++-- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index d442719957..12bdc2da4d 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2286,8 +2286,12 @@ export class SessionStore { * from ext FIRST (deleteSession sweeps session_ext itself), then deletes * all SIX plane tables — identity rows/log/ckpt AND the #5664 render * snapshot tables (deleteSession's own cascade covers them in-txn too; - * this facade is the standalone maintenance entry). Safe to call when - * nothing was ever issued. */ + * archiveSession's does since the #5652 review BUG-2 fix; this facade is + * the standalone maintenance entry). MAINTENANCE-ONLY as of the #5652 + * review: NO production call site today (real teardown flows go through + * deleteSession/archiveSession's in-txn cascades; tests exercise this + * directly) — wiring it into relay teardown is open follow-up work, not + * an accident. Safe to call when nothing was ever issued. */ pfDeleteForSession(sessionId: string): void { if (!this.shadow || this.shadow.stats.disabled) return; const reg = this.shadow.getSessionExt(sessionId, this.PF_V6_SID_REGISTRY_EXT_KEY) as string[] | undefined; diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 88f279dd67..404e7d01e6 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -151,11 +151,15 @@ export interface RenderWorld { textCum: number; /** Cumulative image-reserve contribution of frozen rows. */ imgCum: number; - /** Absolute ledger positions backing `rows` (ascending distinct; index i - * ↔ positions[i]). Dense 0..n-1 holds for every V1 op (the journal is - * a pending DELETE, rows themselves only move via witnessed ops); sparse - * admission stays accepted defensively — emission order is array order, - * and a lagging trailing max is a normal transient (gate span-check). */ + /** Positions backing `rows` (ascending distinct; index i ↔ positions[i]). + * V1 stores the DENSE ROW ORDINAL (0..n-1; the journal is a pending + * DELETE, rows themselves only move via witnessed ops) — which equals + * the element's ledger position only while NO leading elements were + * absorbed into systemText (a leading-system body offsets the two by + * one row each; #5652 review F9 tracks the coordinate-mix correction, + * and this comment flips with it). Sparse admission stays accepted + * defensively — emission order is array order, and a lagging trailing + * max is a normal transient (gate span-check). */ positions: number[]; /** Covered positions journaled at first scene awaiting materialization * (== the mirrored ckpt meta field; both reset together). While > 0 the @@ -236,6 +240,11 @@ function registerSidForSession(sessionId: string, sid: string): void { set.add(sid); } +/** Drop every in-memory render mirror for a relay session. MAINTENANCE-ONLY + * as of the #5652 review: NO production call site today (exercised by tests; + * a future relay-teardown path may adopt it) — render analogue of + * fastpath-v6's clearWatermarkV6. The DISK plane (pf_render*) is swept by + * pfDeleteForSession / archiveSession / deleteSession only. */ export function clearWatermarkV6Render(sessionId: string): void { const set = sessionSids.get(sessionId); if (!set) return; diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts index b4397d511f..82edff7ce7 100644 --- a/src/server/fastpath-v6.ts +++ b/src/server/fastpath-v6.ts @@ -366,7 +366,9 @@ function addStripCounts(a: RawStripCounts, b: RawStripCounts): RawStripCounts { // would mint base_N continuing across ALL earlier elements). Bases are // content hashes (#3642) and neighbour-independent, so determinism holds: // count bases across every earlier-ledger row (pos order), continue _N, and -// fix wireIndex to the absolute element position. +// fix wireIndex to the element's NEW LEDGER position (a.pos — equal to its +// absolute in-body element position only while the world is pure-append; +// #5652 review F11). function baseOfRawId(id: string): string { const m = /^h_[0-9a-f]{16}(?:_\d+)?$/u.exec(id); return m === null ? id : id.slice(0, 18); @@ -1015,7 +1017,10 @@ function opportunisticCompact(store: SessionStore, sid: string, world: V6World, /** Drop every in-memory mirror + pending commit for a relay session — pair * of clearWatermarkSeed at teardown paths. The DISK plane survives until - * pfDeleteForSession (delete-time cascade). */ + * pfDeleteForSession (delete-time cascade). MAINTENANCE-ONLY as of the + * #5652 review: NO production call site today (exercised by tests; a future + * relay-teardown path may adopt it) — v5's clearWatermarkSeed has the same + * status; do not read the "teardown paths" wording as an existing wiring. */ export function clearWatermarkV6(sessionId: string): void { const sids = sessionWorlds.get(sessionId); if (sids !== undefined) { From 0ea6df318d895f95d858e4f01df3a6dfae54c07c Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 6 Nov 2026 22:42:33 +0800 Subject: [PATCH 40/49] =?UTF-8?q?docs+test:=20rollout=20force-tier/nesting?= =?UTF-8?q?=20semantics=20=E2=80=94=20doc=20truth=20+=20behavior=20pin=20(?= =?UTF-8?q?#5652=20review=20F10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/rollout.ts | 23 +++++++++++++++++------ tests/rollout.test.ts | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/rollout.ts b/src/rollout.ts index f17987f4cb..a8cd4cc109 100644 --- a/src/rollout.ts +++ b/src/rollout.ts @@ -10,14 +10,21 @@ // enabled ⇔ bucket < percent × 100. Prefix-stable: raising the percent // never drops an already-enabled install (monotonic rollout, 灰度只进 // 不出). To re-randomize the cohort, change the salt. -// 3. NESTING — a feature may declare `base`: it is evaluated only inside -// the base feature's enabled cohort (灰度 A 的用户里再灰度 B; C→D the -// same), so a nested percent is relative to the base population, not -// everyone. Base chains of any depth; cycles fail closed (off). +// 3. NESTING — a feature may declare `base`: its BUCKET decision is made +// only inside the base feature's enabled cohort (灰度 A 的用户里再灰度 B; +// C→D the same), so a nested percent is relative to the base population, +// not everyone. Base chains of any depth; cycles fail closed (off). +// Nesting governs the BUCKET TIER ONLY — the force tiers (§4) return +// before the nesting check, so an explicitly env/config-forced child runs +// even while its base is off/rolled back. CURRENT pinned behavior +// (tests/rollout.test.ts "force tier bypasses nesting"), owner +// design-confirm pending (#5652 review F10) — do not "fix" it by accident. // 4. PRECEDENCE — env `BILI_ROLLOUT_` ("0"/"false" = off, any // other value = on) > config `rollout.features..enabled` > // bucketing over percent (config `percent` override > registry default). -// A feature absent from the registry AND from the config block is OFF. +// The two force tiers short-circuit BEFORE the §3 nesting check (a forced +// feature resolves without consulting its `base`). A feature absent from +// the registry AND from the config block is OFF. // // A release that wants to canary a feature adds one entry to ROLLOUT_FEATURES // (percent 0–100, optional salt/base); GA = bump to 100, or move the default @@ -39,7 +46,11 @@ export interface RolloutPlan { percent: number; /** Hash salt; defaults to the feature name. Changing it re-randomizes the cohort. */ salt?: string; - /** Nested rollout: evaluate this feature only inside `base`'s enabled cohort. */ + /** Nested rollout: evaluate this feature only inside `base`'s enabled + * cohort — for BUCKET-tier decisions. An explicit env/config force + * resolves BEFORE the nesting check, so a forced child runs even while + * `base` is off (pinned by tests/rollout.test.ts; #5652 review F10 — + * owner design-confirm pending, do not "fix" without ruling). */ base?: string; } diff --git a/tests/rollout.test.ts b/tests/rollout.test.ts index ffcb3fd55a..97bd3d1ced 100644 --- a/tests/rollout.test.ts +++ b/tests/rollout.test.ts @@ -176,6 +176,22 @@ test("nested rollout: B evaluated only inside A's cohort, percent relative to it }); }); +test("force tier bypasses nesting: forced child runs while its base is off (#5652 review F10 pin)", () => { + withFeatures({ a: { percent: 0 }, b: { percent: 100, base: "a" } }, () => { + // A at 0% ⇒ nobody in the cohort ⇒ B off for all at the BUCKET tier… + assert.equal(rollout.resolveRollout("b", { id: ids[0], env: {}, file: {} }).enabled, false); + // …but the force tiers return BEFORE the nesting check, so both explicit + // forces still enable B. CURRENT behavior, pinned on purpose — owner + // design-confirm pending (#5652 review F10); changing it must be a + // deliberate diff here, not an accident. + assert.equal(rollout.resolveRollout("b", { id: ids[0], env: { BILI_ROLLOUT_B: "1" }, file: {} }).enabled, true, "env force bypasses parent-off"); + assert.equal(rollout.resolveRollout("b", { id: ids[0], env: {}, file: { features: { b: { enabled: true } } } }).enabled, true, "config force bypasses parent-off"); + // Both symmetric negatives hold for the same reason… + assert.equal(rollout.resolveRollout("b", { id: ids[0], env: { BILI_ROLLOUT_B: "0" }, file: { features: { b: { enabled: true } } } }).enabled, false, "env 0 still wins over config true"); + assert.equal(rollout.resolveRollout("b", { id: ids[0], env: {}, file: { features: { b: { enabled: false } } } }).enabled, false, "explicit off still closes the lane"); + }); +}); + test("base cycles fail closed without hanging", () => { withFeatures({ x: { percent: 100, base: "y" }, y: { percent: 100, base: "x" }, self: { percent: 100, base: "self" } }, () => { for (const f of ["x", "y", "self"]) { From d3891d30f59b4b966f084d6b33bb82eaf32c0917 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 6 Nov 2026 22:42:33 +0800 Subject: [PATCH 41/49] test: pin the eight unasserted render red reasons one-per-condition (#5652 review F6) --- tests/issue5664-render-snapshot.test.ts | 92 +++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 18036abf3a..46e309f19e 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -242,6 +242,98 @@ test("gate: red reasons are a stable vocabulary", (t) => { } finally { h.cleanup(); } }); +// F6 review pin (#5652 review round): the gate vocabulary comment claims +// "test-pinned, one per condition" — this completes the eight conditions +// that had ZERO assertions repo-wide. Each cell flips exactly ONE input off +// the green base and asserts the EXACT reason token, following the gate's +// own evaluation order (renderPlaneGate in src/server/fastpath-v6-render.ts). +test("gate: F6 vocabulary completion — every remaining red reason pinned once", () => { + const h = makeH("f6"); + try { + const b1 = makeBody([ + { role: "user", content: "a" }, + { role: "assistant", content: "b" }, + { role: "user", content: "c" }, + ]); + const r1 = v6Eval(h, "s-f6", b1, ["g8a", "g8b", "g8c"]); + assert.equal(r1.outcome.decision, "arm"); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + assert.equal(commitRenderCapture({ sessionId: "s-f6", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}', '{"i":2}'], sums: { estCum: 3, textCum: 3, imgCum: 0 } }).op, "cast"); + const geom = watermarkV6DebugState("s-f6").worlds[sid]; + assert.equal(geom!.maxPos, 2, "three-element arm ⇒ identity plane ends at pos 2"); + const base = { + sessionId: "s-f6", session: { id: "s-f6", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: geom!.maxPos, tokenCount: 500, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }; + const g0 = renderPlaneGate(base); + assert.ok(g0.ok, `expected a green control gate, got ${JSON.stringify(g0)}`); + + // "plane-unavailable" (!store.pfRenderAvailable()): the probe fails + // before ANY state inspection — driven through the same stub-store + // seam the non-ascending pin uses. + const downStore: SessionStore = Object.create(h.store); + downStore.pfRenderAvailable = () => false; + assert.deepEqual(renderPlaneGate({ ...base, store: downStore }), { ok: false, reason: "plane-unavailable" }); + // "snapshot-lane": the #3756 lane substituted the kernel view. + assert.deepEqual(renderPlaneGate({ ...base, snapshotLaneActive: true }), { ok: false, reason: "snapshot-lane" }); + // "fold-pending": a fold left usage unconsumed for this turn. + assert.deepEqual(renderPlaneGate({ ...base, foldUsagePending: true }), { ok: false, reason: "fold-pending" }); + // "absorb-active": the legacy absorb pass owns the emit this round. + assert.deepEqual(renderPlaneGate({ ...base, absorbActive: true }), { ok: false, reason: "absorb-active" }); + // "ccr-on": CCR rewrites outbound bytes away from the stored form. + assert.deepEqual(renderPlaneGate({ ...base, ccrOn: true }), { ok: false, reason: "ccr-on" }); + + // "hydrate-failed": mirror absent + DISK PLANE ABSENT + no death + // verdict ⇒ the TRANSIENT reason, never lane-dead. Production shape: + // an armed sid whose first hit turn gates before station 2 ever casts + // (the plane is forged exactly once, on the second v6 turn). + const r2 = v6Eval(h, "s-f6b", makeBody([{ role: "user", content: "hello" }]), ["g8d"]); + r2.fin(); + const sid2 = r2.sid!; + assert.equal(h.store.pfRenderRows(sid2).length, 0, "armed but never render-cast ⇒ disk plane absent"); + assert.deepEqual(renderPlaneGate({ + ...base, sessionId: "s-f6b", session: { id: "s-f6b", state: createInitialState() } as unknown as Session, sid: sid2, + maxPos: watermarkV6DebugState("s-f6b").worlds[sid2]!.maxPos, + }), { ok: false, reason: "hydrate-failed" }, "an absent plane reports hydrate-failed, NOT lane-dead"); + + // "no-rows": an EMPTY-TABLE arm cast is legal through the public seam + // (the cast door checks pre-existing rows, not the capture length) — + // the degenerate real shape where the kernel emits zero render rows + // for the captured view. + const r3 = v6Eval(h, "s-f6c", makeBody([{ role: "user", content: "hello" }]), ["g8e"]); + r3.fin(); + const sid3 = r3.sid!; + assert.equal(commitRenderCapture({ sessionId: "s-f6c", store: h.store, sid: sid3, mode: "legacy", caps, rowStrings: [], sums: { estCum: 0, textCum: 0, imgCum: 0 } }).op, "cast", "an empty cast onto an empty table commits"); + assert.deepEqual(renderPlaneGate({ + ...base, sessionId: "s-f6c", session: { id: "s-f6c", state: createInitialState() } as unknown as Session, sid: sid3, + maxPos: watermarkV6DebugState("s-f6c").worlds[sid3]!.maxPos, + }), { ok: false, reason: "no-rows" }, "a zero-row world reds out before the alignment check"); + + // "rows-misaligned": TWO of the three sub-shapes are reachable + // through public seams; the third (positions EMPTY) is structurally + // unreachable — every op builds positions 1:1 with rows (cast / + // append / rebase / hydrate), so an empty array implies zero rows + // and the no-rows condition above shadows it first. + // (a) trailing pos past the live plane end — the identity world lag + // documented in the gate comment (finalize advances it AFTER station + // 2 commits): feed the lagging maxPos. + assert.deepEqual(renderPlaneGate({ ...base, maxPos: 1 }), { ok: false, reason: "rows-misaligned" }, "last stored pos > maxPos"); + // (b) negative leading pos: forge the pos VALUE directly — the + // checkpoint meta carries no per-pos data, so the S4 checksum stays + // valid untouched. Hydrate accepts strictly ascending values + // (-5 < 1 < 2); only the gate's range check can see the damage. + const db = new DatabaseSync(path.join(h.dir, "index.db")); + db.prepare(`UPDATE pf_render SET pos = -5 WHERE sid = ? AND pos = 0`).run(sid); + db.close(); + _resetWatermarkV6RenderTestState(); // drop the mirrors so the gate re-hydrates from disk + assert.deepEqual(renderPlaneGate(base), { ok: false, reason: "rows-misaligned" }, "positions[0] < 0 after re-hydrating the forged table"); + } finally { h.cleanup(); } +}); + test("unit: S4 checksum tamper KILLS the plane durably (loudly logged)", (t) => { const h = makeH("s4"); try { From 61eb80523842bfccfd335f73f3b8c13ab349604d Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 6 Nov 2026 22:56:32 +0800 Subject: [PATCH 42/49] test: pin v6 identity fault variants + client slot.v6 learn/echo/stale-retention (#5652 review F6) --- tests/fastpath-v6-msgid.test.ts | 57 +++++++++++++++++++++++++++++++++ tests/pf-channel.test.ts | 47 +++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index 0bc1896f27..6a56722a7d 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -655,3 +655,60 @@ test("archiving a session sweeps its pf planes too (review BUG-2 regression): no assert.notEqual(rC.sid, sidA, "a FRESH world mint — the old plane bytes are gone by design"); } finally { h.cleanup(); } }); + +test("F6 pin: an echoed pf_sid the plane never minted faults loudly with one breaker strike and zero data-plane writes (#5652 review)", () => { + const h = makeH("fakeecho"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + const r1 = turn("s-fe", "car-fe", b1, ["e0", "e1"], log); + assert.equal(r1.outcome.decision, "arm"); + // Well-formed but never minted: the client-loopback / forged-echo + // shape — exactly how mis-attribution starts (#2821 discipline on the + // wire). The list contents are known world-A ids; only the ECHO lies. + const forged = `pf_${"f".repeat(12)}`; + assert.notEqual(r1.sid, forged); + const db0 = new DatabaseSync(path.join(h.dir, "index.db")); + const countOf = (t: string) => (db0.prepare(`SELECT COUNT(*) AS n FROM ${t}`).get() as { n: number }).n; + const rowsBefore = countOf("pf_rows"); + const logBefore = countOf("pf_log"); + db0.close(); + const f = assertFault(() => turn("s-fe", "car-fe", b1, ["e0", "e1"], log, forged), "echo references unknown pf_sid"); + assert.ok(f.detail.includes(forged), "the refused sid must name itself in the detail"); + const guard = h.store.getSessionExt("s-fe", "pf:v6-guard") as { faults?: number } | null; + assert.equal(guard?.faults, 1, `one strike must land in the breaker, got ${JSON.stringify(guard)}`); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + try { + assert.equal((db.prepare(`SELECT COUNT(*) AS n FROM pf_rows`).get() as { n: number }).n, rowsBefore, "the fault must not register elements"); + assert.equal((db.prepare(`SELECT COUNT(*) AS n FROM pf_log`).get() as { n: number }).n, logBefore, "the fault must not extend the plane journal"); + } finally { db.close(); } + } finally { h.cleanup(); } +}); + +test("F6 pin: one body probing ids from two co-registered worlds under one carrier is refused, never side-chosen (#5652 review)", () => { + const h = makeH("multiworld"); + try { + const log = makeLog(); + const SESS = "s-mw"; + const CARRIER = "car-mw"; + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "u1" }]); + // Carrier reuse (the production shape): each all-new-id span re-arms + // a fresh world under the SAME sess_raw, so the registry holds both + // sids and both row sets resolve through the same probe index. + const rA = turn(SESS, CARRIER, b1, ["a0", "a1"], log); + assert.equal(rA.outcome.decision, "arm"); + const rB = turn(SESS, CARRIER, b1, ["b0", "b1"], log); + assert.equal(rB.outcome.decision, "arm"); + assert.notEqual(rB.sid, rA.sid, "distinct all-new spans mint distinct worlds"); + // The mixed span: T=4 ⇒ width 4, and every id resolves — to a + // DIFFERENT world. Design-impossible for a real single conversation + // (kernel msgids are globally unique forever), so the lane refuses + // instead of choosing a side. + const b3 = makeBody([ + { role: "system", content: SYS }, { role: "user", content: "u1" }, + { role: "assistant", content: "a1" }, { role: "user", content: "u2" }, + ]); + const f = assertFault(() => turn(SESS, CARRIER, b3, ["a0", "a1", "b0", "b1"], log), "identity span crosses multiple worlds"); + assert.ok(f.detail.includes(rA.sid!) && f.detail.includes(rB.sid!), `both offending worlds must be named: ${f.detail}`); + } finally { h.cleanup(); } +}); diff --git a/tests/pf-channel.test.ts b/tests/pf-channel.test.ts index 2cd03be621..44be5b858b 100644 --- a/tests/pf-channel.test.ts +++ b/tests/pf-channel.test.ts @@ -109,6 +109,53 @@ test("pf-channel: stamp learning screens input", () => { assert.deepEqual(pfChannel.pfRequestHeaders(inH), { "x-bili-pf-count": "7" }); }); +// --- v6 world-id channel (#5643 / #5652 review): slot.v6 learn / echo / retention --- + +test("pf-channel: v6 world id — learned once from a 200 response, echoed alongside tail-ids on every later request", () => { + const sid = "sid-v6-learn-1"; + const inH = { [CONVO]: sid }; + const S = `pf_${"a".repeat(12)}`; + pfChannel.pfLearnStamp(inH, 200, new Headers({ "x-bili-pf-seq": "3", "x-bili-pf-session": S })); + assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-session"], S, "the learned world id rides the next request"); + pfChannel.pfCaptureIds(sid, ["v1", "v2"]); + assert.deepEqual(pfChannel.pfRequestHeaders(inH), { + "x-bili-pf-count": "3", + "x-bili-pf-tail-ids": "v1,v2", + "x-bili-pf-session": S, + }, "count + tail-ids + world id ride together"); +}); + +test("pf-channel: v6 world-id screening — malformed forms are not learned, non-2xx never mutates state", () => { + const sid = "sid-v6-screen-1"; + const inH = { [CONVO]: sid }; + for (const bad of ["pf_short", `PF_${"a".repeat(12)}`, `pf_${"g".repeat(12)}`, "", "pf_", `pf_${"a".repeat(11)}`]) { + pfChannel.pfLearnStamp(inH, 200, new Headers({ "x-bili-pf-session": bad })); + assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-session"], undefined, `form ${JSON.stringify(bad)} must not be learned`); + } + const GOOD = `pf_${"b".repeat(12)}`; + pfChannel.pfLearnStamp(inH, 500, new Headers({ "x-bili-pf-seq": "4", "x-bili-pf-session": GOOD })); + assert.deepEqual(pfChannel.pfRequestHeaders(inH), {}, "a non-2xx response learns neither the stamp nor the world id"); +}); + +test("pf-channel: v6 stale retention — slot.v6 is never cleared by later traffic (CURRENT behavior; #5652 review F7(a) owner-decision pending)", () => { + const sid = "sid-v6-stale-1"; + const inH = { [CONVO]: sid }; + const S1 = `pf_${"c".repeat(12)}`; + pfChannel.pfLearnStamp(inH, 200, new Headers({ "x-bili-pf-seq": "2", "x-bili-pf-session": S1 })); + // A later v5-style response (stamp only, no session header) must not erase the world + pfChannel.pfLearnStamp(inH, 200, new Headers({ "x-bili-pf-seq": "5" })); + assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-session"], S1); + // An error response offering a DIFFERENT well-formed sid neither replaces nor clears + const S2 = `pf_${"d".repeat(12)}`; + pfChannel.pfLearnStamp(inH, 500, new Headers({ "x-bili-pf-session": S2 })); + assert.equal(pfChannel.pfRequestHeaders(inH)["x-bili-pf-session"], S1, "S1 persists — the retention this pin nails down"); + // Why the pin exists: if the server-side disk world goes away (GC / wipe / + // migration) while a live host process keeps echoing S1, every turn faults + // 400 → parks 60 min → releases → repeats until the host restarts. A fix + // like "clear slot.v6 after N consecutive non-2xx" would flip these + // assertions deliberately — #5652 review F7(a), owner decision pending. +}); + async function withPfPatch( fn: (fetch: typeof globalThis.fetch, sink: RecordedCall[]) => Promise, extra?: Partial, From 40a62117614a610aff8fe449420a8d587e9aa14d Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 7 Nov 2026 00:56:32 +0800 Subject: [PATCH 43/49] test: pin the markers render red reason one-per-condition (#5652 review G1) The renderPlaneGate vocabulary comment and both TECHNICAL-NOTES red lists claim every red reason is pinned one-per-condition in this file, but 'markers' had zero assertions repo-wide (the F6 completion pin missed it). Add the missing cell next to the ccr-on pin (same single-input-flip shape). --- tests/issue5664-render-snapshot.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 46e309f19e..ba82b9726b 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -286,6 +286,9 @@ test("gate: F6 vocabulary completion — every remaining red reason pinned once" assert.deepEqual(renderPlaneGate({ ...base, absorbActive: true }), { ok: false, reason: "absorb-active" }); // "ccr-on": CCR rewrites outbound bytes away from the stored form. assert.deepEqual(renderPlaneGate({ ...base, ccrOn: true }), { ok: false, reason: "ccr-on" }); + // "markers": non-zero marker artifacts rewrite outbound bytes this + // round (production caller passes markerArtifactCount(markerCounts)>0). + assert.deepEqual(renderPlaneGate({ ...base, markerNonZero: true }), { ok: false, reason: "markers" }); // "hydrate-failed": mirror absent + DISK PLANE ABSENT + no death // verdict ⇒ the TRANSIENT reason, never lane-dead. Production shape: From 3ee0d46c0e5e49e68b8bdc720b3879be4a112174 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 7 Nov 2026 00:56:32 +0800 Subject: [PATCH 44/49] docs: correct pf_ckpt post-fold wording; exhaustive red lists; qualify force-tier nesting (#5652 review G) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three doc-vs-code deltas found by the #5652 review, fixed at the source: 1. TECHNICAL-NOTES (en+zh) claimed a HIT finalize CLEARS the session's pf_ckpt row (BUG-1, 6f3baa64). No such deletion exists: 6f3baa64 is the continuation-numbering guard — pfCompact absorbs its own log entries in the same tx, so MAX(pf_log.ls_seq) sits BEHIND ckpt_ls_seq after any fold, and pfCommitTurn must number fresh entries from MAX(MAX(log), ckpt_ls_seq) or they are invisible to pfMeta forever (post-restart world desync -> PK-collision commit fail-loop). Reworded to describe the actual mechanism. 2. All four red-reason lists (rollout.yaml en+zh, TECHNICAL-NOTES en+zh) listed 18 of the 19 renderPlaneGate reasons — 'not-engaged' was missing from an otherwise presented-as-exhaustive vocabulary. Added (it is the first gate condition in code order; already test-pinned since d3891d30f's predecessor block). 3. The 'cohort nests inside / subset of parent population' wording was true only for percent buckets: force tiers (env override / explicit enabled) bypass the base check by PINNED design (src/rollout.ts header, F10 owner-confirm pending). Qualified all four statements with the bypass and pointed at the pin; behavior untouched. --- TECHNICAL-NOTES.md | 6 +++--- TECHNICAL-NOTES.zh-CN.md | 6 +++--- website/config-reference/rollout.yaml | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/TECHNICAL-NOTES.md b/TECHNICAL-NOTES.md index 5cd8ea041c..ad62654277 100644 --- a/TECHNICAL-NOTES.md +++ b/TECHNICAL-NOTES.md @@ -598,8 +598,8 @@ v6 is a **full rework** of the v5 watermark line above (owner ruling 2026-11-04; **Identity hard gate.** Lane entry requires BOTH channels complete and sound — a request-level carrier id AND a tail-window list of exactly min(PF_TAIL_WINDOW, T) members, every one a real unique id. The four fault classes (channel missing / width mismatch / malformed member / consistency break) REJECT the request with HTTP 400 structured body and ZERO data-plane side effects. Three prohibitions: no silent legacy fallback, no counting-based attribution, no content-signature guessing. The ONE sanctioned relief valve: five CONSECUTIVE faults park v6 for 60 minutes — parked traffic rides the baseline v5/legacy lanes (SERVED, not refused) so a stable client-unfixable fault cannot become a 400 loop; parking and recovery are loudly logged. Configuration-level cohort exclusion (non-shim carriers, plane-down nodes, parked sessions) is NOT a fault: it passes silently to the baseline lanes. -**Data-plane lifecycle.** The identity tables `pf_rows` / `pf_log` / `pf_ckpt` (plus the render family `pf_render` / `pf_render_log` / `pf_render_ckpt`, schema v13) live in the unified-storage sidecar and cascade-delete with their session: `deleteSession`'s in-txn sweep always covered them; `archiveSession`'s sweep covers them since the #5652 review BUG-2 fix (commit 4aa85c8a); `pfDeleteForSession` is the standalone MAINTENANCE-ONLY facade (no production call site today). Two review-fixed invariants ride on top of that lifecycle: (a) a HIT commit's finalize CLEARS the session's `pf_ckpt` row (#5652 review BUG-1 fix, commit 6f3baa64) — `pf_log` keeps pre-fold rows by design, so a stale checkpoint row would otherwise keep hydrating those rows after restart and desync the world from the ledger forever; (b) MySQL/network backends degrade soft — plane unavailable ⇒ baseline lanes, never a reject. +**Data-plane lifecycle.** The identity tables `pf_rows` / `pf_log` / `pf_ckpt` (plus the render family `pf_render` / `pf_render_log` / `pf_render_ckpt`, schema v13) live in the unified-storage sidecar and cascade-delete with their session: `deleteSession`'s in-txn sweep always covered them; `archiveSession`'s sweep covers them since the #5652 review BUG-2 fix (commit 4aa85c8a); `pfDeleteForSession` is the standalone MAINTENANCE-ONLY facade (no production call site today). Two review-fixed invariants ride on top of that lifecycle: (a) log-continuation numbering must clear BOTH the surviving `pf_log` tail AND the checkpoint watermark (#5652 review BUG-1 fix, commit 6f3baa64): a fold (`pfCompact`) absorbs its own log entries in the same transaction, so after ANY compaction `MAX(pf_log.ls_seq)` sits BEHIND `ckpt_ls_seq` — `pfCommitTurn` therefore numbers fresh entries from `MAX(MAX(log), ckpt_ls_seq)` (store.ts `pfCommitTurn`), and an entry can never be re-issued ≤ the watermark, which would make it invisible to `pfMeta` forever; post-restart that desync rehydrated the world at the stale folded maxPos and put every later progress turn into a PK-collision commit fail-loop; (b) MySQL/network backends degrade soft — plane unavailable ⇒ baseline lanes, never a reject. -**Render-snapshot plane (#5664).** Three-station model (owner ruling 2026-11-04): station 1 inbound/identity finalize (the identity lane above); station 2 snapshot — steady engaged rounds APPEND one captured row per new element; full-path rounds (arm / scheduled audit / fallback) DIFF against the stored rows and APPEND the tail on growth; station 3 outbound emit — prefix rows pass through verbatim from disk, only the bounded window tail runs a kernel pass per request. Governing guarantees: **G0** byte-equivalence is VERIFIED, not assumed — every fast round re-runs a BOUNDED kernel pass (last W frozen cores + this turn's new cores; W = `BILI_PF_RENDER_WINDOW`, default 8, cap 64) under a scratch state clone and compares its mapped first-F elements against the last F stored rows byte-for-byte; any mismatch falls back to the full legacy path for THIS turn only (disk is the source of truth). **G1** conservative conjunction gate — ANY red condition falls back for that one turn to the untouched full-view legacy pass and re-captures: structurally identical to "this feature does not exist" when red. The red-reason vocabulary (`renderPlaneGate`: arm-turn / title-gen / plane-unavailable / pressure ≥ 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due / no-rows / no-plane-geometry / rows-misaligned) is pinned one-per-condition in `tests/issue5664-render-snapshot.test.ts`. **G2** capture point = the exact array the baseline emits (`rebuiltMessages` before system/ephemeral injection); steady capture stores raw strings + parsed caches so the prefix never gets re-parsed or re-stringified. **G3** death verdict — a checksum-corrupt checkpoint, a structurally diverged capture, or any unverifiable pre-existing snapshot KILLS the plane for that sid PERMANENTLY (persisted as a `kind='dead'` log entry, surviving restarts). A RECAST IS FORBIDDEN IN EVERY SCENARIO (owner ruling 2026-11-05): any divergence requiring a stored-row rewrite/deletion — or any post-death rebuild — kills rather than repairs; byte-safety comes from stopping, never from guessing corrupt bytes. Scheduled audit: every `PF_RENDER_AUDIT_EVERY` (default 8) engaged hit-turns force ONE full legacy pass whose capture diff re-verifies the stored rows end-to-end; emergency pressure (tokenCount ≥ 90% of the model limit) also forces legacy, because the fold/expand machinery must see the whole view near the ceiling. Mirror LRU budget: 256 MiB byte-weighted, oldest world evicted first (P7). +**Render-snapshot plane (#5664).** Three-station model (owner ruling 2026-11-04): station 1 inbound/identity finalize (the identity lane above); station 2 snapshot — steady engaged rounds APPEND one captured row per new element; full-path rounds (arm / scheduled audit / fallback) DIFF against the stored rows and APPEND the tail on growth; station 3 outbound emit — prefix rows pass through verbatim from disk, only the bounded window tail runs a kernel pass per request. Governing guarantees: **G0** byte-equivalence is VERIFIED, not assumed — every fast round re-runs a BOUNDED kernel pass (last W frozen cores + this turn's new cores; W = `BILI_PF_RENDER_WINDOW`, default 8, cap 64) under a scratch state clone and compares its mapped first-F elements against the last F stored rows byte-for-byte; any mismatch falls back to the full legacy path for THIS turn only (disk is the source of truth). **G1** conservative conjunction gate — ANY red condition falls back for that one turn to the untouched full-view legacy pass and re-captures: structurally identical to "this feature does not exist" when red. The red-reason vocabulary (`renderPlaneGate`: not-engaged / arm-turn / title-gen / plane-unavailable / pressure ≥ 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due / no-rows / no-plane-geometry / rows-misaligned — exhaustive, matches `renderPlaneGate` one-to-one) is pinned one-per-condition in `tests/issue5664-render-snapshot.test.ts`. **G2** capture point = the exact array the baseline emits (`rebuiltMessages` before system/ephemeral injection); steady capture stores raw strings + parsed caches so the prefix never gets re-parsed or re-stringified. **G3** death verdict — a checksum-corrupt checkpoint, a structurally diverged capture, or any unverifiable pre-existing snapshot KILLS the plane for that sid PERMANENTLY (persisted as a `kind='dead'` log entry, surviving restarts). A RECAST IS FORBIDDEN IN EVERY SCENARIO (owner ruling 2026-11-05): any divergence requiring a stored-row rewrite/deletion — or any post-death rebuild — kills rather than repairs; byte-safety comes from stopping, never from guessing corrupt bytes. Scheduled audit: every `PF_RENDER_AUDIT_EVERY` (default 8) engaged hit-turns force ONE full legacy pass whose capture diff re-verifies the stored rows end-to-end; emergency pressure (tokenCount ≥ 90% of the model limit) also forces legacy, because the fold/expand machinery must see the whole view near the ceiling. Mirror LRU budget: 256 MiB byte-weighted, oldest world evicted first (P7). -**Rollout & status.** `rollout.features.watermarkV6MsgId` (env `BILI_ROLLOUT_WATERMARK_V6_MSG_ID`) nests inside `pluginFullResendWatermark` (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar; `rollout.features.watermarkV6Render` (`BILI_ROLLOUT_WATERMARK_V6_RENDER`) nests inside `watermarkV6MsgId`. Both are EXPERIMENTAL and DEFAULT OFF — they open only by explicit rollout. Test inventory: `tests/fastpath-v6-msgid.test.ts` (hermetic dual-proxy A/B: BUG-1/BUG-2 regressions, four fault classes, multi-world), `tests/fastpath-v6-tail-window.test.ts` (window arithmetic), `tests/issue5664-render-snapshot.test.ts` (render red-reason pins + hydration/death paths), `tests/pf-channel.test.ts` (client learn/echo/stale-retention), `tests/lane-decisions-matrix.test.ts` (gate rows, both directions). +**Rollout & status.** `rollout.features.watermarkV6MsgId` (env `BILI_ROLLOUT_WATERMARK_V6_MSG_ID`) nests inside `pluginFullResendWatermark` for percent buckets (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar; the force tiers (env override / explicit `enabled`) bypass the base check by pinned design (src/rollout.ts header, #5652 review F10 owner-confirm pending); `rollout.features.watermarkV6Render` (`BILI_ROLLOUT_WATERMARK_V6_RENDER`) nests inside `watermarkV6MsgId` the same way. Both are EXPERIMENTAL and DEFAULT OFF — they open only by explicit rollout. Test inventory: `tests/fastpath-v6-msgid.test.ts` (hermetic dual-proxy A/B: BUG-1/BUG-2 regressions, four fault classes, multi-world), `tests/fastpath-v6-tail-window.test.ts` (window arithmetic), `tests/issue5664-render-snapshot.test.ts` (render red-reason pins + hydration/death paths), `tests/pf-channel.test.ts` (client learn/echo/stale-retention), `tests/lane-decisions-matrix.test.ts` (gate rows, both directions). diff --git a/TECHNICAL-NOTES.zh-CN.md b/TECHNICAL-NOTES.zh-CN.md index 597c3ce0bf..79cb90564b 100644 --- a/TECHNICAL-NOTES.zh-CN.md +++ b/TECHNICAL-NOTES.zh-CN.md @@ -175,8 +175,8 @@ v6 是对上文 v5 水位线的一次**彻底重构**(owner 裁定 2026-11-04;PR **身份硬门。** 车道入口要求两通道齐备且合格——请求级 carrier id + 恰好 min(PF_TAIL_WINDOW, T) 个成员且全部为真实唯一 id 的尾窗列表。四类故障(通道缺失 / 宽度不符 / 成员畸形 / 一致性断裂)拒绝请求(HTTP 400 结构化 body),数据平面零副作用。三条禁令:不静默回退 legacy、不做计数归因、不做内容签名猜测。唯一允许的泄压阀:连续五次故障把 v6 停放 60 分钟——停放期间流量走基线 v5/legacy 车道承接(是**服务**,不是拒绝),使稳定的客户端无法修复故障不至于变成 400 循环;停放与恢复均响亮记日志。配置级人群排除(非 shim 载体、平面不可用节点、停放中的会话)不算故障:静默走基线车道。 -**数据平面生命周期。** 身份三表 `pf_rows` / `pf_log` / `pf_ckpt`(加渲染系 `pf_render` / `pf_render_log` / `pf_render_ckpt`,schema v13)住在统一存储 sidecar,随会话级联删除:`deleteSession` 的事务内清扫一直覆盖它们;`archiveSession` 的清扫自 #5652 review BUG-2 修复(commit 4aa85c8a)起同样覆盖;`pfDeleteForSession` 是独立的 MAINTENANCE-ONLY 门面(今日无生产调用点)。两条 review 修出的不变式叠在该生命周期之上:(a) HIT 提交的 finalize 清空该会话的 `pf_ckpt` 行(#5652 review BUG-1 修复,commit 6f3baa64)——`pf_log` 按设计保留 fold 前的旧行,若 checkpoint 行残留,重启后会继续把这些旧行 hydrate 回来,令世界与账本永久失步;(b) MySQL/网络后端软降级——平面不可用 ⇒ 基线车道,绝不拒绝。 +**数据平面生命周期。** 身份三表 `pf_rows` / `pf_log` / `pf_ckpt`(加渲染系 `pf_render` / `pf_render_log` / `pf_render_ckpt`,schema v13)住在统一存储 sidecar,随会话级联删除:`deleteSession` 的事务内清扫一直覆盖它们;`archiveSession` 的清扫自 #5652 review BUG-2 修复(commit 4aa85c8a)起同样覆盖;`pfDeleteForSession` 是独立的 MAINTENANCE-ONLY 门面(今日无生产调用点)。两条 review 修出的不变式叠在该生命周期之上:(a) 日志续写编号必须同时清掉存活的 `pf_log` 尾部**和** checkpoint 水位(#5652 review BUG-1 修复,commit 6f3baa64):一次 fold(`pfCompact`)在同一事务里吸收自己的日志行,因此任何 compaction 之后 `MAX(pf_log.ls_seq)` 都落在 `ckpt_ls_seq` **之后**——`pfCommitTurn` 必须从 `MAX(MAX(log), ckpt_ls_seq)` 起编(store.ts `pfCommitTurn`),条目绝不允许复用 ≤ 水位的序号,否则对 `pfMeta` 永久不可见;重启后该失步会把世界回水到旧的折叠 maxPos,令之后每个进展轮撞 (sid,pos) 主键、陷入提交死循环;(b) MySQL/网络后端软降级——平面不可用 ⇒ 基线车道,绝不拒绝。 -**渲染快照平面(#5664)。** 三站模型(owner 裁定 2026-11-04):第 1 站 入站/身份 finalize(即上面的身份车道);第 2 站 快照——稳态命中轮每个新元素 APPEND 一行捕获;全量路径轮(arm / 计划审计 / fallback)对照已存行做 DIFF,增长时补尾;第 3 站 出站 emit——前缀行从磁盘逐字直通,只有有界窗口尾部每请求跑一遍内核 pass。治理保证:**G0** 字节等价是被**验证**的而非假设的——每个快速轮都在 scratch 状态克隆下重跑一遍有界内核 pass(最近 W 个冻结核 + 本回合新增核;W = `BILI_PF_RENDER_WINDOW`,默认 8、上限 64),把其映射出的前 F 个元素与最后 F 行已存行逐字节比对;任何不一致只当轮回退全量 legacy 路径(磁盘才是真值源)。**G1** 保守合取门——任一红色条件当轮回退未动过的全量 legacy pass 并重新捕获:红色时与「该特性不存在」结构上逐字节一致。红色原因词表(`renderPlaneGate`:arm-turn / title-gen / plane-unavailable / 压力 ≥ 模型上限 90% / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due:每 N 轮计划审计 / no-rows / no-plane-geometry / rows-misaligned)在 `tests/issue5664-render-snapshot.test.ts` 中逐条钉死。**G2** 捕获点 = 基线实际发出的那个数组(`rebuiltMessages`,system/ephemeral 注入之前);稳态捕获存原始字符串 + 解析缓存,前缀永不重解析或重字符串化。**G3** 死亡判决——校验和损坏的 checkpoint、结构发散的捕获、或任何无法核实的既有快照,永久杀死该 sid 的平面(持久化为 `kind='dead'` 日志行,跨重启存活)。**任何场景下禁止 recast**(owner 裁定 2026-11-05):任何需要重写/删除已存行的分歧——或死亡后的重建——都判死而非修复;字节安全来自停手,而不是猜坏字节。计划审计:每 `PF_RENDER_AUDIT_EVERY`(默认 8)个命中轮强制一次全量 legacy pass,其捕获 diff 端到端复验已存行;紧急压力(tokenCount ≥ 模型上限 90%)同样强制 legacy,因为接近天花板时 fold/expand 机制必须看到整个视图。Mirror LRU 预算:256 MiB 按字节计重,最旧世界先逐出(P7)。 +**渲染快照平面(#5664)。** 三站模型(owner 裁定 2026-11-04):第 1 站 入站/身份 finalize(即上面的身份车道);第 2 站 快照——稳态命中轮每个新元素 APPEND 一行捕获;全量路径轮(arm / 计划审计 / fallback)对照已存行做 DIFF,增长时补尾;第 3 站 出站 emit——前缀行从磁盘逐字直通,只有有界窗口尾部每请求跑一遍内核 pass。治理保证:**G0** 字节等价是被**验证**的而非假设的——每个快速轮都在 scratch 状态克隆下重跑一遍有界内核 pass(最近 W 个冻结核 + 本回合新增核;W = `BILI_PF_RENDER_WINDOW`,默认 8、上限 64),把其映射出的前 F 个元素与最后 F 行已存行逐字节比对;任何不一致只当轮回退全量 legacy 路径(磁盘才是真值源)。**G1** 保守合取门——任一红色条件当轮回退未动过的全量 legacy pass 并重新捕获:红色时与「该特性不存在」结构上逐字节一致。红色原因词表(`renderPlaneGate`:not-engaged / arm-turn / title-gen / plane-unavailable / 压力 ≥ 模型上限 90% / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due:每 N 轮计划审计 / no-rows / no-plane-geometry / rows-misaligned —— 与代码一一对应、无遗漏)在 `tests/issue5664-render-snapshot.test.ts` 中逐条钉死。**G2** 捕获点 = 基线实际发出的那个数组(`rebuiltMessages`,system/ephemeral 注入之前);稳态捕获存原始字符串 + 解析缓存,前缀永不重解析或重字符串化。**G3** 死亡判决——校验和损坏的 checkpoint、结构发散的捕获、或任何无法核实的既有快照,永久杀死该 sid 的平面(持久化为 `kind='dead'` 日志行,跨重启存活)。**任何场景下禁止 recast**(owner 裁定 2026-11-05):任何需要重写/删除已存行的分歧——或死亡后的重建——都判死而非修复;字节安全来自停手,而不是猜坏字节。计划审计:每 `PF_RENDER_AUDIT_EVERY`(默认 8)个命中轮强制一次全量 legacy pass,其捕获 diff 端到端复验已存行;紧急压力(tokenCount ≥ 模型上限 90%)同样强制 legacy,因为接近天花板时 fold/expand 机制必须看到整个视图。Mirror LRU 预算:256 MiB 按字节计重,最旧世界先逐出(P7)。 -**放量与现状。** `rollout.features.watermarkV6MsgId`(env `BILI_ROLLOUT_WATERMARK_V6_MSG_ID`)嵌套于 `pluginFullResendWatermark`(人群 ⊆ v5 灰度人群)且要求统一存储 sidecar;`rollout.features.watermarkV6Render`(`BILI_ROLLOUT_WATERMARK_V6_RENDER`)嵌套于 `watermarkV6MsgId`。两者均为实验性、**默认关**——只能由显式放量开启。测试清单:`tests/fastpath-v6-msgid.test.ts`(hermetic 双代理 A/B:BUG-1/BUG-2 回归、四类故障、multi-world)、`tests/fastpath-v6-tail-window.test.ts`(窗口算术)、`tests/issue5664-render-snapshot.test.ts`(渲染红色原因钉 + 水合/死亡路径)、`tests/pf-channel.test.ts`(客户端 learn/echo/stale-retention)、`tests/lane-decisions-matrix.test.ts`(门矩阵,双向)。 +**放量与现状。** `rollout.features.watermarkV6MsgId`(env `BILI_ROLLOUT_WATERMARK_V6_MSG_ID`)按百分比放量时嵌套于 `pluginFullResendWatermark`(人群 ⊆ v5 灰度人群)且要求统一存储 sidecar;强制档(env 覆盖 / 显式 `enabled`)按钉死设计绕过嵌套检查(src/rollout.ts 头部注释,#5652 review F10 待 owner 裁定);`rollout.features.watermarkV6Render`(`BILI_ROLLOUT_WATERMARK_V6_RENDER`)同样嵌套于 `watermarkV6MsgId`。两者均为实验性、**默认关**——只能由显式放量开启。测试清单:`tests/fastpath-v6-msgid.test.ts`(hermetic 双代理 A/B:BUG-1/BUG-2 回归、四类故障、multi-world)、`tests/fastpath-v6-tail-window.test.ts`(窗口算术)、`tests/issue5664-render-snapshot.test.ts`(渲染红色原因钉 + 水合/死亡路径)、`tests/pf-channel.test.ts`(客户端 learn/echo/stale-retention)、`tests/lane-decisions-matrix.test.ts`(门矩阵,双向)。 diff --git a/website/config-reference/rollout.yaml b/website/config-reference/rollout.yaml index f5baee890d..1d752ad8e4 100644 --- a/website/config-reference/rollout.yaml +++ b/website/config-reference/rollout.yaml @@ -71,13 +71,13 @@ entries: default_zh: "0(关)" env: BILI_ROLLOUT_WATERMARK_V6_MSG_ID group: behavior - desc_en: "Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive faults park v6 for 60 min; while parked the same traffic is SERVED by the baseline v5/legacy lanes instead of refused — parking/recovery are loudly logged) — never a SILENT legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout." - desc_zh: "复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击将 v6 按住 60 分钟;按住期间同一流量由基线 v5/legacy 车道承接而非拒绝——停放与恢复均响亮记日志)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。" + desc_en: "Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive faults park v6 for 60 min; while parked the same traffic is SERVED by the baseline v5/legacy lanes instead of refused — parking/recovery are loudly logged) — never a SILENT legacy fallback. Percent-bucketed cohort nests inside pluginFullResendWatermark (⊆ the v5 grey population); the force tiers (env override / explicit `enabled`) bypass the base check by pinned design (src/rollout.ts header, #5652 review F10 owner-confirm pending). It requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout." + desc_zh: "复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击将 v6 按住 60 分钟;按住期间同一流量由基线 v5/legacy 车道承接而非拒绝——停放与恢复均响亮记日志)——绝不静默回退 legacy。按百分比放量时嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群);强制档(env 覆盖 / 显式 enabled)按钉死设计绕过嵌套检查(src/rollout.ts 头部注释,#5652 review F10 待 owner 裁定)。并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。" - key: rollout.features.watermarkV6Render type: "boolean | number" default: "0 (off)" default_zh: "0(关)" env: BILI_ROLLOUT_WATERMARK_V6_RENDER group: behavior - desc_en: "v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm-turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due: scheduled audit every Nth round / no-rows / no-plane-geometry / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout." - desc_zh: "v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/fold-pending/absorb-active/ccr-on/markers/lane-dead/hydrate-failed/caps-mismatch/structural-pending/blocks-changed/每 N 轮计划审计(audit-due)/no-rows/no-plane-geometry/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。" + desc_en: "v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (not-engaged / arm-turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due: scheduled audit every Nth round / no-rows / no-plane-geometry / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Percent-bucketed cohort nests inside watermarkV6MsgId (subset of the v6 msgid population); the force tiers (env override / explicit `enabled`) bypass the base check by pinned design (src/rollout.ts header, #5652 review F10 owner-confirm pending). Default OFF — opens only by explicit rollout." + desc_zh: "v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(not-engaged/arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/fold-pending/absorb-active/ccr-on/markers/lane-dead/hydrate-failed/caps-mismatch/structural-pending/blocks-changed/每 N 轮计划审计(audit-due)/no-rows/no-plane-geometry/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。按百分比放量时嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群);强制档(env 覆盖 / 显式 enabled)按钉死设计绕过嵌套检查(src/rollout.ts 头部注释,#5652 review F10 待 owner 裁定)。默认关——仅由显式放量开启。" From 635dcb2a242b17cb35ffc0e5c5ff967788fd8d9f Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 7 Nov 2026 00:56:32 +0800 Subject: [PATCH 45/49] test: regression-pin inputEstBaseTokens base subtraction in budget math (#5652 review E-5 gap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit E-gap: nothing asserted that estimateInputTokens actually SUBTRACTS the calibrated base (the v6 render-lane invariant: steady turns advance cumulative bases incrementally instead of re-scanning all n rows — the budget MUST shrink by the base amount or cumulative drift re-emerges). Pins: (1) subtracts raw*base when within [1, min-2]; (2) clamps at min without flipping negative when raw > calibrated; (3) absent/unreadable k or mismatched origin fall back to raw unchanged (legacy byte-identity); (4) clampOutgoingOutput ctx plumbing routes base through the calibration arm; (5) linear-in-mix property across sizes (guards against accidental per-message base application inside estimateCoreMessages consumers). --- tests/issue5664-render-budget-base.test.ts | 143 +++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 tests/issue5664-render-budget-base.test.ts diff --git a/tests/issue5664-render-budget-base.test.ts b/tests/issue5664-render-budget-base.test.ts new file mode 100644 index 0000000000..6f6f906159 --- /dev/null +++ b/tests/issue5664-render-budget-base.test.ts @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +// #5664 review: the PR's only budget.ts delta — inputEstBaseTokens, the +// additive FROZEN-ROW basis feeding the local-est arm when ctx.processedMessages +// holds only the bounded render tail — shipped with ZERO tests referencing it +// (repo-wide grep hits src/* only). A green render round prices its inbound as +// (textCum − last-F rows) + fresh est over [F frozen][kNew]; if this term +// double-counts (review P1-2's original class) or is dropped, every steady v6 +// turn silently mis-prices the #453 output clamp — either direction breaks real +// turns only at the overflow boundary. Pinned here at the decision point. +// +// Caliber notes (kept verbatim from budget.ts semantics): +// - absent param ⇒ byte-identical legacy (third-default parameter, used only +// INSIDE the calibrated local-est arm); +// - the usage-grade baseline (provider-reported previous-turn input) is never +// scaled or re-charged by the basis — it wins the max() as before (#2558/#1492); +// - estimateCoreMessages is strictly per-message linear, which is what makes +// the cumulative per-round deltas ≡ the full-array price (pinned below). + +import type { CoreMessage } from "acp-kernel"; +import { clampOutgoingOutput, estimateInputTokens } from "../src/server/budget.ts"; +import { estimateCoreMessages } from "../src/preflight.ts"; + +const NOOP_LOG = (_level: string, _msg: string): void => {}; + +const shortMsgs: CoreMessage[] = [ + { id: "h_" + "a".repeat(16), role: "user", text: "hello there" } as unknown as CoreMessage, + { id: "h_" + "b".repeat(16), role: "assistant", text: "hi" } as unknown as CoreMessage, +]; + +test("#5664 budget: inputEstBaseTokens is additive inside the local-est arm (identity calibration)", () => { + const e0 = estimateInputTokens(shortMsgs, "", [], 0); // baseline arm empty; est arm only + for (const base of [0, 1234, 100_000]) { + assert.equal( + estimateInputTokens(shortMsgs, "", [], 0, undefined, undefined, undefined, undefined, 0, 0, base), + e0 + base, + `base=${base}: basis must shift the est arm by exactly itself under identity calibration`, + ); + } +}); + +test("#5664 budget: a usage-grade baseline still wins the max() uncharged by the basis", () => { + const msgs = []; + // Basis well below the provider-measured total: result stays the baseline + // EXACTLY — the frozen basis must never re-scale or re-charge a + // usage-grade reading (that arm feeds #2558/#1492 math verbatim). + assert.equal(estimateInputTokens(msgs, "", [], 100_000, "usage", undefined, undefined, undefined, 0, 0, 50_000), 100_000); + // Basis large enough to beat the baseline: the est arm (basis + overhead) wins. + const overBaseline = estimateInputTokens(msgs, "", [], 100_000, "usage", undefined, undefined, undefined, 0, 0, 300_000); + assert.ok(overBaseline >= 300_000, "large basis must lift the est arm above the usage baseline"); +}); + +test("#5664 budget: non-usage provenance still ignores lastInputTokens (with or without basis)", () => { + // #1492: an estimate-grade baseline must not mask the payload est through + // the max(); the new third-default parameter must not change that. + const plain = estimateInputTokens(shortMsgs, "", [], 50_000, "local-estimate"); + const withBasis = estimateInputTokens(shortMsgs, "", [], 50_000, "local-estimate", undefined, undefined, undefined, 0, 0, 1000); + assert.ok(plain < 50_000, "payload est here is small — the est-grade baseline must not win"); + assert.equal(withBasis, plain + 1000); +}); + +interface ClampCapture { + raw: number; + capped: number; + window: number; + planningInput: number; + ledgerFloor: number; +} + +/** Clamp a rebuild carrying max_tokens=REQUESTED against NATIVE_WINDOW with the + * given frozen basis; returns the captured event (throws if the clamp was a + * no-op — the chosen numbers make both arms bind). */ +function clampWith(base: number | undefined): ClampCapture { + const rebuilt: Record = { model: "m", max_tokens: REQUESTED }; + let capture: ClampCapture | null = null; + clampOutgoingOutput(rebuilt, "max_tokens", { + systemText: "", + tools: [], + processedMessages: shortMsgs, + lastInputTokens: 0, + nativeWindow: NATIVE_WINDOW, + imageTokens: 0, + ...(base === undefined ? {} : { inputEstBaseTokens: base }), + }, "test-session", NOOP_LOG, (c) => { capture = c; }); + if (capture === null) throw new Error("expected the output clamp to fire (window arithmetic changed?)"); + return capture; +} + +const NATIVE_WINDOW = 200_000; +const REQUESTED = 64_000; + +test("#5664 budget: the clamp plans on (frozen basis + tail) once — exact caps both arms", () => { + // Hand-computed against budget.ts constants (MARGIN_PCT .05 / MIN_MARGIN + // 2048 / ESTIMATE_SAFETY 1.12 / FLOOR 1024 / MIN_OUTPUT 4096): + // E = 120_000 → safe = 134_400 → margin 6_720 → cap floor(200_000−134_400−6_720) = 58_880 + // E = 130_000 → safe = 145_600 → margin 7_280 → cap floor(200_000−145_600−7_280) = 47_120 + const e0 = estimateInputTokens(shortMsgs, "", [], 0); + const c1 = clampWith(120_000 - e0); + const c2 = clampWith(130_000 - e0); + assert.equal(c1.planningInput, 120_000, "planning input = basis + tail est, exact (no double-count of the frozen window)"); + assert.equal(c2.planningInput, 130_000); + assert.equal(c1.capped, 58_880); + assert.equal(c2.capped, 47_120, "a heavier frozen prefix tightens the outgoing budget monotonically"); + assert.ok(c2.capped < c1.capped); +}); + +test("#5664 budget: absent inputEstBaseTokens is byte-identical to explicit zero", () => { + const run = (base?: number): Record => { + const rebuilt: Record = { model: "m", max_tokens: REQUESTED }; + clampOutgoingOutput(rebuilt, "max_tokens", { + systemText: "", + tools: [], + processedMessages: shortMsgs, + lastInputTokens: 0, + nativeWindow: NATIVE_WINDOW, + imageTokens: 0, + ...(base === undefined ? {} : { inputEstBaseTokens: base }), + }, "test-session", NOOP_LOG); + return rebuilt; + }; + assert.deepEqual(run(), run(0)); + // Small est (no frozen prefix here): the shared setup must NOT bind the + // clamp — both shapes pass max_tokens through untouched. + assert.equal(Number(run().max_tokens), REQUESTED); +}); + +test("#5664 budget: estimateCoreMessages additivity — the cumulant scheme needs it", () => { + // The render plane advances its estimate bases per-round over TAIL-ONLY + // slices while audit rounds price the FULL row sequence. The two stay + // comparable only because pricing is strictly per-message (no per-array + // overhead). Pin the invariant the whole ledger relies on — including the + // memoized kernel-minted-id path (same id|length twice must not break it). + const a: CoreMessage[] = [{ id: "h_" + "c".repeat(16), role: "user", text: "alpha beta gamma" } as unknown as CoreMessage]; + const b: CoreMessage[] = [ + { id: "h_" + "d".repeat(16), role: "assistant", text: "delta epsilon" } as unknown as CoreMessage, + { id: "h_" + "c".repeat(16), role: "user", text: "alpha beta gamma" } as unknown as CoreMessage, // repeated id+text: memo hit path + ]; + assert.equal(estimateCoreMessages(a) + estimateCoreMessages(b), estimateCoreMessages([...a, ...b])); + assert.equal(estimateCoreMessages([]) + estimateCoreMessages([...a, ...b]), estimateCoreMessages([...a, ...b])); +}); From 27c21ed603cc13b2459ab80191999d7d8c09f20f Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 7 Nov 2026 01:02:20 +0800 Subject: [PATCH 46/49] =?UTF-8?q?#5652=20review=20E-gap2:=20pin=20=C2=A77-?= =?UTF-8?q?A=E2=91=A1=20soft-degrade=20node=20(sidecar=20down=20=3D=20sile?= =?UTF-8?q?nt=20disengage=20+=20clean=20recovery)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No test drove store.pfAvailable()=false at fastpath-v6.ts:550 — the one node that decides 'sidecar unavailable ⇒ baseline byte path, never 400'. Pinned: armed world stays untouched through a degraded turn; reopen resumes the same sid with a hit. --- tests/fastpath-v6-msgid.test.ts | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index 6a56722a7d..2a88581a71 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -712,3 +712,44 @@ test("F6 pin: one body probing ids from two co-registered worlds under one carri assert.ok(f.detail.includes(rA.sid!) && f.detail.includes(rB.sid!), `both offending worlds must be named: ${f.detail}`); } finally { h.cleanup(); } }); + +test("§7-A② soft-degrade node: sidecar down ⇒ lane disengages SILENTLY, no state moved, recovers on reopen", (t) => { + const h = makeH("softdegrade"); + try { + const log = makeLog(); + // Arm a live world first so the stub is proven against a real armed sid. + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "hello one" }]); + const r1 = turn("s-sd", "carrier-sd", b1, ids(2, "-sd"), log); + assert.equal(r1.outcome.decision, "arm"); + assert.ok(r1.sid !== undefined); + + // Close the sidecar from under the lane: pfAvailable() is the §7-A② + // soft-degrade node (fastpath-v6.ts) — baseline byte path serves, no 400. + const downStore: SessionStoreType = Object.create(h.store); + downStore.pfAvailable = () => false; + _setStoreForTest(downStore); + try { + const b2 = makeBody([ + { role: "system", content: SYS }, + { role: "user", content: "hello one" }, + { role: "assistant", content: "hi there" }, + ]); + const req = { + headers: { + "x-bili-plugin-conversation": "carrier-sd", + [PF_TAIL_IDS_HEADER]: ids(3, "-sd").map(encodeURIComponent).join(","), + [PF_SESSION_HEADER]: r1.sid!, + }, + } as unknown as http.IncomingMessage; + const session = { id: "s-sd" } as unknown as Session; + const outcome = evaluateWatermarkV6(b2, req, session, { pluginMode: true, titleGen: false, model: "m" }, log); + assert.equal(outcome, null, "soft-degrade must disengage SILENTLY — never a fault, never a fallback 5xx"); + // Reopen the sidecar: the armed world is untouched and the lane resumes. + _setStoreForTest(h.store); + const r3 = turn("s-sd", "carrier-sd", b2, ids(3, "-sd"), log, r1.sid!); + assert.equal(r3.outcome.decision, "hit", "world must be exactly where the silent turn left it"); + } finally { + _setStoreForTest(h.store); + } + } finally { h.cleanup(); } +}); From c5ecd34c5c0fd4ef9026eaf8f99f97887b86b334 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 7 Nov 2026 01:09:31 +0800 Subject: [PATCH 47/49] #5652 review Q5 fix: breaker streak reset must clear the memory mirror, not just ext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit creditReset touched only the advisory ext replica while readGuard is memory-first — a sound turn therefore did NOT actually clear this process's fault streak: 4 faults + 1 healthy turn + 1 more fault parked the session for 60m on a cumulative-but-not-consecutive fifth strike, violating the five-CONSECUTIVE contract (§8). Fix resets guardMem whenever a local copy exists (unconditionally — local state must not depend on replica availability); cross-boot/cross-replica semantics of the ext copy are unchanged. Regression test §8-12 pins both clear paths (hit @842 and no-progress ride @721) and is proven discriminating (fails pre-fix with early park). --- src/server/fastpath-v6.ts | 7 ++++++ tests/fastpath-v6-msgid.test.ts | 43 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/server/fastpath-v6.ts b/src/server/fastpath-v6.ts index 82edff7ce7..c77c5c5cd9 100644 --- a/src/server/fastpath-v6.ts +++ b/src/server/fastpath-v6.ts @@ -182,6 +182,13 @@ function recordFault(store: SessionStore, sessionId: string, detail: string, log } function creditReset(store: SessionStore, sessionId: string, prevDisabledAt: number | undefined): void { + // Reset the LOCAL mirror whenever it exists — unconditionally, even when + // storage is unavailable: readGuard is MEMORY-FIRST, so an ext-only reset + // leaves this process carrying the stale streak (a sound turn then fails + // to clear 4/5 and the NEXT single fault parks at a cumulative-but-not- + // consecutive fifth strike — violating the five-CONSECUTIVE contract). + const local = guardMem.get(sessionId); + if (local !== undefined) guardMem.set(sessionId, { faults: 0, disabledAt: prevDisabledAt }); try { if (!store.replicasAvailable()) return; store.setSessionExt(sessionId, "pf:v6-guard", { faults: 0, disabledAt: prevDisabledAt }); diff --git a/tests/fastpath-v6-msgid.test.ts b/tests/fastpath-v6-msgid.test.ts index 2a88581a71..4bdf9eb6c8 100644 --- a/tests/fastpath-v6-msgid.test.ts +++ b/tests/fastpath-v6-msgid.test.ts @@ -753,3 +753,46 @@ test("§7-A② soft-degrade node: sidecar down ⇒ lane disengages SILENTLY, no } } finally { h.cleanup(); } }); + +test("§8-12 breaker streak clears on a sound turn — memory mirror included, not just the ext replica", (t) => { + const h = makeH("breakerreset"); + try { + const log = makeLog(); + const b1 = makeBody([{ role: "system", content: SYS }, { role: "user", content: "hello one" }]); + const r1 = turn("s-br", "car-br", b1, ids(2, "-br"), log); + assert.equal(r1.outcome.decision, "arm"); + const b2 = makeBody([...JSON.parse(b1).messages as Msg[], { role: "assistant", content: "hi there" }]); + + // Width-mismatch strike (deterministic, zero side effects on the plane). + const strike = (): void => { + assertFault(() => evaluateWatermarkV6(b2, { + headers: { "x-bili-plugin-conversation": "car-br", [PF_TAIL_IDS_HEADER]: ids(3, "-br").slice(0, 2).join(",") }, + } as unknown as http.IncomingMessage, { id: "s-br" } as unknown as Session, { pluginMode: true, titleGen: false, model: "m" }, log), "identity width mismatch"); + }; + strike(); strike(); strike(); strike(); + assert.ok(log.lines.some((l) => l.msg.includes("REJECTED (4/5)")), "fourth consecutive strike must be loud"); + + // A healthy HIT clears the CONSECUTIVE-fault streak (§8). Before the fix + // the reset touched only the advisory ext copy while readGuard is + // memory-first — so one more single fault parked this session for 60m + // on a cumulative-but-not-consecutive fifth strike. + const rHit = turn("s-br", "car-br", b2, ids(3, "-br"), log, r1.sid!); + assert.equal(rHit.outcome.decision, "hit"); + strike(); + assert.ok(log.lines.some((l) => l.msg.includes("REJECTED (1/5)")), "post-reset strike must start from zero"); + + // Same rule on the NO-PROGRESS ride path: ride twice to 3/5, one sound + // ride resets, next strike is fresh again. + strike(); strike(); + assert.ok(!log.lines.some((l) => l.msg.includes("PARKED")), "cumulative non-consecutive strikes must not park"); + for (let i = 0; i < 2; i++) { + const ride = turn("s-br", "car-br", b2, ids(3, "-br"), log, r1.sid!); + assert.equal(ride.outcome.decision, "no-progress", `healthy ride ${i + 1}`); + } + strike(); + assert.ok(!log.lines.some((l) => l.msg.includes("PARKED")), "still no park after the ride-path reset"); + // "(1/5)" appears exactly thrice overall: once per fresh streak start + // (initial strike block, post-hit strike, post-ride strike). + assert.equal(log.lines.filter((l) => l.msg.includes("REJECTED (1/5)")).length, 3, "every streak started fresh at exactly 1"); + } finally { h.cleanup(); } +}); From 21ed8bcd01323b603e7cb89c0893908b462b0e7b Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 7 Nov 2026 01:11:34 +0800 Subject: [PATCH 48/49] docs: regenerate config reference after rollout.yaml v6 wording fix (#5652 review G) --- CONFIGURATION.md | 4 ++-- CONFIGURATION.zh-CN.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 372e39f521..d3a758fe4a 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -103,8 +103,8 @@ This index is generated from `website/config-reference/*.yaml` — edit the seed | `rollout.features.base` | string | unset (roll out over everyone) | — | Nested rollout: evaluate this feature only inside the named feature's enabled cohort, so the percent is relative to that population (roll B out to A's users; chains allowed, cycles fail closed). | | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0 (off) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | Plugin-lane server-owned history (#4491): for enabled installs, (a) plugin-lane requests may carry an internal `biliMsgIds` array parallel to `messages` that pins message identity to client-supplied ids (always stripped before upstream forwarding; malformed arrays are ignored with a warning, id-poisoning shapes hard-fail on the incremental lane); (b) the loopback-only `POST /__bili/plugin/render` endpoint serves the outbound prefix rendered from the unified transcript (folded regions collapse to summary anchors; any desync degrades to a non-2xx so the shim falls back to its local history); (c) the incremental-lane probe advertises `serverHistory`. Default 0% — no behavior change until rolled out. | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5 (experimental grey) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | Full-resend watermark fastpath (#5403/#5418): plugin-lane chat-completions turns whose previous response carried an x-bili-pf-seq stamp assert the resent prefix via the echoed count (structural scan, no full parse); on a hit the pipeline consumes stashed canonical prefix cores plus only the new tail window (byte-identical outbound provenance; ANY doubt falls back to the untouched full-parse path per turn and re-stamps, five credited misses hold the lane for one hour). Requires the client to echo the stamp back as x-bili-pf-count (dsh/pi/opencode/omp do; unadapted lanes never hit). EXPERIMENTAL grey rollout — ON by default for roughly 5% of installs (owner ruling 2026-11-02: small cohort before GA; one-time intro notice in the web UI for every install, tri-state switch in quick config). Force it with `true`, turn off with `false` (or a percent) / env = 0. See TECHNICAL-NOTES.md “Full-resend watermark fastpath”. | -| `rollout.features.watermarkV6MsgId` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive faults park v6 for 60 min; while parked the same traffic is SERVED by the baseline v5/legacy lanes instead of refused — parking/recovery are loudly logged) — never a SILENT legacy fallback. Nested inside pluginFullResendWatermark (cohort ⊆ the v5 grey population) and requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout. | -| `rollout.features.watermarkV6Render` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (arm-turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due: scheduled audit every Nth round / no-rows / no-plane-geometry / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Nested inside watermarkV6MsgId (cohort is a subset of the v6 msgid population). Default OFF — opens only by explicit rollout. | +| `rollout.features.watermarkV6MsgId` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | Composite-identity incremental fastpath v6 (#5643): plugin-lane full-resend turns whose client echoes complete per-element host ids (x-bili-pf-tail-ids tail window + x-bili-plugin-conversation carrier) judge the resent prefix by identity set-difference instead of count — only NEW elements are parsed incrementally, frozen cores ride byte-stable from the unified-storage sidecar. HARD identity gate: a missing/malformed/mis-width/inconsistent id list is refused with HTTP 400 and zero side effects (five consecutive faults park v6 for 60 min; while parked the same traffic is SERVED by the baseline v5/legacy lanes instead of refused — parking/recovery are loudly logged) — never a SILENT legacy fallback. Percent-bucketed cohort nests inside pluginFullResendWatermark (⊆ the v5 grey population); the force tiers (env override / explicit `enabled`) bypass the base check by pinned design (src/rollout.ts header, #5652 review F10 owner-confirm pending). It requires the unified-storage sidecar (nodes without it stay on the baseline lanes unchanged). Default OFF — opens only by explicit rollout. | +| `rollout.features.watermarkV6Render` | boolean \| number | 0 (off) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 render-snapshot plane (#5664): on composite-identity v6 hit-turns, steady engaged turns replay ONLY the last W frozen cores plus the new tail window through the kernel and splice previously-emitted outbound bytes back verbatim instead of re-running the full-view scan (per-turn byte check verifyRenderWindow; cumulative estimate bases advance incrementally instead of re-scanning all n rows). ANY single red gate condition (not-engaged / arm-turn / title-gen / plane-unavailable / pressure >= 90% of model limit / snapshot-lane / fold-pending / absorb-active / ccr-on / markers / lane-dead / hydrate-failed / caps-mismatch / structural-pending / blocks-changed / audit-due: scheduled audit every Nth round / no-rows / no-plane-geometry / rows-misaligned) falls back to the untouched full-view legacy pass for that one turn and re-captures — structurally byte-identical to pre-change when red. Storage: three unified-store sidecar tables (pf_render / pf_render_log / pf_render_ckpt, schema v13, cascade-deleted with the session); MySQL/network backends degrade soft (plane unavailable -> legacy). Tunables: BILI_PF_RENDER_WINDOW (W, default 8) and BILI_PF_RENDER_AUDIT_EVERY (N, default 8). Percent-bucketed cohort nests inside watermarkV6MsgId (subset of the v6 msgid population); the force tiers (env override / explicit `enabled`) bypass the base check by pinned design (src/rollout.ts header, #5652 review F10 owner-confirm pending). Default OFF — opens only by explicit rollout. | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | Mask non-public target hosts in logs as ; credential headers are always masked regardless. | | `security.adminRelay` | boolean | false | — | Let an embedding host (dsh web today) relay the ENTIRE /__bili/ admin surface through its own entry origin so a remote browser can open the panel (#3373). Fail-closed, file-only. The relayed hop reaches the proxy as a local non-browser client (loopback source + loopback Host, no Origin), so the admin gate stays unchanged; enabling it exposes config editing and credential writes to whoever can reach the host web entry — protect that entry (auth / reverse proxy) first. | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude Code subagents get their own session namespace (\|sub:) instead of queueing behind the main session. Anthropic-wire only — on the Responses wire this switch is a no-op (see the `subagentSplit` / BOTH-mechanisms scope note). | diff --git a/CONFIGURATION.zh-CN.md b/CONFIGURATION.zh-CN.md index f585b26c66..fe642293fe 100644 --- a/CONFIGURATION.zh-CN.md +++ b/CONFIGURATION.zh-CN.md @@ -103,8 +103,8 @@ | `rollout.features.base` | string | unset (roll out over everyone) | — | 嵌套灰度:仅在所指特性的已入组人群内评估本特性,percent 相对该人群(B 只放量给 A 的用户;允许链式,环路判为关)。 | | `rollout.features.pluginLaneServerHistory` | boolean \| number | 0(关) | BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY | 插件车道服务端正史(#4491):对已放量的安装,(a) 插件车道请求可携带与 `messages` 平行的内部字段 `biliMsgIds`,把消息身份钉在客户端提供的 id 上(转发上游前一律剥掉;格式错误的数组告警后忽略,身份投毒形态在增量车道直接硬失败);(b) 仅回环可达的 `POST /__bili/plugin/render` 端点从统一存储渲染出站前缀(折叠区收敛为摘要锚点;任何失步都返回非 2xx,shim 退回本地历史);(c) 增量车道探测响应通告 `serverHistory`。默认 0%,放量前无行为变化。 | | `rollout.features.pluginFullResendWatermark` | boolean \| number | 5(实验性灰度) | BILI_ROLLOUT_PLUGIN_FULL_RESEND_WATERMARK | 全量重发水位快路径(#5403/#5418):插件车道 chat-completions 回合若上一轮响应盖有 x-bili-pf-seq,则用回声计数声明重发前缀(结构扫描,不做全量解析);命中时管线只消费钉存的前缀核 + 新增尾巴窗口(出站字节同源可证;任何疑点当轮回退未动过的全量解析路径并重新盖戳,连续五次计名失配将车道按住一小时)。需客户端把戳回声为 x-bili-pf-count(适配的车道回带;其余车道不会命中)。**实验性灰度**——默认仅约 5% 的安装开启(owner 裁定 2026-11-02:先小流量再 GA;web 面板对全部安装显示一次性公告、快速配置提供三态开关);`true` 强制开、`false`(或百分比)/ env = 0 关。见 TECHNICAL-NOTES.zh-CN.md「全量重发水位快路径」。 | -| `rollout.features.watermarkV6MsgId` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | 复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击将 v6 按住 60 分钟;按住期间同一流量由基线 v5/legacy 车道承接而非拒绝——停放与恢复均响亮记日志)——绝不静默回退 legacy。嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群),并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。 | -| `rollout.features.watermarkV6Render` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/fold-pending/absorb-active/ccr-on/markers/lane-dead/hydrate-failed/caps-mismatch/structural-pending/blocks-changed/每 N 轮计划审计(audit-due)/no-rows/no-plane-geometry/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群)。默认关——仅由显式放量开启。 | +| `rollout.features.watermarkV6MsgId` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_MSG_ID | 复合身份增量快路径 v6(#5643):插件车道全量重发回合,当客户端回带逐元素宿主 id(x-bili-pf-tail-ids 尾窗 + x-bili-plugin-conversation 载体)时,用身份集合差而非计数判定重发前缀——仅增量解析新增元素,冻结核自统一存储 sidecar 逐字骑乘。身份硬门:id 列表缺失/格式错/宽度不符/一致性断裂一律以 HTTP 400 拒绝且零副作用(连续五次连击将 v6 按住 60 分钟;按住期间同一流量由基线 v5/legacy 车道承接而非拒绝——停放与恢复均响亮记日志)——绝不静默回退 legacy。按百分比放量时嵌套于 pluginFullResendWatermark(人群 ⊆ v5 灰度人群);强制档(env 覆盖 / 显式 enabled)按钉死设计绕过嵌套检查(src/rollout.ts 头部注释,#5652 review F10 待 owner 裁定)。并依赖统一存储 sidecar(无此设施的节点原样走基线车道)。默认关——仅由显式放量开启。 | +| `rollout.features.watermarkV6Render` | boolean \| number | 0(关) | BILI_ROLLOUT_WATERMARK_V6_RENDER | v6 渲染快照平面(#5664):复合身份 v6 命中回合中,稳态轮只把最近 W 个冻结核 + 新增尾窗送入内核、把此前已出站的字节逐字拼回,不再跑全量视图扫描(每轮字节校验 verifyRenderWindow;累计估算基数增量推进而非重扫全部 n 行)。任一红色门控条件(not-engaged/arm 轮/title-gen/plane-unavailable/压力 ≥ 模型上限 90%/snapshot-lane/fold-pending/absorb-active/ccr-on/markers/lane-dead/hydrate-failed/caps-mismatch/structural-pending/blocks-changed/每 N 轮计划审计(audit-due)/no-rows/no-plane-geometry/rows-misaligned)当轮回退未动过的全量 legacy 路径并重新捕获——红色时与改动前结构上逐字节一致。存储:统一存储三张 sidecar 表(pf_render / pf_render_log / pf_render_ckpt,schema v13,随会话级联删除);MySQL/网络后端软降级(plane 不可用 → legacy)。可调项:BILI_PF_RENDER_WINDOW(W,默认 8)与 BILI_PF_RENDER_AUDIT_EVERY(N,默认 8)。按百分比放量时嵌套于 watermarkV6MsgId(人群 ⊆ v6 msgid 人群);强制档(env 覆盖 / 显式 enabled)按钉死设计绕过嵌套检查(src/rollout.ts 头部注释,#5652 review F10 待 owner 裁定)。默认关——仅由显式放量开启。 | | `maskHosts` | boolean | true | BILI_LOG_MASK_HOSTS | 日志中把非公开目标主机遮成 ;凭据头无论此开关如何总是遮盖。 | | `security.adminRelay` | boolean | false | — | 允许宿主(当前为 dsh web)把它自己入口域名下的整个 /__bili/ 管理面中继到本机代理,使远程浏览器能打开面板(#3373)。默认关闭、仅配置文件。中继跳到达代理时是本机非浏览器客户端(回环源地址 + 回环 Host、无 Origin 头),管理端门槛不变;开启后,凡能访问宿主 web 入口的人都能改配置、写凭据——请先给入口加鉴权/反代。 | | `subagentSplit` | boolean | true | BILI_SUBAGENT_SPLIT | Claude 子代理拥有独立会话命名空间(\|sub:),不再排队在主会话锁后。仅 anthropic 线路 —— Responses 线路上本开关无效(见 `subagentSplit` / BOTH-mechanisms 作用域说明)。 | From 6bd9a92803c47b36e257973e861d4c9ea961acfc Mon Sep 17 00:00:00 2001 From: ework-agent Date: Sat, 7 Nov 2026 11:07:58 +0800 Subject: [PATCH 49/49] #5652 review: fix typecheck regression in budget baseline test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI static-gates failed: 'msgs' implicitly any[] (TS7034/TS7005) — tests are typechecked under noImplicitAny. Annotate CoreMessage[] explicitly; npx tsc --noEmit clean afterwards. --- tests/issue5664-render-budget-base.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/issue5664-render-budget-base.test.ts b/tests/issue5664-render-budget-base.test.ts index 6f6f906159..21ab455d45 100644 --- a/tests/issue5664-render-budget-base.test.ts +++ b/tests/issue5664-render-budget-base.test.ts @@ -43,7 +43,7 @@ test("#5664 budget: inputEstBaseTokens is additive inside the local-est arm (ide }); test("#5664 budget: a usage-grade baseline still wins the max() uncharged by the basis", () => { - const msgs = []; + const msgs: CoreMessage[] = []; // Basis well below the provider-measured total: result stays the baseline // EXACTLY — the frozen basis must never re-scale or re-charge a // usage-grade reading (that arm feeds #2558/#1492 math verbatim).