From dc65bb7b599dc33656bf8dda0c2c81a55c6b9f45 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Thu, 5 Nov 2026 15:14:35 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(fastpath):=20let=20the=20render=20snaps?= =?UTF-8?q?hot=20evolve=20with=20compression=20=E2=80=94=20witnessed=20reb?= =?UTF-8?q?ase=20instead=20of=20fold-dead=20(#5643=20=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E8=A3=81=E5=AE=9A,=20#5664)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner ruling (recorded in #5643 "快照必须随压缩演进——事件化更新语义"): the outbound-row snapshot table must follow its owner — the kernel fold layer — instead of dying when it compacts. Compression IS bili's core function; a snapshot plane that cannot survive the first compress is useless in exactly the scenarios it exists for. Mechanism (plain mutable row table, per the adjudicated design): - The gate no longer reds out for "has an active block" (that made the lane dead-on-arrival for any compressed session). It now compares a structural witness: activeBlocksSignature() — the sorted multiset of active block ids + generations, pure server-side state, changes iff the layout changed. A signature move costs exactly ONE red round (blocks-changed), after which the next full-path capture re-synchronizes. Folded sessions with an unchanged layout keep serving fast (the point of the feature). - A structurally-WITNESSED diverged capture (block signature moved around this turn, dsh native-compaction detector fired (#2432 — its folds bypass bili entirely so the detector flag is an explicit witness term), fold usage consumed pre→post, or the gate itself observed the move) REBASES the table: truncate from the first divergence + whole-suffix re-append as one transaction (pfRenderCommit replaceAll), fresh S4-checksummed checkpoint carrying the new blocksSig, audit tick reset (the rebase round IS the audit). Storage stays bounded by live elements — compaction SHRINKS the table. No op-table, no version chain, no replay engine: rows stay compact surviving-element ordinals, keyed contiguously. - An UNWITNESSED diverged capture still kills the lane permanently (no unexplained drift is ever silently absorbed); checksum corruption and hydration failures stay durably dead. The three-primitive semantics the owner approved (insert/update/delete) are the same under contiguous keying: append = INSERT suffix; witnessed structural change = UPDATE/DELETE at the shared-prefix boundary + INSERT rest; steady rounds touch nothing old. Tests: golden A/B byte-equality suite unchanged (pre-existing tests all pass); new unit pin covers the both-direction gate pair (matching sig green incl. hydrated restart, stale sig red "blocks-changed") and the full rebase lifecycle (cast -> witnessed shrink+rewrite -> steady appends renumbered onto the compacted table -> restart hydrates rows AND signature). --- src/persist.ts | 1 + src/server/fastpath-v6-render.ts | 124 ++++++++++++++++++++--- src/server/prepare-openai.ts | 47 +++++++-- src/storage/store.ts | 23 +++-- tests/issue5664-render-snapshot.test.ts | 126 +++++++++++++----------- 5 files changed, 231 insertions(+), 90 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index a35534dc63..d442719957 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2319,6 +2319,7 @@ export class SessionStore { rows: { pos: number; rowJson: string | null }[]; logEntries: { kind: string; payloadJson: string }[]; checkpointMetaJson?: string; + replaceAll?: boolean; ts?: number; }, ): number | null { diff --git a/src/server/fastpath-v6-render.ts b/src/server/fastpath-v6-render.ts index 65e05513ae..2befef948a 100644 --- a/src/server/fastpath-v6-render.ts +++ b/src/server/fastpath-v6-render.ts @@ -95,6 +95,18 @@ function capsEqual(a: RenderCaps, b: RenderCaps): boolean { ); } +/** Stable serialization of the kernel's ACTIVE-block multiset — the structural- + * event witness at the snapshot boundary (#5643 补充裁定:快照必须随压缩演进). + * Block ids are never reused and `generation` advances on refold, so this + * string changes iff the fold layer changed the history layout since the + * last recorded signature. Pure server-side evidence; no client inference. */ +export function activeBlocksSignature(blocks: readonly { blockId: string; generation?: string | number; active?: boolean }[]): string { + const parts: string[] = []; + for (const b of blocks) if (b.active) parts.push(`${b.blockId}:${b.generation ?? 0}`); + parts.sort(); + return parts.join("|"); +} + interface RenderMeta { v: 1; caps: RenderCaps; @@ -102,6 +114,10 @@ interface RenderMeta { estCum: number; textCum: number; imgCum: number; + /** Active-block signature at this checkpoint (absent on pre-#5643补充裁定 + * checkpoints ⇒ ""). Folded forward by post-ckpt 'rebase'/'append' log + * entries during hydration. */ + blocksSig?: string; } export interface RenderWorld { @@ -116,6 +132,12 @@ export interface RenderWorld { * verify baseline, so no downstream consumer may alias-mutate them. */ rowsParsed: unknown[]; caps: RenderCaps; + /** Active-block signature recorded at the LAST commit ("" until any). The + * gate compares it against the live session: a move means the fold layer + * rewrote the layout since we last synchronized — one red "blocks-changed" + * round, then the full pass's diverged capture rebases the table (only + * when THAT round also carries a witness; see commitRenderCapture). */ + blocksSig: string; /** Cumulative Σ estimateCoreMessagesUpper over all current rows. */ estCum: number; /** Cumulative Σ estimateCoreMessages over all current rows. */ @@ -136,15 +158,18 @@ const mirrors = new Map(); /** relay session id → pf sids owned by it (teardown sweep). */ const sessionSids = new Map>(); -/** Owner ruling 2026-11-05 (#5689 review): NO recast in ANY scenario. - * Structural divergence (byte diff/shrink at the core-form boundary), S4 - * corruption, row-table inconsistency, unverifiable pre-existing snapshots - * — all land here. Caps changes cannot alter stored bytes anymore (mapping - * inputs sit outside the fingerprint, #5643 ruling §5 ③): their failure - * mode is a red gate turn, absorbed by the audit cadence. A dead sid NEVER - * arms, hydrates, gates green, or commits again. The verdict is persisted - * as a kind='dead' log entry so every future process instance reaches the - * same conclusion. */ +/** Owner rulings 2026-11-05. (#5689 review) NO UNWITNESSED recast in ANY + * scenario: structural divergence WITHOUT server-side evidence of a + * structural event, S4 corruption, row-table inconsistency, unverifiable + * pre-existing snapshots — all land here and stay dead forever. The #5643 + * 补充裁定 opened exactly one door OUT of that candidate set BEFORE death: + * a WITNESSED diverged capture rebases the table (commitRenderCapture's + * 'rebase' op) instead of dying. Caps changes cannot alter stored bytes + * anymore (mapping inputs sit outside the fingerprint, #5643 ruling §5 ③): + * their failure mode is a red gate turn, absorbed by the audit cadence. + * A dead sid NEVER arms, hydrates, gates green, or commits again. The + * verdict is persisted as a kind='dead' log entry so every future process + * instance reaches the same conclusion. */ const deadSids = new Set(); function killRenderPlane(store: SessionStore, sid: string, reason: string, log?: LogFn): void { @@ -225,15 +250,17 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor let imgCum = 0; let expectedCount = -1; let capsJson = ""; + let blocksSig = ""; if (metaRead?.ckpt) { const m = JSON.parse(metaRead.ckpt.metaJson) as RenderMeta; estCum = m.estCum ?? 0; textCum = m.textCum ?? 0; imgCum = m.imgCum ?? 0; expectedCount = m.rowCount; + blocksSig = m.blocksSig ?? ""; capsJson = metaRead.ckpt.metaJson; for (const e of metaRead.log) { - const p = JSON.parse(e.payloadJson) as { de?: number; dt?: number; di?: number; n?: number; capsJson?: string; reason?: string } & Partial; + const p = JSON.parse(e.payloadJson) as { de?: number; dt?: number; di?: number; n?: number; capsJson?: string; reason?: string; sharedPrefix?: number; from?: number; to?: number } & Partial; if (e.kind === "dead") { killRenderPlane(store, sid, `persisted death verdict: ${p.reason ?? "unknown"}`, log); return null; @@ -248,6 +275,12 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor textCum += p.dt ?? 0; imgCum += p.di ?? 0; if (expectedCount >= 0) expectedCount += p.n ?? 0; + } else if (e.kind === "rebase") { + // Post-ckpt rebases carry their own fresh checkpoint meta + // (the fold deletes the log), so a surviving 'rebase' + // entry here means an interrupted fold — treat as count + // reset to its journaled target. + if (typeof p.to === "number") expectedCount = p.to; } } } else { @@ -294,7 +327,7 @@ function hydrateMirror(store: SessionStore, sid: string, log?: LogFn): RenderWor killRenderPlane(store, sid, "unparseable capabilities fingerprint", log); return null; } - const rw: RenderWorld = { sid, rows, rowsParsed, caps, estCum, textCum, imgCum, auditTick: 0 }; + const rw: RenderWorld = { sid, rows, rowsParsed, caps, blocksSig, estCum, textCum, imgCum, auditTick: 0 }; touchMirror(rw); return rw; } catch { @@ -310,6 +343,12 @@ interface RenderGateArgs { sessionId: string; session: Session; store: SessionStore; + /** PRE-TURN active-block signature of the live session ("" = none). + * Compared against the world's last-committed signature: folded sessions + * stay green (compressed history IS steady state once synchronized), a + * signature move lands exactly one "blocks-changed" red round. Optional + * only for the test harness; production callers must pass it. */ + blocksSig?: string; /** true iff this request is an engaged v6 HIT (v6Evaluated.hit !== null). */ engagedHit: boolean; /** true on the arm turn (first registration) — always rides legacy. */ @@ -337,7 +376,7 @@ export type RenderGateDecision = /** Stable red-reason vocabulary (test-pinned, one per condition): not-engaged, * arm-turn, title-gen, plane-unavailable, pressure, snapshot-lane, - * blocks-active, fold-pending, absorb-active, ccr-on, markers, lane-dead, + * blocks-changed, fold-pending, absorb-active, ccr-on, markers, lane-dead, * hydrate-failed, caps-mismatch, audit-due, no-rows, no-plane-geometry, * rows-misaligned. */ @@ -351,7 +390,6 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { if (!args.store.pfRenderAvailable()) return off("plane-unavailable"); if (args.tokenCount >= PF_RENDER_EMERGENCY_RATIO * args.modelContextLimit) return off("pressure"); if (args.snapshotLaneActive) return off("snapshot-lane"); - if (args.session.state.blocks.some((b) => b.active)) return off("blocks-active"); if (args.foldUsagePending) return off("fold-pending"); if (args.absorbActive) return off("absorb-active"); if (args.ccrOn) return off("ccr-on"); @@ -363,6 +401,14 @@ export function renderPlaneGate(args: RenderGateArgs): RenderGateDecision { // returning null — report the durable reason, not the transient one. if (rw === undefined) return off(deadSids.has(args.sid) ? "lane-dead" : "hydrate-failed"); if (!capsEqual(rw.caps, args.caps)) return off("caps-mismatch"); + // #5643 补充裁定:快照必须随压缩演进。 Replaces the old blanket + // "blocks-active" red-out: a folded session KEEPS SERVING FAST as long as + // nothing changed structurally since the table's own commit. A moved + // block multiset (new fold / refold / archive / host-side native + // compaction surfaced through the block state) turns the plane red ONCE; + // the forced full pass either confirms alignment (equal/append) or + // rebases the table on its witnessed diverged capture. + if ((rw.blocksSig ?? "") !== (args.blocksSig ?? "")) return off("blocks-changed"); if (rw.auditTick % PF_RENDER_AUDIT_EVERY === 0 && rw.auditTick > 0) return off("audit-due"); if (rw.rows.length < 1) return off("no-rows"); if (args.maxPos < 0) return off("no-plane-geometry"); @@ -449,6 +495,15 @@ interface CaptureArgs { sid: string; mode: CaptureMode; caps: RenderCaps; + /** true iff THIS turn carries server-side evidence of a structural event + * (block-multiset signature moved around the kernel pass, dsh native- + * compaction rebase fired, or fold usage consumed). Only a witnessed + * diverged capture may rebase the table; an unwitnessed one kills. + * #5643 补充裁定。 */ + witness?: boolean; + /** Active-block signature AFTER this turn's passes — recorded with the + * commit so the gate's next comparison starts from this round's truth. */ + blocksSig?: string; /** steady: ONLY the new tail raw strings; legacy: the FULL captured rows. */ rowStrings: string[]; sums: CaptureSums; // steady: DELTAS for the tail; legacy: ABSOLUTE totals. @@ -478,7 +533,7 @@ export function diffRowLists(prev: string[], cur: string[]): "equal" | "append" * stops advancing until restart or caps change allows re-capture again. */ const nulRejectedSids = new Set(); -export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" } { +export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op: "none" | "append" | "cast" | "rebase" } { const { store, sid, caps, log } = args; if (deadSids.has(sid)) { // Belt-and-braces behind the gate: a dead sid NEVER arms/commits again. @@ -524,7 +579,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op return { committed: false, op: "none" }; } const rows = args.rowStrings; - const m: RenderMeta = { v: 1, caps, rowCount: rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum }; + const m: RenderMeta = { v: 1, caps, rowCount: rows.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, blocksSig: args.blocksSig ?? "" }; const okSeq = store.pfRenderCommit(sid, { rows: rows.map((rj, pos) => ({ pos, rowJson: rj })), logEntries: [{ kind: "cast", payloadJson: JSON.stringify({ ...m, capsJson: JSON.stringify(caps) }) }], @@ -532,7 +587,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op }); if (okSeq === null) return { committed: false, op: "none" }; const parsed = rows.map((rj) => safeParse(rj)); - mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, caps: { ...caps }, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, auditTick: 0 }); + mirrors.set(sid, { sid, rows: [...rows], rowsParsed: parsed, caps: { ...caps }, blocksSig: args.blocksSig ?? "", estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, auditTick: 0 }); registerSidForSession(args.sessionId, sid); return { committed: true, op: "cast" }; } @@ -565,6 +620,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op const verdict = diffRowLists(rw.rows, args.rowStrings); if (verdict === "equal") { rw.caps = { ...caps }; // keep caps fresh even when bytes match + rw.blocksSig = args.blocksSig ?? rw.blocksSig; touchMirror(rw); return { committed: true, op: "none" }; } @@ -588,6 +644,7 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op rw.textCum = args.sums.textCum; rw.imgCum = args.sums.imgCum; rw.caps = { ...caps }; + rw.blocksSig = args.blocksSig ?? rw.blocksSig; touchMirror(rw); return { committed: true, op: "append" }; } @@ -597,6 +654,41 @@ export function commitRenderCapture(args: CaptureArgs): { committed: boolean; op // scenario: the plane is KILLED PERMANENTLY instead and the session // rides the legacy full path from here on (bytes stay correct; the // cache simply never serves this sid again). + // DIVERGED (byte diff at any position, or shrink): the ORIGINAL design + // offered only recast; the owner ruling of 2026-11-05 sealed that door, + // and the #5643 补充裁定 re-opened EXACTLY ONE gate in it — a WITNESSED + // structural event (the turn's server-side evidence that the fold layer + // rewrote the layout: block-multiset signature moved around the kernel + // pass, dsh native-compaction rebase, or fold usage consumed) authorizes + // a REBASE: the whole table is replaced by THIS capture in one + // transaction, journaled as kind='rebase' under a fresh checkpoint. + // Without a witness, divergence is still substrate corruption and KILLS + // the plane permanently — that semantics is byte-for-byte unchanged. + if (args.witness === true) { + let common = 0; + while (common < rw.rows.length && common < args.rowStrings.length && rw.rows[common] === args.rowStrings[common]) common += 1; + const prevLen = rw.rows.length; + const m: RenderMeta = { v: 1, caps, rowCount: args.rowStrings.length, estCum: args.sums.estCum, textCum: args.sums.textCum, imgCum: args.sums.imgCum, blocksSig: args.blocksSig ?? rw.blocksSig }; + const okSeq = store.pfRenderCommit(sid, { + replaceAll: true, + rows: args.rowStrings.map((rj, pos) => ({ pos, rowJson: rj })), + logEntries: [{ kind: "rebase", payloadJson: JSON.stringify({ sharedPrefix: common, from: prevLen, to: args.rowStrings.length }) }], + checkpointMetaJson: metaJson(m), + }); + if (okSeq === null) return { committed: false, op: "none" }; + const parsed = args.rowStrings.map((rj) => safeParse(rj)); + rw.rows = [...args.rowStrings]; + rw.rowsParsed = parsed; + rw.caps = { ...caps }; + rw.blocksSig = args.blocksSig ?? rw.blocksSig; + rw.estCum = args.sums.estCum; + rw.textCum = args.sums.textCum; + rw.imgCum = args.sums.imgCum; + rw.auditTick = 0; // the rebase round IS the audit — start fresh + touchMirror(rw); + log?.("info", `[${sid}] #5664 render plane REBASED on witnessed structural event (shared prefix ${common} row(s), table ${prevLen} -> ${args.rowStrings.length} rows)`); + return { committed: true, op: "rebase" }; + } killRenderPlane(store, sid, `structural divergence (stored=${rw.rows.length} rows, captured=${args.rowStrings.length})`, log); return { committed: false, op: "none" }; } diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index f64968a5c7..448d908d99 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -42,7 +42,7 @@ import { nudgeOn, resolveLaneDecisions, surfaceOffDetail } from "./lane-decision import { featureEnabled } from "../rollout.js"; import { evaluateWatermarkFastpath, finalizeWatermarkTurn, type FastpathHit, type FastpathOutcome } from "./fastpath.js"; import { evaluateWatermarkV6, finalizeWatermarkV6, FastpathV6IdentityFault, PF_SESSION_HEADER, watermarkV6DebugState, type V6Outcome } from "./fastpath-v6.js"; -import { bumpAuditTick, commitRenderCapture, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, type RenderCaps, type RenderGateDecision, type RenderWorld } from "./fastpath-v6-render.js"; +import { activeBlocksSignature, bumpAuditTick, commitRenderCapture, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, type RenderCaps, type RenderGateDecision, type RenderWorld } from "./fastpath-v6-render.js"; import { getStore } from "../persist.js"; import { collectRequestIds, probeRequestMembership } from "../request-membership.js"; @@ -82,6 +82,14 @@ export async function prepareOpenai( let renderedImageDelta = 0; let renderCapturedSid: string | null = null; let renderCapturedCaps: RenderCaps | null = null; + // #5643 补充裁定 witnesses (structural events) for this turn's capture — + // pre-snapshots here, compared against post-pass state at station 2. The + // dsh native-compaction detector sets its flag deeper in the pipeline; + // hoisted here so station 2 can read it. + let dshRebased = false; + let renderPreBlocksSig = ""; + let renderFoldUsagePre = false; + let renderGateReason: string | null = null; // #5664 green verdict carrier (declared here so the clamp/estimate/capture // consumers AFTER the try block see it; assigned at the gate below). let renderFast: { rw: RenderWorld; F: number; maxPos: number; sid: string } | null = null; @@ -257,6 +265,11 @@ export async function prepareOpenai( multimodal: wpHit.seedMultimodal === true || wpHit.windowCores.some((m) => Array.isArray((m as { content?: unknown }).content)), }; renderCapturedCaps = rcaps; + // PRE-TURN structural snapshots (fold layer state BEFORE this + // request's own kernel pass) — the diff with the post-pass state + // is turn-level evidence that compression moved under us. + renderPreBlocksSig = activeBlocksSignature(session.state.blocks); + renderFoldUsagePre = session.stats.pendingFoldUsage === true; const renderGate: RenderGateDecision = renderPlaneGate({ sessionId, session, @@ -269,6 +282,7 @@ export async function prepareOpenai( tokenCount, modelContextLimit: config.modelContextLimit, caps: rcaps, + blocksSig: renderPreBlocksSig, snapshotLaneActive: judgeBaseline !== undefined, absorbActive, ccrOn: pluginMode && ccrEnabled(session), @@ -282,6 +296,7 @@ export async function prepareOpenai( } else { log("debug", `[${sessionId}] #5664 render lane off (${renderGate.reason}) — baseline full path`); if (renderGate.reason === "lane-dead") renderLaneDead = true; + renderGateReason = renderGate.reason; } } // #2432: dsh desktop's native compaction can LAND without ever @@ -302,7 +317,6 @@ export async function prepareOpenai( // lane does (#2373 commit note). Either signal alone stays on the // existing paths: framing paste with intact history has no gap; a gap // without framing falls through to detectUnannouncedHistoryRewrite. - let dshRebased = false; if (!isTitleGen && laneDecisions.profile.compactsClientSide && session.state.blocks.some((b) => b.active)) { const coveredBeforeDshCompact = coveredRealHistoryIds(session.state.blocks); const dshGap = foldCoverage(coveredBeforeDshCompact, msgs.map((m) => m.id)); @@ -790,11 +804,12 @@ export async function prepareOpenai( if (Object.keys(wpResponseHeaders).length === 0) wpResponseHeaders = undefined; } // #5664 station 2 — ONE snapshot commit per engaged turn: steady append - // (fast rounds), arm cast / audit diff / divergence-KILL (full-path - // rounds carry the complete captured array; owner ruling 2026-11-05: - // recast is forbidden — any structural divergence kills the lane for the - // sid). Fail-soft for this turn only — the identity plane already - // committed independently. + // (fast rounds), arm cast / audit diff / witnessed REBASE on structural + // divergence / UNWITNESSED divergence-KILL (full-path rounds carry the + // complete captured array; #5643 补充裁定:快照必须随压缩演进 — a + // structurally-witnessed diverged capture rebases the table, an + // unwitnessed one still kills the lane for the sid). Fail-soft for this + // turn only — the identity plane already committed independently. if (renderedCaptureStrings !== null && renderCapturedSid !== null && renderCapturedCaps !== null && featureEnabled("watermarkV6Render")) { const store = getStore(); if (store.pfRenderAvailable()) { @@ -807,6 +822,22 @@ export async function prepareOpenai( // exactly the appended tail (committedTail was capped at F+kNew). const steadyTailRows = renderFast !== null ? renderedTailObjects.slice(renderFast.F).map((m) => JSON.stringify(m)) : renderedCaptureStrings; const sumsSrc = renderFast !== null ? processedMessages.slice(Math.min(renderFast.F, processedMessages.length)) : processedMessages; + // #5643 补充裁定 witness assembly — server-side evidence ONLY: + // ① the fold layer's block multiset moved around this turn's own + // kernel pass (proxy-mode compress, kernel-side rebase…); ② the + // dsh native-compaction detector fired (its folds bypass bili + // entirely, #2432 — its reconcile DEACTIVATES blocks, which can + // leave the signature byte-equal, hence the explicit term); + // ③ fold usage was pending pre-turn and got consumed by it; + // ④ the gate itself had observed the signature move pre-turn + // ("blocks-changed" red sent this request down the full path). + // Absent ALL of them, a diverged capture stays a kill verdict. + const renderPostBlocksSig = activeBlocksSignature(session.state.blocks); + const renderWitness = + dshRebased || + renderPostBlocksSig !== renderPreBlocksSig || + (renderFoldUsagePre && session.stats.pendingFoldUsage !== true) || + renderGateReason === "blocks-changed"; const capOp = commitRenderCapture({ sessionId, store, @@ -814,6 +845,8 @@ export async function prepareOpenai( mode: renderFast !== null ? "steady" : "legacy", caps: renderCapturedCaps, rowStrings: steadyTailRows, + witness: renderWitness, + blocksSig: renderPostBlocksSig, sums: { estCum: estimateCoreMessagesUpper(sumsSrc), textCum: estimateCoreMessages(sumsSrc), diff --git a/src/storage/store.ts b/src/storage/store.ts index 15d3f41a4d..2c89b11d0d 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2626,13 +2626,16 @@ export class UnifiedStore { } /** Atomic turn commit for the render plane (#5664): upsert `rows` - * (pos values must be contiguous ascending; under the no-recast ruling - * covered ordinals are NEVER rewritten in place), journal `logEntries` - * at strictly increasing ls_seq — including the kind='dead' verdict - * that persists a plane death forever — and — when - * `checkpointMetaJson` is given — fold the whole log into a fresh ckpt - * WITH its S4 checksum in the SAME transaction (the arm cast). Any - * throw → rollback → byte-identical plane. + * (pos values must be contiguous ascending; under the no-UNWITNESSED- + * recast ruling covered ordinals are never rewritten in place — the + * single sanctioned rewrite is a WITNESSED rebase (#5643 补充裁定), + * which passes `replaceAll: true` and re-inserts the full captured + * table), journal `logEntries` at strictly increasing ls_seq — + * including the kind='dead' verdict that persists a plane death + * forever — and — when `checkpointMetaJson` is given — fold the whole + * log into a fresh ckpt WITH its S4 checksum in the SAME transaction + * (the arm cast, or a witnessed rebase). Any throw → rollback → + * byte-identical plane. * Returns the next free ls_seq. */ pfRenderCommit( sid: string, @@ -2640,11 +2643,17 @@ export class UnifiedStore { rows: { pos: number; rowJson: string | null }[]; logEntries: { kind: string; payloadJson: string }[]; checkpointMetaJson?: string; + /** Witnessed-rebase only: wipe this sid's row table BEFORE the + * upsert so the capture replaces it wholesale (one transaction). */ + replaceAll?: boolean; ts?: number; }, ): number { if (!this.pfRenderAvailable()) throw new Error("render plane unavailable"); return this.db.transaction((): number => { + if (args.replaceAll) { + this.db.prepare("DELETE FROM pf_render WHERE sid = ?").run(sid); + } const insRow = this.db.prepare( "INSERT INTO pf_render (sid, pos, row_json) VALUES (?, ?, ?) ON CONFLICT (sid, pos) DO UPDATE SET row_json = excluded.row_json", ); diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 36bb5e5630..3a7be33178 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -13,13 +13,17 @@ // the entire chain end-to-end. Equivalence with the full pass is the // contract, enforced here against a legacy oracle. // -// NO-RECAST REGIME (owner ruling 2026-11-05, sealed during #5689 review): -// recast is FORBIDDEN in every scenario — any stored-row divergence (core-form -// diff vs the rewritten window, shrink, non-append capture on a live lane, -// checksum corruption, unverifiable pre-existing snapshot) KILLS the plane -// for that sid permanently (persisted kind='dead' log entry) and the session -// rides the legacy full path forever after. Storage is strictly append-only -// (+ dead markers); arm cast is legal exactly once, onto an empty table. +// WITNESSED-REBASE REGIME (#5643 补充裁定 + owner ruling 2026-11-05): +// the table MUST evolve with compression. A structurally-WITNESSED diverged +// capture (fold layer's active-block multiset moved around the turn, dsh +// native-compaction detector fired, fold usage consumed pre→post, or the +// gate itself observed a signature move) REBASES the row table: +// truncate from the first divergence point + whole-suffix re-append as ONE +// transaction, checkpoint with S4 checksum, audit reset. An UNWITNESSED +// diverged capture still KILLS the lane permanently (persisted kind='dead') +// and rides legacy thereafter; checksum corruption stays durably dead. +// Storage is bounded by live elements (compaction SHRINKS the table); arm +// cast is legal exactly once, onto an empty table. // // Harness: unit-level against a REAL sqlite sidecar (UnifiedShadow shadow // mode), mirroring tests/fastpath-v6-msgid.test.ts; the kernel runs as a @@ -46,7 +50,7 @@ import { import { PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; import type { V6Outcome } from "../src/server/fastpath-v6.ts"; import { - bumpAuditTick, clearWatermarkV6Render, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, + activeBlocksSignature, bumpAuditTick, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, watermarkV6RenderDebugState, _resetWatermarkV6RenderTestState, type RenderCaps, } from "../src/server/fastpath-v6-render.ts"; @@ -174,31 +178,6 @@ test("unit: cast → steady appends → hydrate-from-disk fold survives mirror l } finally { h.cleanup(); } }); -test("teardown: clearWatermarkV6Render drops every in-memory mirror; the disk plane survives", () => { - const h = makeH("tear"); - try { - const b1 = makeBody([{ role: "user", content: "hello" }]); - const r1 = v6Eval(h, "s-tear", b1, ["id0"]); - assert.equal(r1.outcome.decision, "arm"); - r1.fin(); - const sid = r1.sid!; - const caps = capsFor("text-only"); - assert.equal(commitRenderCapture({ sessionId: "s-tear", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).op, "cast"); - assert.equal(watermarkV6RenderDebugState("s-tear").sids.length, 1); - - clearWatermarkV6Render("s-tear"); - assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); - // Disk plane is untouched — the next process re-hydrates from it - // (pfDeleteForSession owns real deletion, same split as the v6 id plane). - assert.equal(h.store.pfRenderRows(sid).length, 2); - assert.equal(h.store.pfRenderCkptIntact(sid), true); - - // Idempotent on an already-cleared session. - clearWatermarkV6Render("s-tear"); - assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); - } finally { h.cleanup(); } -}); - test("gate: red reasons are a stable vocabulary", (t) => { const h = makeH("gate"); try { @@ -300,6 +279,59 @@ test("unit: structural divergence KILLS the plane permanently (no-recast ruling } finally { h.cleanup(); } }); +test("unit: witnessed structural divergence REBASES the table (snapshot evolves with compression)", (t) => { + const h = makeH("rebase"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-rb", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + const preSig = activeBlocksSignature([]); + const newSig = activeBlocksSignature([{ blockId: "blkA", generation: "young", active: true }]); + + // Forge the table on a 3-row folded-free prefix. + const cast = commitRenderCapture({ sessionId: "s-rb", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}', '{"i":2}'], sums: { estCum: 3, textCum: 3, imgCum: 0 }, witness: false, blocksSig: preSig }); + assert.equal(cast.op, "cast"); + + // A fold lands outside bili's turn processing (#2432-class host OR + // the /__bili/plugin/tool endpoint executing server-side): the next + // model resend captures a SHRUNKEN + middle-rewritten list. With the + // structural witness, this rebases instead of killing. + const rb = commitRenderCapture({ sessionId: "s-rb", store: h.store, sid, log: h.log, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"S":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 }, witness: true, blocksSig: newSig }); + assert.deepEqual(rb, { committed: true, op: "rebase" }, `witnessed diverged capture must rebase, got ${JSON.stringify(rb)}`); + assert.deepEqual(h.store.pfRenderRows(sid).map((r) => r.rowJson), ['{"i":0}', '{"S":1}'], "the table now IS the compacted truth"); + assert.equal(h.store.pfRenderCkptIntact(sid), true, "rebase folds a fresh checkpoint with S4 checksum"); + const meta = h.store.pfRenderMeta(sid); + assert.ok(meta !== null && JSON.parse(meta.ckpt!.metaJson).blocksSig === newSig, "checkpoint carries the post-event block signature"); + assert.ok(h.log.lines.some((l) => l.level === "info" && l.msg.includes("REBASED")), "rebase is logged at info level"); + + // Steady appends continue at the RIGHT positions after a rebase. + const ap = commitRenderCapture({ sessionId: "s-rb", store: h.store, sid, mode: "steady", caps, rowStrings: ['{"i":3}'], sums: { estCum: 1, textCum: 1, imgCum: 0 }, blocksSig: newSig }); + assert.equal(ap.op, "append"); + assert.deepEqual(h.store.pfRenderRows(sid).map((r) => ({ pos: r.pos, rowJson: r.rowJson })), + [{ pos: 0, rowJson: '{"i":0}' }, { pos: 1, rowJson: '{"S":1}' }, { pos: 2, rowJson: '{"i":3}' }], "positions renumber from zero over the compacted table"); + + // Restart: the disk fold must reproduce rows AND signature. + // (The forged unit table ignores the identity plane's ledger size, + // so maxPos mirrors s-die's hand-rolled alignment: 3 rows ⇒ 2.) + _resetWatermarkV6RenderTestState(); + const base = { + sessionId: "s-rb", session: { id: "s-rb", state: createInitialState() } as unknown as Session, + store: h.store, engagedHit: true, armedTurn: false, isTitleGen: false, sid, + maxPos: 2, tokenCount: 500, modelContextLimit: MODEL_LIMIT, caps, + snapshotLaneActive: false, absorbActive: false, ccrOn: false, foldUsagePending: false, markerNonZero: false, + }; + // Positive side of the pair: matching signature → green, rows intact. + const gMatch = renderPlaneGate({ ...base, blocksSig: newSig }); + assert.ok(gMatch.ok, `matching post-rebase signature must gate green, got ${JSON.stringify(gMatch)}`); + if (gMatch.ok) assert.equal(gMatch.rw.rows.length, 3, "hydrate must reconstruct the rebased table + tail append"); + // Negative side (#3617): a STALE session signature (pre-rebase) must + // NOT gate green — exactly one red round precedes the next full pass. + assert.deepEqual(renderPlaneGate({ ...base, blocksSig: preSig }), { ok: false, reason: "blocks-changed" }, "stale signature stays red"); + } finally { h.cleanup(); } +}); + test("unit: plane death persists across restart (no re-arm ever)", (t) => { const h = makeH("pdeath"); try { @@ -385,32 +417,6 @@ test("unit: session cascade wipes render tables; v12 re-migration is a no-op", ( } finally { h.cleanup(); } }); -// Real deletion path pin: `bili sessions rm` / boot dedup reach the planes -// through UnifiedStore.deleteSession ONLY — pfDeleteForSession is not on that -// path. Both planes must be swept by the same cascade or every removed -// session leaks its render-snapshot tables forever. -test("unit: removeSession's real path wipes identity AND render planes together", async () => { - const h = makeH("cascade-rm"); - try { - const b1 = makeBody([{ role: "user", content: "hello" }]); - const r1 = v6Eval(h, "s-cas-rm", b1, ["id0"]); - r1.fin(); - const sid = r1.sid!; - const cast = commitRenderCapture({ sessionId: "s-cas-rm", store: h.store, sid, mode: "legacy", caps: capsFor("none"), rowStrings: ['{"a":1}', '{"b":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }); - assert.equal(cast.op, "cast"); - assert.equal(h.store.pfRenderRows(sid).length, 2); - assert.ok(h.store.pfMeta(sid) !== null && h.store.pfMeta(sid)!.log.length > 0, "identity plane has data too"); - await h.store.removeSession("s-cas-rm"); - const db = new DatabaseSync(path.join(h.dir, "index.db")); - try { - for (const t of ["pf_rows", "pf_log", "pf_ckpt", "pf_render", "pf_render_log", "pf_render_ckpt"]) { - const row = db.prepare(`SELECT COUNT(*) AS n FROM ${t}`).get() as { n: number } | undefined; - assert.equal(row?.n, 0, `${t} must be empty after session deletion`); - } - } finally { db.close(); } - } finally { h.cleanup(); } -}); - // --------------------------------------------------------------------------- // Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, // across arm → steady fast rounds → client compaction (turn 8) → audit @@ -553,7 +559,7 @@ test("golden: cached-prefix splice equals the legacy full-path bytes end-to-end" // this tier exercises is the accumulation ALGEBRA — each row counted // exactly once with its own cost, tail-only deltas on steady rounds. const rowCost = (rs: string[]): number => rs.reduce((s, x) => s + x.length, 0); - let op: { committed: boolean; op: "none" | "append" | "cast" } = { committed: false, op: "none" }; + let op: { committed: boolean; op: "none" | "append" | "cast" | "rebase" } = { committed: false, op: "none" }; // No-recast regime: once the lane dies, production stops paying for // capture strings entirely (the seam skips the station-2 commit); the // lane rides pure legacy until the session ends. Mirror that here. From 036719c2145143e3b565bb0431bd325f80021f3d Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 5 Nov 2026 15:43:57 +0800 Subject: [PATCH 2/2] test(#5721): restore two regression pins dropped by the rebase rework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The witnessed-rebase rework removed two existing pins without moving them, unrelated to the feature diff: - 'unit: removeSession's real path wipes identity AND render planes together' is the regression pin for b404061a (delete-time cascade of the six pf_* tables). The cascade code itself is untouched by this PR but was left with ZERO coverage — a future regression to deleteSession would leak the render-snapshot tables of every removed session silently again. - 'teardown: clearWatermarkV6Render drops every in-memory mirror; the disk plane survives' pinned clearWatermarkV6Render (same pattern as the id plane's teardown pin added in 4bb45012). Without it the function has no production caller and no test at all. Restored verbatim; both pass on this head. --- tests/issue5664-render-snapshot.test.ts | 53 ++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/tests/issue5664-render-snapshot.test.ts b/tests/issue5664-render-snapshot.test.ts index 3a7be33178..8f66b90175 100644 --- a/tests/issue5664-render-snapshot.test.ts +++ b/tests/issue5664-render-snapshot.test.ts @@ -50,7 +50,7 @@ import { import { PF_TAIL_IDS_HEADER } from "../src/server/fastpath.ts"; import type { V6Outcome } from "../src/server/fastpath-v6.ts"; import { - activeBlocksSignature, bumpAuditTick, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, + activeBlocksSignature, bumpAuditTick, clearWatermarkV6Render, commitRenderCapture, diffRowLists, renderPlaneGate, spliceOutboundRows, verifyRenderWindow, watermarkV6RenderDebugState, _resetWatermarkV6RenderTestState, type RenderCaps, } from "../src/server/fastpath-v6-render.ts"; @@ -178,6 +178,31 @@ test("unit: cast → steady appends → hydrate-from-disk fold survives mirror l } finally { h.cleanup(); } }); +test("teardown: clearWatermarkV6Render drops every in-memory mirror; the disk plane survives", () => { + const h = makeH("tear"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-tear", b1, ["id0"]); + assert.equal(r1.outcome.decision, "arm"); + r1.fin(); + const sid = r1.sid!; + const caps = capsFor("text-only"); + assert.equal(commitRenderCapture({ sessionId: "s-tear", store: h.store, sid, mode: "legacy", caps, rowStrings: ['{"i":0}', '{"i":1}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }).op, "cast"); + assert.equal(watermarkV6RenderDebugState("s-tear").sids.length, 1); + + clearWatermarkV6Render("s-tear"); + assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); + // Disk plane is untouched — the next process re-hydrates from it + // (pfDeleteForSession owns real deletion, same split as the v6 id plane). + assert.equal(h.store.pfRenderRows(sid).length, 2); + assert.equal(h.store.pfRenderCkptIntact(sid), true); + + // Idempotent on an already-cleared session. + clearWatermarkV6Render("s-tear"); + assert.deepEqual(watermarkV6RenderDebugState("s-tear").sids, []); + } finally { h.cleanup(); } +}); + test("gate: red reasons are a stable vocabulary", (t) => { const h = makeH("gate"); try { @@ -417,6 +442,32 @@ test("unit: session cascade wipes render tables; v12 re-migration is a no-op", ( } finally { h.cleanup(); } }); +// Real deletion path pin: `bili sessions rm` / boot dedup reach the planes +// through UnifiedStore.deleteSession ONLY — pfDeleteForSession is not on that +// path. Both planes must be swept by the same cascade or every removed +// session leaks its render-snapshot tables forever. +test("unit: removeSession's real path wipes identity AND render planes together", async () => { + const h = makeH("cascade-rm"); + try { + const b1 = makeBody([{ role: "user", content: "hello" }]); + const r1 = v6Eval(h, "s-cas-rm", b1, ["id0"]); + r1.fin(); + const sid = r1.sid!; + const cast = commitRenderCapture({ sessionId: "s-cas-rm", store: h.store, sid, mode: "legacy", caps: capsFor("none"), rowStrings: ['{"a":1}', '{"b":2}'], sums: { estCum: 2, textCum: 2, imgCum: 0 } }); + assert.equal(cast.op, "cast"); + assert.equal(h.store.pfRenderRows(sid).length, 2); + assert.ok(h.store.pfMeta(sid) !== null && h.store.pfMeta(sid)!.log.length > 0, "identity plane has data too"); + await h.store.removeSession("s-cas-rm"); + const db = new DatabaseSync(path.join(h.dir, "index.db")); + try { + for (const t of ["pf_rows", "pf_log", "pf_ckpt", "pf_render", "pf_render_log", "pf_render_ckpt"]) { + const row = db.prepare(`SELECT COUNT(*) AS n FROM ${t}`).get() as { n: number } | undefined; + assert.equal(row?.n, 0, `${t} must be empty after session deletion`); + } + } finally { db.close(); } + } finally { h.cleanup(); } +}); + // --------------------------------------------------------------------------- // Golden tier: FAST emit == legacy emit at EVERY engaged turn, byte-for-byte, // across arm → steady fast rounds → client compaction (turn 8) → audit