From e9b71de4299366d86aa0e049c20d897c10ad26e5 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:25:01 +0800 Subject: [PATCH 1/2] fix: surface D-lane degrade reason and guard activation loudly (#3771) The D-lane e2e (tests/snapshot-render.test.ts) failed non-deterministically in CI while passing locally: most functional asserts also hold in legacy mode, so a run where the D lane never activated read as "flaky latch" instead of "lane never activated". The fixed-window nap replacement now lives on master as settleRows (f296b0e0d: wait until the session's tail view CHANGES); this commit lands the two pieces that remain: - assertDActive(): loud guard in every D-on test (openai AND google variants). If the lane did not actually render, it throws naming the degrade reason plus store state (replicasAvailable/rows/seqs), closing the silent-legacy blind spot so a future genuine degrade fails informatively instead of tripping an opaque latch assert. - src/server/snapshot-render.ts: additive session.metadata['snapshotRenderDegradeReason'] marker set on each post-arm degrade (store-unavailable/transcript-empty/ row-unparsable/view-pointer-missing-row/transcript-error/chain-not-started) and cleared on success. Pre-arm exits (rollout off / nothing armed) are untouched. No wire or message-shape change: degrades still return clientMsgs unchanged. - Contract pins both directions (#3617 gate discipline): the storage-unavailable test asserts the surfaced reason ('transcript-empty'); steady-state asserts the marker clears after successful D renders. --- src/server/snapshot-render.ts | 29 ++++++++++++++++++++--------- tests/snapshot-render.test.ts | 30 +++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/server/snapshot-render.ts b/src/server/snapshot-render.ts index a7933cc6fa..eb904eefc3 100644 --- a/src/server/snapshot-render.ts +++ b/src/server/snapshot-render.ts @@ -77,6 +77,15 @@ function noteRender(session: Session, deps: SnapshotRenderDeps): void { if (prior === 0) deps.log?.("info", `[snapshot-render] active: outbound prefix now rendered from the unified transcript (issue #3756)`); } +/** Post-arm degrade marker (observability only — never touches outbound bytes). + * Set here, cleared on a successful render. Pre-arm exits (rollout off / nothing + * armed) must NOT route through this: an unrendered lane is not a degradation. */ +function legacyFor(session: Session, deps: SnapshotRenderDeps, clientMsgs: BiliMessage[], reason: string): BiliMessage[] { + session.metadata["snapshotRenderDegradeReason"] = reason; + deps.log?.("debug", `[snapshot-render] legacy fallback (${reason})`); + return clientMsgs; +} + function parseRow(content: string, seq: number): BiliMessage | null { try { const parsed = JSON.parse(content) as { id?: unknown; role?: unknown }; @@ -89,35 +98,36 @@ function parseRow(content: string, seq: number): BiliMessage | null { /** * Returns the merged view, or `clientMsgs` unchanged when the lane is - * inactive/degraded. Pure with respect to the transcript: the only mutation - * is the observability latch on `session.metadata`. + * inactive/degraded. Pure with respect to the transcript: the only mutations + * are two observability fields on `session.metadata` (the render latch and the + * last post-arm degrade reason) — neither touches outbound bytes. */ export function maybeSnapshotView(session: Session, clientMsgs: BiliMessage[], deps: SnapshotRenderDeps): BiliMessage[] { if (!featureEnabled("snapshotRender")) return clientMsgs; const activeBlocks = session.state.blocks.filter((b) => b.active); if (activeBlocks.length === 0) return clientMsgs; const store = getStore(); - if (!store) return clientMsgs; + if (!store) return legacyFor(session, deps, clientMsgs, "store-unavailable"); let rows: ViewRow[] = []; try { const rawRows = store.unifiedTailViewRows(session.id); const seqs = store.unifiedTailSeqs(session.id); - if (!rawRows || rawRows.length === 0 || !seqs || seqs.length === 0) return clientMsgs; + if (!rawRows || rawRows.length === 0 || !seqs || seqs.length === 0) return legacyFor(session, deps, clientMsgs, "transcript-empty"); const bySeq = new Map(); for (const row of rawRows) { const msg = parseRow(row.content, row.seq); - if (!msg) return clientMsgs; // untrustworthy transcript: degrade, never guess + if (!msg) return legacyFor(session, deps, clientMsgs, "row-unparsable"); // untrustworthy transcript: degrade, never guess bySeq.set(row.seq, { seq: row.seq, msg }); } for (const seq of seqs) { const v = bySeq.get(seq); - if (!v) return clientMsgs; // view pointer to a missing row: degrade + if (!v) return legacyFor(session, deps, clientMsgs, "view-pointer-missing-row"); // view pointer to a missing row: degrade rows.push(v); } } catch { - return clientMsgs; + return legacyFor(session, deps, clientMsgs, "transcript-error"); } - if (rows.length === 0) return clientMsgs; + if (rows.length === 0) return legacyFor(session, deps, clientMsgs, "transcript-empty"); // Backward chain: from the client's end, match view rows (id fast path, // then identity signature). The chain must be contiguous; the first @@ -168,7 +178,7 @@ export function maybeSnapshotView(session: Session, clientMsgs: BiliMessage[], d } cursor = hit; } - if (!started) return clientMsgs; // nothing chains: epoch-like rewrite, legacy + if (!started) return legacyFor(session, deps, clientMsgs, "chain-not-started"); // nothing chains: epoch-like rewrite, legacy } const d = joinRow + 1; // prefix = rows[0..d) — everything up to the join; rows beyond (rewound-away tails) never render @@ -230,6 +240,7 @@ export function maybeSnapshotView(session: Session, clientMsgs: BiliMessage[], d while (ai < anchors.length) merged.push(anchors[ai++].msg); for (let k = clientJoin; k < clientMsgs.length; k++) merged.push(clientMsgs[k]); + delete session.metadata["snapshotRenderDegradeReason"]; noteRender(session, deps); return merged; } diff --git a/tests/snapshot-render.test.ts b/tests/snapshot-render.test.ts index b4ae4accc1..c96fa3c0a9 100644 --- a/tests/snapshot-render.test.ts +++ b/tests/snapshot-render.test.ts @@ -31,7 +31,7 @@ import { defaultConfig } from "acp-kernel"; import { __setMessageIdSaltForTests } from "acp-kernel/wire"; import { startServer } from "../src/server.ts"; import type { ProxyOptions } from "../src/config.ts"; -import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { SessionStore, _setStoreForTest, getStore } from "../src/persist.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; import { listSessions, _resetSessionsForTest } from "../src/session.ts"; import { rmrf } from "./tmp-rm.ts"; @@ -261,6 +261,21 @@ function assertPrefixStable(a: { role: string; content: unknown }[], b: { role: } } +/** Loud guard for the #3771 blind spot: a D-on test must prove the lane + * ACTUALLY rendered, not silently run the legacy path (whose behavior also + * satisfies most functional asserts, so a silent fallback read as green). On + * short activation, throw naming the degrade reason + store state. */ +function assertDActive(h: Harness, label: string, minTurns: number, ctx: string): void { + const s = h.sessionOf(label)!; + const turns = s.metadata["snapshotRenderTurns"] as number | undefined; + if (typeof turns === "number" && turns >= minTurns) return; + const st = getStore(); + throw new Error( + `[D-lane] did not activate (${ctx}): snapshotRenderTurns=${turns ?? "undefined"} (want >= ${minTurns}); degradeReason=${s.metadata["snapshotRenderDegradeReason"] ?? "(none - lane off or nothing armed)"}; ` + + `replicasAvailable=${String(st.replicasAvailable())} rows=${st.unifiedTailViewRows(label)?.length ?? "null"} seqs=${st.unifiedTailSeqs(label)?.length ?? "null"}`, + ); +} + async function armBlock(h: Harness, label: string): Promise { const r = await turn(h, h.openaiUrl, label, conversation()); assert.equal(r.status, 200, r.body); @@ -298,8 +313,8 @@ test("D on: steady state — anchor rendered once, covered rows absent, prefix b // tail = the new user message + the assistant run that absorbed the // turn-2 reply (#3658 openai assistant run-merge discipline). assertPrefixStable(out3, out2, 2, "steady-state"); - const s = h.sessionOf("s-steady")!; - assert.ok((s.metadata["snapshotRenderTurns"] as number) >= 2, "D observability latch counted turns"); + assertDActive(h, "s-steady", 2, "steady-state"); + assert.equal(h.sessionOf("s-steady")!.metadata["snapshotRenderDegradeReason"], undefined, "degrade reason cleared after successful D renders"); }, true); }); @@ -327,6 +342,7 @@ test("D on: identity-seed upgrade — outbound prefix byte-identical, block aliv assert.equal(anchorCount(joined3), 1, "exactly one anchor copy"); // prefix (everything except the live tail) byte-identical to pre-upgrade assertPrefixStable(out3, out2, 1, "seed-upgrade"); + assertDActive(h, "s-upg", 1, "seed-upgrade"); const s = h.sessionOf("s-upg")!; assert.ok(s.state.blocks.some((b) => b.active), "block survived the upgrade"); assert.equal(s.metadata["foldDriftEscalated"], undefined, "no destroy verdict"); @@ -372,6 +388,7 @@ test("D on: client drops/checkpoints the covered region — server-authoritative assert.equal(r4.status, 200, r4.body); const out4 = sent(h.bodies, r4.idx); assertPrefixStable(out4, out3, 1, "client-checkpoint"); + assertDActive(h, "s-drop", 1, "client-checkpoint"); }, true); }); @@ -391,6 +408,7 @@ test("D on: rewind — dropped tail messages do not come back", async () => { assert.ok(!joined3.includes("RW_A_"), "rewound message not rendered"); assert.ok(joined3.includes("RW_B_"), "new question present"); assert.ok(joined3.includes(SUMMARY), "anchor still rendered"); + assertDActive(h, "s-rw", 1, "rewind"); }, true); }); @@ -414,6 +432,7 @@ test("D on: protocol switch mid-session — anchors render on the new wire, sess assert.ok(joined2.includes("SW_A_"), "tail present"); const s = h.sessionOf("s-switch"); assert.ok(s, "session survived the protocol switch (not forked away)"); + assertDActive(h, "s-switch", 1, "protocol-switch"); }, true); }); @@ -452,6 +471,7 @@ test("D on but storage unavailable: legacy fallback, request still served", asyn const joined2 = sent(h.bodies, r2.idx).map(textOf).join("\n"); assert.ok(joined2.includes("NS_A_"), "served"); assert.equal(h.sessionOf("s-nostore")!.metadata["snapshotRenderTurns"], undefined, "D inactive without rows"); + assert.equal(h.sessionOf("s-nostore")!.metadata["snapshotRenderDegradeReason"], "transcript-empty", "degrade reason surfaced on post-arm degrade"); }, false); }); @@ -531,8 +551,7 @@ test("D on · google: steady state — anchor rendered once, covered rows suppre const p3 = prefixThroughSummary(j3); assert.ok(p2 !== null && p3 !== null, "summary present in both turns"); assert.equal(p3, p2, "google prefix through the summary byte-stable"); - const s = h.sessionOf("g-steady")!; - assert.ok((s.metadata["snapshotRenderTurns"] as number) >= 2, "D observability latch counted turns"); + assertDActive(h, "g-steady", 2, "google-steady-state"); }, true); }); @@ -562,6 +581,7 @@ test("D on · google: identity-seed upgrade — prefix byte-identical, block ali const s = h.sessionOf("g-upg")!; assert.ok(s.state.blocks.some((b) => b.active), "block survived the upgrade"); assert.equal(s.metadata["foldDriftEscalated"], undefined, "no destroy verdict"); + assertDActive(h, "g-upg", 1, "google-seed-upgrade"); }, true); }); From 3b45e67de2f64b57242971111672c9a6632aceda Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:40:42 +0800 Subject: [PATCH 2/2] fix: pin snapshot-render D-lane test to unified shadow mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test never pinned its storage mode. An ambient BILI_STORAGE_UNIFIED of legacy/0/false (or a config value) resolves mode to "legacy", where maybeOpen returns null SILENTLY and the harness store has no unified sidecar. Every D-on test then silently ran the legacy path and failed on the D latch / anchor / prefix asserts — CI-only red because local runs had the var unset (default shadow). Pinning shadow makes the sidecar open deterministically; verified green under =0/=false/unset incl. the google lane. --- tests/snapshot-render.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/snapshot-render.test.ts b/tests/snapshot-render.test.ts index c96fa3c0a9..2ab8a410a3 100644 --- a/tests/snapshot-render.test.ts +++ b/tests/snapshot-render.test.ts @@ -25,6 +25,11 @@ import path from "node:path"; process.env.NODE_ENV = "test"; process.env.BILI_PERSIST_ZSTD = "0"; +// The D-lane renders from the unified store's tail view, so its sidecar must be +// open. Pin shadow explicitly: an ambient BILI_STORAGE_UNIFIED of legacy/0/false +// (or a config value) otherwise resolves mode to "legacy", where maybeOpen returns +// null SILENTLY and every D-on test degrades to the legacy path (#3771). +process.env.BILI_STORAGE_UNIFIED = "shadow"; process.env.BILI_CONFIG_FILE = path.join(mkdtempSync(path.join(tmpdir(), "bili-snap-hermetic-")), "no-config.json"); import { defaultConfig } from "acp-kernel";