From 7caaed5f77a03939fe68f6418d81fb8d09e7662c Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 12 Nov 2026 02:36:19 +0800 Subject: [PATCH 1/8] =?UTF-8?q?feat(#5874=202=C3=972):=202=C3=972=20snapsh?= =?UTF-8?q?ot=20egress=20modes=20=E2=80=94=20stage-0=20world=20match,=20ke?= =?UTF-8?q?rnel-input=20assembly,=20delta=20outbound=20(#6181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the 2×2 brief (discussion 5874 comment 19544722) on top of the P2 plane, kernel unchanged. The snapshot table (l1_world + row rows) stays the single truth; the two selection points are independent: - Inbound (stage 0, prepare-openai lane): the client's (sess_raw, msgid[]) pair is matched against the stored world canon (matchWorld: echo/probe/scatter/issue/diverge/resync). Probe rounds assemble the kernel input from plane rows + only the NEW span, so M1 (full resend) and M2 (incremental) reach byte-identical kernel cores. R3 identity faults (declared msgids vs element count) 400 with zero side effects, 3-strike window latch degrades to the legacy path. - Outbound (egress stage, snap-egress + prepare-responses): the snapshot view carries a per-element side channel; the responses lane (provider retains context via previous_response_id) selects seed/delta/no-op via selectOutbound — first round full (incl. anchors), next rounds only new rows, isFullResend trips a reseed. Degrade to legacy resets the delta frontier (one-round convergence, #7). Files: src/snapshot/{assemble,checksums,delta,identity,plane,types}.ts (new), store.ts snapStage0/snapStage0Cores/snapStage0Reset facades, snap.ts l1 envelope + clearSnapPlaneDead, msgid-pin.ts seqs channel, session.ts responsesDelta persistence, architecture.md map. Tests: tests/issue5874-2x2-p3.test.ts (33: verdicts ±, M1/M2 byte pin, T5 scatter pin, #7 resync, #8 delta pin, R3 red gates, store round trip). Regression: p1/p2/fastpath 23/23, snapshot-render 24/24, storage-engine-proofs 12/12, opencode-rolling 10/10. tsc clean, unused-export gate clean, architecture-docs guard green. --- reference/architecture.md | 7 + src/msgid-pin.ts | 14 + src/persist.ts | 25 ++ src/server/prepare-openai.ts | 124 +++++- src/server/prepare-responses.ts | 51 ++- src/server/snap-egress.ts | 87 ++++- src/session.ts | 6 + src/snapshot/assemble.ts | 193 ++++++++++ src/snapshot/checksums.ts | 62 +++ src/snapshot/delta.ts | 137 +++++++ src/snapshot/identity.ts | 285 ++++++++++++++ src/snapshot/plane.ts | 248 ++++++++++++ src/snapshot/types.ts | 68 ++++ src/storage/schema.ts | 17 +- src/storage/shadow.ts | 24 ++ src/storage/snap.ts | 89 ++++- src/storage/store.ts | 78 ++++ tests/issue5874-2x2-p3.test.ts | 661 ++++++++++++++++++++++++++++++++ 18 files changed, 2152 insertions(+), 24 deletions(-) create mode 100644 src/snapshot/assemble.ts create mode 100644 src/snapshot/checksums.ts create mode 100644 src/snapshot/delta.ts create mode 100644 src/snapshot/identity.ts create mode 100644 src/snapshot/plane.ts create mode 100644 src/snapshot/types.ts create mode 100644 tests/issue5874-2x2-p3.test.ts diff --git a/reference/architecture.md b/reference/architecture.md index e81d6f6e9e..280637123f 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -145,6 +145,13 @@ billion-context/ │ ├── session-gc.ts # Session-file GC, opt-in (sessions.gc.* / BILI_SESSION_GC*) (#1082) │ ├── persist.ts # On-disk session persistence (kernel StateStore) │ ├── persist-eperm.ts # EPERM/EACCES write-failure alert layer (#362) +│ ├── snapshot/ # 2×2 snapshot plane logic: stage-0 world match, assembly, delta egress (#6181) +│ │ ├── assemble.ts # Stage-0 kernel-input assembly: plane rows + new spans + anchors, envelope order (#6181) +│ │ ├── checksums.ts # Snapshot checksums (lowercase hex sha256) + canon digests (#5874) +│ │ ├── delta.ts # Outbound delta selection: seed/delta/no-op + full-resend detection (#6181) +│ │ ├── identity.ts # matchWorld verdicts (echo/probe/scatter/issue/diverge/resync) + canon parse (#5874) +│ │ ├── plane.ts # Stage-0 decision glue: R3 identity check, 400 fault breaker, lane decide (#6181) +│ │ └── types.ts # Shared stage-0 types (type-only, kernel-public-types only) (#6181) │ ├── storage/ # Unified storage layer (schema, SQLite engine ladder, backends) (#2671) │ │ ├── backend.ts # Storage backend contract: sqlite + mysql operation seam (#3931) │ │ ├── canonical.ts # Shared canonicalization for storage digests (#2671) diff --git a/src/msgid-pin.ts b/src/msgid-pin.ts index 6f93e72a64..55993cfa86 100644 --- a/src/msgid-pin.ts +++ b/src/msgid-pin.ts @@ -82,6 +82,20 @@ export function incomingMsgIdsHeader(headerValue: string | string[] | undefined) type SeqValidation = { ok: true; seqs: number[] } | { ok: false; reason: string }; +// #5874 2×2 (brief 19544722, Part A): the SEQS half of the dual-channel +// identity (biliLedgerSeqs, #4316 layer 3 transport). Lenient read like +// incomingMsgIdsOf — absent/malformed → undefined → callers take exactly +// the pre-2×2 path (the strict pin stays with validateMsgSeqs). +export function incomingSeqsOf(body: unknown): number[] | undefined { + if (typeof body !== "object" || body === null) return undefined; + const raw = (body as Record)[MSG_SEQS_BODY_FIELD]; + if (!Array.isArray(raw) || raw.length === 0) return undefined; + for (const v of raw) { + if (typeof v !== "number" || !Number.isSafeInteger(v) || v < 1) return undefined; + } + return raw as number[]; +} + /** Strict validation for the seq stamp array: 1-based positive integers, * exactly one per message. Non-monotonic or non-contiguous values are * ACCEPTED on purpose — the branch pointers pick arbitrary row subsets diff --git a/src/persist.ts b/src/persist.ts index 956cb2b607..5146b96835 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2442,6 +2442,31 @@ export class SessionStore { } } + // #5874 2×2 stage 0 (brief 19544722): errors PROPAGATE (the lane maps the + // R3 reject to a hard 400 and any other fault to the legacy full path) — + // the egress-view swallow above must not be copied here. + + /** Stage 0 (element world match) for one turn. null when the unified + * store is unavailable — the lane keeps the legacy path. */ + snapStage0(sessionId: string, incoming: string[], opts?: { resync?: boolean, seqs?: number[] | null }): import("./snapshot/plane.js").Stage0Result | null { + if (!this.replicasAvailable()) return null; + return this.liveShadow().snapStage0(sessionId, incoming, opts); + } + + /** The plane's ACTIVE cores, pos-ordered (the "old cores from the + * table" half of the kernel-input assembly). */ + snapStage0Cores(sessionId: string): { rows: { pos: number; ids: string[]; postJson: string }[]; coverMax: Record } | null { + if (!this.replicasAvailable()) return null; + return this.liveShadow().snapStage0Cores(sessionId); + } + + /** The stage-0 resync (R4 / table behind): reset the session plane so + * the next turn re-issues from the client's current array. */ + snapStage0Reset(sessionId: string): void { + if (!this.replicasAvailable()) return; + this.liveShadow().snapStage0Reset(sessionId); + } + /** #3928: physical index.db shrink after deletions (`bili sessions rm`). * Throws on failure — the deletion itself is already committed, callers * report the shrink as partial. Throws in legacy-only lanes — check diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 33821287d8..616379d9b5 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -7,7 +7,7 @@ import type { ProxyOptions } from "../config.js"; import { effectivePromptCacheRouting } from "../config.js"; import { currentCalibrationFactor, hardenOpenaiAssistantContent, systemToUser } from "../util.js"; import { conversationHeaderSource, resolveRelayAffinityPck } from "../session-id.js"; -import { MSG_IDS_BODY_FIELD, applyMsgIdPin, validateMsgIds } from "../msgid-pin.js"; +import { MSG_IDS_BODY_FIELD, MSG_SEQS_BODY_FIELD, applyMsgIdPin, applyMsgSeqStamps, incomingSeqsOf, validateMsgIds, validateMsgSeqs } from "../msgid-pin.js"; import { pluginAgentHeader } from "../plugin.js"; import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, warnDroppedOpenaiParts, warnReasoningPairs, type Prepared } from "../server.js"; import { recordNudgeObservation } from "../context-analysis.js"; @@ -40,7 +40,10 @@ import { isStrictReasoningEcho, modelIdOf, normalizeStrictEchoReasoning } from " import { reconcileSystemAnchor } from "../system-anchor.js"; import { logRawStripCounts, markerArtifactCount, stripRawStage, type RawStripCounts } from "../marker/engine.js"; import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js"; -import { SessionStore } from "../persist.js"; +import { SessionStore, getStore } from "../persist.js"; +import { FoundationRejectError, r3FaultTracker } from "../snapshot/plane.js"; +import { assembleKernelInput, shouldResync } from "../snapshot/assemble.js"; +import { ANCHOR_ID_PREFIX } from "../storage/snap.js"; import { applyImageRescalePass } from "../image-rescale.js"; import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed } from "./budget.js"; import { effectiveAbsorbBlock, effectiveBlockOpsEnabled } from "./prepare-responses.js"; @@ -77,6 +80,13 @@ export async function prepareOpenai( // host body stamps (pi incremental lane) or the #5825 handle-seam // re-stamp of stamped-header/harvested ids. const incomingMsgIds = (parsed as Record)[MSG_IDS_BODY_FIELD]; + // #5874 2×2: the seqs carrier (biliLedgerSeqs) — the M2 incremental + // join evidence. An optimization channel, not an identity channel: + // malformed/absent → the M2 fast path is disabled (the full-resend + // matchWorld path still works); it never 400s. (Stamped once msgs + // exists, below.) + const incomingSeqsRaw = incomingSeqsOf(parsed); + delete parsed[MSG_SEQS_BODY_FIELD]; delete (parsed as Record)[MSG_IDS_BODY_FIELD]; // Real wire element count; a watermark hit overrides it with the measured // total T (see the #5403/#5418 block below). @@ -171,6 +181,14 @@ export async function prepareOpenai( // otherwise the proxy would forward payloads without any system. const converted = wpHit !== undefined ? { msgs: wpHit.view, systemText: wpHit.systemText } : openaiToCore(parsed); let msgs = converted.msgs; + // #5874 2×2: stamp the M2 seqs carrier onto the converted array + // (parallel to messages; malformed/absent → null → M2 fast path + // disabled, the full-resend path still works). + const incomingSeqs: number[] | null = + incomingSeqsRaw !== undefined && incomingSeqsRaw.length === msgs.length && validateMsgSeqs(incomingSeqsRaw, msgs.length).ok + ? incomingSeqsRaw + : null; + if (incomingSeqs !== null) applyMsgSeqStamps(msgs, incomingSeqs); let systemText = converted.systemText; openaiSystemText = systemText; wpSeedSystemText = systemText; @@ -192,12 +210,35 @@ export async function prepareOpenai( // materialized, and the decoded tail window is SHORTER than the // stamped list, so count-parity there would fail by design — pinning // only adds noise. + // #5874 2×2 stage 0: cohort (dual-channel DECLARED) plugin traffic + // assembles the kernel input from the snapshot plane (old cores — + // the kernel's own tagged output, invariant v) + the NEW spans + // (P5). Non-cohort traffic keeps the D-lane view below unchanged. + const twoX2Armed0 = wpHit === undefined && pluginMode && Array.isArray(incomingMsgIds) && incomingMsgIds.length > 0; + let twoX2Armed = twoX2Armed0 && !r3FaultTracker.isLatched(sessionId); + if (twoX2Armed0 && r3FaultTracker.isLatched(sessionId)) { + log("debug", `[${sessionId}] [2x2] sid is R3-latched — legacy path this turn`); + } if (wpHit === undefined && pluginMode && incomingMsgIds !== undefined) { if (featureEnabled("pluginLaneServerHistory")) { const v = validateMsgIds(incomingMsgIds, msgs.length); if (v.ok) { msgs = applyMsgIdPin(msgs, v.ids); originalMessages = msgs; + } else if (twoX2Armed) { + // #5874 2×2 red gate R3: a cohort host that DECLARED the + // dual-channel (per-element msgids) but did not collect + // them all → hard 400 BEFORE any kernel side effect, and + // the fault feeds the breaker (10-minute window, 3 + // strikes → the sid latches to the legacy path instead of + // bricking the session on a misbehaving host). + const { latched } = r3FaultTracker.recordFault(sessionId); + if (latched) { + twoX2Armed = false; + log("error", `[${sessionId}] [2x2] R3 identity fault ×3 in window (${v.reason}) — sid latched to the legacy path`); + } else { + throw new FoundationRejectError(`inbound msgid envelope failed validation (${v.reason})`); + } } else { log("warn", `[${sessionId}] [msgid] openai-wire pin rejected (${v.reason}) — falling back to content-derived ids (#4491/#5825)`); } @@ -205,12 +246,89 @@ export async function prepareOpenai( log("info", `[${sessionId}] [msgid] openai-wire biliMsgIds present but pluginLaneServerHistory rollout is off for this install — field stripped, ids not pinned (#5825)`); } } + // #5874 2×2 stage 0 (brief 19544722): world match + kernel-input + // assembly for cohort traffic. Runs BEFORE processTurn, so a thrown + // FoundationRejectError (R3) carries zero kernel side effects. Every + // other fault → the legacy full path (degrade, never guess — + // one-round convergence). + let twoX2KernelInput: BiliMessage[] | undefined; + if (twoX2Armed) { + try { + const store = getStore(); + const stage0 = store?.snapStage0(session.id, msgs.map((m) => m.id), { seqs: incomingSeqs }); + const cores = stage0 ? (store?.snapStage0Cores(session.id) ?? null) : null; + if (stage0 && cores) { + if (shouldResync(stage0.decision, cores.rows.length)) { + // Structural disagreement (diverge) or an explicit + // re-baseline: reset the plane; THIS turn runs the + // legacy full path and re-seeds it through the + // in-request flush — the NEXT turn re-issues the + // envelope from the client's current array. + store!.snapStage0Reset(session.id); + log("warn", `[${sessionId}] [2x2] stage-0 ${stage0.decision.mode} — plane reset, this turn legacy (one-round convergence)`); + } else { + // newCores = the converted spans of the NEW ids, in + // envelope order. The front stage already converted + // the whole array (front-stage invariance), so the + // new spans carry their full-parse bytes — T5 holds + // by construction. + const newIdSet = new Set(stage0.decision.newIdPos.map((n) => n.id)); + const newCores: BiliMessage[] = []; + const taken = new Set(); + for (const e of stage0.decision.envelope) { + if (!newIdSet.has(e.id) || taken.has(e.id)) continue; + taken.add(e.id); + const m = msgs.find((x) => x.id === e.id); + if (m === undefined) throw new Error(`new span ${e.id} absent from the converted array`); + newCores.push(m); + } + // The plane's ANCHOR rows (active acp_summary_) + // ride the kernel input at their covered-run boundary + // — the D-view placement parity. snapStage0Cores + // returns both the rows and the coverMax map. + const anchors = cores.rows + .filter((r) => r.ids.some((id) => id.startsWith(ANCHOR_ID_PREFIX))) + .map((r) => { + const id = r.ids.find((x) => x.startsWith(ANCHOR_ID_PREFIX))!; + return { + msg: JSON.parse(r.postJson) as BiliMessage, + blockId: id.slice(ANCHOR_ID_PREFIX.length), + }; + }); + const asm = assembleKernelInput({ + decision: stage0.decision, + rows: cores.rows + .filter((r) => !r.ids.some((id) => id.startsWith(ANCHOR_ID_PREFIX))) + .map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })), + newCores, + skipIds: stage0.explainedGone, + anchors, + coverMax: cores.coverMax, + }); + twoX2KernelInput = asm.kernelInput; + log("info", `[${sessionId}] [2x2] stage-0 ${stage0.decision.mode} — kernel input from plane (old=${asm.oldCount}, new=${newCores.length}, anchors=${anchors.length})`); + } + } + } catch (err) { + if (err instanceof FoundationRejectError) throw err; // R3 outranks the fallback + const why = err instanceof Error ? err.message : String(err); + if (why.startsWith("[snapshot 2×2]")) { + // The assembly could not address the envelope against the + // plane (table behind — R4-shaped): reset and take the + // legacy path this turn (one-round convergence). + try { + getStore()?.snapStage0Reset(session.id); + } catch { /* the reset is best-effort — the legacy path is the floor */ } + } + log("warn", `[${sessionId}] [2x2] stage-0 failed (${why}) — legacy full path`); + } + } // #3756 snapshot render: the outbound prefix renders from the unified // transcript (server-authoritative, fold-time ids — immune to identity // re-mints #3752); only the live tail keeps the client's bytes. // No-op returns `msgs` when the lane is inactive/degraded. // reconcileFoldCoverage below still observes the CLIENT view. - const renderMsgs = maybeSnapshotView(session, msgs, { sessionId, pluginMode, log }); + const renderMsgs = twoX2KernelInput ?? maybeSnapshotView(session, msgs, { sessionId, pluginMode, log }); // #4241: the render lane replaced the view (reference-inequality is the // discriminator — maybeSnapshotView returns the SAME array when idle), // so the judged baseline is the rendered transcript, not the inbound. diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index b13a9672b2..13c8ef3e4a 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -17,6 +17,7 @@ import { dropWhitespaceResponsesMessages, normalizeResponsesMessageItems, saniti import { METADATA_LAST_RESPONSES_MODE, noteCanonicalSystemFingerprint, reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; import { buildAnchoredView, maybeSnapshotView, noteAnchoredRender, SnapshotAnchorFallbackError, wsAnchoredPayload } from "./snapshot-render.js"; import { applySnapEgress } from "./snap-egress.js"; +import type { DeltaState } from "../snapshot/delta.js"; import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed, pluginLaneDegraded } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; @@ -112,6 +113,10 @@ export async function prepareResponses( let toolsOut = parsed.tools; let transformOk = false; let responsesDevContent: string | undefined; + // 2×2 Part B: true when this round's outbound is a PLANE DELTA (the + // provider's retained chain carries the rest) — gates the developer + // message re-injection and the previous_response_id strip below. + let deltaServed = false; let sysNotes: string[] = []; // #3713: billed tokens of the #1638-masked mid-history sys/dev items — they // ride the rebuilt input verbatim yet enter neither projection.msgs nor @@ -447,15 +452,46 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr const ordered = repairResponsesAssistantOrdering(applyReasoningDrop(stripKernelSummaries(turn.messages, turn.state), reasoningPlan, log, sessionId), originalMessages); processedMessages = ordered.messages; separatorSlots = ordered.slots; - // #5874 P2 (mapEgress): tripwire-guarded plane egress — the lane's - // exact chain (strip → drop → repair) reapplied to the plane view. + // #5874 P2 (mapEgress) direct-authority plane egress. 2×2 Part B + // (出站增量): the Responses protocol is the only cohort with + // provider-side retention (previous_response_id chaining), so this is + // the only lane with an incremental outbound mode. Delta mode requires + // the client to chain (previous_response_id present) AND the owner to + // opt in via the EXISTING keepResponseId knob (PR config-surface + // section: no new field — the knob's documented "diagnostic only" + // status is what this upgrades to: keep + delta input). + // state===null = no persisted frontier (first round / resync / lost + // state) → reseed (full send, frontier reset). + const chains = knobKeepResponseId() && typeof parsed.previous_response_id === "string" && parsed.previous_response_id.length > 0; + const deltaState: DeltaState | null | undefined = chains ? (session.responsesDelta ?? null) : undefined; + let selectionFired = false; + // #5874 P2 (mapEgress): the plane view is the outbound source; + // the lane's exact chain (strip → drop → repair) is reapplied to it. processedMessages = applySnapEgress(turn.state, processedMessages, { sessionId, protocol: "responses", log, isCompressTool: loopConfig.isCompressTool, transform: (m) => repairResponsesAssistantOrdering(applyReasoningDrop(stripKernelSummaries(attachSubagentSessions(applyAbsorbView(m, session.state, loopConfig, tokenCount), session), turn.state), reasoningPlan, log, sessionId), originalMessages).messages, + ...(deltaState !== undefined ? { + deltaState, + onSelection: (sel) => { + selectionFired = true; + session.responsesDelta = sel.nextState; + deltaServed = !sel.reseed && sel.delta.length > 0; + }, + } : {}), }); + if (deltaState !== undefined && !selectionFired) { + // Delta was requested but the plane degraded to legacy this round + // (the legacy bytes are the FULL replay and the id was stripped + // below) — the provider's chain no longer matches the stored + // frontier. Forget it: the next chained round re-seeds. + session.responsesDelta = null; + } + if (deltaServed) { + log("info", `[${sessionId}] [snaplane] outbound=delta frontier=${(session.responsesDelta as { frontierPos: number })?.frontierPos} — input carries only the unsent suffix; the retained chain carries the rest (re-injected developer message / forged summaries skipped to avoid duplicating what the provider keeps)`); + } // #3477: consume the announced native-compaction boundary (pi session_compact // → /__bili/plugin/compact → markCompactionBoundary) exactly once, after the // post-compaction history was processed: archives the deactivated blocks, @@ -497,7 +533,9 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr const devContent = devParts.join("\n\n---\n\n"); responsesDevContent = devContent; if (forgedSummaries.length > 0) log("debug", `[${sessionId}] [inject] ${forgedSummaries.length} captured summary block(s) re-injected into developer message`); - rebuiltInput = injectResponsesDeveloperMessage(rebuiltInput, devContent, projection.headRole ?? "developer"); + // 2×2 Part B: on a delta round the provider KEEPS the seed round's + // developer message — re-injecting would duplicate it. + if (!deltaServed) rebuiltInput = injectResponsesDeveloperMessage(rebuiltInput, devContent, projection.headRole ?? "developer"); if (!knobNoInjectTool() && injectTools) { // #1712: decompress's startId/endId execute only on CCR-armed // sessions (#1179) — serve the no-range schema otherwise. @@ -511,7 +549,7 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr const devContent = [...projection.systemParts, ...forgedSummaries, ...(tagsOnlyPrompt !== "" ? [tagsOnlyPrompt] : []), ...(genBoundaryActive && !isCompactionTrigger ? [GENERATION_BOUNDARY_NOTE] : [])].join("\n\n---\n\n"); responsesDevContent = devContent; if (forgedSummaries.length > 0) log("debug", `[${sessionId}] [inject] ${forgedSummaries.length} captured summary block(s) re-injected into developer message`); - rebuiltInput = injectResponsesDeveloperMessage(rebuiltInput, devContent, projection.headRole ?? "developer"); + if (!deltaServed) rebuiltInput = injectResponsesDeveloperMessage(rebuiltInput, devContent, projection.headRole ?? "developer"); } if (sysNotes.length > 0) { const inputItems: ResponseInputItem[] = typeof rebuiltInput === "string" @@ -692,7 +730,10 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // #1954: stripping is only lossless when input already holds the full // conversation. Warn when we strip a non-empty id so a native-chaining // (delta) continuation that loses its history is visible, not silent 200s. - if (!knobKeepResponseId()) { + // 2×2 Part B: a plane DELTA round KEEPS the id (the input is the unsent + // suffix — chaining is the whole point); a reseed round STRIPS it (the + // full input would otherwise duplicate the retained chain). + if (!knobKeepResponseId() || !deltaServed) { const chainWarn = strippedResponseIdWarning(rebuilt.previous_response_id); if (chainWarn) log("warn", `[${sessionId}] ${chainWarn}`); delete rebuilt.previous_response_id; diff --git a/src/server/snap-egress.ts b/src/server/snap-egress.ts index 93c08ef0f3..b2fcab9d66 100644 --- a/src/server/snap-egress.ts +++ b/src/server/snap-egress.ts @@ -28,6 +28,7 @@ import type { BiliMessage } from "acp-kernel/wire"; import { loadConfigFile } from "../config.js"; import { getStore } from "../persist.js"; import { isSnapPlaneDead, type SnapView } from "../storage/snap.js"; +import { selectOutbound, isFullResend, type DeltaState, type DeltaSelection, type EgressElement } from "../snapshot/delta.js"; export type SnapEgressProtocol = "openai" | "anthropic" | "google" | "responses"; @@ -44,6 +45,16 @@ interface SnapEgressDeps { /** #4311: which tool names are compress calls (kernel config * isCompressTool). Gates the visible-compress-call anchor skip. */ isCompressTool?: (name: string) => boolean; + /** 2×2 Part B (出站增量): the persisted delta state. Present only for + * lanes whose target protocol has provider-side retention (Responses + * chaining); undefined = snapshot outbound (send the whole active view, + * the existing #5901 behavior). See src/snapshot/delta.ts. */ + deltaState?: DeltaState | null; + /** Called once per successful plane egress with the selection outcome so + * the lane can persist the new frontier (only after the outbound + * succeeds — a failed round keeps the old state and re-sends the same + * delta: at-least-once). */ + onSelection?: (sel: DeltaSelection) => void; } // --------------------------------------------------------------------------- @@ -186,6 +197,22 @@ interface AnchorSeed { insertAt: number; unresolved: boolean; msg: BiliMessage; + /** The stored anchor row's (negative) plane pos — 0 when synthesized. + * Carried to the side channel so the incremental-outbound selector can + * track which anchors were already sent. */ + pos: number; +} + +/** Per-element bookkeeping for the assembled egress view (parallel to + * `msgs`). The snapshot outbound ignores it; the incremental outbound + * (2×2 Part B, src/snapshot/delta.ts) uses it to pick the unsent suffix. */ +interface EgressElementSide { + /** The plane row pos (append-only). Anchors carry their negative pos. */ + pos: number; + /** true for rendered summary anchors. */ + isAnchor: boolean; + /** The anchor summary id (isAnchor only). */ + anchorId?: string; } /** Assemble the outbound view from the S4-verified plane (kernel prune/ @@ -194,7 +221,7 @@ export function buildSnapEgressView( view: SnapView, state: CompressionState, isCompressTool: (name: string) => boolean, -): { ok: true; msgs: BiliMessage[]; n: number } | { ok: false; reason: string } { +): { ok: true; msgs: BiliMessage[]; n: number; side: EgressElementSide[] } | { ok: false; reason: string } { if (!view.rows.length) return { ok: false, reason: "no-plane" }; // Parse EVERY row (active and covered): the covered rows stay in the walk @@ -223,10 +250,10 @@ export function buildSnapEgressView( // whose core_ids carried a covered base id — the #4052 fresh-repeat // message vanished from egress while the kernel served it). const nonAnchor = rows.filter((r) => !r.isAnchor && r.active === 1); - const anchorMsgById = new Map(); + const anchorMsgById = new Map(); for (const r of rows) { if (!r.isAnchor) continue; - for (const id of r.ids) if (id.startsWith("acp_summary_")) anchorMsgById.set(id, r.msg); + for (const id of r.ids) if (id.startsWith("acp_summary_")) anchorMsgById.set(id, { msg: r.msg, pos: r.pos }); } const n = nonAnchor.length; @@ -237,7 +264,12 @@ export function buildSnapEgressView( const coveredIds = new Set(); for (const b of state.blocks) if (b.active) for (const id of b.effectiveMessageIds) coveredIds.add(id); if (coveredIds.size === 0) { - return { ok: true, msgs: nonAnchor.map((r) => r.msg), n }; + return { + ok: true, + msgs: nonAnchor.map((r) => r.msg), + n, + side: nonAnchor.map((r) => ({ pos: r.pos, isAnchor: false })), + }; } // #4311: a compress tool-call visible in the input suppresses its block's // separate summary render. @@ -325,9 +357,11 @@ export function buildSnapEgressView( if (b.compressCallId !== undefined && visibleCompressCallIds.has(b.compressCallId)) continue; const sid = summaryMessageId(b.blockId); const existing = summaryIndexById.get(sid); - const msg = anchorMsgById.get(sid) ?? renderAnchor(b); + const stored = anchorMsgById.get(sid); + const msg = stored?.msg ?? renderAnchor(b); + const anchorPos = stored?.pos ?? 0; if (existing !== undefined) { - anchors.push({ insertAt: existing, unresolved: false, msg }); + anchors.push({ insertAt: existing, unresolved: false, msg, pos: anchorPos }); continue; } const anchorIds = b.directMessageIds && b.directMessageIds.length > 0 ? b.directMessageIds : b.effectiveMessageIds; @@ -349,7 +383,7 @@ export function buildSnapEgressView( insertAt = earliest ?? fallbackIndex; unresolved = earliest === null; } - anchors.push({ insertAt, unresolved, msg }); + anchors.push({ insertAt, unresolved, msg, pos: anchorPos }); } // Pair-safe adjustment + pin/fallback clamp (kernel rebuildMessages // parity: non-head-anchors clamp past the pin when resolved, to the first @@ -364,11 +398,20 @@ export function buildSnapEgressView( // The walk (kernel rebuildMessages parity): anchors at their index, the // pinned run verbatim, covered / positional / stale-summary skips. const merged: BiliMessage[] = []; + const side: EgressElementSide[] = []; + const pushAnchor = (a: AnchorSeed) => { + merged.push(a.msg); + side.push({ pos: a.pos, isAnchor: true, anchorId: a.msg.id }); + }; + const pushRow = (r: EgressRow) => { + merged.push(r.msg); + side.push({ pos: r.pos, isAnchor: false }); + }; let ai = 0; for (let i = 0; i < n; i++) { - while (ai < anchors.length && anchors[ai]!.insertAt === i) merged.push(anchors[ai++]!.msg); + while (ai < anchors.length && anchors[ai]!.insertAt === i) pushAnchor(anchors[ai++]!); if (i === firstUserIndex && firstUserIndex >= 0) { - for (let k = i; k < firstUserPinEnd && k < n; k++) merged.push(nonAnchor[k]!.msg); + for (let k = i; k < firstUserPinEnd && k < n; k++) pushRow(nonAnchor[k]!); i = firstUserPinEnd - 1; continue; } @@ -377,10 +420,10 @@ export function buildSnapEgressView( // nonAnchor is the active set already — covered rows never reach // the walk (see the filter above). if (isRenderedSummaryMessage(m) && knownSummaryIds.has(m.id)) continue; - merged.push(m); + pushRow(r); } - while (ai < anchors.length) merged.push(anchors[ai++]!.msg); - return { ok: true, msgs: merged, n: merged.length }; + while (ai < anchors.length) pushAnchor(anchors[ai++]!); + return { ok: true, msgs: merged, n: merged.length, side }; } // Entry point — called by each protocol lane right after the lane computed @@ -419,7 +462,25 @@ export function applySnapEgress( noteDegrade(built.reason); return legacyProcessed; } - const planeProcessed = deps.transform(built.msgs); + // 2×2 Part B: incremental outbound. Both modes read the SAME plane + // view — the difference is only how many elements go over the wire. + let outMsgs = built.msgs; + let selection: DeltaSelection | null = null; + if (deps.deltaState !== undefined) { + const elements: EgressElement[] = built.msgs.map((m, i) => ({ + msg: m, + pos: built.side[i]!.pos, + isAnchor: built.side[i]!.isAnchor, + anchorId: built.side[i]!.anchorId, + })); + selection = selectOutbound(elements, deps.deltaState); + if (!selection.reseed && !isFullResend(selection, elements)) { + const keep = new Set(selection.delta.map((e) => e.msg)); + outMsgs = built.msgs.filter((m) => keep.has(m)); + } + deps.onSelection?.(selection); + } + const planeProcessed = deps.transform(outMsgs); // DIRECT AUTHORITY: the plane view is the outbound source. The legacy // bytes exist only as the crash-grade fallback above. egressStats.plane++; diff --git a/src/session.ts b/src/session.ts index 744275f9f6..4c56897a57 100644 --- a/src/session.ts +++ b/src/session.ts @@ -694,6 +694,12 @@ export type Session = { * only the light transform (absorb/attach/strip/drop) — no seam walk, * no re-tagging, no id re-keying. Cleared per turn like lastMessages. */ lastPlaneMessages?: CoreMessage[]; + /** 2×2 Part B (出站增量, responses lane only): the incremental-outbound + * frontier — the highest plane row pos + anchor summary ids the provider + * already has via its retained chain. Persisted with the session; lost + * state → next round reseeds (full send), one-round convergence. See + * src/snapshot/delta.ts. */ + responsesDelta?: { frontierPos: number; servedAnchors: string[] } | null; pluginSnapshot?: CoreMessage[]; /** #3301: original message decompositions supplied by an external state * converter (e.g. opencode-acp migration) at import time. Transient diff --git a/src/snapshot/assemble.ts b/src/snapshot/assemble.ts new file mode 100644 index 0000000000..2eb3edbb25 --- /dev/null +++ b/src/snapshot/assemble.ts @@ -0,0 +1,193 @@ +// 2×2 stage 0 — kernel-input assembly (brief 19544722, Part A core). +// +// The stage-0 verdict (identity.ts) tells us WHICH envelope elements are new +// and where every element sits (append-only pos). This module assembles the +// kernel input for the turn: +// +// kernelInput = the envelope in CLIENT ORDER, where each element resolves to +// (a) a FRESH core — the parsed NEW span, converted, in this same turn +// (the only parse of this turn: P5 — non-new rounds never re-parse), or +// (b) an OLD core — the plane row's stored kernel byte (post_json), +// hydrated, verbatim. The kernel re-renders its own tag idempotently +// (render-refs.ts:124-131) and the token count is a first-render +// snapshot, so a table-old core rides byte-stable (prefix cache). +// +// The kernel input order is the ENVELOPE (client) order, NOT the plane pos +// order: a scattered NEW span (a mid-history edit) lands at its conversation +// position, not at the tail. That is exactly the kernel input the full-parse +// path (M1) would have produced for the same client array — the byte +// identity M1 vs M2 (acceptance #2) falls out by construction. +// +// R4 (table gap): an envelope element that is neither new nor plane-resolvable +// is a stage-0 verdict we cannot honor — the table is behind the client. +// The lane catches the throw, resets the plane (snapStage0Reset) and serves +// the round from the legacy full path: one-round convergence, never a half +// conversation. + +import type { BiliMessage } from "acp-kernel/wire"; +import type { WorldDecision } from "./types.ts"; + +/** A plane row's kernel-input contribution: the stored tagged core, hydrated. */ +interface PlaneRowCore { + pos: number; + /** The row's bound element ids (core_ids, alias-chain included). */ + ids: string[]; + /** post_json hydrated (media CAS resolved) — the kernel byte. */ + msg: BiliMessage; +} + +interface AssembleInput { + /** The stage-0 verdict (identity.ts). */ + decision: WorldDecision; + /** The plane's ACTIVE rows, pos-ordered (the old cores). Inactive + * (fold-covered) rows are NOT part of the kernel input — the kernel + * folds them; their summary anchor is carried by the kernel state. */ + rows: readonly PlaneRowCore[]; + /** The NEW spans, parsed: the kernel-input form of each new element, in + * envelope order. Length must equal decision.newIdPos.length. */ + newCores: readonly BiliMessage[]; + /** Envelope ids that may legally have NO plane row and are NOT new: + * the fold-explained gone set (a fold absorbed the row — the kernel + * state carries the block, the anchor row carries the summary). These + * envelope entries are SKIPPED in the walk (the R4 table-gap throw is + * suppressed for them and only). */ + skipIds?: ReadonlySet; + /** The plane's ANCHOR rows (id = acp_summary_, active), in + * kernel-input form. Inserted after the walk at the covered-run + * boundary (coverMax). */ + anchors?: readonly EgressAnchor[]; + /** blockId → highest covered row pos (store.snapStage0Cores). The + * anchor renders before the first served element whose pos exceeds + * the covered run (the D-view placement parity). */ + coverMax?: Record; +} + +/** A plane anchor row in kernel-input form (the rendered summary message). + * `blockId` is the suffix of the anchor id (acp_summary_). */ +interface EgressAnchor { + msg: BiliMessage; + blockId: string; +} + +interface AssembleResult { + /** The assembled kernel input: envelope-ordered elements (old cores and + * fresh cores interleaved at their conversation positions) with the + * anchor rows inserted at their covered-run boundary. Feed this to + * processTurn. */ + kernelInput: BiliMessage[]; + /** The number of OLD element cores taken from the plane (anchor rows + * excluded — they are not ledger elements). */ + oldCount: number; +} + +/** + * Assemble the kernel input for a stage-0-armed turn. + * + * @throws when the plane cannot satisfy the verdict (R4-shaped): the caller + * resets the plane and serves the round from the legacy full path. + */ +export function assembleKernelInput(input: AssembleInput): AssembleResult { + const { decision, rows, newCores, skipIds, anchors, coverMax } = input; + if (newCores.length !== decision.newIdPos.length) { + throw new Error( + `[snapshot 2×2] assembleKernelInput: newCores length ${newCores.length} != decision.newIdPos length ${decision.newIdPos.length} (P5: parse exactly the NEW spans)`, + ); + } + // Anchor rows are plane rows in their own right — their ids must not be + // mistaken for element cores (an anchor id can never match an envelope + // element; excluding them keeps rowById element-only). + const anchorIds = new Set(); + for (const a of anchors ?? []) anchorIds.add(`acp_summary_${a.blockId}`); + const newRowById = new Map(); + decision.newIdPos.forEach(({ id }, i) => { + if (!newRowById.has(id)) newRowById.set(id, newCores[i]!); + }); + // Element id → active plane row core (first row in pos order whose core + // ids contain the element id — alias-safe: a row bound to several ids + // serves whichever the client echoes first). + const rowById = new Map(); + for (const row of rows) { + if (row.ids.some((id) => anchorIds.has(id))) continue; // anchor row + for (const id of row.ids) if (!rowById.has(id)) rowById.set(id, row.msg); + } + + // The walk in CLIENT (envelope) order, recording each element's plane + // pos so the anchors can be placed against the surviving sequence. + const kernelInput: BiliMessage[] = []; + const elementPos: number[] = []; // parallel to kernelInput + let oldCount = 0; + for (const e of decision.envelope) { + const fresh = newRowById.get(e.id); + if (fresh !== undefined) { + kernelInput.push(fresh); + elementPos.push(e.pos); + continue; + } + const old = rowById.get(e.id); + if (old !== undefined) { + kernelInput.push(old); + elementPos.push(e.pos); + oldCount++; + continue; + } + if (skipIds?.has(e.id)) { + // Fold-covered and fold-explained: the kernel state carries the + // block, the anchor row (inserted below) carries the summary. + continue; + } + // An envelope element that is neither new nor plane-resolvable: + // the stage-0 verdict says it is incoming, so this is a plane/table + // gap (R4-shaped) — refuse to assemble half a conversation. + throw new Error( + `[snapshot 2×2] assembleKernelInput: envelope element ${e.id} (pos ${e.pos}) is neither a new span nor a plane row (table behind the client — resync required)`, + ); + } + + // Anchor insertion (D-view placement parity): an anchor renders before + // the first served element whose pos exceeds its block's covered run + // (coverMax). No coverMax evidence (the covered rows were GC'd) → the + // view start (the kernel's "earliest covered message" fallback). The pin + // run (first user) is served verbatim by the kernel on top of this input, + // so the anchor never needs a pin clamp here. + const placed: Array<{ at: number, anchor: EgressAnchor }> = []; + for (const anchor of anchors ?? []) { + const coverHi = coverMax?.[anchor.blockId] ?? -1; + let at = 0; // default: view start + for (let i = 0; i < kernelInput.length; i++) { + if ((elementPos[i] ?? -1) > coverHi) { + at = i; + break; + } + } + placed.push({ at, anchor }); + } + if (placed.length > 0) { + placed.sort((a, b) => a.at - b.at); + const out: BiliMessage[] = []; + let ki = 0; + for (const p of placed) { + while (ki < p.at) out.push(kernelInput[ki++]!); + out.push(p.anchor.msg); + } + while (ki < kernelInput.length) out.push(kernelInput[ki++]!); + return { kernelInput: out, oldCount }; + } + return { kernelInput, oldCount }; +} + +/** + * The resync predicate (brief: the resync safety net). + * + * - `diverge` — the client's array structurally disagrees with the stored + * canon (head moved / order broken): the table cannot be trusted for + * THIS round → reset + re-issue from the client's current array. + * - `resync` — an explicit mode switch (the lane requested it). + * + * A plain `issue` with an EMPTY plane is the NORMAL seed (first turn of a + * fresh session) — NOT a resync: resetting there would wipe the just-issued + * canon and re-issue forever. R4 (table gap) surfaces through + * assembleKernelInput's throw instead — it is per-round, not structural. + */ +export function shouldResync(decision: WorldDecision, _planeRowCount: number): boolean { + return decision.mode === "diverge" || decision.mode === "resync"; +} diff --git a/src/snapshot/checksums.ts b/src/snapshot/checksums.ts new file mode 100644 index 0000000000..7d90b38c68 --- /dev/null +++ b/src/snapshot/checksums.ts @@ -0,0 +1,62 @@ +/** + * #5874 — snapshot-table compression foundation: checksums (S4 + canon). + * + * Pure, dependency-light. Every checksum in the foundation is a lowercase + * hex sha256 so the values are grep-able across the audit stream and the + * storage layer (they are stored verbatim in snap_rows / snap_ckpt / + * l1_world). The S4 per-row aggregate is the red-line invariant (iii): a + * mismatch means the base material is corrupt and the sid plane is PERMANENTLY + * dead (R1). + */ +import { createHash } from "node:crypto"; + +/** Lowercase-hex sha256 of a UTF-8 string. */ +function sha256hex(text: string): string { + return createHash("sha256").update(text, "utf8").digest("hex"); +} + +/** Canonical JSON for checksum inputs: sorted-object-key, no whitespace. This + * is the ONLY serialization the foundation checksums over, so a key-order + * shuffle (JSON.stringify non-determinism, client churn) cannot move a + * checksum. */ +export function canonicalJson(value: unknown): string { + return stableStringify(value); +} + +function stableStringify(value: unknown): string { + if (value === null || typeof value !== "object") { + if (typeof value === "string") return JSON.stringify(value); + return JSON.stringify(value) ?? "null"; + } + if (Array.isArray(value)) { + return "[" + value.map((v) => stableStringify(v)).join(",") + "]"; + } + const keys = Object.keys(value as Record).sort(); + const parts: string[] = []; + for (const k of keys) { + const v = (value as Record)[k]; + if (v === undefined) continue; + parts.push(JSON.stringify(k) + ":" + stableStringify(v)); + } + return "{" + parts.join(",") + "}"; +} + +/** sha256 of the canonical envelope JSON (the L1 world's fast-mismatch + * sentinel, §4.0.1). */ +export function envelopeChecksum(envelope: readonly string[]): string { + return sha256hex(canonicalJson(Array.from(envelope))); +} + +/** sha256 of one row's post form (S4 unit). The post form is the common-core + * JSON; checksumming the canonical form keeps it byte-churn-proof. */ +function rowPostChecksum(postJson: string): string { + let parsed: unknown; + try { + parsed = JSON.parse(postJson); + } catch { + // A non-JSON post form is itself a fault — checksum the raw bytes so + // the mismatch is still detected (and grep-able) rather than masked. + return sha256hex(postJson); + } + return sha256hex(canonicalJson(parsed)); +} diff --git a/src/snapshot/delta.ts b/src/snapshot/delta.ts new file mode 100644 index 0000000000..38e854a929 --- /dev/null +++ b/src/snapshot/delta.ts @@ -0,0 +1,137 @@ +/** + * #5874 / 2×2 — Part B: incremental outbound (seed / delta selection). + * + * The 出站协议改写 stage reads the SAME snapshot plane for every mode + * (brief 19544722): 出站 snapshot (M1/2) sends ALL active rows (the existing + * #5901 direct-authority egress — prefix bytes stable by construction); + * 出站增量 (M3/4) sends only what the provider has NOT seen yet: + * + * - the SEED round (the first outbound for the session, and the first + * after a resync / re-arm) must send the FULL view — the provider + * (Responses-style retention) has nothing to chain from. + * - a DELTA round sends only the new elements: active rows with + * pos > frontier (never sent) + anchor rows whose summary id was never + * sent. + * + * WHY THE PROVIDER'S CONTEXT GROWS MONOTONICALLY: a Responses-style + * provider retains every element it was ever sent (the chain is + * append-only — there is no delete API). Rows that later get fold-covered + * stay at the provider; the provider's effective conversation is therefore + * a SUPERSET of the current egress view. That is the documented trade-off + * of 出站增量 (the brief's acceptance #3: the CUMULATIVE rows sent — + * including the seed round — are byte-identical to the active-row set the + * snapshot mode sends). When the provider's window overflows, the next + * outbound RE-SEEDS (full view) and converges in one round. + * + * LCP-based delta selection (diff the assembled views) is REJECTED: a fold + * deletes covered rows from the view's prefix, so the new and old views no + * longer share a prefix and the "delta" would re-send the tail (duplicating + * it at the provider). The row-level frontier (append-only pos) is the + * only correct basis. + */ + +/** One element of the assembled egress view, with the bookkeeping the + * selector needs (buildSnapEgressView returns plain messages; the lane + * provides this side channel in the same order). */ +export interface EgressElement { + /** The wire message (anchor = the rendered summary message, data row = + * the transformed plane row). */ + msg: unknown; + /** The plane row pos (append-only). Anchor elements carry their + * NEGATIVE anchor pos. */ + pos: number; + /** true for rendered summary anchors (id = acp_summary_). */ + isAnchor: boolean; + /** The anchor's summary id (isAnchor only). */ + anchorId?: string; +} + +export interface DeltaState { + /** The highest row pos sent to the provider so far (-1 = nothing). */ + frontierPos: number; + /** The anchor summary ids already sent (acp_summary_*). */ + servedAnchors: string[]; +} + +export interface DeltaSelection { + /** true = the provider has nothing (or we re-armed) → send the FULL + * view (seed). false = send `delta`. */ + reseed: boolean; + /** The elements to send, in view order (anchors at their pair-safe + * positions, rows pos-ordered). Empty when reseed. */ + delta: readonly EgressElement[]; + /** The new state to persist after this outbound (no-op when reseed + * failed upstream — the lane keeps the old state on error). */ + nextState: DeltaState; +} + +const ANCHOR_ID_PREFIX = "acp_summary_"; + +/** The frontier state after the WHOLE view has been sent (seed / re-seed). */ +function stateFromElements(elements: readonly EgressElement[]): DeltaState { + let maxPos = -1; + const served: string[] = []; + for (const el of elements) { + if (el.isAnchor) { + const id = el.anchorId ?? anchorIdOf(el.msg); + if (id !== null && !served.includes(id)) served.push(id); + } else if (el.pos > maxPos) { + maxPos = el.pos; + } + } + return { frontierPos: maxPos, servedAnchors: served }; +} + +/** Select the outbound elements for one round. + * + * `state` is the persisted delta state (session.metadata); pass null for a + * fresh session → reseed. `elements` is the full egress view (snapshot + * mode's output), element side-channel included, in send order. + * + * The selector is PURE (no storage, no clock) — the lane persists + * `nextState` only AFTER the outbound succeeds (a failed round keeps the + * old frontier and re-sends the same delta next round: at-least-once, the + * provider dedupes by the chain). + */ +export function selectOutbound(elements: readonly EgressElement[], state: DeltaState | null): DeltaSelection { + if (state === null || elements.length === 0) { + // Reseed: the provider has nothing to chain from — send the FULL view + // and record everything as served (the frontier moves to the max pos; + // a later delta sends only what lands above it). + return { reseed: true, delta: [], nextState: stateFromElements(elements) }; + } + const served = new Set(state.servedAnchors); + const delta: EgressElement[] = []; + let maxPos = state.frontierPos; + const newAnchors: string[] = []; + for (const el of elements) { + if (el.isAnchor) { + const id = el.anchorId ?? anchorIdOf(el.msg); + if (id !== null && !served.has(id)) { + delta.push(el); + newAnchors.push(id); + } + continue; + } + if (el.pos > state.frontierPos) { + delta.push(el); + if (el.pos > maxPos) maxPos = el.pos; + } + } + return { + reseed: false, + delta, + nextState: { frontierPos: maxPos, servedAnchors: [...state.servedAnchors, ...newAnchors] }, + }; +} + +/** A delta that grew to the whole view (everything unsent) is a re-seed in + * disguise — the provider's chain is empty, so send it as a full seed. */ +export function isFullResend(sel: DeltaSelection, elements: readonly EgressElement[]): boolean { + return !sel.reseed && sel.delta.length >= elements.length; +} + +function anchorIdOf(msg: unknown): string | null { + const m = msg as { id?: unknown }; + return typeof m?.id === "string" && m.id.startsWith(ANCHOR_ID_PREFIX) ? m.id : null; +} diff --git a/src/snapshot/identity.ts b/src/snapshot/identity.ts new file mode 100644 index 0000000000..aae9be5ffc --- /dev/null +++ b/src/snapshot/identity.ts @@ -0,0 +1,285 @@ +/** + * #5874 — snapshot-table compression foundation: stage 0 (identity / world + * match). + * + * The host's `(sess_raw, msgid[])` pair is the identity channel (§4.0). The + * `l1_world` table is the server's standing canon: for a raw session id it + * records the canonical host-msgid ENVELOPE — an ordered list of + * `{id, pos}` where `pos` is the APPEND-ONLY ledger position the msgid was + * first issued at (never reused, the DDL invariant) — plus the ledger + * identity it was issued under. + * + * Every turn the incoming envelope (host msgids in host order) is matched + * against the stored canon: + * + * - echo — the id sequence matches positionally → all known. + * - probe — the stored id sequence is a strict PREFIX of the incoming → + * tail append (the common, v5-proven shape). + * - scatter — known ids keep their relative order but NEW ids are inserted + * at any position → the span-parse case (T5 byte pin). + * - issue — no stored world → issue a fresh ledger identity + arm. + * - diverge — the LIVE head changed, or a KNOWN id changed its relative + * order (history rewrite) → this turn legacy + re-stamp; the + * canon is NOT updated (the client converges in one turn). + * - resync — an explicit re-baseline (mode switch). + * + * Deletions (a stored id absent from the incoming) are NOT a divergence while + * the live head and the survivors' relative order are unchanged: they are + * reported via `goneIds` and the reconcile turns them into `delete` events. + * + * THE LIVE HEAD: after a fold the client's envelope starts at the first + * SURVIVING element while the canon still lists the covered head entries + * (their pos is never reused). A head change is a divergence only when the + * first stored id that is NOT fold-explained (`explainedGone`) differs from + * the incoming head — a legal post-fold shrink therefore keeps the plane + * green, while a host that drops its head WITHOUT fold evidence diverges. + * (Middle deletions without fold evidence stay legal — reconcile emits the + * `delete` events and the egress skips the row.) + * + * NEW ids are issued fresh append-only pos (maxPos+1.., in incoming order), so + * a middle insertion never shifts a stored row — the rows (and the prefix + * cache) stay put. + */ +import { envelopeChecksum } from "./checksums.ts"; +import type { WorldDecision, WorldEnvelopeEntry } from "./types.ts"; + +interface WorldMatchInput { + /** The stored L1 world for this sess_raw, or null when none exists. */ + stored: { + ledgerId: string; + /** The stored canon (ordered {id, pos}). */ + envelope: WorldEnvelopeEntry[]; + envelopeChecksum: string; + issuedAt: number; + armedAt?: number; + } | null; + /** The incoming host envelope (per-element msgids, in host order). For a + * non-cohort host this is the sequence-derived pseudo-envelope. */ + incoming: string[]; + /** Stored ids whose absence a fold already explains (their row is + * inactive with a coverRef). Only such ids may legally vanish from the + * HEAD of the envelope; any other gone id is a client-side drop + * (legal in the middle, divergent at the head). */ + explainedGone?: ReadonlySet; + /** The raw session key (host value, never truncated, P9). */ + sessRaw: string; + /** The bili canonical ledger identity to issue when no world exists + * (ruling D: the existing sid keyspace). */ + ledgerId: string; + now: number; + /** True when this turn is an explicit re-baseline (mode switch). */ + resync?: boolean; +} + +/** Match the incoming envelope against the stored canon. Pure — returns the + * verdict; the persistence (upsertL1World) is done by the caller so a + * `diverge` turn has ZERO side effects (the canon is not touched). */ +export function matchWorld(input: WorldMatchInput): WorldDecision { + const incoming = Array.from(input.incoming); + const incomingCk = envelopeChecksum(incoming); + + // Explicit re-baseline always re-issues the canon (the mode changed). + if (input.resync) { + const env: WorldEnvelopeEntry[] = incoming.map((id, i) => ({ id, pos: i + 1 })); + return { + mode: "resync", + envelope: env, + envelopeChecksum: incomingCk, + newIdPos: env.map((e) => ({ id: e.id, pos: e.pos })), + goneIds: [], + issuedLedgerId: input.ledgerId, + issuedAt: input.now, + armedAt: input.now, + }; + } + + if (input.stored === null) { + // No world yet: issue a fresh ledger identity, pay one arming turn. + const env: WorldEnvelopeEntry[] = incoming.map((id, i) => ({ id, pos: i + 1 })); + return { + mode: "issue", + envelope: env, + envelopeChecksum: incomingCk, + newIdPos: env.map((e) => ({ id: e.id, pos: e.pos })), + goneIds: [], + issuedLedgerId: input.ledgerId, + issuedAt: input.now, + armedAt: input.now, + }; + } + + const stored = input.stored; + const storedIds = stored.envelope.map((e) => e.id); + + // 1. echo: the id sequence matches positionally → all known. + if (stored.envelopeChecksum === incomingCk && stored.envelope.length === incoming.length) { + return { + mode: "echo", + envelope: stored.envelope.map((e) => ({ id: e.id, pos: e.pos })), + envelopeChecksum: incomingCk, + newIdPos: [], + goneIds: [], + issuedAt: stored.issuedAt, // B1 trap: origin never moves on echo + }; + } + + // Known-id bookkeeping. + const storedPos = new Map(); + for (const e of stored.envelope) storedPos.set(e.id, e.pos); + const incomingSet = new Set(incoming); + const goneIds: string[] = []; + for (const id of storedIds) if (!incomingSet.has(id)) goneIds.push(id); + + // A KNOWN id that changed its relative order (or the LIVE head changed) + // is a history rewrite → diverge (the canon is not updated; the turn + // runs legacy and re-stamps so the client converges in one turn). + // The live head = the first stored id that a fold did NOT absorb: + // after a post-fold shrink the incoming envelope legitimately starts at + // the first surviving element, not at the (covered) canon head. + const aliveStored = storedIds.filter((id) => !input.explainedGone?.has(id)); + const headMoved = aliveStored.length > 0 && incoming.length > 0 && incoming[0] !== aliveStored[0]; + let orderBroken = false; + if (!headMoved) { + // The survivor ids must appear in the same relative order as stored. + const survivorsStored = storedIds.filter((id) => incomingSet.has(id)); + const survivorsIncoming = incoming.filter((id) => storedPos.has(id)); + if (survivorsStored.length !== survivorsIncoming.length) { + orderBroken = true; + } else { + for (let i = 0; i < survivorsStored.length; i++) { + if (survivorsStored[i] !== survivorsIncoming[i]) { + orderBroken = true; + break; + } + } + } + } + if (headMoved || orderBroken) { + return { + mode: "diverge", + envelope: stored.envelope.map((e) => ({ id: e.id, pos: e.pos })), + envelopeChecksum: stored.envelopeChecksum, + newIdPos: [], + goneIds: [], + issuedAt: stored.issuedAt, + witness: `head ${storedIds[0] ?? "-"} -> ${incoming[0] ?? "-"}; orderBroken=${orderBroken}`, + }; + } + + // Head stable + survivor order preserved → a legal extension (probe or + // scatter). Issue fresh pos to every NEW id (maxPos+1.., incoming order). + let maxPos = 0; + for (const e of stored.envelope) maxPos = Math.max(maxPos, e.pos); + const newIdPos: Array<{ id: string; pos: number }> = []; + const posMap = new Map(); + for (const e of stored.envelope) posMap.set(e.id, e.pos); + for (const id of incoming) { + if (!posMap.has(id)) { + maxPos += 1; + posMap.set(id, maxPos); + newIdPos.push({ id, pos: maxPos }); + } + } + const envelope: WorldEnvelopeEntry[] = incoming.map((id) => ({ id, pos: posMap.get(id)! })); + + // probe = stored is a strict prefix of incoming (tail append only); + // scatter = new ids are interleaved. + let isPrefix = storedIds.length < incoming.length; + if (isPrefix) { + for (let i = 0; i < storedIds.length; i++) { + if (incoming[i] !== storedIds[i]) { + isPrefix = false; + break; + } + } + } + const mode = newIdPos.length > 0 + ? (isPrefix ? "probe" : "scatter") + : goneIds.length > 0 + ? "probe" // canon shrink (deletions only) + : "echo"; + + return { + mode, + envelope, + envelopeChecksum: incomingCk, + newIdPos, + goneIds, + issuedAt: stored.issuedAt, // B1 trap: origin never moves on extend + }; +} + +/** + * #5874 2×2 (brief 19544722, Part A) — the M2 INCREMENTAL envelope: + * the client sends ONLY the new elements (with msgids), so the incoming + * list shares NO ids with the stored canon (a pure addition). The caller + * has ALREADY located the sequence position (世界匹配定位序列位置) via the + * seqs carrier — the incoming ledger seqs are contiguous and start exactly + * after the plane's max stamped seq — so every incoming id gets a fresh + * append-only pos (maxPos+1.., in incoming order) and the envelope is the + * stored canon ++ the new tail. + * + * This is the "probe" shape of a client that never re-sends: the covered + * (fold-explained) canon entries stay in the canon (the caller prunes them + * from the kernel-input walk via explainedGone — the plane keeps serving + * them as covered rows until GC). + */ +export function matchAppend( + stored: { + ledgerId: string; + envelope: WorldEnvelopeEntry[]; + envelopeChecksum: string; + issuedAt: number; + armedAt?: number; + }, + incoming: string[], +): WorldDecision { + const incomingCk = envelopeChecksum(incoming); + let maxPos = 0; + const posMap = new Map(); + for (const e of stored.envelope) { + maxPos = Math.max(maxPos, e.pos); + posMap.set(e.id, e.pos); + } + const newIdPos: Array<{ id: string, pos: number }> = []; + for (const id of incoming) { + if (posMap.has(id)) continue; // defensive: the caller guarantees a pure addition + maxPos += 1; + posMap.set(id, maxPos); + newIdPos.push({ id, pos: maxPos }); + } + const envelope: WorldEnvelopeEntry[] = [ + ...stored.envelope.map((e) => ({ id: e.id, pos: e.pos })), + ...incoming.map((id) => ({ id, pos: posMap.get(id)! })), + ]; + return { + mode: "probe", + envelope, + // The stored canon's checksum covers the WHOLE envelope sequence + // (the next M1 full resend must echo against it), not just the new + // tail. + envelopeChecksum: envelopeChecksum(envelope.map((e) => e.id)), + newIdPos, + goneIds: [], + issuedAt: stored.issuedAt, // B1 trap: origin never moves on extend + }; +} + +/** Parse the persisted element envelope (the l1_world envelope_json column — + * `JSON.stringify(WorldEnvelopeEntry[])`, host order). Validated: a + * malformed canon is a storage-corruption signal and throws (the lane + * maps it to a resync — the client converges in one round) rather than + * being silently re-issued over a half-read canon. */ +export function parseEnvelope(envelopeJson: string | null): { envelope: WorldEnvelopeEntry[] } { + if (envelopeJson === null) return { envelope: [] }; + const parsed: unknown = JSON.parse(envelopeJson); + if (!Array.isArray(parsed)) throw new Error("parseEnvelope: the canon is not an array (corrupted l1_world row)"); + const envelope: WorldEnvelopeEntry[] = parsed.map((e, i) => { + const entry = e as { id?: unknown, pos?: unknown }; + if (typeof entry.id !== "string" || !Number.isInteger(entry.pos) || (entry.pos as number) < 0) { + throw new Error(`parseEnvelope: canon entry ${i} is not a {id, pos} pair (corrupted l1_world row)`); + } + return { id: entry.id, pos: entry.pos as number }; + }); + return { envelope }; +} diff --git a/src/snapshot/plane.ts b/src/snapshot/plane.ts new file mode 100644 index 0000000000..b2be5d1afa --- /dev/null +++ b/src/snapshot/plane.ts @@ -0,0 +1,248 @@ +/** + * #5874 / 2×2 — stage 0 (identity / world match) decision glue. + * + * Sits at the head of the six-stage chain (raw bytes → **stage 0** → protocol + * conversion → kernel → plane write → egress). It runs the pure `matchWorld` + * (identity.ts) on the l1_world element envelope and carries the R3 identity + * fault policy (brief 19544722 red gate R3): + * + * - a cohort host that DECLARED the dual-channel (per-element msgids) but + * did not collect them all → the request is HARD-REJECTED (400) BEFORE + * any table/kernel side effect, and the fault feeds a fastpath-style + * breaker (10-minute window, 3 strikes → a persistent per-sid latch that + * keeps rejecting until an operator clears it). + * + * The decision itself is pure (identity.ts); this module adds the storage + * glue (read the stored envelope, build the fold-explained-gone set from the + * full row set, persist the new canon) — and ONLY for non-diverge verdicts + * (a diverge turn has zero side effects by construction). + */ +import { matchAppend, matchWorld, parseEnvelope } from "./identity.ts"; +import { snapL1EnvelopeRead, snapL1EnvelopeUpsert, snapL1Lookup, snapRowsAll } from "../storage/snap.ts"; +import type { DbLike } from "../storage/snap.ts"; +import type { WorldDecision } from "./types.ts"; + +/** R3 identity fault — the lane rejects the request with HTTP 400 and takes + * NO side effect (no plane event, no kernel run, no session ext write other + * than the breaker counter). */ +export class FoundationRejectError extends Error { + readonly status = 400; + constructor(public readonly reason: string) { + super(`[snapshot] R3 identity fault: ${reason}`); + this.name = "FoundationRejectError"; + } +} + +/** R3 breaker: a per-process fault window (10 minutes) with a strike cap + * (3). On the third strike within the window the sid gets a PERSISTENT + * latch (the caller persists it, e.g. session_ext) and every subsequent + * R3 fault for that sid rejects immediately without re-counting. Mirrors + * the incremental-lane breaker discipline (the brief's R3 reference). */ +export class R3FaultTracker { + private readonly faults = new Map(); + private readonly latched = new Set(); + constructor( + private readonly windowMs: number = 10 * 60 * 1000, + private readonly strikeCap: number = 3, + ) {} + + /** Record an R3 fault. Returns the persistent-latch verdict. */ + recordFault(sid: string, now: number = Date.now()): { latched: boolean } { + if (this.latched.has(sid)) return { latched: true }; + let entry = this.faults.get(sid); + if (!entry || now - entry.firstAt > this.windowMs) { + entry = { count: 0, firstAt: now }; + this.faults.set(sid, entry); + } + entry.count += 1; + if (entry.count >= this.strikeCap) { + this.latched.add(sid); + return { latched: true }; + } + return { latched: false }; + } + + isLatched(sid: string): boolean { + return this.latched.has(sid); + } + + /** Operator escape hatch (the latch is persistent across the window but + * not across a process restart unless the caller also persists it). */ + clearLatched(sid: string): void { + this.latched.delete(sid); + this.faults.delete(sid); + } +} + +/** The process-wide R3 fault tracker (one per process; the lane consults it + * before every stage-0 run and records on every fault). Exported so tests + * can inspect/reset it. */ +export const r3FaultTracker = new R3FaultTracker(); + +/** The stage-0 input: the raw identity channel for one turn. + * @param db the storage db (snap.ts DbLike surface). + * @param sessRaw the raw host session id (P9: never truncated). + * @param ledgerId the bili canonical ledger identity (the sid). + * @param incoming the per-element msgids in host order. + * @param resync true for an explicit re-baseline (mode switch). + * @param now clock (injectable for tests). + */ +interface Stage0Input { + db: DbLike; + sessRaw: string; + ledgerId: string; + incoming: string[]; + /** The `biliLedgerSeqs` carrier (M2 position locating, #4316 layer-3 + * transport). When it is contiguous and starts exactly after the + * plane's max stamped seq, an ALL-UNKNOWN incoming is a pure M2 + * append (matchAppend); without it, an all-unknown incoming is + * treated as a full (possibly re-minted) envelope → the normal + * matchWorld verdict (usually diverge → resync, one-round + * convergence). */ + seqs?: number[] | null; + resync?: boolean; + now?: number; +} + +export interface Stage0Result { + /** The pure world-match verdict (identity.ts). */ + decision: WorldDecision; + /** True when the verdict is `diverge` — the caller runs this turn on the + * legacy path with ZERO plane side effects (the canon is untouched; the + * client converges one turn later). */ + diverge: boolean; + /** The fold-explained gone set (stored ids whose row is inactive with a + * coverRef) — exposed for the reconcile (delete events) and tests. */ + explainedGone: Set; +} + +/** Run stage 0: read the stored element envelope + the full row set, match + * the incoming ids, and (for every verdict except `diverge`) persist the + * new canon. A `diverge` turn writes NOTHING (zero side effects). + * + * The pos values the verdict issues are the LOGICAL append-only ledger + * positions (maxPos+1..); the physical snap_rows pos (ledger seq) is + * assigned by the ingest path in the same incoming order, so the two agree + * for tail-appends. For scatter (middle insertion) the reconcile re-homes + * the new rows to their envelope pos in the same transaction. */ +export function stage0Decide(input: Stage0Input): Stage0Result { + const now = input.now ?? Date.now(); + const l1 = snapL1Lookup(input.db, input.sessRaw); + const env = snapL1EnvelopeRead(input.db, input.sessRaw); + + // Fold-explained gone: a stored id may legally vanish from the envelope + // head when its row was deactivated by a fold (coverRef set). Read the + // FULL row set (active + inactive) — the egress-side view filters the + // inactive rows out and would under-explain. + const rows = snapRowsAll(input.db, input.ledgerId); + const explainedGone = new Set(); + for (const r of rows) { + if (r.active === 0 && r.coverRef !== null) { + for (const id of r.coreIds) explainedGone.add(id); + } + } + + const stored = + l1 && env && env.envelopeJson !== null && env.envelopeChecksum !== null + ? { + ledgerId: l1.ledgerId, + envelope: parseEnvelope(env.envelopeJson).envelope, + envelopeChecksum: env.envelopeChecksum, + issuedAt: l1.issuedAt, + armedAt: l1.armedAt ?? undefined, + } + : null; + + // M2 pure append (brief 19544722): the client sent ONLY new elements — + // every incoming id is unknown to the canon — and the seqs carrier says + // they land as a contiguous run starting exactly after the plane's max + // stamped seq. That is the 世界匹配定位序列位置 evidence: issue append- + // only pos (maxPos+1..) without any head/order risk. Anything less + // certain (missing seqs, a gap, a known id mixed in) falls through to + // matchWorld — an all-unknown full array there is usually a re-mint and + // diverges → resync, one-round convergence. Never guess. + let decision: WorldDecision; + const seqs = input.seqs ?? null; + const m2Candidate = + stored !== null && + !input.resync && + input.incoming.length > 0 && + seqs !== null && + seqs.length === input.incoming.length; + if (m2Candidate) { + const storedSet = new Set(stored.envelope.map((e) => e.id)); + const allUnknown = input.incoming.every((id) => !storedSet.has(id)); + // The plane's max stamped seq: rows are stamped ledgerSeq = pos+1 + // (the D-lane contract, snapshot-render.ts:154), so the frontier is + // derivable from the row pos alone — no post_json parse. + let planeMaxSeq = 0; + for (const r of rows) { + if (r.active === 1 && r.pos + 1 > planeMaxSeq) planeMaxSeq = r.pos + 1; + } + let contig = seqs[0] === planeMaxSeq + 1; + if (contig) { + for (let i = 1; i < seqs.length; i++) { + if (seqs[i] !== seqs[i - 1] + 1) { + contig = false; + break; + } + } + } + decision = allUnknown && contig + ? matchAppend(stored, input.incoming) + : matchWorld({ + stored, + incoming: input.incoming, + explainedGone, + sessRaw: input.sessRaw, + ledgerId: input.ledgerId, + now, + resync: input.resync, + }); + } else { + decision = matchWorld({ + stored, + incoming: input.incoming, + explainedGone, + sessRaw: input.sessRaw, + ledgerId: input.ledgerId, + now, + resync: input.resync, + }); + } + + // Persist the canon for every verdict EXCEPT diverge (zero side effects). + if (decision.mode !== "diverge") { + snapL1EnvelopeUpsert( + input.db, + input.sessRaw, + JSON.stringify(decision.envelope), + decision.envelopeChecksum, + ); + } + + return { decision, diverge: decision.mode === "diverge", explainedGone }; +} + +/** The R3 identity check for a cohort host that DECLARED the dual-channel: + * the per-element msgid list must cover EVERY element of the body. A short + * (or empty, when the body is non-empty) collection is the identity gate + * fault → the caller throws FoundationRejectError (400, zero side effects) + * and records the fault on the tracker. + * + * @param declared the host profile declares per-element msgids (cohort). + * @param elementCount the body element count (the msgs to be parsed). + * @param msgids the collected per-element msgids (null/undefined = + * the host sent none). + * @returns true when the turn may proceed (non-cohort, or full collection). + */ +export function checkR3Identity( + declared: boolean, + elementCount: number, + msgids: string[] | null | undefined, +): boolean { + if (!declared) return true; // non-cohort: sticky order>hash degrade, same tables + if (elementCount === 0) return true; + if (!msgids || msgids.length !== elementCount) return false; + return true; +} diff --git a/src/snapshot/types.ts b/src/snapshot/types.ts new file mode 100644 index 0000000000..255380534e --- /dev/null +++ b/src/snapshot/types.ts @@ -0,0 +1,68 @@ +/** + * #5874 / 2×2 — snapshot-foundation stage-0 shared types. + * + * TYPE-ONLY surface. Depends on nothing but the kernel's public types so it + * can be imported from any layer (storage, server, tests) without a cycle. + * + * Aligned to the reference implementation (#6155 `src/snapshot/`) — the stage-0 + * world-match / hydrate / lane decision vocabulary is shared verbatim. The + * PLANE itself stays on the #5901 message-level model (one `snap_rows` row per + * ledger element, `post_json` = the post-rewrite common-core BiliMessage JSON, + * `pos` = the append-only ledger position), so the reconcile/egress event + * types live in `src/storage/snap.ts` (unchanged) and are not re-declared here. + */ + +/** One entry in the standing canon envelope: a host msgid bound to the + * append-only ledger pos it was first issued at (pos is never reused — the + * DDL invariant). The canon is stored as an ordered [{id, pos}] list so the + * L1 table is self-contained (world = id + pos, §4.0 ruling G). */ +export interface WorldEnvelopeEntry { + id: string; + pos: number; +} + +/** World-match verdicts for stage 0 (§4.0). Executed in order; first hit + * wins. `pos` is append-only: a middle insertion does NOT shift stored rows. + * - `echo`: the incoming id sequence matches the stored one positionally. + * - `probe`: the stored id sequence is a strict PREFIX of the incoming one + * (tail append — the common, v5-proven shape) or a canon shrink. + * - `scatter`: known ids keep their relative order but NEW ids are inserted + * in the middle / at any position (the span-parse case, T5 byte pin). + * - `issue`: no stored world for this sess_raw → fresh ledger identity + arm. + * - `diverge`: the live head changed, or a KNOWN id changed its relative + * order (history rewrite) → this turn runs legacy and re-stamps; the canon + * is NOT updated (the client converges in one turn). + * - `resync`: an explicit re-baseline (mode switch / ledger re-issue). + * + * Deletions (a stored id absent from the incoming) are orthogonal to the mode + * and are reported via `goneIds` → the reconcile turns them into `delete` + * events (not a divergence while the head + survivor order are unchanged). */ +type WorldMode = "echo" | "probe" | "scatter" | "issue" | "diverge" | "resync"; + +export interface WorldDecision { + mode: WorldMode; + /** The canon envelope to persist after this turn (host ids in host order, + * each bound to its append-only pos). For `diverge` this is the UNCHANGED + * stored canon. */ + envelope: WorldEnvelopeEntry[]; + /** sha256 of the id sequence (host order) — the fast mismatch before the + * positional walk. */ + envelopeChecksum: string; + /** The NEW ids issued this turn (id → fresh pos), in incoming order. + * Empty for echo / diverge. */ + newIdPos: Array<{ id: string; pos: number }>; + /** The stored ids ABSENT from the incoming (client deletions) — the + * reconcile turns these into `delete` events. Empty for issue / echo. */ + goneIds: string[]; + /** For `issue`: the freshly issued ledger identity (the existing sid + * keyspace, ruling D). Absent otherwise. */ + issuedLedgerId?: string; + /** The world origin (issued_at) — carried so the caller can preserve it + * on a plain update (B1 trap: it never moves except on issue/resync). */ + issuedAt: number; + /** The arming-turn marker set on `issue`/`resync`. */ + armedAt?: number; + /** Diagnostic witness sample (echo spot-check) — warn stream only. */ + witness?: string; +} + diff --git a/src/storage/schema.ts b/src/storage/schema.ts index dcda19d760..2fb918de61 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -360,7 +360,9 @@ CREATE TABLE IF NOT EXISTS l1_world ( ledger_id TEXT NOT NULL, witness_probe TEXT NOT NULL, issued_at INTEGER NOT NULL, - armed_at INTEGER + armed_at INTEGER, + envelope_json TEXT, + envelope_checksum TEXT ); CREATE INDEX IF NOT EXISTS idx_l1_ledger ON l1_world(ledger_id); CREATE TABLE IF NOT EXISTS snap_rows ( @@ -415,6 +417,19 @@ export function migrateSchema(db: { .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'snap_rows'") .get(); if (!hasSnapRows) db.transaction(() => db.exec(SNAPBASE_DDL_V12)); + // 2×2 (#5874 brief, element-level world): the L1 world carries the + // per-element msgid ENVELOPE ({id, pos} canon, §4.0) in addition to the + // session-level witness_probe. Pure additive probe-first ALTER (a dev + // window that created l1_world before this column exists upgrades with + // one ALTER; a fresh DB already has it from the DDL above). The 2×2 + // stage-0 (src/snapshot/identity.ts matchWorld) reads/writes these; the + // session-level v1 path (#5901) leaves them NULL and is unaffected. + const l1Cols = (db.prepare("PRAGMA table_info(l1_world)").all() as unknown as { name?: unknown }[]) + .map((c) => c.name); + if (l1Cols.length > 0) { + if (!l1Cols.includes("envelope_json")) db.exec("ALTER TABLE l1_world ADD COLUMN envelope_json TEXT"); + if (!l1Cols.includes("envelope_checksum")) db.exec("ALTER TABLE l1_world ADD COLUMN envelope_checksum TEXT"); + } if (from >= STORAGE_SCHEMA_VERSION) return { from, to: from }; let needV8Backfill = false; db.transaction(() => { diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index b0f026b682..05ad7ecac3 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -988,6 +988,30 @@ export class UnifiedShadow { return this.sync.snapEgressView(sessionId); } + // #5874 2×2 stage 0 (brief 19544722): the element world-match accessors. + // Errors propagate (the lane maps the R3 reject to a hard 400 and every + // other fault to the legacy full path) — unlike snapEgressView these may + // NOT swallow. + + /** Stage 0 for one turn (see UnifiedStore.snapStage0). null when there + * is no sync store. */ + snapStage0(sessionId: string, incoming: string[], opts?: { resync?: boolean, seqs?: number[] | null }): import("../snapshot/plane.js").Stage0Result | null { + if (this.sync === null) return null; + return this.sync.snapStage0(sessionId, incoming, opts); + } + + /** The plane's ACTIVE cores, pos-ordered (old cores from the table). */ + snapStage0Cores(sessionId: string): { rows: { pos: number; ids: string[]; postJson: string }[]; coverMax: Record } | null { + if (this.sync === null) return null; + return this.sync.snapStage0Cores(sessionId); + } + + /** The stage-0 resync (R4 / table-behind): reset the session plane. */ + snapStage0Reset(sessionId: string): void { + if (this.sync === null) return; + this.sync.snapStage0Reset(sessionId); + } + vacuum(mode: "full" | "incremental"): void { if (this.sync === null) throw new Error("[storage] sync vacuum unavailable on a network backend"); return this.sync.vacuum(mode); diff --git a/src/storage/snap.ts b/src/storage/snap.ts index 823c6f1645..79aadeb3d1 100644 --- a/src/storage/snap.ts +++ b/src/storage/snap.ts @@ -114,6 +114,12 @@ export function isSnapPlaneDead(sid: string): boolean { export function markSnapPlaneDead(sid: string): void { deadSids.add(sid); } +/** 2×2 stage-0 resync: clear ONE sid's sticky dead latch (the plane is + * being re-baselined from the client's current array — a stale R1 verdict + * about the OLD rows must not shadow the new plane). */ +export function clearSnapPlaneDead(sid: string): void { + deadSids.delete(sid); +} /** Test-only: clear sticky flags between hermetic fixtures. */ export function _resetSnapPlaneDeadForTests(): void { deadSids.clear(); @@ -168,8 +174,13 @@ function snapAggregate(rows: { * count, not its message count, allocates anchor positions). */ const ANCHOR_POS_MIN = -100000; +/** #5874 2×2: the stored anchor-row id prefix (acp_summary_). The + * write side, the D view and the 2×2 stage-0 assembly all key on it — + * one constant, one source of truth. */ +export const ANCHOR_ID_PREFIX = "acp_summary_"; + function isAnchorPosDomain(pos: number, coreIds: string[] | undefined): boolean { - return pos >= ANCHOR_POS_MIN && pos < 0 && coreIds !== undefined && coreIds.some((id) => id.startsWith("acp_summary_")); + return pos >= ANCHOR_POS_MIN && pos < 0 && coreIds !== undefined && coreIds.some((id) => id.startsWith(ANCHOR_ID_PREFIX)); } /** Validate event payloads (invariant v + addressing sanity). Throws on @@ -245,7 +256,10 @@ function requireCoreShape(sid: string, ev: string, pos: number, json: string): v } } -interface DbLike { +/** The minimal db surface the plane functions need (exported so the stage-0 + * decision module (src/snapshot/plane.ts) can be type-coupled to it without + * importing a driver). */ +export interface DbLike { exec(sql: string, ...params: unknown[]): unknown; prepare(sql: string): { run(...params: unknown[]): unknown; @@ -677,6 +691,77 @@ export function snapGcCoveredRows(db: DbLike, sid: string): number { }); } +// --------------------------------------------------------------------------- +// 2×2 (proposal §4.0, brief 19544722) — the element-level WORLD ENVELOPE. +// +// The session-level v1 path above (snapL1Record/snapL1Lookup) stays the +// identity ANCHOR (sess_raw → ledger_id + one-round arming). The 2×2 stage 0 +// adds, on the same row, the per-element msgid envelope: an ordered +// [{id, pos}] canon binding every host msgid to its append-only ledger pos +// (ruling G — the table is self-contained). The stage-0 decision function +// (src/snapshot/identity.ts matchWorld) runs on this envelope; the plane rows +// (snap_rows) keep the #5901 message-level model (pos = ledger seq). +// --------------------------------------------------------------------------- + +interface L1Envelope { + /** The canon envelope JSON: an array of {id, pos} in host order. + * Null when the session is still on the session-level v1 path (the + * 2×2 element world has not been issued for it yet). */ + envelopeJson: string | null; + /** sha256 of the canonical envelope (fast mismatch before the walk). */ + envelopeChecksum: string | null; +} + +/** Read the element-level envelope for a raw session id (null when the l1 + * row is absent; envelopeJson null when present but not yet 2×2-issued). */ +export function snapL1EnvelopeRead(db: DbLike, sessRaw: string): L1Envelope | null { + const r = db + .prepare("SELECT envelope_json, envelope_checksum FROM l1_world WHERE sess_raw = ?") + .get(sessRaw) as { envelope_json: string | null; envelope_checksum: string | null } | undefined; + if (!r) return null; + return { envelopeJson: r.envelope_json, envelopeChecksum: r.envelope_checksum }; +} + +/** Persist the element-level envelope for a raw session id (UPDATE only — + * the l1 row itself is created by snapL1Record; a missing row is a + * programming error and is surfaced, not silently created). */ +export function snapL1EnvelopeUpsert(db: DbLike, sessRaw: string, envelopeJson: string, envelopeChecksum: string): void { + const res = db + .prepare("UPDATE l1_world SET envelope_json = ?, envelope_checksum = ? WHERE sess_raw = ?") + .run(envelopeJson, envelopeChecksum, sessRaw) as { changes: number }; + if (res.changes === 0) throw new Error(`snapL1EnvelopeUpsert: no l1_world row for sess_raw ${sessRaw.slice(0, 12)}… (record it first)`); +} + +// --------------------------------------------------------------------------- +// 2×2 read side — the FULL row set (active AND inactive), for the stage-0 +// hydrate: S4 drift checks, explainedGone (fold-covered ids may legally +// vanish from the head) and the coverage evidence all need the deactivated +// rows, which snapRowsView/readSnapView filter out of the egress path. +// --------------------------------------------------------------------------- + +interface SnapRowAll { + pos: number; + coreIds: string[]; + postJson: string | null; + active: 0 | 1; + coverRef: string | null; +} + +/** Every snap_rows entry for a sid, pos-ordered (active + inactive). Unlike + * readSnapView this does NOT verify S4 and does NOT latch R1 — it is a raw + * evidence read for the stage-0 decision; the verification path (readSnapView) + * is unchanged and remains the only S4 authority. */ +export function snapRowsAll(db: DbLike, sid: string): SnapRowAll[] { + const rows = readRawRows(db, sid); + return rows.map((r) => ({ + pos: r.pos, + coreIds: JSON.parse(r.core_ids) as string[], + postJson: r.post_json, + active: (r.active ? 1 : 0) as 0 | 1, + coverRef: r.cover_ref, + })); +} + // --------------------------------------------------------------------------- // Status (wire-safe, for the /__bili/status attribution surface). // --------------------------------------------------------------------------- diff --git a/src/storage/store.ts b/src/storage/store.ts index b70a9b99dd..f2a127e792 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -25,16 +25,19 @@ import { migrateSchema, storagePragmas, WAL_JOURNAL_SIZE_LIMIT } from "./schema. import { identityOfStoredContent, splitMediaContent } from "./identity.js"; import { applySnapInTransaction, + clearSnapPlaneDead, isSnapPlaneDead, markSnapPlaneDead, readSnapView, snapGcCoveredRows, snapL1Record, snapplaneStatusWire, + snapRowsAll, SnapPlaneDeadError, type SnapEvent, type SnapView, } from "./snap.js"; +import { FoundationRejectError, stage0Decide, type Stage0Result } from "../snapshot/plane.js"; import { recordIntegrityOutcome, recordSecondProcessOwner, recordBackupOutcome } from "./runtime-status.js"; import { integrityModeFor, writeIntegrityCache, scheduleBackgroundIntegrityCheck, backgroundIntegrityInflightForTests, noteIntegrityCheckRun } from "./integrity-check.js"; import { sqliteCorruptionDetail } from "./sqlite-errors.js"; @@ -2411,6 +2414,81 @@ export class UnifiedStore { } } + // #5874 2×2 stage 0 (brief 19544722): the element world match lives at the + // head of the chain, BEFORE protocol conversion. These accessors are the + // storage side of that stage — the pure decision (snapshot/plane.ts) and + // the kernel-input assembly (snapshot/assemble.ts) are lane-side. + + private snapSessRawOf(sid: string): string { + // Raw host session id (P9: never truncated) → the l1_world key; the + // conversation binding when the host supplied one, the sid otherwise. + // (Same resolution as the save path below — stage 0 and the plane + // commit must address the SAME l1 row.) + return ( + (this.db.prepare("SELECT conversation_id FROM conversations WHERE session_id = ?").get(sid) as { conversation_id: string } | undefined) + ?.conversation_id ?? sid + ); + } + + /** Stage 0 for one turn: ensure the l1 row exists (idempotent session- + * level v1 record), run the element world match on the stored envelope, + * and persist the new canon (every verdict except `diverge` — that one + * has zero side effects by construction). A session-level identity + * conflict ("red") throws — the lane maps it to the R3 hard 400 BEFORE + * any kernel side effect. + */ + snapStage0(sid: string, incoming: string[], opts?: { resync?: boolean, seqs?: number[] | null }): Stage0Result { + const sessRaw = this.snapSessRawOf(sid); + const now = Date.now(); + const l1 = snapL1Record(this.db, sessRaw, sid, this.snapWitnessProbe(sid), now); + if (l1 === "red") { + // Session-level identity conflict (R3): the raw host id maps to a + // DIFFERENT ledger. Hard-reject BEFORE any kernel side effect — + // snapL1Record wrote nothing on this path. + throw new FoundationRejectError(`l1 ledger conflict: sess_raw=${sessRaw} maps to another ledger (R3)`); + } + return stage0Decide({ db: this.db, sessRaw, ledgerId: sid, incoming, seqs: opts?.seqs ?? null, resync: opts?.resync, now }); + } + + /** The plane's ACTIVE rows, pos-ordered — the "old cores from the table" + * half of the kernel-input assembly. post_json is the tagged kernel + * byte (invariant v) in its STORED (media-offloaded) form; the caller + * hydrates before feeding the kernel. `coverMax` maps each active + * block id to the HIGHEST row pos it covers (over the FULL row set — + * the covered rows themselves) — the kernel-input anchor placement + * input (an anchor renders before the first surviving element above + * its covered run). */ + snapStage0Cores(sid: string): { rows: { pos: number; ids: string[]; postJson: string }[]; coverMax: Record } { + const all = snapRowsAll(this.db, sid); + const coverMax: Record = {}; + for (const r of all) { + if (r.coverRef !== null && r.pos > (coverMax[r.coverRef] ?? -1)) { + coverMax[r.coverRef] = r.pos; + } + } + const rows = all + .filter((r) => r.active === 1 && r.postJson !== null) + .map((r) => ({ pos: r.pos, ids: r.coreIds, postJson: this.hydrateMedia(r.postJson as string) })); + return { rows, coverMax }; + } + + /** The stage-0 resync (brief red gate R4 / the resync ruling): reset the + * session's plane (rows + log + ckpt + the l1 element envelope + the + * sticky R1 latch) in one txn. This turn then runs the legacy full path + * and re-seeds the plane through the normal in-request flush; the NEXT + * turn re-issues the envelope from the client's current array (one-round + * convergence). */ + snapStage0Reset(sid: string): void { + const sessRaw = this.snapSessRawOf(sid); + this.db.transaction(() => { + this.db.prepare("DELETE FROM snap_rows WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM snap_log WHERE sid = ?").run(sid); + this.db.prepare("DELETE FROM snap_ckpt WHERE sid = ?").run(sid); + this.db.prepare("UPDATE l1_world SET envelope_json = NULL, envelope_checksum = NULL WHERE sess_raw = ?").run(sessRaw); + }); + clearSnapPlaneDead(sid); + } + /** #3250: cadence-tick WAL reclamation (see WAL_CHECKPOINT_EVERY_INGESTS). * Never throws into the ingest path — a failed checkpoint just leaves the * WAL longer than ideal until the next tick. */ diff --git a/tests/issue5874-2x2-p3.test.ts b/tests/issue5874-2x2-p3.test.ts new file mode 100644 index 0000000000..f6270e709c --- /dev/null +++ b/tests/issue5874-2x2-p3.test.ts @@ -0,0 +1,661 @@ +/** + * #5874 2×2 — stage-0 identity enrollment (M1/M2 inbound) + seed/delta + * egress (M1–M4 outbound), acceptance items from the brief (comment + * 19544722, tracking #6181). + * + * M1 full-inbound + snapshot egress (the #5901 behavior, extended) + * M2 incremental-inbound + snapshot (this PR: matchAppend + seqs carrier) + * M3 incremental-inbound + delta (this PR: selectOutbound, responses) + * M4 full-inbound + delta (this PR: selectOutbound, responses) + * + * Unit scope (the lane wiring is e2e-verified by CI; these are the brief's + * acceptance pins 1–8 + the red-gate matrix): + * W* the world verdicts — echo/probe/scatter/issue/diverge/resync ± pairs + * (acceptance 5), the B1 origin-immutability trap, matchAppend. + * A* assembly — #2 M1-vs-M2 kernel-input byte identity, T5 scattered NEW + * (acceptance 4), #7 resync one-round convergence (acceptance 7), + * R4 table-gap throw, anchor placement. + * D* delta egress — #8 seed/delta/no-op pins (acceptance 8). + * R* red gates — R3 identity fault (400 + zero side effects + breaker), + * R2 diverge zero side effects. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import type { BiliMessage } from "acp-kernel/wire"; +import { planIngest, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import { + matchWorld, + matchAppend, +} from "../src/snapshot/identity.ts"; +import type { WorldEnvelopeEntry, WorldDecision } from "../src/snapshot/types.ts"; +import { envelopeChecksum } from "../src/snapshot/checksums.ts"; +import { + assembleKernelInput, + shouldResync, +} from "../src/snapshot/assemble.ts"; +import { + selectOutbound, + isFullResend, + type EgressElement, + type DeltaState, +} from "../src/snapshot/delta.ts"; +import { + FoundationRejectError, + R3FaultTracker, + checkR3Identity, + stage0Decide, +} from "../src/snapshot/plane.ts"; +import { snapL1Lookup, snapL1EnvelopeRead, snapL1Record } from "../src/storage/snap.ts"; + +process.env.BILI_PERSIST_ZSTD = "0"; +const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-5874-p3-")); +const configPath = path.join(hermetic, "config.json"); +fs.writeFileSync(configPath, "{}"); + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +type StoredCanon = { + ledgerId: string; + envelope: WorldEnvelopeEntry[]; + envelopeChecksum: string; + issuedAt: number; + armedAt?: number; +}; + +interface WorldMatchInputLocal { + stored: StoredCanon | null; + incoming: string[]; + explainedGone?: Set; + sessRaw: string; + ledgerId: string; + now: number; + resync?: boolean; +} + +const now0 = 1_700_000_000_000; + +/** A stored element envelope (canon) for ids 0..n-1 (1-based ids m1.. + * 1-based pos, the issue/resync shape). */ +function canon(n: number): StoredCanon { + const ids = Array.from({ length: n }, (_, i) => `m${i + 1}`); + return { + ledgerId: "sid-t", + envelope: ids.map((id, i) => ({ id, pos: i + 1 })), + envelopeChecksum: envelopeChecksum(ids), + issuedAt: now0, + }; +} + +function matchInput(stored: StoredCanon | null, incoming: string[], extra: Partial = {}): WorldMatchInputLocal { + return { + stored, + incoming, + sessRaw: "sess-t", + ledgerId: "sid-t", + now: now0 + 1, + ...extra, + }; +} + +function openPlane(p: string): UnifiedStore { + return UnifiedStore.open(p, undefined, { snapBase: { enabled: true, shadowWrite: true } }); +} + +/** A plain (untagged) core message — the lane's converted form. */ +function core(i: number, id?: string): BiliMessage { + return { + id: id ?? `m${i}`, + role: (i % 2 === 0 ? "user" : "assistant") as "user" | "assistant", + contentType: "text", + text: `message ${i}`, + } as BiliMessage; +} + +function plainSession(id: string, n: number): LegacySessionLike { + const msgs = Array.from({ length: n }, (_, i) => ({ + id: `m${i + 1}`, + role: (i % 2 === 0 ? "user" : "assistant") as "user" | "assistant", + contentType: "text" as const, + text: `message ${i + 1}`, + })); + const byRaw: Record = {}; + const byRef: Record = {}; + msgs.forEach((m, i) => { + byRaw[m.id] = `m${String(i + 1).padStart(5, "0")}`; + byRef[`m${String(i + 1).padStart(5, "0")}`] = m.id; + }); + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "openai" }, + stats: { requests: 1 }, + metadata: {}, + state: { blocks: [], nextBlockId: 1, nextRunId: 1, messageRefs: { byRaw, byRef } }, + blockContents: new Map(), + lastMessages: msgs, + lastMessagesFolded: true, + pluginSnapshot: undefined, + contentStore: undefined, + }; +} + +// --------------------------------------------------------------------------- +// W — the world verdicts (acceptance 5: each verdict with paired +/- tests) +// --------------------------------------------------------------------------- + +test("W1 echo: the same envelope checksums → echo, canon untouched, B1 origin frozen", () => { + const stored = canon(4); + const ids = stored.envelope.map((e) => e.id); + const d = matchWorld(matchInput(stored, ids)); + assert.equal(d.mode, "echo"); + assert.deepEqual(d.newIdPos, []); + assert.deepEqual(d.goneIds, []); + assert.equal(d.issuedAt, now0, "B1: the origin never moves on echo"); + assert.equal(d.envelopeChecksum, stored.envelopeChecksum); +}); + +test("W2 echo +/- : one id changed → NOT echo (the checksum is content-binding)", () => { + const stored = canon(4); + const d = matchWorld(matchInput(stored, ["m1", "m2", "m3", "mX"])); + assert.notEqual(d.mode, "echo"); + // mX is a new interleaved span (the head is stable, survivor order holds) + // → scatter; the replaced m4 is reported gone (reconcile: delete). + assert.equal(d.mode, "scatter"); + assert.deepEqual(d.newIdPos, [{ id: "mX", pos: 5 }]); + assert.deepEqual(d.goneIds, ["m4"]); +}); + +test("W3 probe: a tail extension → probe, fresh pos at maxPos+1", () => { + const d = matchWorld(matchInput(canon(4), ["m1", "m2", "m3", "m4", "m5"])); + assert.equal(d.mode, "probe"); + assert.deepEqual(d.newIdPos, [{ id: "m5", pos: 5 }]); + assert.deepEqual(d.goneIds, []); +}); + +test("W4 probe +/- : the extension with a GAP in the middle → scatter", () => { + const d = matchWorld(matchInput(canon(4), ["m1", "m2", "NEW", "m3", "m4"])); + assert.equal(d.mode, "scatter", "an interleaved new id is a scatter, not a probe"); + assert.deepEqual(d.newIdPos, [{ id: "NEW", pos: 5 }]); +}); + +test("W5 issue: no stored canon → issue, fresh pos 0.., armed now", () => { + const d = matchWorld(matchInput(null, ["a", "b"])); + assert.equal(d.mode, "issue"); + assert.deepEqual(d.newIdPos, [{ id: "a", pos: 1 }, { id: "b", pos: 2 }]); + assert.equal(d.armedAt, now0 + 1, "the issue arms the lane now"); +}); + +test("W6 issue +/- : resync with a stored canon → re-issue pos 1.. from the client array", () => { + const d = matchWorld(matchInput(canon(3), ["x", "y", "z"], { resync: true })); + assert.equal(d.mode, "resync"); + assert.deepEqual(d.newIdPos, [{ id: "x", pos: 1 }, { id: "y", pos: 2 }, { id: "z", pos: 3 }]); +}); + +test("W7 diverge: head moved → diverge, canon untouched", () => { + const d = matchWorld(matchInput(canon(4), ["OTHER", "m1", "m2", "m3"])); + assert.equal(d.mode, "diverge"); + assert.equal(d.witness, "head m1 -> OTHER; orderBroken=false"); +}); + +test("W8 diverge: survivor order broken → diverge", () => { + const d = matchWorld(matchInput(canon(4), ["m1", "m3", "m2", "m4"])); + assert.equal(d.mode, "diverge"); + assert.equal(d.witness, "head m1 -> m1; orderBroken=true"); +}); + +test("W9 diverge +/- : a fold-explained gone at the head is NOT a head move", () => { + const d = matchWorld(matchInput(canon(4), ["m2", "m3", "m4"], { explainedGone: new Set(["m1"]) })); + assert.notEqual(d.mode, "diverge"); + assert.equal(d.mode, "probe", "the shrink with a fold-explained gone is a deletions-only probe"); + assert.deepEqual(d.goneIds, ["m1"]); +}); + +test("W10 matchAppend: pure M2 addition — canon order preserved, whole-sequence checksum", () => { + const stored = canon(4); + const d = matchAppend(stored, ["m5", "m6"]); + assert.equal(d.mode, "probe"); + assert.deepEqual( + d.envelope.map((e) => [e.id, e.pos]), + [["m1", 1], ["m2", 2], ["m3", 3], ["m4", 4], ["m5", 5], ["m6", 6]], + ); + assert.deepEqual(d.newIdPos, [{ id: "m5", pos: 5 }, { id: "m6", pos: 6 }]); + // The next M1 full resend (all six ids) must ECHO against this canon. + assert.equal(d.envelopeChecksum, envelopeChecksum(["m1", "m2", "m3", "m4", "m5", "m6"])); + assert.equal(d.issuedAt, now0, "B1: the origin never moves on extend"); +}); + +test("W11 matchAppend +/- : the next M1 full resend echoes (the round-trip)", () => { + const stored = canon(4); + const app = matchAppend(stored, ["m5"]); + const d = matchWorld(matchInput( + { ...stored, envelopeChecksum: app.envelopeChecksum, envelope: app.envelope }, + ["m1", "m2", "m3", "m4", "m5"], + )); + assert.equal(d.mode, "echo"); +}); + +// --------------------------------------------------------------------------- +// A — assembly (acceptance 2, 4, 7 + R4 + anchors) +// --------------------------------------------------------------------------- + +function storeWithRows(n: number, sid: string): UnifiedStore { + const store = openPlane(path.join(hermetic, `a-${sid}-${n}`, "index.db")); + store.ingestSession(planIngest(plainSession(sid, n))); + return store; +} + +test("A1 #2 M1 vs M2: same turn, full-inbound vs incremental-inbound kernel input byte-identical", () => { + const sid = "a1"; + const store = storeWithRows(10, sid); + const cores = store.snapStage0Cores(sid); + assert.equal(cores.rows.length, 10); + + const newCore = core(11); + const rowCores = cores.rows.map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })); + const ten = Array.from({ length: 10 }, (_, i) => `m${i + 1}`); + + // Seed the world (first turn: the client's full array of 10). + const seed = store.snapStage0(sid, ten); + assert.equal(seed.decision.mode, "issue"); + + // M1: the client full-resends all 11 (the plane holds 10 → the 11th is + // the only new span). seqs present but the array is NOT all-unknown → + // the matchWorld path (probe/scatter), new id appended. + const m1 = store.snapStage0(sid, [...ten, newCore.id], { seqs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] }); + assert.equal(m1.decision.mode, "probe"); + assert.deepEqual(m1.decision.newIdPos, [{ id: "m11", pos: 11 }]); + const asm1 = assembleKernelInput({ decision: m1.decision, rows: rowCores, newCores: [newCore] }); + + // A FRESH store for the M2 leg (the same plane state, a different client + // behavior on the SAME turn): the client sends ONLY the new element, + // with the seqs carrier locating it after the plane frontier (10). + const store2 = storeWithRows(10, sid); + // Enroll the canon first (the M2 branch requires a stored canon): the + // seed turn was the M1 full resend of 10. + store2.snapStage0(sid, ten); + const m2 = store2.snapStage0(sid, [newCore.id], { seqs: [11] }); + assert.equal(m2.decision.mode, "probe", "a contiguous all-unknown seqs run is the M2 append"); + assert.deepEqual(m2.decision.newIdPos, [{ id: "m11", pos: 11 }]); + const cores2 = store2.snapStage0Cores(sid); + const asm2 = assembleKernelInput({ + decision: m2.decision, + rows: cores2.rows.map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })), + newCores: [newCore], + }); + + assert.equal(asm1.oldCount, 10); + assert.equal(asm2.oldCount, 10); + assert.equal( + JSON.stringify(asm1.kernelInput), + JSON.stringify(asm2.kernelInput), + "#2: the assembled kernel input cores are byte-identical (M1 vs M2)", + ); +}); + +test("A1b #2 +/- : the M2 branch REQUIRES the seqs carrier (no seqs → matchWorld path)", () => { + const sid = "a1b"; + const store = storeWithRows(10, sid); + store.snapStage0(sid, Array.from({ length: 10 }, (_, i) => `m${i + 1}`)); + // All-unknown WITHOUT seqs: a full (re-minted) envelope → matchWorld → + // the old canon ids are gone → head moved → diverge (never guess). + const d = store.snapStage0(sid, ["m11"]); + assert.equal(d.decision.mode, "diverge"); +}); + +test("A1c #2 +/- : a non-contiguous seqs run falls back to matchWorld", () => { + const sid = "a1c"; + const store = storeWithRows(10, sid); + store.snapStage0(sid, Array.from({ length: 10 }, (_, i) => `m${i + 1}`)); + const d = store.snapStage0(sid, ["m11"], { seqs: [15] }); + assert.notEqual(d.decision.mode, "probe", "seqs[0]=15 != frontier+1=11 → not the M2 append"); +}); + +test("A2 T5: a scattered NEW span lands at its CONVERSATION position (envelope order, not pos order)", () => { + const sid = "a2"; + const store = storeWithRows(10, sid); + const ten = Array.from({ length: 10 }, (_, i) => `m${i + 1}`); + store.snapStage0(sid, ten); // seed (issue) + const d = store.snapStage0(sid, ["m1", "m2", "m3", "NEW", "m4", "m5", "m6", "m7", "m8", "m9", "m10"]); + assert.equal(d.decision.mode, "scatter"); + assert.deepEqual(d.decision.newIdPos, [{ id: "NEW", pos: 11 }]); + const cores = store.snapStage0Cores(sid); + const newCore = { ...core(1), id: "NEW", text: "edited message" } as BiliMessage; + const asm = assembleKernelInput({ + decision: d.decision, + rows: cores.rows.map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })), + newCores: [newCore], + }); + const ids = asm.kernelInput.map((m) => m.id); + assert.deepEqual(ids, ["m1", "m2", "m3", "NEW", "m4", "m5", "m6", "m7", "m8", "m9", "m10"]); + assert.equal(asm.kernelInput[3]!.text, "edited message"); + assert.equal(asm.oldCount, 10); +}); + +test("A3 R4: an envelope element that is neither new nor a plane row → throw (table gap)", () => { + const stored = canon(4); + const d = matchWorld(matchInput(stored, ["m1", "m2", "m3", "m4", "m5"])); + // Only rows 1-3 exist (the table is behind the client). + assert.throws( + () => + assembleKernelInput({ + decision: d, + rows: [ + { pos: 0, ids: ["m1"], msg: core(1) }, + { pos: 1, ids: ["m2"], msg: core(2) }, + { pos: 2, ids: ["m3"], msg: core(3) }, + ], + newCores: [core(5, "m5")], + }), + /table behind the client/, + ); +}); + +test("A3b R4 +/- : a fold-explained gone in the envelope is SKIPPED (skipIds), no throw", () => { + const stored = canon(4); + // m1 was fold-covered (explained gone); the client still lists it. + const d = matchWorld(matchInput(stored, ["m1", "m2", "m3", "m4"], { explainedGone: new Set(["m1"]) })); + assert.equal(d.mode, "echo", "checksum match is positional — the gone is reconciled out of band"); + const asm = assembleKernelInput({ + decision: d, + rows: [ + { pos: 2, ids: ["m2"], msg: core(2) }, + { pos: 3, ids: ["m3"], msg: core(3) }, + { pos: 4, ids: ["m4"], msg: core(4) }, + ], + newCores: [], + skipIds: new Set(["m1"]), + }); + assert.deepEqual(asm.kernelInput.map((m) => m.id), ["m2", "m3", "m4"]); +}); + +test("A4 #7 resync: diverge → zero side effects (the canon is untouched), shouldResync true", () => { + const sid = "a4"; + const store = storeWithRows(10, sid); + const ten = Array.from({ length: 10 }, (_, i) => `m${i + 1}`); + const seed = store.snapStage0(sid, ten); + assert.equal(seed.decision.mode, "issue"); + const db = (store as unknown as { db: import("../src/storage/snap.ts").DbLike }).db; + const sessRaw = sid; // plainSession id == conversation id + const before = snapL1EnvelopeRead(db, sessRaw); + assert.ok(before && before.envelopeChecksum); + + const d = store.snapStage0(sid, ["OTHER", "m1", "m2"]); + assert.equal(d.decision.mode, "diverge"); + assert.equal(d.diverge, true); + assert.ok(shouldResync(d.decision, 10)); + + // Zero side effects: the diverge turn persisted nothing. + const after = snapL1EnvelopeRead(db, sessRaw); + assert.equal(after!.envelopeChecksum, before!.envelopeChecksum, "the diverge wrote nothing"); + + // The resync: reset + re-issue from the client's CURRENT array → + // one-round convergence. + store.snapStage0Reset(sid); + const re = store.snapStage0(sid, ["OTHER", "m1", "m2"], { resync: true }); + assert.equal(re.decision.mode, "resync"); + assert.deepEqual( + re.decision.envelope.map((e) => e.id), + ["OTHER", "m1", "m2"], + ); + assert.equal(shouldResync(re.decision, 0), true, "the resync turn itself resets (the lane re-seeds from it)"); +}); + +test("A4b #7 +/- : a plain issue on an EMPTY plane is the NORMAL seed (not a resync)", () => { + const sid = "a4b"; + const store = openPlane(path.join(hermetic, "a4b", "index.db")); + const d = store.snapStage0(sid, ["a", "b", "c"]); + assert.equal(d.decision.mode, "issue"); + assert.equal(shouldResync(d.decision, 0), false, "seed must not loop the reset"); +}); + +test("A5 anchor placement: the anchor renders before the first element past the covered run", () => { + const stored = canon(6); + const d = matchWorld(matchInput(stored, ["m1", "m2", "m3", "m4", "m5", "m6"])); + const rows = Array.from({ length: 6 }, (_, i) => ({ pos: i, ids: [`m${i + 1}`], msg: core(i + 1) })); + const anchor = { id: "acp_summary_b1", role: "system", contentType: "text", text: "[Compressed conversation section]" } as BiliMessage; + // The block covered rows through pos 3 (1-based: m1..m3); m1 (the pin) + // stays active, m2/m3 flipped. The served elements: pos 1..6 → the + // anchor sits before the first served element with pos > 3. + const asm = assembleKernelInput({ + decision: d, + rows, + newCores: [], + anchors: [{ msg: anchor, blockId: "b1" }], + coverMax: { b1: 3 }, + }); + const ids = asm.kernelInput.map((m) => m.id); + assert.equal(ids.indexOf("acp_summary_b1"), 3, "the anchor precedes m4 (pos 4, the first past the run)"); + assert.deepEqual(ids, ["m1", "m2", "m3", "acp_summary_b1", "m4", "m5", "m6"]); +}); + +test("A5b anchor +/- : no coverMax evidence (covered rows GC'd) → the view start", () => { + const stored = canon(3); + const d = matchWorld(matchInput(stored, ["m1", "m2", "m3"])); + const anchor = { id: "acp_summary_b1", role: "system", contentType: "text", text: "[Compressed conversation section]" } as BiliMessage; + const asm = assembleKernelInput({ + decision: d, + rows: [ + { pos: 1, ids: ["m1"], msg: core(1) }, + { pos: 2, ids: ["m2"], msg: core(2) }, + { pos: 3, ids: ["m3"], msg: core(3) }, + ], + newCores: [], + anchors: [{ msg: anchor, blockId: "b1" }], + }); + assert.deepEqual(asm.kernelInput.map((m) => m.id), ["acp_summary_b1", "m1", "m2", "m3"]); +}); + +test("A6 P5: the new-core count must match the verdict (a parse of a non-new span throws)", () => { + const stored = canon(2); + const d = matchWorld(matchInput(stored, ["m1", "m2"])); + assert.throws( + () => + assembleKernelInput({ + decision: d, + rows: [{ pos: 0, ids: ["m1"], msg: core(1) }, { pos: 1, ids: ["m2"], msg: core(2) }], + newCores: [core(9)], + }), + /newCores length/, + ); +}); + +// --------------------------------------------------------------------------- +// D — delta egress (acceptance 8) +// --------------------------------------------------------------------------- + +function elementsOf(n: number): EgressElement[] { + const els: EgressElement[] = Array.from({ length: n }, (_, i) => ({ + msg: core(i + 1), + pos: i, + isAnchor: false, + })); + els.splice(2, 0, { + msg: { id: "acp_summary_b1", role: "system", contentType: "text", text: "s" } as BiliMessage, + pos: -1, + isAnchor: true, + anchorId: "b1", + }); + return els; +} + +test("D1 #8 seed: the first round (null state) is a FULL resend of the whole view", () => { + const els = elementsOf(5); + const sel = selectOutbound(els, null); + assert.equal(sel.reseed, true); + // A reseed IS the full view (the lane sends the whole view; `delta` is + // empty and isFullResend only flags the delta-grown-to-full case). + assert.equal(isFullResend(sel, els), false); + assert.equal(sel.nextState.frontierPos, 4, "the seed frontier covers every sent data row"); + assert.deepEqual(sel.nextState.servedAnchors, ["b1"]); +}); + +test("D2 #8 delta: the next round sends ONLY rows past the frontier (plus unsent anchors)", () => { + const els = elementsOf(5); + const seed = selectOutbound(els, null); + // One new tail row (pos 5) + the anchor re-sent? NO — the anchor was + // already served: the delta is exactly the one new row. + const els2 = [...els, { msg: core(6), pos: 5, isAnchor: false }]; + const sel = selectOutbound(els2, seed.nextState); + assert.equal(sel.reseed, false); + assert.deepEqual(sel.delta.map((e) => e.pos), [5]); + assert.equal(sel.nextState.frontierPos, 5); + assert.deepEqual(sel.nextState.servedAnchors, ["b1"]); +}); + +test("D3 #8 no-op: a steady round with nothing new sends an empty delta (the plane round rides the provider cache)", () => { + const els = elementsOf(5); + const seed = selectOutbound(els, null); + const sel = selectOutbound(els, seed.nextState); + assert.equal(sel.reseed, false); + assert.deepEqual(sel.delta, []); +}); + +test("D4 #8 +/- : an unsent anchor (a NEW fold) rides the next delta", () => { + const els = elementsOf(5); + const seed = selectOutbound(els, null); + const els2 = [ + ...els, + { msg: { id: "acp_summary_b2", role: "system", contentType: "text", text: "s2" } as BiliMessage, pos: -2, isAnchor: true, anchorId: "b2" }, + { msg: core(6), pos: 5, isAnchor: false }, + ]; + const sel = selectOutbound(els2, seed.nextState); + assert.deepEqual(sel.delta.map((e) => e.anchorId ?? e.pos), ["b2", 5]); + assert.deepEqual(sel.nextState.servedAnchors.sort(), ["b1", "b2"]); +}); + +test("D5 isFullResend: the delta grown to the whole view is a re-seed in disguise", () => { + // A view with NO anchors: 5 data rows, state frontier -1 (nothing sent) + // → the delta covers every element → full resend. + const els: EgressElement[] = Array.from({ length: 5 }, (_, i) => ({ + msg: core(i + 1), + pos: i, + isAnchor: false, + })); + const state: DeltaState = { frontierPos: -1, servedAnchors: [] }; + const sel = selectOutbound(els, state); + assert.equal(isFullResend(sel, els), true, "5 of 5 unsent → the delta IS the full view"); + // +/- : the frontier already past the data, an UNSENT anchor rides alone + // → the delta is 1 element, NOT a full resend. + const els2 = [...els, { + msg: { id: "acp_summary_b9", role: "system", contentType: "text", text: "s" } as BiliMessage, + pos: -1, + isAnchor: true, + anchorId: "b9", + }]; + const sel2 = selectOutbound(els2, { frontierPos: 4, servedAnchors: [] }); + assert.deepEqual(sel2.delta.map((e) => e.anchorId), ["b9"]); + assert.equal(isFullResend(sel2, els2), false, "one anchor alone is not a full resend"); +}); + +// --------------------------------------------------------------------------- +// R — red gates +// --------------------------------------------------------------------------- + +test("R1 R3: a declared host that fails to collect msgids → the fault (400 source)", () => { + assert.equal(checkR3Identity(true, 5, undefined), false, "declared, no msgids"); + assert.equal(checkR3Identity(true, 5, ["a", "b", "c"]), false, "declared, wrong length"); + assert.equal(checkR3Identity(true, 5, ["a", "b", "c", "d", "e"]), true, "declared, complete"); + assert.equal(checkR3Identity(false, 5, undefined), true, "non-cohort: never a fault"); + assert.equal(checkR3Identity(true, 0, undefined), true, "no elements: nothing to collect"); +}); + +test("R2 R3: FoundationRejectError is the 400 control channel (zero side effects are the caller's, the error is the signal)", () => { + const err = new FoundationRejectError("envelope fault"); + assert.ok(err instanceof Error); + assert.equal(err.status, 400); + assert.match(err.message, /R3 identity fault/); +}); + +test("R3 R3 breaker: 3 faults in the 10-minute window latch; the latch rejects further faults and outlives the window", () => { + const t = new R3FaultTracker(60_000, 3); + const t0 = 1_000_000; + assert.equal(t.recordFault("s", t0).latched, false); + assert.equal(t.recordFault("s", t0 + 1_000).latched, false); + assert.equal(t.recordFault("s", t0 + 2_000).latched, true, "the third strike latches"); + assert.equal(t.isLatched("s"), true); + // Latched sids reject immediately — even after the window would expire. + assert.equal(t.recordFault("s", t0 + 999_999).latched, true); + // A different sid is unaffected. + assert.equal(t.isLatched("other"), false); + t.clearLatched("s"); + assert.equal(t.isLatched("s"), false); +}); + +test("R4 R3 +/- : the window expires (a new window restarts the strike count)", () => { + const t = new R3FaultTracker(1000, 3); + t.recordFault("s", 0); + t.recordFault("s", 100); + t.recordFault("s", 200); // latched? strikeCap 3 → yes + assert.equal(t.isLatched("s"), true); + t.clearLatched("s"); + // Fresh window: two faults do NOT re-latch. + t.recordFault("s", 5_000); + t.recordFault("s", 6_000); + assert.equal(t.isLatched("s"), false); +}); + +test("R5 R2: diverge = zero side effects (stage0Decide writes nothing on a structural disagreement)", () => { + const dir = path.join(hermetic, "r5"); + fs.mkdirSync(dir, { recursive: true }); + const db = (openPlane(path.join(dir, "index.db")) as unknown as { db: import("../src/storage/snap.ts").DbLike }).db; + // The l1 row is recorded by the store facade before stage0 (the lane + // path) — do it here so the pure function has its world to match. + snapL1Record(db, "s5", "s5", "witness", now0); + // Seed the world: issue a 3-id envelope. + const seed = stage0Decide({ db, sessRaw: "s5", ledgerId: "s5", incoming: ["a", "b", "c"], now: now0 }); + assert.equal(seed.decision.mode, "issue"); + // A structural disagreement (head moved): the verdict is diverge and the + // stored envelope is UNCHANGED (zero side effects — the lane runs the + // legacy round and the client converges next turn). + const d = stage0Decide({ db, sessRaw: "s5", ledgerId: "s5", incoming: ["OTHER", "a", "b"], now: now0 + 1 }); + assert.equal(d.diverge, true); + const l1 = snapL1Lookup(db, "s5"); + assert.ok(l1, "the seed l1 row exists"); + const env = snapL1EnvelopeRead(db, "s5"); + assert.equal(env!.envelopeChecksum, seed.decision.envelopeChecksum, "the diverge persisted nothing"); +}); + +// --------------------------------------------------------------------------- +// S — the stage-0 store seam (M1 seed → M2 append round trip through SQLite) +// --------------------------------------------------------------------------- + +test("S1 the full M1→M2 round trip: seed (issue) → M1 probe → M2 append → M1 echo", () => { + const dir = path.join(hermetic, "s1"); + fs.mkdirSync(dir, { recursive: true }); + const store = openPlane(path.join(dir, "index.db")); + const sid = "s1"; + store.ingestSession(planIngest(plainSession(sid, 3))); + const rows = () => store.snapStage0Cores(sid).rows.map((r) => r.ids[0]!); + + // Seed: the client's first full array (3 ids). + const seed = store.snapStage0(sid, ["m1", "m2", "m3"]); + assert.equal(seed.decision.mode, "issue"); + + // M1 tail extension: full resend + one new id (1-based pos: seed is 1..3 + // → m4 lands at pos 4). + const m1 = store.snapStage0(sid, ["m1", "m2", "m3", "m4"]); + assert.equal(m1.decision.mode, "probe"); + assert.deepEqual(m1.decision.newIdPos, [{ id: "m4", pos: 4 }]); + + // M2: ONLY the new id + the seqs carrier (4 rows pos 0..3 → frontier 4, + // the next seq is 5). + store.ingestSession(planIngest(plainSession(sid, 4))); + const m2 = store.snapStage0(sid, ["m5"], { seqs: [5] }); + assert.equal(m2.decision.mode, "probe", "the M2 append"); + assert.deepEqual(m2.decision.newIdPos, [{ id: "m5", pos: 5 }]); + assert.deepEqual(m2.decision.envelope.map((e) => e.id), ["m1", "m2", "m3", "m4", "m5"]); + + // The next M1 full resend must ECHO the matchAppend canon. + const echo = store.snapStage0(sid, ["m1", "m2", "m3", "m4", "m5"]); + assert.equal(echo.decision.mode, "echo", "M1 full resend echoes the M2-persisted canon"); + assert.deepEqual(echo.decision.newIdPos, []); + void rows; +}); From da6ffdda834cceb4496c6ecac08441d149d3a67b Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 12 Nov 2026 03:01:30 +0800 Subject: [PATCH 2/8] =?UTF-8?q?fix(#5874=202=C3=972):=20restore=20the=20#1?= =?UTF-8?q?954=20KEEP=3D1=20contract=20=E2=80=94=20strip=20previous=5Fresp?= =?UTF-8?q?onse=5Fid=20only=20after=20this=20proxy=20established=20a=20del?= =?UTF-8?q?ta=20frontier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI run 43079490144 red: 'ACP_KEEP_RESPONSE_ID=1 keeps the id forwarded and stays silent' — the Part B strip condition (!deltaServed) removed the id on EVERY non-delta round with the knob on, including the native case (no snapshot frontier ever established) where the pinned #1954 contract says the id is forwarded intact. Fix: capture hadFrontier (a persisted delta state existed before this round's selection). The id is now stripped only when hadFrontier && the round served no partial delta — i.e. the provider's retained context equals exactly the sent prefix, so a full replay would duplicate it. Before the first frontier the native semantics hold regardless of the outbound (seed / full view / legacy). deltaServed tightened to mean 'a PARTIAL delta went out': a reseed and a full-resend (delta grew to the whole view) both send the full view and now read as no-delta. onSelection carries the total element count so the lane can tell them apart. The dev-message gate (skip on partial delta) keeps its meaning. --- src/server/prepare-responses.ts | 41 +++++++++++++++++++++++++-------- src/server/snap-egress.ts | 4 ++-- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 13c8ef3e4a..602194ad11 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -117,6 +117,10 @@ export async function prepareResponses( // provider's retained chain carries the rest) — gates the developer // message re-injection and the previous_response_id strip below. let deltaServed = false; + // 2×2 Part B: a persisted delta frontier existed BEFORE this round's + // outbound selection (the provider's retained context equals exactly the + // elements sent up to it) — see the strip site below for the contract. + let hadFrontier = false; let sysNotes: string[] = []; // #3713: billed tokens of the #1638-masked mid-history sys/dev items — they // ride the rebuilt input verbatim yet enter neither projection.msgs nor @@ -464,6 +468,13 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // state) → reseed (full send, frontier reset). const chains = knobKeepResponseId() && typeof parsed.previous_response_id === "string" && parsed.previous_response_id.length > 0; const deltaState: DeltaState | null | undefined = chains ? (session.responsesDelta ?? null) : undefined; + // True once THIS proxy has established a delta chain with the provider + // (a persisted frontier exists). Only then may a non-delta round strip + // the id: the provider's retained context equals exactly the elements + // up to the frontier, so a full replay would duplicate it. Before the + // first frontier the native #1954 KEEP=1 contract holds (id forwarded + // intact, whatever the outbound is). + hadFrontier = deltaState !== undefined && deltaState !== null; let selectionFired = false; // #5874 P2 (mapEgress): the plane view is the outbound source; // the lane's exact chain (strip → drop → repair) is reapplied to it. @@ -475,18 +486,25 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr transform: (m) => repairResponsesAssistantOrdering(applyReasoningDrop(stripKernelSummaries(attachSubagentSessions(applyAbsorbView(m, session.state, loopConfig, tokenCount), session), turn.state), reasoningPlan, log, sessionId), originalMessages).messages, ...(deltaState !== undefined ? { deltaState, - onSelection: (sel) => { + onSelection: (sel, totalElements) => { selectionFired = true; session.responsesDelta = sel.nextState; - deltaServed = !sel.reseed && sel.delta.length > 0; + // deltaServed = a PARTIAL delta went out: new content only. + // A reseed (state lost / first round) and a full-resend + // (delta grew to the whole view — isFullResend) both send + // the FULL view, which must read as "no delta" for the id + // handling below. + deltaServed = !sel.reseed && sel.delta.length > 0 && sel.delta.length < totalElements; }, } : {}), }); if (deltaState !== undefined && !selectionFired) { // Delta was requested but the plane degraded to legacy this round - // (the legacy bytes are the FULL replay and the id was stripped - // below) — the provider's chain no longer matches the stored - // frontier. Forget it: the next chained round re-seeds. + // (the legacy bytes are the FULL replay). With an established + // frontier the id is stripped below and the provider's chain no + // longer matches the stored frontier → forget it: the next chained + // round re-seeds. Without one (native KEEP=1 client, plane never + // seeded) nothing to forget — the native contract stays intact. session.responsesDelta = null; } if (deltaServed) { @@ -730,10 +748,15 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // #1954: stripping is only lossless when input already holds the full // conversation. Warn when we strip a non-empty id so a native-chaining // (delta) continuation that loses its history is visible, not silent 200s. - // 2×2 Part B: a plane DELTA round KEEPS the id (the input is the unsent - // suffix — chaining is the whole point); a reseed round STRIPS it (the - // full input would otherwise duplicate the retained chain). - if (!knobKeepResponseId() || !deltaServed) { + // 2×2 Part B: the id is governed by the delta chain this proxy established + // (hadFrontier). NO prior frontier → native #1954 semantics: KEEP=1 + // forwards the id intact regardless of what the outbound is (seed / full + // view / legacy — the pinned contract). WITH a prior frontier the provider + // retains exactly the sent elements, so only a PARTIAL delta keeps the id; + // a reseed / full-resend / degraded-legacy round STRIPS it (the chain + // breaks, the client re-chains from this round's fresh response id and the + // frontier re-seeds — one-round convergence, acceptance #7). + if (!knobKeepResponseId() || (hadFrontier && !deltaServed)) { const chainWarn = strippedResponseIdWarning(rebuilt.previous_response_id); if (chainWarn) log("warn", `[${sessionId}] ${chainWarn}`); delete rebuilt.previous_response_id; diff --git a/src/server/snap-egress.ts b/src/server/snap-egress.ts index b2fcab9d66..6ebcb068dd 100644 --- a/src/server/snap-egress.ts +++ b/src/server/snap-egress.ts @@ -54,7 +54,7 @@ interface SnapEgressDeps { * the lane can persist the new frontier (only after the outbound * succeeds — a failed round keeps the old state and re-sends the same * delta: at-least-once). */ - onSelection?: (sel: DeltaSelection) => void; + onSelection?: (sel: DeltaSelection, totalElements: number) => void; } // --------------------------------------------------------------------------- @@ -478,7 +478,7 @@ export function applySnapEgress( const keep = new Set(selection.delta.map((e) => e.msg)); outMsgs = built.msgs.filter((m) => keep.has(m)); } - deps.onSelection?.(selection); + deps.onSelection?.(selection, elements.length); } const planeProcessed = deps.transform(outMsgs); // DIRECT AUTHORITY: the plane view is the outbound source. The legacy From ac2a510d9955a5f05ab4afc00d34a4ad9ebf73d0 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 12 Nov 2026 12:41:33 +0800 Subject: [PATCH 3/8] =?UTF-8?q?fix(#5874=202x2):=20review=20round=201=20?= =?UTF-8?q?=E2=80=94=20B0-B4=20+=20M1-M6=20(plane=20bypass,=20400=20funnel?= =?UTF-8?q?,=20frontier=20commit-on-accept,=20pure=20continuation,=20share?= =?UTF-8?q?d=20stage-0,=20codec-level=20pin)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of PR #6299 found four blockers and six majors; this is the complete fix round. B0 (root): a chained turn on the responses lane has no full-view reconstruction (the D lane is skipped for pluginMode and #4804 augmentation is WS-only), so its kernel view is a delta, not the world. Flushing that view to the plane deleted every row it did not contain. Now a chained turn that is neither cohort (msgid-pinned stage-0) nor WS-anchored sets session.skipPlaneFlush (one-shot, consumed by the persist build — snapShadowSync returns before any plane work) and passes bypass:true to applySnapEgress (the outbound is the legacy bytes; the plane stays stale-but-intact until the next full/cohort turn, which re-syncs it one-round). B1: FoundationRejectError was swallowed by the lane 'kernel transform failed, forwarding unchanged' catch -> 200. Both lane catches now rethrow it first; the server funnel maps it to a 400 with x-bili-snapshot: identity and {error:{type:'snapshot_identity_fault'}} before the generic 503 path. B3: a plugin chained round with EMPTY input (a pure provider-chain continuation) is routed to the raw-body passthrough (prepareResponsesCompact) at the dispatcher — no dev-message injection, no chain-id strip, no kernel round. The injected dev message + stripped id on a no-content round produced a blind provider response that the host then chained from (permanent context loss). B4: the delta frontier advanced at egress-build time with no rollback on upstream failure. The selection now stashes pendingResponsesDelta on the Prepared and commits session.responsesDelta only in the relay when the upstream response status is < 400 (4xx -> null = reseed next round; never advances on failure). M1: stage-0 arming now requires the pluginLaneServerHistory rollout (featureEnabled in runStage0), tying R3 to the rollout that makes it reachable. M2: checkR3Identity is now the single R3 arbiter, called in runStage0. M3: the R3 breaker latch persists (l1_world.r3_latched_at, probe-first ALTER, upsert — the first fault precedes the session's first l1 row) and survives a process restart; a latched session degrades forever, never 400s again. M4: A1d pins the kernel-core byte-identity at the CODEC level (the new element's core is byte-identical between the M1 full-array parse and the M2 standalone parse, wireIndex aside). M6: session.responsesDelta is cleared at every stage-0 resync (s0.resynced). M5: PR body rewritten to match the code. Shared stage-0: the pin + R3 + stage-0 decision + assembly moved out of the openai lane into src/snapshot/stage0.ts (runStage0), wired into BOTH the openai and responses lanes (the responses lane reads biliMsgIds/biliLedgerSeqs from its body; cohort = pluginMode + declared). Tests: tests/issue5874-2x2-p3.test.ts 40/40 (was 33): new A1d (M4 codec pin), B0a (skipPlaneFlush leaves the plane untouched), B0b (egress bypass ref-identity), M3 (latch persistence across store instances), and HTTP e2e B1 (400 + header + zero upstream forwards), B3 (pure continuation raw passthrough), B4 (400 turn does not commit the frontier; the next round re-seeds the full view). Regression: snaplane-p2+snapshot-render+chain+shadow 41/41, storage-engine-proofs +opencode-rolling 22/22, snapbase-p0+snaplane-p1+fastpath+ responses-previous-response-id 23/23, tsc clean. --- reference/architecture.md | 1 + src/persist.ts | 13 ++ src/server.ts | 23 +++ src/server/handle.ts | 25 ++- src/server/prepare-openai.ts | 158 ++++------------ src/server/prepare-responses.ts | 99 +++++++++- src/server/relay.ts | 13 ++ src/server/snap-egress.ts | 8 + src/session.ts | 7 + src/snapshot/stage0.ts | 207 +++++++++++++++++++++ src/storage/ingest.ts | 8 +- src/storage/schema.ts | 7 +- src/storage/shadow.ts | 12 ++ src/storage/snap.ts | 26 +++ src/storage/store.ts | 35 ++++ tests/issue5874-2x2-p3.test.ts | 308 ++++++++++++++++++++++++++++++++ 16 files changed, 815 insertions(+), 135 deletions(-) create mode 100644 src/snapshot/stage0.ts diff --git a/reference/architecture.md b/reference/architecture.md index 280637123f..2f6c711014 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -151,6 +151,7 @@ billion-context/ │ │ ├── delta.ts # Outbound delta selection: seed/delta/no-op + full-resend detection (#6181) │ │ ├── identity.ts # matchWorld verdicts (echo/probe/scatter/issue/diverge/resync) + canon parse (#5874) │ │ ├── plane.ts # Stage-0 decision glue: R3 identity check, 400 fault breaker, lane decide (#6181) +│ │ ├── stage0.ts # Lane-agnostic stage-0 entry: pin + R3 + stage-0 decision + assembly, shared by the openai and responses lanes (#6181) │ │ └── types.ts # Shared stage-0 types (type-only, kernel-public-types only) (#6181) │ ├── storage/ # Unified storage layer (schema, SQLite engine ladder, backends) (#2671) │ │ ├── backend.ts # Storage backend contract: sqlite + mysql operation seam (#3931) diff --git a/src/persist.ts b/src/persist.ts index 5146b96835..20868ec1f0 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2467,6 +2467,19 @@ export class SessionStore { this.liveShadow().snapStage0Reset(sessionId); } + /** #6181 M3: whether this sid's R3 breaker latch persisted from a + * previous process (a restart must not re-arm a misbehaving host). */ + snapR3Latched(sessionId: string): boolean { + if (!this.replicasAvailable()) return false; + return this.liveShadow().snapR3Latched(sessionId); + } + + /** #6181 M3: persist the R3 breaker latch (best-effort). */ + snapR3Latch(sessionId: string): void { + if (!this.replicasAvailable()) return; + this.liveShadow().snapR3Latch(sessionId); + } + /** #3928: physical index.db shrink after deletions (`bili sessions rm`). * Throws on failure — the deletion itself is already committed, callers * report the shrink as partial. Throws in legacy-only lanes — check diff --git a/src/server.ts b/src/server.ts index 0f5aa38456..b66136c166 100644 --- a/src/server.ts +++ b/src/server.ts @@ -165,6 +165,7 @@ import { artifactSeedHit, detectAcpArtifacts } from "./server/chain-artifacts.js import { droppedOpenaiParts } from "./wire-drop-warn.js"; import { BP_MAX, ROLLOUT_FEATURES, rolloutDecisions, rolloutInstallId, warnRolloutConfigProblems } from "./rollout.js"; import { SnapshotAnchorFallbackError, WS_ANCHORED_REQ_PROP } from "./server/snapshot-render.js"; +import { FoundationRejectError } from "./snapshot/plane.js"; export { forward } from "./server/relay.js"; // #1086/#1218: the per-session chain-verdict memory moved to plugin.ts @@ -780,6 +781,21 @@ export async function startServer(opts: ProxyOptions): Promise { // The rethrow rides ONLY the request the WS session stamped // (WS_ANCHORED_REQ_PROP); plain-HTTP lanes keep the 503 shape. if (err instanceof SnapshotAnchorFallbackError && (req as unknown as Record)[WS_ANCHORED_REQ_PROP] !== undefined) throw err; + // #6181 B1/B2: an R3 identity fault (the client declared a + // snapshot envelope that fails the identity gates) is a HARD 400 + // with the identity marker — NOT the generic 503, NOT a silent + // raw forward: the host must learn that the envelope is broken. + // Both lanes rethrow FoundationRejectError into this funnel; it + // throws before processTurn, so the session is untouched. + if (err instanceof FoundationRejectError) { + if (!res.headersSent) { + res.writeHead(400, { "content-type": "application/json", "x-bili-snapshot": "identity" }); + res.end(JSON.stringify({ error: { message: err.message, type: "snapshot_identity_fault" } })); + } else { + res.end(); + } + return; + } const msg = String(err); const e = err as { name?: string; message?: string }; // #411: a client cancel aborts the upstream fetch via @@ -2001,6 +2017,13 @@ export type Prepared = { responsesTextProtocol?: boolean; resetAfterSuccess?: boolean; responsesProjection?: ResponsesProjection; + /** #6181 B4: the delta frontier selected for THIS turn's outbound — + * COMMITTED to the session only AFTER the upstream ACCEPTS the request + * (relay.ts, first forward, status < 400). A failed/4xx forward drops + * it (null) so the next round RE-SEEDS (at-least-once: over-serving is + * safe, under-serving is the context loss). Absent (undefined) = no + * delta was selected this turn. */ + pendingResponsesDelta?: { frontierPos: number; servedAnchors: string[] } | null; anthropicSystem?: AnthropicRequestBody["system"]; /** #2189: the client's billing-attribution block captured from the INBOUND * anthropic system (pre-anchor). Preflight summary calls must carry it — diff --git a/src/server/handle.ts b/src/server/handle.ts index 95e5d5d171..9824409eb0 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -2989,11 +2989,34 @@ async function handleRequest( // a real model (the loop's synthetic frames read it back). return await prepareGoogle(work as GoogleRequestBody, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, nativeWindow, googleModel ?? v1Model, googlePathKind(urlPath) === "stream-generate", visibilityMarkers, upstreamOrigin, req, decide); } + // #6181 B3: a PLUGIN-lane chained round whose input is + // EMPTY is a pure chain continuation — the provider + // already has the full world via its retained chain. The + // kernel path for such a round is HARMFUL: it injects the + // standing dev message into an otherwise-empty input and + // (on the no-delta egress path) strips the chain id, so + // the provider answers from nothing and the host then + // chains from that BLIND response → permanent context + // loss. Forward the client's bytes verbatim instead (the + // same passthrough the /responses/compact endpoint + // already owns: no kernel, no dev injection, no plane + // side effects — the response-side ingest still updates + // the session from the answer). + const pureChainContinuation = + protocol === "responses" && + !responsesCompact && + !wsLaneEnvelope && + pluginMode && + typeof (work as ResponsesRequestBody).previous_response_id === "string" && + (work as ResponsesRequestBody).previous_response_id!.length > 0 && + (!Array.isArray((work as ResponsesRequestBody).input) || (work as ResponsesRequestBody).input!.length === 0); const preparedOut = protocol === "anthropic" ? await prepareAnthropic(work as AnthropicRequestBody, req, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, upstreamOrigin, reasoningCfg, visibilityMarkers, requestModel, decide) : protocol === "openai" ? await prepareOpenai(work as OpenAIRequestBody, req, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, upstreamOrigin, nativeWindow, reasoningCfg, visibilityMarkers, route?.rewrittenUrl, decide, bodyText) - : responsesCompact + : pureChainContinuation + ? prepareResponsesCompact(bodyBuffer, work as ResponsesRequestBody, session, req, core, reqConfig, log) + : responsesCompact // #618: when no bili compaction item is present, // prepareResponsesCompact falls back to the raw bodyBuffer — // forward it unchanged (byte-identical passthrough). diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 616379d9b5..bf80481eec 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -7,7 +7,7 @@ import type { ProxyOptions } from "../config.js"; import { effectivePromptCacheRouting } from "../config.js"; import { currentCalibrationFactor, hardenOpenaiAssistantContent, systemToUser } from "../util.js"; import { conversationHeaderSource, resolveRelayAffinityPck } from "../session-id.js"; -import { MSG_IDS_BODY_FIELD, MSG_SEQS_BODY_FIELD, applyMsgIdPin, applyMsgSeqStamps, incomingSeqsOf, validateMsgIds, validateMsgSeqs } from "../msgid-pin.js"; +import { MSG_IDS_BODY_FIELD, MSG_SEQS_BODY_FIELD, applyMsgSeqStamps, incomingSeqsOf, validateMsgSeqs } from "../msgid-pin.js"; import { pluginAgentHeader } from "../plugin.js"; import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, warnDroppedOpenaiParts, warnReasoningPairs, type Prepared } from "../server.js"; import { recordNudgeObservation } from "../context-analysis.js"; @@ -40,10 +40,9 @@ import { isStrictReasoningEcho, modelIdOf, normalizeStrictEchoReasoning } from " import { reconcileSystemAnchor } from "../system-anchor.js"; import { logRawStripCounts, markerArtifactCount, stripRawStage, type RawStripCounts } from "../marker/engine.js"; import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js"; -import { SessionStore, getStore } from "../persist.js"; -import { FoundationRejectError, r3FaultTracker } from "../snapshot/plane.js"; -import { assembleKernelInput, shouldResync } from "../snapshot/assemble.js"; -import { ANCHOR_ID_PREFIX } from "../storage/snap.js"; +import { SessionStore } from "../persist.js"; +import { FoundationRejectError } from "../snapshot/plane.js"; +import { runStage0 } from "../snapshot/stage0.js"; import { applyImageRescalePass } from "../image-rescale.js"; import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed } from "./budget.js"; import { effectiveAbsorbBlock, effectiveBlockOpsEnabled } from "./prepare-responses.js"; @@ -202,133 +201,35 @@ export async function prepareOpenai( openaiSystemText = outcome.outbound; } originalMessages = msgs; - // #4491/#5825 parity: pin identity to client-supplied ids (plugin - // lanes only — pure proxy traffic has no msgid channel and stays on - // snapshot render D), the SAME gated contract as the anthropic twin. - // Skipped on watermark hits on purpose: fastpath turns reuse the - // seed stash's prefix view whose rows were already pinned when it was - // materialized, and the decoded tail window is SHORTER than the - // stamped list, so count-parity there would fail by design — pinning - // only adds noise. - // #5874 2×2 stage 0: cohort (dual-channel DECLARED) plugin traffic - // assembles the kernel input from the snapshot plane (old cores — - // the kernel's own tagged output, invariant v) + the NEW spans - // (P5). Non-cohort traffic keeps the D-lane view below unchanged. - const twoX2Armed0 = wpHit === undefined && pluginMode && Array.isArray(incomingMsgIds) && incomingMsgIds.length > 0; - let twoX2Armed = twoX2Armed0 && !r3FaultTracker.isLatched(sessionId); - if (twoX2Armed0 && r3FaultTracker.isLatched(sessionId)) { - log("debug", `[${sessionId}] [2x2] sid is R3-latched — legacy path this turn`); - } - if (wpHit === undefined && pluginMode && incomingMsgIds !== undefined) { - if (featureEnabled("pluginLaneServerHistory")) { - const v = validateMsgIds(incomingMsgIds, msgs.length); - if (v.ok) { - msgs = applyMsgIdPin(msgs, v.ids); - originalMessages = msgs; - } else if (twoX2Armed) { - // #5874 2×2 red gate R3: a cohort host that DECLARED the - // dual-channel (per-element msgids) but did not collect - // them all → hard 400 BEFORE any kernel side effect, and - // the fault feeds the breaker (10-minute window, 3 - // strikes → the sid latches to the legacy path instead of - // bricking the session on a misbehaving host). - const { latched } = r3FaultTracker.recordFault(sessionId); - if (latched) { - twoX2Armed = false; - log("error", `[${sessionId}] [2x2] R3 identity fault ×3 in window (${v.reason}) — sid latched to the legacy path`); - } else { - throw new FoundationRejectError(`inbound msgid envelope failed validation (${v.reason})`); - } - } else { - log("warn", `[${sessionId}] [msgid] openai-wire pin rejected (${v.reason}) — falling back to content-derived ids (#4491/#5825)`); - } - } else { - log("info", `[${sessionId}] [msgid] openai-wire biliMsgIds present but pluginLaneServerHistory rollout is off for this install — field stripped, ids not pinned (#5825)`); - } - } - // #5874 2×2 stage 0 (brief 19544722): world match + kernel-input - // assembly for cohort traffic. Runs BEFORE processTurn, so a thrown - // FoundationRejectError (R3) carries zero kernel side effects. Every - // other fault → the legacy full path (degrade, never guess — - // one-round convergence). - let twoX2KernelInput: BiliMessage[] | undefined; - if (twoX2Armed) { - try { - const store = getStore(); - const stage0 = store?.snapStage0(session.id, msgs.map((m) => m.id), { seqs: incomingSeqs }); - const cores = stage0 ? (store?.snapStage0Cores(session.id) ?? null) : null; - if (stage0 && cores) { - if (shouldResync(stage0.decision, cores.rows.length)) { - // Structural disagreement (diverge) or an explicit - // re-baseline: reset the plane; THIS turn runs the - // legacy full path and re-seeds it through the - // in-request flush — the NEXT turn re-issues the - // envelope from the client's current array. - store!.snapStage0Reset(session.id); - log("warn", `[${sessionId}] [2x2] stage-0 ${stage0.decision.mode} — plane reset, this turn legacy (one-round convergence)`); - } else { - // newCores = the converted spans of the NEW ids, in - // envelope order. The front stage already converted - // the whole array (front-stage invariance), so the - // new spans carry their full-parse bytes — T5 holds - // by construction. - const newIdSet = new Set(stage0.decision.newIdPos.map((n) => n.id)); - const newCores: BiliMessage[] = []; - const taken = new Set(); - for (const e of stage0.decision.envelope) { - if (!newIdSet.has(e.id) || taken.has(e.id)) continue; - taken.add(e.id); - const m = msgs.find((x) => x.id === e.id); - if (m === undefined) throw new Error(`new span ${e.id} absent from the converted array`); - newCores.push(m); - } - // The plane's ANCHOR rows (active acp_summary_) - // ride the kernel input at their covered-run boundary - // — the D-view placement parity. snapStage0Cores - // returns both the rows and the coverMax map. - const anchors = cores.rows - .filter((r) => r.ids.some((id) => id.startsWith(ANCHOR_ID_PREFIX))) - .map((r) => { - const id = r.ids.find((x) => x.startsWith(ANCHOR_ID_PREFIX))!; - return { - msg: JSON.parse(r.postJson) as BiliMessage, - blockId: id.slice(ANCHOR_ID_PREFIX.length), - }; - }); - const asm = assembleKernelInput({ - decision: stage0.decision, - rows: cores.rows - .filter((r) => !r.ids.some((id) => id.startsWith(ANCHOR_ID_PREFIX))) - .map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })), - newCores, - skipIds: stage0.explainedGone, - anchors, - coverMax: cores.coverMax, - }); - twoX2KernelInput = asm.kernelInput; - log("info", `[${sessionId}] [2x2] stage-0 ${stage0.decision.mode} — kernel input from plane (old=${asm.oldCount}, new=${newCores.length}, anchors=${anchors.length})`); - } - } - } catch (err) { - if (err instanceof FoundationRejectError) throw err; // R3 outranks the fallback - const why = err instanceof Error ? err.message : String(err); - if (why.startsWith("[snapshot 2×2]")) { - // The assembly could not address the envelope against the - // plane (table behind — R4-shaped): reset and take the - // legacy path this turn (one-round convergence). - try { - getStore()?.snapStage0Reset(session.id); - } catch { /* the reset is best-effort — the legacy path is the floor */ } - } - log("warn", `[${sessionId}] [2x2] stage-0 failed (${why}) — legacy full path`); - } +// #4491/#5825 + #5874 2×2 stage 0 (brief 19544722 / #6181): the + // SHARED pin + world-match + kernel-input assembly for cohort + // traffic (src/snapshot/stage0.ts — the responses lane runs the + // SAME helper, same red gate, same one-round-convergence + // contract). Runs BEFORE processTurn, so a thrown + // FoundationRejectError (R3) carries zero kernel side effects + // (the server funnel maps it to the hard 400); every other fault + // → the legacy full path (one-round convergence). + const s0 = runStage0({ + sessionId, + lane: "openai", + msgs, + incomingMsgIds, + incomingSeqs, + pluginMode, + wpHit, + log, + }); + if (s0.pinned !== null) { + msgs = s0.pinned; + originalMessages = msgs; } // #3756 snapshot render: the outbound prefix renders from the unified // transcript (server-authoritative, fold-time ids — immune to identity // re-mints #3752); only the live tail keeps the client's bytes. // No-op returns `msgs` when the lane is inactive/degraded. // reconcileFoldCoverage below still observes the CLIENT view. - const renderMsgs = twoX2KernelInput ?? maybeSnapshotView(session, msgs, { sessionId, pluginMode, log }); + const renderMsgs = s0.kernelInput ?? maybeSnapshotView(session, msgs, { sessionId, pluginMode, log }); + // #4241: the render lane replaced the view (reference-inequality is the // discriminator — maybeSnapshotView returns the SAME array when idle), // so the judged baseline is the rendered transcript, not the inbound. @@ -716,6 +617,11 @@ export async function prepareOpenai( attachedRetrievalNoteIds = retrNotes.map((n) => n.id); } } catch (err) { + // #6181 B1: the R3 identity fault is a HARD reject, not a degrade — + // rethrow it to the server funnel (400 + x-bili-snapshot:identity) + // BEFORE the generic 503; zero kernel side effects by construction + // (it throws before processTurn). + if (err instanceof FoundationRejectError) throw err; log("warn", `[${sessionId}] kernel transform failed, forwarding unchanged: ${String(err)}`); if (attachedRetrievals.length > 0) dropRetrievals(session, attachedRetrievals.map((i) => i.ref), "prepare failed; forwarded unprocessed"); processedMessages = []; diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 602194ad11..5858a8ac3c 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -18,6 +18,9 @@ import { METADATA_LAST_RESPONSES_MODE, noteCanonicalSystemFingerprint, reconcile import { buildAnchoredView, maybeSnapshotView, noteAnchoredRender, SnapshotAnchorFallbackError, wsAnchoredPayload } from "./snapshot-render.js"; import { applySnapEgress } from "./snap-egress.js"; import type { DeltaState } from "../snapshot/delta.js"; +import { FoundationRejectError } from "../snapshot/plane.js"; +import { runStage0 } from "../snapshot/stage0.js"; +import { MSG_IDS_BODY_FIELD, MSG_SEQS_BODY_FIELD, applyMsgSeqStamps, incomingSeqsOf, validateMsgSeqs } from "../msgid-pin.js"; import { storedMessagesFor, storedRowLookupFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed, pluginLaneDegraded } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; @@ -100,6 +103,14 @@ export async function prepareResponses( } let processedMessages: CoreMessage[] = []; + // #6181 2×2: the msgid + ledger-seq dual-channel the plugin host rides + // on the body (src/msgid-pin.ts) — read and STRIPPED before any + // projection so the real upstream never sees a field it does not know. + // (Same discipline as the openai twin.) + const incomingMsgIds = (parsed as unknown as Record)[MSG_IDS_BODY_FIELD]; + const incomingSeqsRaw = incomingSeqsOf(parsed); + delete (parsed as unknown as Record)[MSG_SEQS_BODY_FIELD]; + delete (parsed as unknown as Record)[MSG_IDS_BODY_FIELD]; // #564/#2627: the ordering separators THIS turn's repair actually inserted, // recorded by the producer so provenance never has to be inferred from item // text later. Empty on the overwhelming majority of turns. @@ -121,6 +132,16 @@ export async function prepareResponses( // outbound selection (the provider's retained context equals exactly the // elements sent up to it) — see the strip site below for the contract. let hadFrontier = false; + // 2×2 Part B (B4): hoisted so the return literal can hand the selected + // frontier to the relay (committed only after the upstream accepts). + let deltaState: DeltaState | null | undefined; + let selectionFired = false; + let pendingDelta: { frontierPos: number; servedAnchors: string[] } | null = null; + // #6181 B0: a chained turn that cannot be addressed against the plane + // (non-cohort — its kernel view is a delta, not the world) skips BOTH + // the plane write for this turn's flush and the plane egress — see the + // flag set site below the processTurn block. + let planeBypass = false; let sysNotes: string[] = []; // #3713: billed tokens of the #1638-masked mid-history sys/dev items — they // ride the rebuilt input verbatim yet enter neither projection.msgs nor @@ -234,8 +255,37 @@ export async function prepareResponses( sysNotes = outcome.notes; if (outcome.outbound !== fresh) projection.systemParts = outcome.outbound ? [outcome.outbound] : []; } - const { msgs } = projection; + let { msgs } = projection; originalMessages = msgs; + // #6181 2×2: stamp the M2 seqs carrier onto the converted array + // (parallel to input; malformed/absent → null → the M2 fast path is + // disabled, the full-resend world-match path still works). + const incomingSeqs: number[] | null = + incomingSeqsRaw !== undefined && incomingSeqsRaw.length === msgs.length && validateMsgSeqs(incomingSeqsRaw, msgs.length).ok + ? incomingSeqsRaw + : null; + if (incomingSeqs !== null) applyMsgSeqStamps(msgs, incomingSeqs); + // #6181 2×2 stage 0 (brief 19544722): the SHARED pin + world-match + // + kernel-input assembly (src/snapshot/stage0.ts — the SAME helper + // as the openai lane, same red gate). Before processTurn, so an R3 + // FoundationRejectError carries zero kernel side effects. + const s0 = runStage0({ + sessionId, + lane: "responses", + msgs, + incomingMsgIds, + incomingSeqs, + pluginMode, + log, + }); + if (s0.pinned !== null) { + msgs = s0.pinned; + originalMessages = msgs; + } + // #6181 M6: a stage-0 resync re-issues the pos space — a stale delta + // frontier against the re-issued pos is a dead loop (empty delta + // forever); clear it so the next chained round reseeds. + if (s0.resynced) session.responsesDelta = null; // #3756 snapshot render: outbound prefix from the unified transcript // (fold-time ids, immune to identity re-mints); live tail keeps the // client's bytes. No-op when the lane is inactive/degraded; the @@ -258,7 +308,10 @@ export async function prepareResponses( noteAnchoredRender(session, log); renderMsgs = anchored.view; } else { - renderMsgs = maybeSnapshotView(session, msgs, { sessionId, pluginMode, log }); + // #6181 2×2: the stage-0 assembly outranks the D-lane view for + // cohort turns (the plane IS the kernel-input authority); the + // D lane keeps serving everyone else unchanged. + renderMsgs = s0.kernelInput ?? maybeSnapshotView(session, msgs, { sessionId, pluginMode, log }); } // #4241: the render lane replaced the view (reference-inequality is the // discriminator — maybeSnapshotView returns the SAME array when idle), @@ -374,6 +427,17 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // stored rows must stay the pre-mutation bytes. The anchor rows ride // the copy too: they are processTurn output. session.lastPlaneMessages = JSON.parse(JSON.stringify(turn.messages)); + // #6181 B0: set BEFORE the in-request flush below — a chained turn + // that was not assembled from the plane (non-cohort: its kernel view + // is a delta, not the world) must not flush its collapsed view into + // the plane (snapViewEvents is a final-state diff — it would DELETE + // every row the view does not contain). One-shot: cleared right + // after the flush; the plane egress below gets the same verdict as + // `bypass` (the plane is one turn stale — serving it would drop the + // new input). + const isChained = typeof parsed.previous_response_id === "string" && parsed.previous_response_id.length > 0; + planeBypass = isChained && !s0.cohortRan && !wsAnchored; + if (planeBypass) session.skipPlaneFlush = true; // #5874 P2-final: snapshot application (in-request) — the canonical // transcript up to THIS user message plus the kernel's post-fold state // must be on the snapshot plane before the egress stage reads it @@ -387,6 +451,10 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // path. snapshotMessages(session, originalMessages); snapFlush = await SessionStore.flushOrSkip(session); + // #6181 B0: the in-request flush above either skipped the plane + // write (flag consumed) or ran it — either way the one-shot flag is + // done for this turn. + session.skipPlaneFlush = false; // #4528: split the kernel pass from its post-processing scan passes so // each is a bounded synchronous stretch rather than one long block. await yieldToEventLoop(); @@ -467,7 +535,7 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // state===null = no persisted frontier (first round / resync / lost // state) → reseed (full send, frontier reset). const chains = knobKeepResponseId() && typeof parsed.previous_response_id === "string" && parsed.previous_response_id.length > 0; - const deltaState: DeltaState | null | undefined = chains ? (session.responsesDelta ?? null) : undefined; + deltaState = chains ? (session.responsesDelta ?? null) : undefined; // True once THIS proxy has established a delta chain with the provider // (a persisted frontier exists). Only then may a non-delta round strip // the id: the provider's retained context equals exactly the elements @@ -475,7 +543,17 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // first frontier the native #1954 KEEP=1 contract holds (id forwarded // intact, whatever the outbound is). hadFrontier = deltaState !== undefined && deltaState !== null; - let selectionFired = false; + // #6181 B0 (decided at the flush site, above): a CHAINED turn that was + // NOT assembled from the plane (non-cohort — no msgids — or a cohort + // turn whose stage 0 did not run) cannot be addressed against the + // plane: its kernel view is a DELTA, not the world. It must touch + // neither the plane write (deriving events from the collapsed view + // would DELETE every row the view does not contain) nor the plane + // egress (the plane is one turn stale — serving it would drop the + // new input). Outbound = the legacy bytes verbatim, exactly what + // master serves for this turn; the plane stays stale-but-intact + // until the next full/cohort turn re-syncs it (one-round + // convergence). // #5874 P2 (mapEgress): the plane view is the outbound source; // the lane's exact chain (strip → drop → repair) is reapplied to it. processedMessages = applySnapEgress(turn.state, processedMessages, { @@ -488,7 +566,7 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr deltaState, onSelection: (sel, totalElements) => { selectionFired = true; - session.responsesDelta = sel.nextState; + pendingDelta = sel.nextState; // deltaServed = a PARTIAL delta went out: new content only. // A reseed (state lost / first round) and a full-resend // (delta grew to the whole view — isFullResend) both send @@ -497,6 +575,7 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr deltaServed = !sel.reseed && sel.delta.length > 0 && sel.delta.length < totalElements; }, } : {}), + ...(planeBypass ? { bypass: true } : {}), }); if (deltaState !== undefined && !selectionFired) { // Delta was requested but the plane degraded to legacy this round @@ -508,7 +587,7 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr session.responsesDelta = null; } if (deltaServed) { - log("info", `[${sessionId}] [snaplane] outbound=delta frontier=${(session.responsesDelta as { frontierPos: number })?.frontierPos} — input carries only the unsent suffix; the retained chain carries the rest (re-injected developer message / forged summaries skipped to avoid duplicating what the provider keeps)`); + log("info", `[${sessionId}] [snaplane] outbound=delta frontier=${(pendingDelta as { frontierPos: number } | null)?.frontierPos} — input carries only the unsent suffix; the retained chain carries the rest (re-injected developer message / forged summaries skipped to avoid duplicating what the provider keeps)`); } // #3477: consume the announced native-compaction boundary (pi session_compact // → /__bili/plugin/compact → markCompactionBoundary) exactly once, after the @@ -668,6 +747,11 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr } transformOk = true; } catch (err) { + // #6181 B1: the R3 identity fault is a HARD reject, not a degrade — + // rethrow it to the server funnel (400 + x-bili-snapshot:identity) + // BEFORE the generic 503; zero kernel side effects by construction + // (it throws before processTurn). + if (err instanceof FoundationRejectError) throw err; log("warn", `[${sessionId}] kernel transform failed, forwarding unchanged: ${String(err)}`); processedMessages = []; } @@ -834,6 +918,9 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr judgeBaseline, ...(separatorManifest !== undefined ? { separatorManifest } : {}), responsesProjection, + // #6181 B4: commit the selected frontier only after the upstream + // accepts the request (relay.ts); a failed forward drops it. + ...(deltaState !== undefined && selectionFired ? { pendingResponsesDelta: pendingDelta } : {}), systemNotes: sysNotes, protocol: "responses", stream, diff --git a/src/server/relay.ts b/src/server/relay.ts index 7e34790cf8..4f1772e2e1 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1274,6 +1274,19 @@ async function forwardCaptured( try { upstreamResult = await fetchPromise; firstAttemptClear = upstreamResult.clearTimer; + // #6181 B4: commit the delta frontier ONLY after the upstream + // accepts the request (fetch resolves on 4xx too — check the + // status). A 4xx/failed forward DROPS the pending state: the + // next round re-seeds (at-least-once — over-serving the full + // view is safe, under-serving is the context loss). The pending + // field is consumed at this single point (the first forward of + // the turn; compress-loop re-requests carry full views). + if (prepared && prepared.pendingResponsesDelta !== undefined) { + const accepted = upstreamResult.response.status < 400; + prepared.session.responsesDelta = accepted ? prepared.pendingResponsesDelta : null; + markDirty(prepared.session); + prepared.pendingResponsesDelta = undefined; + } } finally { clearTimeout(firstByteTimer); } diff --git a/src/server/snap-egress.ts b/src/server/snap-egress.ts index 6ebcb068dd..9ca14df30c 100644 --- a/src/server/snap-egress.ts +++ b/src/server/snap-egress.ts @@ -55,6 +55,13 @@ interface SnapEgressDeps { * succeeds — a failed round keeps the old state and re-sends the same * delta: at-least-once). */ onSelection?: (sel: DeltaSelection, totalElements: number) => void; + /** #6181 B0: this turn cannot be addressed against the plane (a + * non-cohort chained delta whose kernel view is a delta, not the + * world) — serve the LEGACY bytes verbatim for this turn: the plane + * is one turn stale (its write was skipped), and serving it would + * drop the new input element. The legacy bytes are exactly what + * master serves for this turn (the delta the provider chains from). */ + bypass?: boolean; } // --------------------------------------------------------------------------- @@ -440,6 +447,7 @@ export function applySnapEgress( const flags = snapEgressFlags(); if (!flags.egress || !flags.cohorts.has(deps.protocol)) return legacyProcessed; if (isSnapPlaneDead(deps.sessionId)) return legacyProcessed; + if (deps.bypass) return legacyProcessed; // #6181 B0 — one turn; the plane stays stale-but-intact let view: SnapView | null; try { diff --git a/src/session.ts b/src/session.ts index 4c56897a57..d9948beaf4 100644 --- a/src/session.ts +++ b/src/session.ts @@ -694,6 +694,13 @@ export type Session = { * only the light transform (absorb/attach/strip/drop) — no seam walk, * no re-tagging, no id re-keying. Cleared per turn like lastMessages. */ lastPlaneMessages?: CoreMessage[]; + /** #6181 B0: one-shot "skip the snapshot-plane write on the next + * save" — set by the lane for a turn that CANNOT be addressed against + * the plane (a non-cohort chained delta whose kernel view is a delta, + * not the world): deriving events from it would delete every row the + * view does not contain. snapShadowSync clears it when it consumes it + * (one save, then the flag is gone). */ + skipPlaneFlush?: boolean; /** 2×2 Part B (出站增量, responses lane only): the incremental-outbound * frontier — the highest plane row pos + anchor summary ids the provider * already has via its retained chain. Persisted with the session; lost diff --git a/src/snapshot/stage0.ts b/src/snapshot/stage0.ts new file mode 100644 index 0000000000..612831ebdf --- /dev/null +++ b/src/snapshot/stage0.ts @@ -0,0 +1,207 @@ +// #5874 2×2 stage 0 (#6181): world match + kernel-input assembly for +// cohort (dual-channel DECLARED) plugin traffic. SHARED by the openai and +// responses lanes so both carry the same red gate and the same +// one-round-convergence contract — the HTTP responses lane previously had +// NO assembly layer at all, which is how a chained (delta) turn's collapsed +// kernel view reached the plane and destroyed it (#6181 B0: the plane +// invariant is "the resent array IS the world", and only a cohort turn +// proves that — a non-cohort chained turn cannot be addressed against the +// plane at all). +// +// Runs BEFORE processTurn in the lane, so a thrown FoundationRejectError +// (R3) carries ZERO kernel side effects. Every other fault → the legacy +// full path (degrade, never guess — one-round convergence). +// +// Contract (brief 19544722 + review 2026-11-12): +// - armed = pluginMode + dual-channel DECLARED (msgids present) + the +// pluginLaneServerHistory rollout (M1: the red gate must be unreachable +// wherever the pin is unreachable) + no R3 latch on the sid (in-memory +// window OR the persisted latch, M3). +// - R3: declared but incomplete → checkR3Identity is the single arbiter +// (M2: it is CALLED here, in both lanes); an unlatched fault throws +// FoundationRejectError → the server maps it to 400 (B1); 3 faults in a +// 10-minute window latch the sid to the legacy path and the latch +// PERSISTS across proxy restarts (snapR3Latch). +// - stage 0: snapStage0 → resync (diverge / explicit re-baseline) → plane +// reset + legacy this turn + the lane clears session.responsesDelta +// (M6: a stale frontier against re-issued pos is a dead loop); +// otherwise assemble the kernel input from the plane rows (old cores — +// the kernel's own tagged output, invariant v) + the NEW spans (P5) + +// anchor rows at their covered-run boundary. + +import type { BiliMessage } from "acp-kernel/wire"; +import { getStore } from "../persist.js"; +import { featureEnabled } from "../rollout.js"; +import { applyMsgIdPin, validateMsgIds } from "../msgid-pin.js"; +import { ANCHOR_ID_PREFIX } from "../storage/snap.js"; +import { assembleKernelInput, shouldResync } from "./assemble.js"; +import { checkR3Identity, FoundationRejectError, r3FaultTracker } from "./plane.js"; + +export interface Stage0Input { + sessionId: string; + /** Lane name for log prefixes only ("openai" | "responses"). */ + lane: string; + /** The converted array for this turn (seq-stamped by the lane). */ + msgs: BiliMessage[]; + /** The raw declared biliMsgIds field value (whatever shape it had). */ + incomingMsgIds: unknown; + /** Validated ledger seqs (null when absent/mismatched — the lane + * already applied them to msgs when non-null). */ + incomingSeqs: number[] | null; + pluginMode: boolean; + /** The openai fast-path watermark hit; the responses lane has none. */ + wpHit?: unknown; + log: (level: string, msg: string) => void; +} + +export interface Stage0Result { + /** Pinned msg array when the pin applied — the lane adopts it + * (`msgs = pinned; originalMessages = msgs`). null otherwise. */ + pinned: BiliMessage[] | null; + /** Assembled kernel input (stage 0 hit). undefined → the lane renders + * its usual view (anchored / D lane). */ + kernelInput: BiliMessage[] | undefined; + /** True when the cohort armed and RAN (assembled, resynced, or + * degraded) — the lane gates the chained-non-assembled plane bypass + * on its NEGATION: a non-cohort chained turn's kernel view is a + * delta, not the world, and must touch neither the plane write nor + * the plane egress for this turn (B0 defensive). */ + cohortRan: boolean; + /** True when stage 0 reset the plane this turn (diverge / explicit + * re-baseline) — the lane must clear session.responsesDelta (M6). */ + resynced: boolean; +} + +export function runStage0(opts: Stage0Input): Stage0Result { + const { sessionId, lane, msgs, incomingMsgIds, incomingSeqs, pluginMode, wpHit, log } = opts; + const rollout = featureEnabled("pluginLaneServerHistory"); + const declared = Array.isArray(incomingMsgIds) && incomingMsgIds.length > 0; + // M1: the arm rides the SAME rollout as the pin — under the default + // (rollout off) the whole 2×2 path is unreachable, so R3 is not dead + // code waiting for a rollout that never arms it. + const armed0 = rollout && wpHit === undefined && pluginMode && declared; + const persistedLatch = (() => { + try { + return getStore()?.snapR3Latched(sessionId) ?? false; + } catch { + return false; // a store fault must not change the arming verdict + } + })(); + let armed = armed0 && !r3FaultTracker.isLatched(sessionId) && !persistedLatch; + if (armed0 && !armed) { + log("debug", `[${sessionId}] [2x2:${lane}] sid is R3-latched — legacy path this turn`); + } + let pinned: BiliMessage[] | null = null; + if (wpHit === undefined && pluginMode && incomingMsgIds !== undefined) { + if (rollout) { + const v = validateMsgIds(incomingMsgIds, msgs.length); + if (v.ok) { + pinned = applyMsgIdPin(msgs, v.ids); + } else if (armed) { + // #6181 M2: checkR3Identity is the single arbiter of the + // identity fault (declared + element count mismatched) — + // validateMsgIds' shape verdicts are its evidence, not a + // second gate. + if (!checkR3Identity(true, msgs.length, Array.isArray(incomingMsgIds) ? (incomingMsgIds as string[]) : null)) { + // R3: a cohort host that DECLARED the dual-channel but + // did not collect every element → hard 400 BEFORE any + // kernel side effect; the fault feeds the breaker + // (10-minute window, 3 strikes → the sid latches to the + // legacy path, PERSISTED across restarts (M3), instead + // of bricking the session on a misbehaving host). + const { latched } = r3FaultTracker.recordFault(sessionId); + if (latched) { + armed = false; + try { + getStore()?.snapR3Latch(sessionId); + } catch { /* the in-memory latch already fires; persistence is best-effort */ } + log("error", `[${sessionId}] [2x2:${lane}] R3 identity fault ×3 in window (${v.reason}) — sid latched to the legacy path (persisted)`); + } else { + throw new FoundationRejectError(`inbound msgid envelope failed validation (${v.reason})`); + } + } + } else { + log("warn", `[${sessionId}] [msgid] ${lane}-wire pin rejected (${v.reason}) — falling back to content-derived ids (#4491/#5825)`); + } + } else { + log("info", `[${sessionId}] [msgid] ${lane}-wire biliMsgIds present but pluginLaneServerHistory rollout is off for this install — field stripped, ids not pinned (#5825)`); + } + } + let kernelInput: BiliMessage[] | undefined; + let resynced = false; + if (armed) { + try { + const store = getStore(); + const stage0 = store?.snapStage0(sessionId, msgs.map((m) => m.id), { seqs: incomingSeqs ?? undefined }); + const cores = stage0 ? (store?.snapStage0Cores(sessionId) ?? null) : null; + if (stage0 && cores) { + if (shouldResync(stage0.decision, cores.rows.length)) { + // Structural disagreement (diverge) or an explicit + // re-baseline: reset the plane; THIS turn runs the + // legacy full path and re-seeds it through the + // in-request flush — the NEXT turn re-issues the + // envelope from the client's current array. + store!.snapStage0Reset(sessionId); + resynced = true; + log("warn", `[${sessionId}] [2x2:${lane}] stage-0 ${stage0.decision.mode} — plane reset, this turn legacy (one-round convergence)`); + } else { + // newCores = the converted spans of the NEW ids, in + // envelope order. The front stage already converted the + // whole array (front-stage invariance), so the new + // spans carry their full-parse bytes — T5 holds by + // construction. + const newIdSet = new Set(stage0.decision.newIdPos.map((n) => n.id)); + const newCores: BiliMessage[] = []; + const taken = new Set(); + for (const e of stage0.decision.envelope) { + if (!newIdSet.has(e.id) || taken.has(e.id)) continue; + taken.add(e.id); + const m = msgs.find((x) => x.id === e.id); + if (m === undefined) throw new Error(`new span ${e.id} absent from the converted array`); + newCores.push(m); + } + // The plane's ANCHOR rows (active acp_summary_) + // ride the kernel input at their covered-run boundary — + // the D-view placement parity. snapStage0Cores returns + // both the rows and the coverMax map. + const anchors = cores.rows + .filter((r) => r.ids.some((id) => id.startsWith(ANCHOR_ID_PREFIX))) + .map((r) => { + const id = r.ids.find((x) => x.startsWith(ANCHOR_ID_PREFIX))!; + return { + msg: JSON.parse(r.postJson) as BiliMessage, + blockId: id.slice(ANCHOR_ID_PREFIX.length), + }; + }); + const asm = assembleKernelInput({ + decision: stage0.decision, + rows: cores.rows + .filter((r) => !r.ids.some((id) => id.startsWith(ANCHOR_ID_PREFIX))) + .map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })), + newCores, + skipIds: stage0.explainedGone, + anchors, + coverMax: cores.coverMax, + }); + kernelInput = asm.kernelInput; + log("info", `[${sessionId}] [2x2:${lane}] stage-0 ${stage0.decision.mode} — kernel input from plane (old=${asm.oldCount}, new=${newCores.length}, anchors=${anchors.length})`); + } + } + } catch (err) { + if (err instanceof FoundationRejectError) throw err; // R3 outranks the fallback + const why = err instanceof Error ? err.message : String(err); + if (why.startsWith("[snapshot 2×2]")) { + // The assembly could not address the envelope against the + // plane (table behind — R4-shaped): reset and take the + // legacy path this turn (one-round convergence). M6: the + // lane clears session.responsesDelta for a resynced sid. + try { + getStore()?.snapStage0Reset(sessionId); + } catch { /* the reset is best-effort — the legacy path is the floor */ } + resynced = true; + } + log("warn", `[${sessionId}] [2x2:${lane}] stage-0 failed (${why}) — legacy full path`); + } + } + return { pinned, kernelInput, cohortRan: armed, resynced }; +} diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 2edb9add46..095d4c0a6e 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -66,6 +66,9 @@ export interface LegacySessionLike { * runs only the light transform. Carried into IngestSessionInput as * planeMessages. */ lastPlaneMessages?: CoreMessage[]; + /** #6181 B0: one-shot plane-write skip — see Session.skipPlaneFlush. + * Consumed (cleared) by snapShadowSync on the save that carries it. */ + skipPlaneFlush?: boolean; pluginSnapshot?: CoreMessage[]; /** #3301: external-converter import payload — original message * decompositions interned BEFORE the snapshot/tail so their rows get the @@ -133,6 +136,8 @@ export function toLegacyLike(s: { lastMessages?: LegacySessionLike["lastMessages"]; lastMessagesFolded?: boolean; lastPlaneMessages?: LegacySessionLike["lastPlaneMessages"]; + /** #6181 B0: one-shot plane-write skip (see Session.skipPlaneFlush). */ + skipPlaneFlush?: LegacySessionLike["skipPlaneFlush"]; pluginSnapshot?: LegacySessionLike["pluginSnapshot"]; decompositionLog?: LegacySessionLike["decompositionLog"]; contentStore?: LegacySessionLike["contentStore"]; @@ -149,6 +154,7 @@ export function toLegacyLike(s: { lastMessages: s.lastMessages, lastMessagesFolded: s.lastMessagesFolded, lastPlaneMessages: s.lastPlaneMessages, + skipPlaneFlush: s.skipPlaneFlush, pluginSnapshot: s.pluginSnapshot, decompositionLog: s.decompositionLog, contentStore: s.contentStore, @@ -514,7 +520,7 @@ function finishPlan( ccrJson: session.contentStore ? JSON.stringify(session.contentStore) : null, }; - return { session: sessionRow, messages, blocks, deadRefs, refs, ccrEntries, pluginSnapshotSeqs, tailSeqs, planeMessages: session.lastPlaneMessages ?? null }; + return { session: sessionRow, messages, blocks, deadRefs, refs, ccrEntries, pluginSnapshotSeqs, tailSeqs, planeMessages: session.lastPlaneMessages ?? null, ...(session.skipPlaneFlush ? { skipPlaneFlush: true } : {}) }; } /** Rebuilds the legacy load view from unified rows alone. */ diff --git a/src/storage/schema.ts b/src/storage/schema.ts index 2fb918de61..b36021aef8 100644 --- a/src/storage/schema.ts +++ b/src/storage/schema.ts @@ -362,7 +362,8 @@ CREATE TABLE IF NOT EXISTS l1_world ( issued_at INTEGER NOT NULL, armed_at INTEGER, envelope_json TEXT, - envelope_checksum TEXT + envelope_checksum TEXT, + r3_latched_at INTEGER ); CREATE INDEX IF NOT EXISTS idx_l1_ledger ON l1_world(ledger_id); CREATE TABLE IF NOT EXISTS snap_rows ( @@ -429,6 +430,10 @@ export function migrateSchema(db: { if (l1Cols.length > 0) { if (!l1Cols.includes("envelope_json")) db.exec("ALTER TABLE l1_world ADD COLUMN envelope_json TEXT"); if (!l1Cols.includes("envelope_checksum")) db.exec("ALTER TABLE l1_world ADD COLUMN envelope_checksum TEXT"); + // #6181 M3: the R3 breaker latch PERSISTS (a proxy restart must not + // re-arm a misbehaving host's sid and re-400 the same broken + // envelope); NULL/0 = not latched, else the epoch-ms of the latch. + if (!l1Cols.includes("r3_latched_at")) db.exec("ALTER TABLE l1_world ADD COLUMN r3_latched_at INTEGER"); } if (from >= STORAGE_SCHEMA_VERSION) return { from, to: from }; let needV8Backfill = false; diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 05ad7ecac3..ab1e017cfa 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1012,6 +1012,18 @@ export class UnifiedShadow { this.sync.snapStage0Reset(sessionId); } + /** #6181 M3: persisted R3 breaker latch (see UnifiedStore.snapR3Latched). */ + snapR3Latched(sessionId: string): boolean { + if (this.sync === null) return false; + return this.sync.snapR3Latched(sessionId); + } + + /** #6181 M3: persist the R3 breaker latch (best-effort). */ + snapR3Latch(sessionId: string): void { + if (this.sync === null) return; + this.sync.snapR3Latch(sessionId); + } + vacuum(mode: "full" | "incremental"): void { if (this.sync === null) throw new Error("[storage] sync vacuum unavailable on a network backend"); return this.sync.vacuum(mode); diff --git a/src/storage/snap.ts b/src/storage/snap.ts index 79aadeb3d1..372aa2f601 100644 --- a/src/storage/snap.ts +++ b/src/storage/snap.ts @@ -732,6 +732,32 @@ export function snapL1EnvelopeUpsert(db: DbLike, sessRaw: string, envelopeJson: if (res.changes === 0) throw new Error(`snapL1EnvelopeUpsert: no l1_world row for sess_raw ${sessRaw.slice(0, 12)}… (record it first)`); } +/** #6181 M3: read the persisted R3 breaker latch (true when the sid's + * breaker latched in a PREVIOUS process — a restart must not re-arm a + * misbehaving host). Missing row or NULL → false. */ +export function snapR3LatchedRead(db: DbLike, sessRaw: string): boolean { + const r = db + .prepare("SELECT r3_latched_at FROM l1_world WHERE sess_raw = ?") + .get(sessRaw) as { r3_latched_at: number | null } | undefined; + return !!r && !!r.r3_latched_at; +} + +/** #6181 M3: persist the R3 breaker latch (UPDATE only — the l1 row is + * created by snapL1Record before the lane arms; a missing row degrades + * silently, the in-memory latch still fires for this process). */ +export function snapR3LatchWrite(db: DbLike, sessRaw: string, nowMs: number): void { + // Upsert — the first fault of a fresh session happens BEFORE its stage-0 + // snapL1Record call (the R3 check runs at pin validation), so an + // UPDATE-only latch would silently no-op for sessions with no l1 row yet. + // ledger_id takes the placeholder sessRaw and witness_probe/issued_at the + // sentinel values: the latch row's envelope stays NULL until a later + // (repaired) stage-0 snapL1Record upsert carries the real ledger id and + // witness. A latched session never re-arms (runStage0 checks the persisted + // latch before any store call), so the placeholder can never hit the + // "red" ledger-conflict path of snapL1Record. + db.prepare("INSERT INTO l1_world (sess_raw, ledger_id, witness_probe, issued_at, r3_latched_at) VALUES (?, ?, '', ?, ?) ON CONFLICT(sess_raw) DO UPDATE SET r3_latched_at = excluded.r3_latched_at, ledger_id = COALESCE(l1_world.ledger_id, excluded.ledger_id), witness_probe = COALESCE(NULLIF(l1_world.witness_probe, ''), excluded.witness_probe)").run(sessRaw, sessRaw, nowMs, nowMs); +} + // --------------------------------------------------------------------------- // 2×2 read side — the FULL row set (active AND inactive), for the stage-0 // hydrate: S4 drift checks, explainedGone (fold-covered ids may legally diff --git a/src/storage/store.ts b/src/storage/store.ts index f2a127e792..0580d0911a 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -31,6 +31,8 @@ import { readSnapView, snapGcCoveredRows, snapL1Record, + snapR3LatchedRead, + snapR3LatchWrite, snapplaneStatusWire, snapRowsAll, SnapPlaneDeadError, @@ -355,6 +357,12 @@ export interface IngestSessionInput { * from this view; null falls back to the defensive full-transcript diff * (saves that don't carry a turn view). */ planeMessages?: CoreMessage[] | null; + /** #6181 B0: one-shot plane-write skip for THIS save — set by the lane + * for a turn that cannot be addressed against the plane (a non-cohort + * chained delta). snapShadowSync returns before any plane work; the + * flag is consumed at the persist build (the live session flag is + * cleared by the lane after the in-request flush). */ + skipPlaneFlush?: boolean; /** Serialized orphan map (Record) — blockContents * entries whose block is absent from state.blocks (#4216/#4242: the reap * shape is REPRESENTABLE; rides the same transaction so the session row, @@ -1521,6 +1529,13 @@ export class UnifiedStore { const sid = input.session.sessionId; if (this.hasLaneExt(sid)) return; // skip:lane — lane transcripts are not the cohort truth if (isSnapPlaneDead(sid)) return; // dead (R1) — sticky, silent; status surfaces it + // #6181 B0: a turn that CANNOT be addressed against the plane (a + // non-cohort chained delta — its kernel view is a delta, not the + // world) skips the plane write for THIS save: deriving events from + // the collapsed view would DELETE every row the view does not + // contain (snapViewEvents is a final-state diff). The flag is + // one-shot — consumed by the persist build, not here. + if (input.skipPlaneFlush) return; const now = Date.now(); // Stage-0 identity: raw host session id → ledger id (v1 mapping: the // conversation binding, session_id when there is none). @@ -2489,6 +2504,26 @@ export class UnifiedStore { clearSnapPlaneDead(sid); } + /** #6181 M3: whether this sid's R3 breaker latch PERSISTED from a + * previous process (a restart must not re-arm a misbehaving host and + * re-400 the same broken envelope). */ + snapR3Latched(sid: string): boolean { + try { + return snapR3LatchedRead(this.db, this.snapSessRawOf(sid)); + } catch { + return false; // schema/column edge: treat as unlatched, the in-memory breaker still fires + } + } + + /** #6181 M3: persist the R3 breaker latch (best-effort — a failed + * write degrades to the in-memory latch, which still fires this + * process). */ + snapR3Latch(sid: string): void { + try { + snapR3LatchWrite(this.db, this.snapSessRawOf(sid), Date.now()); + } catch { /* best-effort — see above */ } + } + /** #3250: cadence-tick WAL reclamation (see WAL_CHECKPOINT_EVERY_INGESTS). * Never throws into the ingest path — a failed checkpoint just leaves the * WAL longer than ideal until the next tick. */ diff --git a/tests/issue5874-2x2-p3.test.ts b/tests/issue5874-2x2-p3.test.ts index f6270e709c..644173d900 100644 --- a/tests/issue5874-2x2-p3.test.ts +++ b/tests/issue5874-2x2-p3.test.ts @@ -659,3 +659,311 @@ test("S1 the full M1→M2 round trip: seed (issue) → M1 probe → M2 append assert.deepEqual(echo.decision.newIdPos, []); void rows; }); + +// --------------------------------------------------------------------------- +// Fix-round tests: M4 codec-level pin, B0 plane bypass, M3 latch persistence, +// and the HTTP e2e pins (B1 400 funnel, B3 pure continuation, B4 frontier +// commit on upstream accept). +// --------------------------------------------------------------------------- + +import http from "node:http"; +import { once } from "node:events"; +import { defaultConfig } from "acp-kernel"; +import { openaiToCore } from "acp-kernel/wire"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _resetSnapEgressForTests, applySnapEgress } from "../src/server/snap-egress.ts"; + +test("A1d M4: the codec-level pin — parsing the new element ALONE (M2 wire) yields the same core as parsing it inside the full array (M1 wire), wireIndex aside", () => { + const contents = Array.from({ length: 11 }, (_, i) => `codec content ${i + 1}`); + const fullBody = { model: "gpt-m4", messages: contents.map((c) => ({ role: "user", content: c })) }; + const full = openaiToCore(fullBody as never); + assert.equal(full.msgs.length, 11); + // M2 wire: the client sends ONLY the 11th element — a fresh parse with a + // fresh ClusterCounter. The canonical id derives from (role, contentType, + // text), so the counter state cannot change the id for an element whose + // content appears once. + const inc = openaiToCore({ model: "gpt-m4", messages: [fullBody.messages[10]!] } as never); + assert.equal(inc.msgs.length, 1); + const strip = (m: BiliMessage): Record => { + const c = { ...(m as unknown as Record) }; + delete c.wireIndex; // position on the wire — legitimately differs (10 vs 0) + return c; + }; + assert.equal( + JSON.stringify(strip(full.msgs[10]!)), + JSON.stringify(strip(inc.msgs[0]!)), + "the new element's core is byte-identical between the full-array parse (M1) and the standalone parse (M2), so the assembled kernel cores are codec-level identical, not just assembly-level identical", + ); + + // And through the assembly: plane rows m1..m10 + the new core from each + // leg's own parse — the kernel input bytes must match. + const sid = "a1d"; + const store = storeWithRows(10, sid); + store.snapStage0(sid, Array.from({ length: 10 }, (_, i) => `m${i + 1}`)); + const rows10 = store.snapStage0Cores(sid).rows.map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })); + const ten = Array.from({ length: 10 }, (_, i) => `m${i + 1}`); + const m1 = store.snapStage0(sid, [...ten, "codec-11"], { seqs: ten.map((_, i) => i + 1).concat(11) }); + const asm1 = assembleKernelInput({ decision: m1.decision, rows: rows10, newCores: [full.msgs[10]!] }); + const store2 = storeWithRows(10, sid); + store2.snapStage0(sid, ten); + const m2 = store2.snapStage0(sid, ["codec-11"], { seqs: [11] }); + const rows10b = store2.snapStage0Cores(sid).rows.map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })); + const asm2 = assembleKernelInput({ decision: m2.decision, rows: rows10b, newCores: [inc.msgs[0]!] }); + // wireIndex is the parse-time position artifact (10 in the full parse, 0 + // in the standalone parse). It does not change the re-emitted wire: the + // plane rows carry no wireIndex, so the new element groups as its own + // wire message in BOTH legs. The semantic kernel cores must match. + const stripAll = (arr: BiliMessage[]) => JSON.stringify(arr.map(strip)); + assert.equal(stripAll(asm1.kernelInput), stripAll(asm2.kernelInput), "assembled kernel input byte-identical with codec-produced new cores (M1 full parse vs M2 standalone parse), wireIndex aside"); +}); + +test("B0a: a skipPlaneFlush save leaves the plane rows untouched — a collapsed view cannot delete history", () => { + const sid = "b0a"; + const store = storeWithRows(10, sid); + store.snapStage0(sid, Array.from({ length: 10 }, (_, i) => `m${i + 1}`)); + assert.equal(store.snapStage0Cores(sid).rows.length, 10); + + // A chained delta turn: the kernel view holds only the 4 messages it was + // given (a collapse). Without the flag, deriving events from this view + // would DELETE every row it does not contain. With the flag, the save + // runs the legacy ingest but skips ALL plane work. + const collapsed = [core(1, "m1"), core(2, "m2"), core(3, "m3"), core(4, "m4")]; + const plan = planIngest(plainSession(sid, 4)); + plan.planeMessages = collapsed as never; + plan.skipPlaneFlush = true; + store.ingestSession(plan); + const rows = store.snapStage0Cores(sid).rows; + assert.equal(rows.length, 10, "the plane still holds all 10 rows after the flagged save"); + assert.deepEqual(rows.map((r) => r.ids).flat().sort(), Array.from({ length: 10 }, (_, i) => `m${i + 1}`).sort(), "and the ids are exactly the original ten (nothing deleted, nothing added)"); + + // The flag is one-shot: the next save (full view, no flag) runs the plane + // path and stays at 10 rows (idempotent final-state diff). + const full = Array.from({ length: 10 }, (_, i) => core(i + 1, `m${i + 1}`)); + const plan2 = planIngest(plainSession(sid, 10)); + plan2.planeMessages = full as never; + store.ingestSession(plan2); + assert.equal(store.snapStage0Cores(sid).rows.length, 10); +}); + +test("B0b: egress bypass serves the legacy bytes verbatim — a stale plane never leaks into a delta turn", () => { + _resetSnapEgressForTests(); + const sid = "b0b"; + const store = storeWithRows(3, sid); + store.snapStage0(sid, ["m1", "m2", "m3"]); + // Register the store under test through the global seam (egress reads + // getStore() — the P2 G-test pattern). + const shadow = { + stats: { disabled: false }, + snapEgressView: (s: string) => store.snapEgressView(s), + } as never; + _setStoreForTest(new SessionStore({ enabled: true, shadow, log: () => undefined })); + + const state = { blocks: [] } as never; + const legacy = [core(1, "m1"), core(2, "m2"), core(3, "m3")]; + const deps = { sessionId: sid, protocol: "responses" as never, isCompressTool: (name: string) => name === "compress", log: () => undefined, transform: (msgs: BiliMessage[]) => msgs }; + const bypassed = applySnapEgress(state, legacy, { ...deps, bypass: true }); + assert.equal(bypassed, legacy, "bypass returns the legacy array BY REFERENCE (zero plane involvement for the turn)"); +}); + +test("M3: the R3 latch persists across store instances — a latched session degrades forever, never 400s again", () => { + const p = path.join(hermetic, "m3", "index.db"); + const a = openPlane(p); + assert.equal(a.snapR3Latched("sid-m3"), false); + a.snapR3Latch("sid-m3"); + // A FRESH instance over the same file — the process restart case: the + // in-memory breaker is cold, the persisted latch must carry the state. + const b = openPlane(p); + assert.equal(b.snapR3Latched("sid-m3"), true, "the latch survives a new store instance (process restart)"); + assert.equal(b.snapR3Latched("other-sid"), false, "other sessions are unaffected"); +}); + +// ── HTTP e2e (real proxy + fake upstream) ─────────────────────────────────── + +function listenT(server: http.Server): Promise { + if (server.listening) return Promise.resolve(); + return once(server, "listening").then(() => undefined); +} +function closeT(server: http.Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} +function sseT(type: string, data: unknown): string { + return `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; +} +function sseDone(): string { + return [ + sseT("response.output_item.done", { item: { type: "message", id: "msg_1", role: "assistant", content: [{ type: "output_text", text: "ok" }] }, output_index: 0 }), + sseT("response.completed", { response: { id: "resp_done", status: "completed", output: [], usage: { input_tokens: 100, output_tokens: 5, total_tokens: 105 } } }), + ].join(""); +} + +type UpstreamBehavior = (n: number, body: string) => { status: number; payload: string }; + +async function withProxyT( + behavior: UpstreamBehavior, + fn: (url: string, bodies: string[]) => Promise, + opts: { storeDir?: string; env?: Record } = {}, +): Promise { + const bodies: string[] = []; + let reqN = 0; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + const body = Buffer.concat(chunks).toString("utf8"); + bodies.push(body); + const { status, payload } = behavior(++reqN, body); + const streaming = payload.startsWith("event: "); + res.writeHead(status, streaming ? { "content-type": "text/event-stream", "cache-control": "no-cache" } : { "content-type": "application/json" }); + res.write(payload); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await listenT(upstream); + const upstreamPort = (upstream.address() as { port: number }).port; + + const savedEnv: Record = {}; + for (const [k, v] of Object.entries(opts.env ?? {})) { savedEnv[k] = process.env[k]; process.env[k] = v; } + let store: SessionStore | undefined; + try { + if (opts.storeDir) { + process.env.BILI_STORAGE_UNIFIED = "shadow"; + store = new SessionStore({ enabled: true, dir: opts.storeDir, debounceMs: 0 }); + _setStoreForTest(store); + } else { + _setStoreForTest(new SessionStore({ enabled: false })); + } + const proxyOpts: ProxyOptions = { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-e2e": { context: 400_000 } } } }, + modelContextLimit: 400_000, + kernelConfig: defaultConfig(400_000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + logFile: "off", + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + mitm: { enabled: false, domains: [] }, + }; + const proxy = await startServer(proxyOpts); + await listenT(proxy); + const proxyPort = (proxy.address() as { port: number }).port; + try { + await fn(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/responses`, bodies); + } finally { + await closeT(proxy); + } + } finally { + await closeT(upstream); + for (const [k, v] of Object.entries(savedEnv)) { if (v === undefined) delete process.env[k]; else process.env[k] = v; } + delete process.env.BILI_STORAGE_UNIFIED; + _setStoreForTest(new SessionStore({ enabled: false })); + void store; + } +} + +const PLUGIN_HEADERS = { "content-type": "application/json", "x-bili-plugin": "e2e-agent", "x-acp-session": "e2e-sess" } as Record; + +test("B1: a declared host that fails to collect msgids (count mismatch) → 400 + x-bili-snapshot: identity, no upstream forward", async () => { + await withProxyT( + () => ({ status: 200, payload: sseDone() }), + async (url, bodies) => { + const res = await fetch(url, { + method: "POST", + headers: PLUGIN_HEADERS, + body: JSON.stringify({ + model: "gpt-e2e", + stream: true, + input: [ + { type: "message", role: "user", content: "first" }, + { type: "message", role: "user", content: "second" }, + ], + biliMsgIds: ["b1m1"], // 1 declared vs 2 elements — the R3 fault + }), + }); + assert.equal(res.status, 400, "the identity fault is a 400 (the B1 control channel — not the generic 503)"); + assert.equal(res.headers.get("x-bili-snapshot"), "identity"); + const err = JSON.parse(await res.text()) as { error: { type: string; message: string } }; + assert.equal(err.error.type, "snapshot_identity_fault"); + assert.ok(err.error.message.includes("R3"), "the error names the fault class"); + assert.equal(bodies.length, 0, "zero side effects: the upstream was never contacted"); + }, + { env: { BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY: "1" } }, + ); +}); + +test("B3: a plugin chained round with EMPTY input is a pure continuation — the raw body passes through verbatim (no dev injection, id kept)", async () => { + await withProxyT( + () => ({ status: 200, payload: sseDone() }), + async (url, bodies) => { + const reqBody = { model: "gpt-e2e", stream: true, previous_response_id: "resp_b3", input: [] }; + const res = await fetch(url, { method: "POST", headers: PLUGIN_HEADERS, body: JSON.stringify(reqBody) }); + assert.equal(res.status, 200); + await res.text(); + assert.equal(bodies.length, 1); + const fwd = JSON.parse(bodies[0]!) as Record; + assert.equal(fwd.previous_response_id, "resp_b3", "the chain id is forwarded intact"); + assert.deepEqual(fwd.input, [], "the empty input passes through (NOT rebuilt, NOT dropped)"); + assert.ok(!JSON.stringify(fwd).includes("You are bili"), "no developer-message injection on the pure continuation (the B3 blind-response fix)"); + }, + {}, + ); +}); + +test("B4: the delta frontier commits only when the upstream ACCEPTS — a 400 turn leaves the frontier at its previous value (the next round re-seeds)", async () => { + await withProxyT( + (n) => (n === 2 ? { status: 400, payload: JSON.stringify({ error: { message: "upstream refused" } }) } : { status: 200, payload: sseDone() }), + async (url, bodies) => { + // Turn 1: seed (no chain ref yet). + const r1 = await fetch(url, { + method: "POST", + headers: PLUGIN_HEADERS, + body: JSON.stringify({ model: "gpt-e2e", stream: true, input: [{ type: "message", role: "user", content: "one" }], biliMsgIds: ["b4m1"], biliLedgerSeqs: [1] }), + }); + assert.equal(r1.status, 200); + await r1.text(); + + // Turn 2: CHAINED cohort turn (M2 append). Its outbound is the + // SEED round of the delta chain (no frontier yet) — and the + // upstream REFUSES it. The selected frontier must NOT commit. + const r2 = await fetch(url, { + method: "POST", + headers: PLUGIN_HEADERS, + body: JSON.stringify({ model: "gpt-e2e", stream: true, previous_response_id: "resp_t1", input: [{ type: "message", role: "user", content: "two" }], biliMsgIds: ["b4m2"], biliLedgerSeqs: [2] }), + }); + assert.equal(r2.status, 400, "the upstream refusal passes through"); + await r2.text(); + + // Turn 3: another chained cohort turn. With the frontier + // uncommitted, the outbound must be a RE-SEED: the FULL view + // (all 3 world elements over the wire). If the 400 turn had + // advanced the frontier, this round would send a 1-element delta. + const r3 = await fetch(url, { + method: "POST", + headers: PLUGIN_HEADERS, + body: JSON.stringify({ model: "gpt-e2e", stream: true, previous_response_id: "resp_t2", input: [{ type: "message", role: "user", content: "three" }], biliMsgIds: ["b4m3"], biliLedgerSeqs: [3] }), + }); + assert.equal(r3.status, 200); + await r3.text(); + const fwd3 = JSON.parse(bodies[2]!) as { input: { content?: unknown }[] }; + const texts = fwd3.input.flatMap((x) => (Array.isArray(x.content) ? x.content.map((p) => (p as { text?: string }).text ?? "") : [String(x.content ?? "")])); + for (const c of ["one", "two", "three"]) assert.ok(texts.some((t) => t.includes(c)), `turn 3's reseed view carries world element \"${c}\"`); + assert.equal(fwd3.input.length, 4, "the 3 world elements plus the re-injected developer message (a reseed round sends the full self-contained view + dev; a delta would have sent 1 world element and no dev)"); + }, + { storeDir: path.join(hermetic, "b4-store"), env: { BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY: "1", ACP_KEEP_RESPONSE_ID: "1" } }, + ); +}); From 6826c521ef29f0a3a06131144d6bda49afc78e34 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Thu, 12 Nov 2026 23:26:52 +0800 Subject: [PATCH 4/8] =?UTF-8?q?fix(#5874=202x2):=20review=20round=202=20?= =?UTF-8?q?=E2=80=94=20no-op=20rounds=20keep=20the=20chain=20id,=20same-bo?= =?UTF-8?q?dy=20M2=20retries=20no=20longer=20wipe=20the=20plane,=20and=20t?= =?UTF-8?q?he=20first=20chained=20round=20seeds=20its=20frontier=20at=20th?= =?UTF-8?q?e=20stored-world=20boundary=20(gated=20on=20a=20trusted,=20neve?= =?UTF-8?q?r-refused=20chain)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Residual 1a: an empty delta (frontier already covers the world — the host retried the same body after an upstream failure) is now classified as a NOOP: the chain id is kept, no developer message is injected, and the empty input rides the provider's own cache. Before: id stripped + dev injected = a blind response the host would chain from forever. - Residual 1b: stage 0 recognizes a full replay (every incoming id already in the stored canon, carrier seqs at/below the plane frontier) as echo+replay — zero side effects. Before: an M2 retry ([m5] vs head m1) read as headMoved -> diverge -> snapStage0Reset: a harmless retry wiped the plane. - Residual 2: the first CHAINED round of a keepResponseId chain carried no committed frontier (round 1 had no id), so it re-seeded the whole world the provider's chain already holds from this proxy's own round-1 full view (2x the world's tokens, one time). Stage 0 now proves the safe case (probe + nothing gone + new elements -> the chain provably holds the stored world) and seeds the frontier at the stored-world boundary; everything else keeps the plain re-seed. - B4 hardening: a REFUSED selection round now sets a persisted responsesChainBroken marker (committed with the session). While set, the frontier seed is blocked and the round re-seeds the full view — the in-request flush wrote the refused turn's rows before the forward, so the plane holds more than the provider's chain and trusting storedFrontierPos would lose the refused content. The flag clears on a full-view/noop acceptance; the pending delta now carries its kind (full/noop/partial) so the relay can maintain it. - Tests: p3 42/42 (new: residual-2 unit shape pin, residual 1a+2 e2e through the proxy, A1/A1d legs now use isolated DBs — the two legs are ALTERNATIVE client behaviors on the same turn and sharing a file let leg M1's canon contaminate leg M2's verdict). - Regression: p0 10, p1 4, p2 17, fastpath 2, snapshot-render 13+6, snapshot-shadow 5, storage-engine-proofs 12, opencode-rolling 10, responses-previous-response-id 7 — all green. --- src/server.ts | 2 +- src/server/prepare-responses.ts | 66 +++++++++++++-- src/server/relay.ts | 19 ++++- src/session.ts | 11 +++ src/snapshot/plane.ts | 117 +++++++++++++++++--------- src/snapshot/stage0.ts | 28 ++++++- src/snapshot/types.ts | 6 ++ tests/issue5874-2x2-p3.test.ts | 144 ++++++++++++++++++++++++++++++-- 8 files changed, 338 insertions(+), 55 deletions(-) diff --git a/src/server.ts b/src/server.ts index b66136c166..dad4282449 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2023,7 +2023,7 @@ export type Prepared = { * it (null) so the next round RE-SEEDS (at-least-once: over-serving is * safe, under-serving is the context loss). Absent (undefined) = no * delta was selected this turn. */ - pendingResponsesDelta?: { frontierPos: number; servedAnchors: string[] } | null; + pendingResponsesDelta?: { state: { frontierPos: number; servedAnchors: string[] }; kind: "full" | "noop" | "partial" } | null; anthropicSystem?: AnthropicRequestBody["system"]; /** #2189: the client's billing-attribution block captured from the INBOUND * anthropic system (pre-anchor). Preflight summary calls must carry it — diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 5858a8ac3c..ec5057ca39 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -128,6 +128,13 @@ export async function prepareResponses( // provider's retained chain carries the rest) — gates the developer // message re-injection and the previous_response_id strip below. let deltaServed = false; + // 2×2 Part B (review-round-2 Residual 1a): a chained round whose plane + // frontier is ALREADY at the top of the world — nothing new to send. + // The outbound is an EMPTY input with the chain id KEPT and no developer + // message: the provider's retained chain holds the whole world. Without + // this classification the no-op read as "no delta" → id stripped + dev + // injected + empty input = a BLIND response the host would chain from. + let noopServed = false; // 2×2 Part B: a persisted delta frontier existed BEFORE this round's // outbound selection (the provider's retained context equals exactly the // elements sent up to it) — see the strip site below for the contract. @@ -136,7 +143,7 @@ export async function prepareResponses( // frontier to the relay (committed only after the upstream accepts). let deltaState: DeltaState | null | undefined; let selectionFired = false; - let pendingDelta: { frontierPos: number; servedAnchors: string[] } | null = null; + let pendingDelta: { state: { frontierPos: number; servedAnchors: string[] }; kind: "full" | "noop" | "partial" } | null = null; // #6181 B0: a chained turn that cannot be addressed against the plane // (non-cohort — its kernel view is a delta, not the world) skips BOTH // the plane write for this turn's flush and the plane egress — see the @@ -536,6 +543,31 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // state) → reseed (full send, frontier reset). const chains = knobKeepResponseId() && typeof parsed.previous_response_id === "string" && parsed.previous_response_id.length > 0; deltaState = chains ? (session.responsesDelta ?? null) : undefined; + // #6181 review round 2, Residual 2: the first CHAINED round of a + // keepResponseId chain carries a previous_response_id but NO committed + // frontier — round 1 carried no id, so no selection fired and B4 had + // nothing to commit. A plain re-seed would resend the WHOLE world that + // the provider's chain already holds from this proxy's own round-1 + // full view (2× the world's tokens, the conversation twice). Stage 0 + // can PROVE the safe case: the current world is a strict extension of + // the world the plane last held (probe, nothing gone, new elements). + // The chain was built from the client's own resends of exactly that + // world, so initialize the frontier at the stored-world boundary and + // serve the new elements only. Every other shape (scatter / issue / + // resync / degraded stage 0 / no chain) falls through to the plain + // re-seed — worst case there is the one-time duplication the re-seed + // always had, never a loss. Gated on !responsesChainBroken: a rejected + // selection (B4) means the provider never ran that round, so the plane + // holds more than the provider's chain — the stored-world boundary is + // NOT safe and the round must re-seed the full view. + if ( + deltaState === null && chains + && !session.responsesChainBroken + && s0.kernelInput !== undefined && s0.storedFrontierPos !== null + ) { + deltaState = { frontierPos: s0.storedFrontierPos, servedAnchors: [] }; + log("info", `[${sessionId}] [snaplane] frontier seeded at the stored-world boundary (pos ${s0.storedFrontierPos}) — the provider chain from this proxy's own prior round holds the stored world; serving the new elements only instead of re-seeding (Residual 2)`); + } // True once THIS proxy has established a delta chain with the provider // (a persisted frontier exists). Only then may a non-delta round strip // the id: the provider's retained context equals exactly the elements @@ -566,12 +598,26 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr deltaState, onSelection: (sel, totalElements) => { selectionFired = true; - pendingDelta = sel.nextState; + // kind (review round 2, Residual 2 companion): the relay + // needs to know WHAT went out to maintain responsesChainBroken. + // full = reseed/full-resend (the provider now provably holds + // the whole world) · noop = nothing new requested (the chain + // is intact) · partial = the held world extended by the new + // suffix only. + pendingDelta = { + state: sel.nextState, + kind: sel.reseed || sel.delta.length >= totalElements ? "full" : sel.delta.length === 0 ? "noop" : "partial", + }; // deltaServed = a PARTIAL delta went out: new content only. // A reseed (state lost / first round) and a full-resend // (delta grew to the whole view — isFullResend) both send // the FULL view, which must read as "no delta" for the id // handling below. + // noopServed = the frontier already covers the whole world + // (Residual 1a: same-body retry of an M1/M2 turn) — empty + // input goes out WITH the chain id kept and NO developer + // message: pure chain continuation. + noopServed = !sel.reseed && sel.delta.length === 0; deltaServed = !sel.reseed && sel.delta.length > 0 && sel.delta.length < totalElements; }, } : {}), @@ -587,7 +633,7 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr session.responsesDelta = null; } if (deltaServed) { - log("info", `[${sessionId}] [snaplane] outbound=delta frontier=${(pendingDelta as { frontierPos: number } | null)?.frontierPos} — input carries only the unsent suffix; the retained chain carries the rest (re-injected developer message / forged summaries skipped to avoid duplicating what the provider keeps)`); + log("info", `[${sessionId}] [snaplane] outbound=delta frontier=${(pendingDelta as { state: { frontierPos: number } } | null)?.state.frontierPos} — input carries only the unsent suffix; the retained chain carries the rest (re-injected developer message / forged summaries skipped to avoid duplicating what the provider keeps)`); } // #3477: consume the announced native-compaction boundary (pi session_compact // → /__bili/plugin/compact → markCompactionBoundary) exactly once, after the @@ -631,8 +677,11 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr responsesDevContent = devContent; if (forgedSummaries.length > 0) log("debug", `[${sessionId}] [inject] ${forgedSummaries.length} captured summary block(s) re-injected into developer message`); // 2×2 Part B: on a delta round the provider KEEPS the seed round's - // developer message — re-injecting would duplicate it. - if (!deltaServed) rebuiltInput = injectResponsesDeveloperMessage(rebuiltInput, devContent, projection.headRole ?? "developer"); + // developer message — re-injecting would duplicate it. On a no-op + // round (Residual 1a) there is NO new content at all — the provider + // chain holds everything, so injecting dev + sending an empty input + // would be a BLIND response. + if (!deltaServed && !noopServed) rebuiltInput = injectResponsesDeveloperMessage(rebuiltInput, devContent, projection.headRole ?? "developer"); if (!knobNoInjectTool() && injectTools) { // #1712: decompress's startId/endId execute only on CCR-armed // sessions (#1179) — serve the no-range schema otherwise. @@ -646,7 +695,10 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr const devContent = [...projection.systemParts, ...forgedSummaries, ...(tagsOnlyPrompt !== "" ? [tagsOnlyPrompt] : []), ...(genBoundaryActive && !isCompactionTrigger ? [GENERATION_BOUNDARY_NOTE] : [])].join("\n\n---\n\n"); responsesDevContent = devContent; if (forgedSummaries.length > 0) log("debug", `[${sessionId}] [inject] ${forgedSummaries.length} captured summary block(s) re-injected into developer message`); - if (!deltaServed) rebuiltInput = injectResponsesDeveloperMessage(rebuiltInput, devContent, projection.headRole ?? "developer"); + // Residual 1a: a no-op round sends an EMPTY input — injecting the + // developer message there would make the provider answer from the + // dev prompt alone (a blind response the host would chain from). + if (!deltaServed && !noopServed) rebuiltInput = injectResponsesDeveloperMessage(rebuiltInput, devContent, projection.headRole ?? "developer"); } if (sysNotes.length > 0) { const inputItems: ResponseInputItem[] = typeof rebuiltInput === "string" @@ -840,7 +892,7 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // a reseed / full-resend / degraded-legacy round STRIPS it (the chain // breaks, the client re-chains from this round's fresh response id and the // frontier re-seeds — one-round convergence, acceptance #7). - if (!knobKeepResponseId() || (hadFrontier && !deltaServed)) { + if (!knobKeepResponseId() || (hadFrontier && !deltaServed && !noopServed)) { const chainWarn = strippedResponseIdWarning(rebuilt.previous_response_id); if (chainWarn) log("warn", `[${sessionId}] ${chainWarn}`); delete rebuilt.previous_response_id; diff --git a/src/server/relay.ts b/src/server/relay.ts index 4f1772e2e1..efcf2b8aa2 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1283,7 +1283,24 @@ async function forwardCaptured( // the turn; compress-loop re-requests carry full views). if (prepared && prepared.pendingResponsesDelta !== undefined) { const accepted = upstreamResult.response.status < 400; - prepared.session.responsesDelta = accepted ? prepared.pendingResponsesDelta : null; + const pending = prepared.pendingResponsesDelta; + prepared.session.responsesDelta = accepted ? (pending ? pending.state : null) : null; + // #6181 review round 2, Residual 2 companion (B4 hardening): + // a REFUSED selection means the provider never ran that round — + // its chain holds at most the previously-accepted world while + // the plane already holds more (the in-request flush wrote the + // refused turn's rows before the forward). While broken, the + // first-chained-round frontier seed is blocked and the round + // re-seeds the FULL view (at-least-once). A full-view or noop + // acceptance re-proves the chain holds the whole world and + // clears the flag; a partial acceptance leaves it as-is (it + // cannot be set — a partial needs a valid frontier, and the + // broken state forces a reseed instead). + if (accepted) { + if (pending && pending.kind !== "partial") prepared.session.responsesChainBroken = false; + } else { + prepared.session.responsesChainBroken = true; + } markDirty(prepared.session); prepared.pendingResponsesDelta = undefined; } diff --git a/src/session.ts b/src/session.ts index d9948beaf4..49f27cf934 100644 --- a/src/session.ts +++ b/src/session.ts @@ -707,6 +707,17 @@ export type Session = { * state → next round reseeds (full send), one-round convergence. See * src/snapshot/delta.ts. */ responsesDelta?: { frontierPos: number; servedAnchors: string[] } | null; + /** #6181 review round 2, Residual 2 companion: the provider's chain may + * NOT be assumed to hold the stored world. Set when a delta-selection + * round was REFUSED upstream (the provider never ran it — its chain + * holds at most the previously-accepted world while the plane already + * holds more). While set, the first-chained-round frontier seed is + * blocked and the round re-seeds the FULL view (at-least-once — + * over-serving is safe, under-serving is the context loss). Cleared + * when a full-view round is accepted (the provider provably holds the + * whole world again). Transient within a turn? No — persisted with the + * session, like responsesDelta. */ + responsesChainBroken?: boolean; pluginSnapshot?: CoreMessage[]; /** #3301: original message decompositions supplied by an external state * converter (e.g. opencode-acp migration) at import time. Transient diff --git a/src/snapshot/plane.ts b/src/snapshot/plane.ts index b2be5d1afa..d9408934ad 100644 --- a/src/snapshot/plane.ts +++ b/src/snapshot/plane.ts @@ -163,52 +163,89 @@ export function stage0Decide(input: Stage0Input): Stage0Result { // diverges → resync, one-round convergence. Never guess. let decision: WorldDecision; const seqs = input.seqs ?? null; - const m2Candidate = + const storedSet = stored !== null ? new Set(stored.envelope.map((e) => e.id)) : null; + // The plane's max stamped seq: rows are stamped ledgerSeq = pos+1 + // (the D-lane contract, snapshot-render.ts:154), so the frontier is + // derivable from the row pos alone — no post_json parse. + let planeMaxSeq = 0; + for (const r of rows) { + if (r.active === 1 && r.pos + 1 > planeMaxSeq) planeMaxSeq = r.pos + 1; + } + // Review-round-2 Residual 1b: the host re-sent a body this proxy already + // processed — every id is already in the stored canon and the carrier + // seqs (when present) sit at/below the plane frontier (the original + // turn's in-request flush stamped them before its response left). Zero + // side effects: the canon is kept as-is, the lane re-assembles the SAME + // world (an idempotent kernel round), and the egress no-op + // classification keeps the chain id and skips the developer message. + // Without this guard the M2 retry ([m5] vs head m1) would headMove → + // diverge → snapStage0Reset: a harmless retry would WIPE the plane. + if ( stored !== null && + storedSet !== null && !input.resync && input.incoming.length > 0 && - seqs !== null && - seqs.length === input.incoming.length; - if (m2Candidate) { - const storedSet = new Set(stored.envelope.map((e) => e.id)); - const allUnknown = input.incoming.every((id) => !storedSet.has(id)); - // The plane's max stamped seq: rows are stamped ledgerSeq = pos+1 - // (the D-lane contract, snapshot-render.ts:154), so the frontier is - // derivable from the row pos alone — no post_json parse. - let planeMaxSeq = 0; - for (const r of rows) { - if (r.active === 1 && r.pos + 1 > planeMaxSeq) planeMaxSeq = r.pos + 1; - } - let contig = seqs[0] === planeMaxSeq + 1; - if (contig) { - for (let i = 1; i < seqs.length; i++) { - if (seqs[i] !== seqs[i - 1] + 1) { - contig = false; - break; + input.incoming.every((id) => storedSet.has(id)) && + (seqs === null || seqs.every((s) => s <= planeMaxSeq)) + ) { + decision = { + mode: "echo", + replay: true, + envelope: stored.envelope, + envelopeChecksum: stored.envelopeChecksum, + newIdPos: [], + goneIds: [], + issuedAt: stored.issuedAt, + witness: `replay: all ${input.incoming.length} ids known (maxSeq ${seqs ? Math.max(...seqs) : "-"} <= frontier ${planeMaxSeq})`, + }; + } else { + // M2 pure append (brief 19544722): the client sent ONLY new elements — + // every incoming id is unknown to the canon — and the seqs carrier says + // they land as a contiguous run starting exactly after the plane's max + // stamped seq. That is the 世界匹配定位序列位置 evidence: issue append- + // only pos (maxPos+1..) without any head/order risk. Anything less + // certain (missing seqs, a gap, a known id mixed in) falls through to + // matchWorld — an all-unknown full array there is usually a re-mint and + // diverges → resync, one-round convergence. Never guess. + const m2Candidate = + stored !== null && + !input.resync && + input.incoming.length > 0 && + seqs !== null && + seqs.length === input.incoming.length; + if (m2Candidate) { + const allUnknown = input.incoming.every((id) => !storedSet!.has(id)); + let contig = seqs![0] === planeMaxSeq + 1; + if (contig) { + for (let i = 1; i < seqs!.length; i++) { + if (seqs![i] !== seqs![i - 1] + 1) { + contig = false; + break; + } } } + decision = allUnknown && contig + ? matchAppend(stored, input.incoming) + : matchWorld({ + stored, + incoming: input.incoming, + explainedGone, + sessRaw: input.sessRaw, + ledgerId: input.ledgerId, + now, + resync: input.resync, + }); + } else { + decision = matchWorld({ + stored, + incoming: input.incoming, + explainedGone, + sessRaw: input.sessRaw, + ledgerId: input.ledgerId, + now, + resync: input.resync, + }); } - decision = allUnknown && contig - ? matchAppend(stored, input.incoming) - : matchWorld({ - stored, - incoming: input.incoming, - explainedGone, - sessRaw: input.sessRaw, - ledgerId: input.ledgerId, - now, - resync: input.resync, - }); - } else { - decision = matchWorld({ - stored, - incoming: input.incoming, - explainedGone, - sessRaw: input.sessRaw, - ledgerId: input.ledgerId, - now, - resync: input.resync, - }); } // Persist the canon for every verdict EXCEPT diverge (zero side effects). diff --git a/src/snapshot/stage0.ts b/src/snapshot/stage0.ts index 612831ebdf..a9cbb7633b 100644 --- a/src/snapshot/stage0.ts +++ b/src/snapshot/stage0.ts @@ -70,6 +70,16 @@ export interface Stage0Result { /** True when stage 0 reset the plane this turn (diverge / explicit * re-baseline) — the lane must clear session.responsesDelta (M6). */ resynced: boolean; + /** Review-round-2 Residual 2: the highest PLANE row pos of the STORED + * world at decision time (the new rows are not flushed yet) — present + * only when the decision is a strict extension (mode probe, nothing + * gone, new elements only) AND the assembly succeeded. The responses + * lane uses it to INITIALIZE the delta frontier on the first chained + * round of a keepResponseId chain: the provider's chain — built from + * this proxy's own full-view round — already holds exactly the stored + * world, so the outbound is the new elements only, never a re-seed. + * null = the lane re-seeds as before (every other shape). */ + storedFrontierPos: number | null; } export function runStage0(opts: Stage0Input): Stage0Result { @@ -129,6 +139,7 @@ export function runStage0(opts: Stage0Input): Stage0Result { } let kernelInput: BiliMessage[] | undefined; let resynced = false; + let storedFrontierPos: number | null = null; if (armed) { try { const store = getStore(); @@ -184,6 +195,21 @@ export function runStage0(opts: Stage0Input): Stage0Result { coverMax: cores.coverMax, }); kernelInput = asm.kernelInput; + // Residual 2: at THIS moment every active plane row is a + // stored-world row (this round's new rows land only in the + // lane's in-request flush, which runs AFTER stage 0). The + // stored world's boundary is therefore the max active row + // pos — a strict-extension decision (probe, nothing gone, + // new elements) is what makes "the chain holds exactly + // these rows" provable; every other shape stays null and + // the lane re-seeds as before. + if ( + stage0.decision.mode === "probe" + && stage0.decision.goneIds.length === 0 + && stage0.decision.newIdPos.length > 0 + ) { + storedFrontierPos = cores.rows.reduce((mx, r) => (r.pos > mx ? r.pos : mx), -1); + } log("info", `[${sessionId}] [2x2:${lane}] stage-0 ${stage0.decision.mode} — kernel input from plane (old=${asm.oldCount}, new=${newCores.length}, anchors=${anchors.length})`); } } @@ -203,5 +229,5 @@ export function runStage0(opts: Stage0Input): Stage0Result { log("warn", `[${sessionId}] [2x2:${lane}] stage-0 failed (${why}) — legacy full path`); } } - return { pinned, kernelInput, cohortRan: armed, resynced }; + return { pinned, kernelInput, cohortRan: armed, resynced, storedFrontierPos }; } diff --git a/src/snapshot/types.ts b/src/snapshot/types.ts index 255380534e..160335aab0 100644 --- a/src/snapshot/types.ts +++ b/src/snapshot/types.ts @@ -64,5 +64,11 @@ export interface WorldDecision { armedAt?: number; /** Diagnostic witness sample (echo spot-check) — warn stream only. */ witness?: string; + /** Review-round-2 Residual 1b: the host re-sent a body this proxy ALREADY + * processed (every id known, carrier seqs at/below the frontier) — an + * upstream-200 retry. The lane must neither wipe the plane (the diverge + * this would otherwise cause) nor strip the chain id (the egress no-op + * classification does the continuation). Zero side effects. */ + replay?: boolean; } diff --git a/tests/issue5874-2x2-p3.test.ts b/tests/issue5874-2x2-p3.test.ts index 644173d900..130ee57bb7 100644 --- a/tests/issue5874-2x2-p3.test.ts +++ b/tests/issue5874-2x2-p3.test.ts @@ -51,6 +51,7 @@ import { stage0Decide, } from "../src/snapshot/plane.ts"; import { snapL1Lookup, snapL1EnvelopeRead, snapL1Record } from "../src/storage/snap.ts"; +import { runStage0 } from "../src/snapshot/stage0.ts"; process.env.BILI_PERSIST_ZSTD = "0"; const hermetic = fs.mkdtempSync(path.join(os.tmpdir(), "bili-5874-p3-")); @@ -274,10 +275,12 @@ test("A1 #2 M1 vs M2: same turn, full-inbound vs incremental-inbound kernel inpu assert.deepEqual(m1.decision.newIdPos, [{ id: "m11", pos: 11 }]); const asm1 = assembleKernelInput({ decision: m1.decision, rows: rowCores, newCores: [newCore] }); - // A FRESH store for the M2 leg (the same plane state, a different client - // behavior on the SAME turn): the client sends ONLY the new element, - // with the seqs carrier locating it after the plane frontier (10). - const store2 = storeWithRows(10, sid); + // A FRESH store (a different DB — the two legs are ALTERNATIVE client + // behaviors on the same turn; sharing the file would let leg M1's canon + // contaminate leg M2's stage-0 verdict) with the same plane state: + // rows m1..m10 only. + const store2 = openPlane(path.join(hermetic, `a-${sid}-m2-10`, "index.db")); + store2.ingestSession(planIngest(plainSession(sid, 10))); // Enroll the canon first (the M2 branch requires a stored canon): the // seed turn was the M1 full resend of 10. store2.snapStage0(sid, ten); @@ -706,7 +709,10 @@ test("A1d M4: the codec-level pin — parsing the new element ALONE (M2 wire) yi const ten = Array.from({ length: 10 }, (_, i) => `m${i + 1}`); const m1 = store.snapStage0(sid, [...ten, "codec-11"], { seqs: ten.map((_, i) => i + 1).concat(11) }); const asm1 = assembleKernelInput({ decision: m1.decision, rows: rows10, newCores: [full.msgs[10]!] }); - const store2 = storeWithRows(10, sid); + // Isolated DB for the M2 leg (see A1: the legs are alternative client + // behaviors on the same turn and must not share stage-0 canon state). + const store2 = openPlane(path.join(hermetic, `a-${sid}-m2-10`, "index.db")); + store2.ingestSession(planIngest(plainSession(sid, 10))); store2.snapStage0(sid, ten); const m2 = store2.snapStage0(sid, ["codec-11"], { seqs: [11] }); const rows10b = store2.snapStage0Cores(sid).rows.map((r) => ({ pos: r.pos, ids: r.ids, msg: JSON.parse(r.postJson) as BiliMessage })); @@ -967,3 +973,131 @@ test("B4: the delta frontier commits only when the upstream ACCEPTS — a 400 tu { storeDir: path.join(hermetic, "b4-store"), env: { BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY: "1", ACP_KEEP_RESPONSE_ID: "1" } }, ); }); + +test("Residual 2 (unit): storedFrontierPos is set only for a strict extension — probe + nothing gone + new elements; every other shape is null", () => { + const dir = path.join(hermetic, "r2u"); + fs.mkdirSync(dir, { recursive: true }); + const plane = openPlane(path.join(dir, "index.db")); + const sid = "r2-sid"; + const saved = process.env.BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY; + const savedUnified = process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY = "1"; + process.env.BILI_STORAGE_UNIFIED = "shadow"; // the SessionStore facade's snap* delegates need a live shadow + // The facade's replicasAvailable() reads shadow.stats.disabled — the raw + // UnifiedStore has no such surface, so shim it (the P2 G-test pattern). + const shadow = new Proxy(plane, { + get: (t: object, p: string | symbol) => (p === "stats" ? { disabled: false } : (t as Record)[p]), + }); + _setStoreForTest(new SessionStore({ enabled: true, shadow: shadow as never, log: () => undefined })); + try { + // Seed the plane with a 3-element world (rows pos 0..2). + plane.ingestSession(planIngest(plainSession(sid, 3))); + const run = (ids: string[]) => runStage0({ + sessionId: sid, + lane: "responses", + msgs: ids.map((id) => core(Number(id.slice(1)), id)), + incomingMsgIds: ids, + incomingSeqs: null, + pluginMode: true, + log: () => undefined, + }); + + // First stage-0 call: issue (the l1 envelope is recorded here — the + // plane rows already exist from the ingest above). + const seed = run(["m1", "m2", "m3"]); + assert.equal(seed.storedFrontierPos, null, "issue (no stored world yet) → null"); + + // Strict extension (+m4) → the stored world's boundary = max active + // row pos (the new rows are not flushed at stage-0 time). + const ext = run(["m1", "m2", "m3", "m4"]); + assert.equal(ext.storedFrontierPos, 2, "probe + nothing gone + new → the stored-world boundary (max active row pos)"); + + // A full resend with nothing new → echo → null (a re-seed is the + // honest answer; there is no provable chain boundary). + const echo = run(["m1", "m2", "m3", "m4"]); + assert.equal(echo.storedFrontierPos, null, "echo → null"); + + // An extension WITH a deletion (m2 gone) is not provable — the + // chain may not hold the hole → null, the lane re-seeds. + const del = run(["m1", "m3", "m4", "m5"]); + assert.equal(del.storedFrontierPos, null, "an extension with a deletion is not a strict prefix → null"); + } finally { + if (saved === undefined) delete process.env.BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY; + else process.env.BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY = saved; + if (savedUnified === undefined) delete process.env.BILI_STORAGE_UNIFIED; + else process.env.BILI_STORAGE_UNIFIED = savedUnified; + _setStoreForTest(new SessionStore({ enabled: false })); + } +}); + +test("Residual 1a+2 (e2e): the first CHAINED round of a keepResponseId chain serves the new elements ONLY, and a same-body RETRY of it is a no-op (id kept, empty input, no dev)", async () => { + await withProxyT( + () => ({ status: 200, payload: sseDone() }), + async (url, bodies) => { + const msg = (role: string, content: string) => ({ type: "message", role, content }); + // Round 1: the full world, NO chain ref. The provider's chain + // starts here; no selection fires (no id in) → no frontier. + const r1 = await fetch(url, { + method: "POST", + headers: PLUGIN_HEADERS, + body: JSON.stringify({ + model: "gpt-e2e", + stream: true, + input: [msg("user", "one"), msg("assistant", "ack one"), msg("user", "two")], + biliMsgIds: ["r2m1", "r2m2", "r2m3"], + }), + }); + assert.equal(r1.status, 200); + await r1.text(); + + // Round 2: CHAINED (KEEP=1 forwards the id) + one new element. + // WITHOUT Residual 2 this would re-seed the full 4-element view; + // WITH it the outbound is exactly the new element, id kept, no + // developer re-injection. + const r2 = await fetch(url, { + method: "POST", + headers: PLUGIN_HEADERS, + body: JSON.stringify({ + model: "gpt-e2e", + stream: true, + previous_response_id: "resp_r1", + input: [msg("user", "one"), msg("assistant", "ack one"), msg("user", "two"), msg("assistant", "ack two")], + biliMsgIds: ["r2m1", "r2m2", "r2m3", "r2m4"], + }), + }); + assert.equal(r2.status, 200); + await r2.text(); + const fwd2 = JSON.parse(bodies[1]!) as Record; + assert.equal(fwd2.previous_response_id, "resp_r1", "the chain id is kept on the delta round"); + const texts2 = ((fwd2.input as unknown as { content?: unknown }[])).flatMap((x) => (Array.isArray(x.content) ? x.content.map((p) => (p as { text?: string }).text ?? "") : [String(x.content ?? "")])); + assert.ok(texts2.some((t) => t.includes("ack two")), "the new element is sent"); + assert.ok(!texts2.some((t) => t.includes("one")), "the stored world is NOT re-sent (the provider chain holds it)"); + assert.equal((fwd2.input as unknown[]).length, 1, "exactly one element on the wire (a re-seed would carry 4)"); + assert.ok(!JSON.stringify(fwd2).includes("You are bili"), "no developer re-injection on the delta round"); + + // Round 3: the host retries the SAME body as round 2 (upstream + // 200 lost in transit — the B4 retry class). Residual 1a: the + // delta comes out EMPTY — the no-op classification keeps the id, + // sends an EMPTY input, and injects NO dev message (the blind- + // response path). The plane must stay intact. + const r3 = await fetch(url, { + method: "POST", + headers: PLUGIN_HEADERS, + body: JSON.stringify({ + model: "gpt-e2e", + stream: true, + previous_response_id: "resp_r2", + input: [msg("user", "one"), msg("assistant", "ack one"), msg("user", "two"), msg("assistant", "ack two")], + biliMsgIds: ["r2m1", "r2m2", "r2m3", "r2m4"], + }), + }); + assert.equal(r3.status, 200); + await r3.text(); + const fwd3 = JSON.parse(bodies[2]!) as Record; + assert.equal(fwd3.previous_response_id, "resp_r2", "the no-op round KEEPS the chain id (stripping it orphans the provider's context)"); + assert.deepEqual(fwd3.input, [], "the no-op round sends an EMPTY input (pure chain continuation)"); + assert.ok(!JSON.stringify(fwd3).includes("You are bili"), "no developer message on the no-op round (it would make a blind response)"); + }, + { storeDir: path.join(hermetic, "r2-store"), env: { BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY: "1", ACP_KEEP_RESPONSE_ID: "1" } }, + ); +}); From 59ce46445dd9fabdf72260b1021240807d5f0aba Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 13 Nov 2026 09:39:12 +0800 Subject: [PATCH 5/8] =?UTF-8?q?fix(#5874=202=C3=972):=20Residual=202=20har?= =?UTF-8?q?dening=20=E2=80=94=20the=20chain=20id=20is=20opaque:=20trust=20?= =?UTF-8?q?a=20delta=20frontier=20only=20for=20an=20id=20this=20proxy=20mi?= =?UTF-8?q?nted=20(#6181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit previous_response_id proves nothing about which world the provider's chain holds. A host forking/rewinding to an older response id (or carrying an id another proxy minted) can pass the strict-extension proof yet lose the gap between the old chain's world and the stored world when we seed at the stored boundary. - session.lastMintedResponse {id, frontierPos}: the latest response id THIS proxy emitted for the session, paired with the plane frontier the chain provably held — recorded in the relay from the id observed on the outbound stream (SSE response.created/completed scan on the exact bytes written to the client, json.id for non-streaming), accepted forwards only (a refused round mints nothing the client chains from). - store facade snapPlaneMaxPos (store/shadow/persist) — the lane records the post-flush plane max pos as prepared.mintFrontierPos. - seed gate: the first-chained-round frontier seed now also requires lastMintedResponse.id === previous_response_id AND lastMintedResponse.frontierPos === storedFrontierPos. - persisted-frontier trust: a chained round whose id this proxy did not mint last drops the persisted frontier and re-seeds the full view (id kept, dev injected — the pre-seed status quo, at-least-once, never a loss); the persisted state is cleared so the next round re-establishes verifiability from a fresh seed. - B3 latent bug: the pure-chain-continuation route reuses the compact-passthrough base, which hard-codes resetAfterSuccess — scheduling a native-compaction rebase (resetSessionCompression) for the next turn although nothing compacted. Dropped for the B3 route. - p3: per-round upstream ids in the e2e harness; the Residual 1a+2 e2e chains from the ids the responses actually carried and gains a round-4 fork case (old id + full current world + new element → full re-seed, id kept). 41/41. --- src/persist.ts | 7 +++++ src/server.ts | 8 +++++ src/server/handle.ts | 11 +++++++ src/server/prepare-responses.ts | 54 +++++++++++++++++++++++++++++++- src/server/relay.ts | 55 +++++++++++++++++++++++++++++++++ src/session.ts | 12 +++++++ src/storage/shadow.ts | 7 +++++ src/storage/store.ts | 12 +++++++ tests/issue5874-2x2-p3.test.ts | 46 ++++++++++++++++++++++----- 9 files changed, 203 insertions(+), 9 deletions(-) diff --git a/src/persist.ts b/src/persist.ts index d98b9b2630..032e2f7d53 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -2460,6 +2460,13 @@ export class SessionStore { return this.liveShadow().snapStage0Cores(sessionId); } + /** #6181 round 2, Residual 2 hardening: the plane's max ACTIVE pos + * (-1 when empty) — see UnifiedStore.snapPlaneMaxPos. */ + snapPlaneMaxPos(sessionId: string): number { + if (!this.replicasAvailable()) return -1; + return this.liveShadow().snapPlaneMaxPos(sessionId); + } + /** The stage-0 resync (R4 / table behind): reset the session plane so * the next turn re-issues from the client's current array. */ snapStage0Reset(sessionId: string): void { diff --git a/src/server.ts b/src/server.ts index dad4282449..9f4a334e71 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2024,6 +2024,14 @@ export type Prepared = { * safe, under-serving is the context loss). Absent (undefined) = no * delta was selected this turn. */ pendingResponsesDelta?: { state: { frontierPos: number; servedAnchors: string[] }; kind: "full" | "noop" | "partial" } | null; + /** #6181 round 2, Residual 2 hardening: the plane max pos (post + * in-request flush) the provider chain will hold once THIS forward is + * accepted — the relay pairs it with the response id it observes on + * the outbound stream and records session.lastMintedResponse (accepted + * forwards only; a refused round mints nothing). Set by the responses + * lane on every round whose accepted forward leaves the provider + * holding the full world. */ + mintFrontierPos?: number; anthropicSystem?: AnthropicRequestBody["system"]; /** #2189: the client's billing-attribution block captured from the INBOUND * anthropic system (pre-anchor). Preflight summary calls must carry it — diff --git a/src/server/handle.ts b/src/server/handle.ts index 9824409eb0..856bd52c27 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -3026,6 +3026,17 @@ async function handleRequest( // Anthropic-on-Vertex wire the body carries none, and the // relay's compress-loop rebuild reads it from here. preparedOut.model = requestModel; + // #6181 B3: the compact-passthrough base hard-codes + // resetAfterSuccess — which schedules a native-compaction + // REBASE for the next Responses turn (reconcileNative- + // CompactionBoundary → resetSessionCompression). A pure + // chain-continuation round compacts nothing: honoring the + // marker would wipe this session's compression state on + // the next turn with no compaction ever having happened. + // Drop it for the B3 route (the next turn's stage-0 + // resync already converges the plane when the host + // resends a genuinely-compact world). + if (pureChainContinuation) preparedOut.resetAfterSuccess = false; return preparedOut; }; // #1295: stamp the envelope meta onto every Prepared for this diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index ec5057ca39..2ffc1dc4e6 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -47,7 +47,7 @@ import { pckKeyAbated } from "../pck-probe.js"; import { reconcilePluginFoldCallIds } from "../plugin.js"; import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker/engine.js"; import { imageRescaleMaxDim, keepResponseId as knobKeepResponseId, noCompressPrompt as knobNoCompressPrompt, noInjectTool as knobNoInjectTool, renderNone as knobRenderNone } from "../knobs.js"; -import { SessionStore } from "../persist.js"; +import { SessionStore, getStore } from "../persist.js"; import { applyImageRescaleItems } from "../image-rescale.js"; import { clampOutgoingOutput, countLoadedToolTokens, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed, modelVisibleTools } from "./budget.js"; import { FORCE_TEXT_PROTOCOL, injectResponsesTool, injectTool } from "./inject.js"; @@ -149,6 +149,14 @@ export async function prepareResponses( // the plane write for this turn's flush and the plane egress — see the // flag set site below the processTurn block. let planeBypass = false; + // #6181 round 2, Residual 2 hardening: the plane max pos the provider + // chain will hold once THIS forward is accepted — the relay pairs it + // with the response id it observes on the outbound stream (see + // relay.ts). Set after the in-request flush: the plane then holds + // exactly this round's world (or, for a planeBypass round, the + // previous world — which is precisely the boundary the chain already + // holds, since the bypass skips the flush). + let mintFrontierPos: number | undefined; let sysNotes: string[] = []; // #3713: billed tokens of the #1638-masked mid-history sys/dev items — they // ride the rebuilt input verbatim yet enter neither projection.msgs nor @@ -462,6 +470,17 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // write (flag consumed) or ran it — either way the one-shot flag is // done for this turn. session.skipPlaneFlush = false; + // #6181 round 2, Residual 2 hardening: record the frontier this + // ACCEPTED round leaves the provider chain holding (post-flush + // plane max pos — see the declaration). Best-effort: a store error + // just leaves the mint unset (no verifiable chain → re-seeds, the + // status quo, never a loss). + try { + const s = getStore(); + if (s !== null && s !== undefined) mintFrontierPos = s.snapPlaneMaxPos(session.id); + } catch { + /* no mint record for this round */ + } // #4528: split the kernel pass from its post-processing scan passes so // each is a bounded synchronous stretch rather than one long block. await yieldToEventLoop(); @@ -564,6 +583,9 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr deltaState === null && chains && !session.responsesChainBroken && s0.kernelInput !== undefined && s0.storedFrontierPos !== null + && session.lastMintedResponse !== null && session.lastMintedResponse !== undefined + && session.lastMintedResponse.id === parsed.previous_response_id + && session.lastMintedResponse.frontierPos === s0.storedFrontierPos ) { deltaState = { frontierPos: s0.storedFrontierPos, servedAnchors: [] }; log("info", `[${sessionId}] [snaplane] frontier seeded at the stored-world boundary (pos ${s0.storedFrontierPos}) — the provider chain from this proxy's own prior round holds the stored world; serving the new elements only instead of re-seeding (Residual 2)`); @@ -575,6 +597,32 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // first frontier the native #1954 KEEP=1 contract holds (id forwarded // intact, whatever the outbound is). hadFrontier = deltaState !== undefined && deltaState !== null; + // #6181 round 2, Residual 2 hardening: a persisted frontier is a + // claim about the PROVIDER's chain, not just the plane — and + // previous_response_id is OPAQUE. A host can fork/rewind to an older + // response id (or carry an id another proxy minted); the strict- + // extension proof bounds only the plane, so a delta built from a + // frontier the chain never provably holds loses the gap. Trust the + // frontier ONLY when the host continues from an id THIS proxy minted + // last for this session; any other chained round drops it (the egress + // then does a plain re-seed: full view, id kept — the status quo, + // at-least-once, never a loss) and the persisted state is cleared so + // the next round re-establishes verifiability from a fresh seed. + // (planeBypass rounds never reach the delta machinery — the egress + // serves the legacy bytes — so this is inert there.) + if ( + deltaState !== undefined && deltaState !== null + && !(session.lastMintedResponse !== null && session.lastMintedResponse !== undefined + && session.lastMintedResponse.id === parsed.previous_response_id) + ) { + log("info", `[${sessionId}] [snaplane] chained round carries a previous_response_id this proxy did not mint last (${session.lastMintedResponse === null || session.lastMintedResponse === undefined ? "no mint record" : "id mismatch / fork / rewind"}) — dropping the persisted frontier and re-seeding the full view (Residual 2 hardening)`); + deltaState = null; + session.responsesDelta = null; + // The re-seed is a "no frontier" round for the strip/dev gates + // too: id kept (native #1954), developer message injected (the + // full view is self-contained) — exactly the pre-seed status quo. + hadFrontier = false; + } // #6181 B0 (decided at the flush site, above): a CHAINED turn that was // NOT assembled from the plane (non-cohort — no msgids — or a cohort // turn whose stage 0 did not run) cannot be addressed against the @@ -973,6 +1021,10 @@ reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStr // #6181 B4: commit the selected frontier only after the upstream // accepts the request (relay.ts); a failed forward drops it. ...(deltaState !== undefined && selectionFired ? { pendingResponsesDelta: pendingDelta } : {}), + // #6181 round 2, Residual 2 hardening: the relay pairs this with + // the response id observed on the outbound stream (accepted forwards + // only) and records session.lastMintedResponse. + ...(mintFrontierPos !== undefined ? { mintFrontierPos } : {}), systemNotes: sysNotes, protocol: "responses", stream, diff --git a/src/server/relay.ts b/src/server/relay.ts index efcf2b8aa2..34af7e9d60 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -2768,6 +2768,40 @@ async function forwardCaptured( clientAbort.signal, ); let protocolFragmentWarned = false; + // #6181 round 2, Residual 2 hardening: capture the response id + // the CLIENT actually sees (response.created / response.completed + // in the outbound SSE — the upstream's id when present, the + // proxy's minted one otherwise, inline-minted included: the scan + // runs on the exact bytes written to the client). The relay pairs + // it with prepared.mintFrontierPos into session.lastMintedResponse. + let mintCreatedId: string | undefined; + let mintCompletedId: string | undefined; + let mintSseTail = ""; + const mintScan = (text: string): void => { + if (prepared.protocol !== "responses" || prepared.mintFrontierPos === undefined) return; + mintSseTail += text; + if (mintSseTail.length > 65536) mintSseTail = mintSseTail.slice(-32768); + if (!mintSseTail.includes("response.completed") && !mintSseTail.includes("response.created")) return; + let cut = mintSseTail.indexOf("\n\n"); + while (cut !== -1) { + const event = mintSseTail.slice(0, cut); + mintSseTail = mintSseTail.slice(cut + 2); + const dataLine = event.split("\n").find((l) => l.startsWith("data: ")); + if (dataLine !== undefined && (dataLine.includes("response.completed") || dataLine.includes("response.created"))) { + try { + const evt = JSON.parse(dataLine.slice(6)) as { type?: string; response?: { id?: unknown } }; + const id = evt.response?.id; + if (typeof id === "string" && id.length > 0) { + if (evt.type === "response.completed") mintCompletedId = id; + else mintCreatedId = id; + } + } catch { + /* partial/corrupt event — ignore */ + } + } + cut = mintSseTail.indexOf("\n\n"); + } + }; for await (const chunk of loop) { if (res.destroyed || res.writableEnded) break; { @@ -2780,12 +2814,25 @@ async function forwardCaptured( } } res.write(chunk); + mintScan(chunk.toString("utf8")); if (res.writableNeedDrain) { await awaitDrain(res); } if (res.destroyed || res.writableEnded) break; } res.end(); + // #6181 round 2, Residual 2 hardening: pair the observed id with + // the frontier the lane recorded — accepted forwards only (a + // refused round mints nothing the client chains from) and only + // when the lane proved this round leaves the provider holding the + // full world (mintFrontierPos set). The markDirty in the finally + // below persists it with the rest of the turn's state. + if (prepared.protocol === "responses" && upstreamResult.response.status < 400 && prepared.mintFrontierPos !== undefined) { + const seen = mintCompletedId ?? mintCreatedId; + if (seen !== undefined) { + prepared.session.lastMintedResponse = { id: seen, frontierPos: prepared.mintFrontierPos }; + } + } } catch (e) { emitStreamError(res, prepared.commandcode ? "commandcode" : prepared.protocol, (e as Error)?.message ?? String(e), (m) => log("error", `[${prepared.session.id}] ${m}`), opts.streamErrorShape); } finally { @@ -2895,6 +2942,14 @@ async function forwardCaptured( await withSessionLock(prepared.session, () => rewriteJsonResponseAsync(json, ctx, clientAbort.signal)); } captureHandoffJson(reply, prepared.protocol, json); + // #6181 round 2, Residual 2 hardening: non-streaming twin of + // the streaming mint capture — json.id here is the final + // client-visible id (post rewriteResponsesJsonResponseAsync). + if (prepared.protocol === "responses" && upstreamResult.response.status < 400 && prepared.mintFrontierPos !== undefined) { + if (typeof json.id === "string" && json.id.length > 0) { + prepared.session.lastMintedResponse = { id: json.id, frontierPos: prepared.mintFrontierPos }; + } + } reply?.save(); res.end(JSON.stringify(json)); } catch { diff --git a/src/session.ts b/src/session.ts index 49f27cf934..5754937147 100644 --- a/src/session.ts +++ b/src/session.ts @@ -718,6 +718,18 @@ export type Session = { * whole world again). Transient within a turn? No — persisted with the * session, like responsesDelta. */ responsesChainBroken?: boolean; + /** #6181 round 2, Residual 2 hardening: the latest response id THIS + * proxy emitted for this session (responses lane), paired with the + * plane frontier the provider chain provably held after that response + * (committed only on an ACCEPTED forward — a refused round mints + * nothing the client chains from). The chain id itself is opaque: + * the first-chained-round frontier seed AND every persisted frontier + * are trusted only when previous_response_id equals this id (the host + * continues from our latest response — a fork/rewind to an older id, + * or an id another proxy minted, can carry a world that chain never + * held, so it re-seeds instead). Persisted with the session, like + * responsesDelta. */ + lastMintedResponse?: { id: string; frontierPos: number } | null; pluginSnapshot?: CoreMessage[]; /** #3301: original message decompositions supplied by an external state * converter (e.g. opencode-acp migration) at import time. Transient diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index ab1e017cfa..73a7d57308 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -1006,6 +1006,13 @@ export class UnifiedShadow { return this.sync.snapStage0Cores(sessionId); } + /** #6181 round 2, Residual 2 hardening: the plane's max ACTIVE pos + * (-1 when empty) — see UnifiedStore.snapPlaneMaxPos. */ + snapPlaneMaxPos(sessionId: string): number { + if (this.sync === null) return -1; + return this.sync.snapPlaneMaxPos(sessionId); + } + /** The stage-0 resync (R4 / table-behind): reset the session plane. */ snapStage0Reset(sessionId: string): void { if (this.sync === null) return; diff --git a/src/storage/store.ts b/src/storage/store.ts index 0580d0911a..5f8fd0b98c 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -2487,6 +2487,18 @@ export class UnifiedStore { return { rows, coverMax }; } + /** #6181 round 2, Residual 2 hardening: the plane's max ACTIVE pos for + * this sid (-1 when the plane has no active row) — the frontier a + * full-world round leaves the provider chain holding. */ + snapPlaneMaxPos(sid: string): number { + const all = snapRowsAll(this.db, sid); + let max = -1; + for (const r of all) { + if (r.active === 1 && r.pos > max) max = r.pos; + } + return max; + } + /** The stage-0 resync (brief red gate R4 / the resync ruling): reset the * session's plane (rows + log + ckpt + the l1 element envelope + the * sticky R1 latch) in one txn. This turn then runs the legacy full path diff --git a/tests/issue5874-2x2-p3.test.ts b/tests/issue5874-2x2-p3.test.ts index 130ee57bb7..20a1d861ac 100644 --- a/tests/issue5874-2x2-p3.test.ts +++ b/tests/issue5874-2x2-p3.test.ts @@ -797,10 +797,10 @@ function closeT(server: http.Server): Promise { function sseT(type: string, data: unknown): string { return `event: ${type}\ndata: ${JSON.stringify(data)}\n\n`; } -function sseDone(): string { +function sseDone(id = "resp_done"): string { return [ sseT("response.output_item.done", { item: { type: "message", id: "msg_1", role: "assistant", content: [{ type: "output_text", text: "ok" }] }, output_index: 0 }), - sseT("response.completed", { response: { id: "resp_done", status: "completed", output: [], usage: { input_tokens: 100, output_tokens: 5, total_tokens: 105 } } }), + sseT("response.completed", { response: { id, status: "completed", output: [], usage: { input_tokens: 100, output_tokens: 5, total_tokens: 105 } } }), ].join(""); } @@ -1032,7 +1032,7 @@ test("Residual 2 (unit): storedFrontierPos is set only for a strict extension test("Residual 1a+2 (e2e): the first CHAINED round of a keepResponseId chain serves the new elements ONLY, and a same-body RETRY of it is a no-op (id kept, empty input, no dev)", async () => { await withProxyT( - () => ({ status: 200, payload: sseDone() }), + (n) => ({ status: 200, payload: sseDone(`resp_done_${n}`) }), async (url, bodies) => { const msg = (role: string, content: string) => ({ type: "message", role, content }); // Round 1: the full world, NO chain ref. The provider's chain @@ -1053,14 +1053,16 @@ test("Residual 1a+2 (e2e): the first CHAINED round of a keepResponseId chain ser // Round 2: CHAINED (KEEP=1 forwards the id) + one new element. // WITHOUT Residual 2 this would re-seed the full 4-element view; // WITH it the outbound is exactly the new element, id kept, no - // developer re-injection. + // developer re-injection. The chain id must be the one round 1's + // response actually carried (resp_done_1) — the frontier seed is + // trusted only for an id THIS proxy minted (Residual 2 hardening). const r2 = await fetch(url, { method: "POST", headers: PLUGIN_HEADERS, body: JSON.stringify({ model: "gpt-e2e", stream: true, - previous_response_id: "resp_r1", + previous_response_id: "resp_done_1", input: [msg("user", "one"), msg("assistant", "ack one"), msg("user", "two"), msg("assistant", "ack two")], biliMsgIds: ["r2m1", "r2m2", "r2m3", "r2m4"], }), @@ -1068,7 +1070,7 @@ test("Residual 1a+2 (e2e): the first CHAINED round of a keepResponseId chain ser assert.equal(r2.status, 200); await r2.text(); const fwd2 = JSON.parse(bodies[1]!) as Record; - assert.equal(fwd2.previous_response_id, "resp_r1", "the chain id is kept on the delta round"); + assert.equal(fwd2.previous_response_id, "resp_done_1", "the chain id is kept on the delta round"); const texts2 = ((fwd2.input as unknown as { content?: unknown }[])).flatMap((x) => (Array.isArray(x.content) ? x.content.map((p) => (p as { text?: string }).text ?? "") : [String(x.content ?? "")])); assert.ok(texts2.some((t) => t.includes("ack two")), "the new element is sent"); assert.ok(!texts2.some((t) => t.includes("one")), "the stored world is NOT re-sent (the provider chain holds it)"); @@ -1086,7 +1088,7 @@ test("Residual 1a+2 (e2e): the first CHAINED round of a keepResponseId chain ser body: JSON.stringify({ model: "gpt-e2e", stream: true, - previous_response_id: "resp_r2", + previous_response_id: "resp_done_2", input: [msg("user", "one"), msg("assistant", "ack one"), msg("user", "two"), msg("assistant", "ack two")], biliMsgIds: ["r2m1", "r2m2", "r2m3", "r2m4"], }), @@ -1094,9 +1096,37 @@ test("Residual 1a+2 (e2e): the first CHAINED round of a keepResponseId chain ser assert.equal(r3.status, 200); await r3.text(); const fwd3 = JSON.parse(bodies[2]!) as Record; - assert.equal(fwd3.previous_response_id, "resp_r2", "the no-op round KEEPS the chain id (stripping it orphans the provider's context)"); + assert.equal(fwd3.previous_response_id, "resp_done_2", "the no-op round KEEPS the chain id (stripping it orphans the provider's context)"); assert.deepEqual(fwd3.input, [], "the no-op round sends an EMPTY input (pure chain continuation)"); assert.ok(!JSON.stringify(fwd3).includes("You are bili"), "no developer message on the no-op round (it would make a blind response)"); + + // Round 4: FORK — the host rewinds to round 1's response id while + // keeping the full current world plus one new element. The strict- + // extension proof would still pass (the plane is a strict prefix), + // but the chain from resp_done_1 may not hold the round-2/3 world + // if it was ever re-seeded — so the id mismatch drops the + // frontier and the round re-seeds the FULL view (id kept, never a + // loss). A 1-element delta here would be the loss edge. + const r4 = await fetch(url, { + method: "POST", + headers: PLUGIN_HEADERS, + body: JSON.stringify({ + model: "gpt-e2e", + stream: true, + previous_response_id: "resp_done_1", + input: [msg("user", "one"), msg("assistant", "ack one"), msg("user", "two"), msg("assistant", "ack two"), msg("assistant", "ack three")], + biliMsgIds: ["r2m1", "r2m2", "r2m3", "r2m4", "r2m5"], + }), + }); + assert.equal(r4.status, 200); + await r4.text(); + const fwd4 = JSON.parse(bodies[3]!) as Record; + assert.equal(fwd4.previous_response_id, "resp_done_1", "the forked id is forwarded (kept, not stripped)"); + const texts4 = ((fwd4.input as unknown as { content?: unknown }[])).flatMap((x) => (Array.isArray(x.content) ? x.content.map((p) => (p as { text?: string }).text ?? "") : [String(x.content ?? "")])); + assert.ok(texts4.some((t) => t.includes("one")), "the fork round RE-SEEDS the full view (the old chain is not provably holding the stored world)"); + assert.ok(texts4.some((t) => t.includes("ack three")), "the new element is in the re-seed view"); + const worldItems4 = (fwd4.input as { role?: string }[]).filter((x) => x.role !== "developer"); + assert.equal(worldItems4.length, 5, "all 5 world elements on the wire (a delta would have sent 1)"); }, { storeDir: path.join(hermetic, "r2-store"), env: { BILI_ROLLOUT_PLUGIN_LANE_SERVER_HISTORY: "1", ACP_KEEP_RESPONSE_ID: "1" } }, ); From be4c27f56f082c2aa153db8fe69e9e88a3434d22 Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 13 Nov 2026 11:06:23 +0800 Subject: [PATCH 6/8] fix(#5874 2x2): use safeSuffix for the mint-scan tail clamp (family gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Residual-2 hardening commit added a raw mintSseTail.slice(-32768) which the text-safe family gate (tests/text-safe.test.ts) rejects — raw negative string slices reintroduce the #816/#1615 lone-surrogate family. Replace with safeSuffix(mintSseTail, 32768) (same last-32768-code-units semantics, surrogate-pair-safe). safeSuffix was already imported. --- src/server/relay.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/relay.ts b/src/server/relay.ts index 34af7e9d60..032628f666 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -2780,7 +2780,7 @@ async function forwardCaptured( const mintScan = (text: string): void => { if (prepared.protocol !== "responses" || prepared.mintFrontierPos === undefined) return; mintSseTail += text; - if (mintSseTail.length > 65536) mintSseTail = mintSseTail.slice(-32768); + if (mintSseTail.length > 65536) mintSseTail = safeSuffix(mintSseTail, 32768); if (!mintSseTail.includes("response.completed") && !mintSseTail.includes("response.created")) return; let cut = mintSseTail.indexOf("\n\n"); while (cut !== -1) { From 59af7056f337af058de4220ef16a403650ac4a74 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 13 Nov 2026 18:12:29 +0800 Subject: [PATCH 7/8] =?UTF-8?q?fix(#6181):=205901-base=20regressions=20?= =?UTF-8?q?=E2=80=94=20untagged-save=20plane=20guard=20(prefix=20rule)=20+?= =?UTF-8?q?=20mint=20tap=20spin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two regressions surfaced when the branch was re-based onto the #5901 head (f0001977a, which carries #6326 durability-before-render): 1. B4 e2e: #6326's ingestRowsNow persists the ledger rows BEFORE render as an UNTAGGED save; its final-state diff then deleted/re-homed the pinned plane rows (ids the tagged plane cannot address — R2 doctrine) and the next tagged save marked the plane sticky-dead. snapShadowSync now guards untagged saves against a sid that holds active plane rows with a PREFIX RULE: the save is allowed only when its ledger-id view prefix-identically extends the stage-0 l1 envelope (that save is the one that materializes the rows stage-0 declared — ids agree, idempotent); a stale view (re-derived kernel ids, pre-turn transcript, compaction) is skipped for the plane, the disk record still lands, and the plane converges at the next tagged save or stage-0 resync. The guard runs BEFORE the transaction (after it the destructive diff is already committed). 2. Residual-2 e2e: the mint-id scan (the record that lets the next chained round seed the delta frontier at the stored-world boundary) existed only in the compress-loop (injected) branch; plugin and non-injected responses turns never entered it, so the first chained round re-seeded the whole world. The scan is factored into MintIdTap and rides a tee'd twin of the plugin-lane stream and of the plain (non-rewrite) proxy stream — both branches where the client-facing bytes ARE the upstream bytes. The tap fix also repairs a latent infinite-spin in the old inline scan: the event-boundary recompute ran only at the loop bottom, which the non-marker skipped — a non-marker event preceding a marker event pinned the boundary on a shrinking tail and spun the loop forever (event loop blocked; no timers fired). The recompute is now unconditional on every pass. p3 42/42 (incl. B4 frontier-commit-on-accept + Residual 1a+2); p0 10/10, p1 4/4, fastpath 2/2, p2 17/17, responses-previous-response-id 7/7, snapshot-render 13/13, render-chain 6/6, shadow 5/5, storage-engine-proofs 12/12 (kernel rebuild for #5901's compress.ts), opencode-rolling 10/10. --- src/server/relay.ts | 144 +++++++++++++++++++++++++++++++++---------- src/storage/store.ts | 49 ++++++++++----- 2 files changed, 145 insertions(+), 48 deletions(-) diff --git a/src/server/relay.ts b/src/server/relay.ts index bd698e15c1..e8523fe53b 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -326,6 +326,51 @@ export function tryAnswerEarlyHold(o: { return true; } +// #6181 round 2, Residual 2 hardening: scans the OUTBOUND SSE for the response +// id the client actually chains from (response.created / response.completed). +// The event type rides in the data JSON for real upstreams +// ({"type":"response.completed",...}); some clients/tests emit it on the +// `event:` line instead — accept both. Feeds are cheap (bounded tail, string +// ops only) so the scan can ride a tee'd twin of any streaming branch. +class MintIdTap { + private tail = ""; + created?: string; + completed?: string; + feed(text: string): void { + this.tail += text; + if (this.tail.length > 65536) this.tail = safeSuffix(this.tail, 32768); + if (!this.tail.includes("response.completed") && !this.tail.includes("response.created")) return; + let cut = this.tail.indexOf("\n\n"); + while (cut !== -1) { + const event = this.tail.slice(0, cut); + this.tail = this.tail.slice(cut + 2); + // The recompute must run on EVERY pass — the marker check below + // `continue`s, and skipping the recompute there pins `cut` on a + // shrinking tail: a non-marker event preceding a marker event + // (e.g. response.output_item.done before response.completed) + // spins the loop forever on an empty tail. + cut = this.tail.indexOf("\n\n"); + if (!event.includes("response.completed") && !event.includes("response.created")) continue; + const lines = event.split("\n"); + const dataLine = lines.find((l) => l.startsWith("data: ")); + const eventLine = lines.find((l) => l.startsWith("event: ")); + if (dataLine !== undefined) { + try { + const evt = JSON.parse(dataLine.slice(6)) as { type?: string; response?: { id?: unknown } }; + const id = evt.response?.id; + if (typeof id === "string" && id.length > 0) { + const type = evt.type ?? (eventLine !== undefined ? eventLine.slice(7).trim() : ""); + if (type === "response.completed") this.completed = id; + else if (type === "response.created") this.created = id; + } + } catch { + /* partial/corrupt event — ignore */ + } + } + } + } +} + export function forward(...args: Parameters): Promise { return withCacheCapture(args[4]?.session, () => forwardCaptured(...args)); } @@ -2291,6 +2336,27 @@ async function forwardCaptured( pluginBody = a; dumpRaw = dumpStreamToFile(b, opts.dumpSse, `${Date.now()}-${safeSessionId(prepared.session.id)}-raw.sse`); } + // #6181 round 2, Residual 2: the mint scan (the record that lets + // the NEXT chained round prove the provider chain holds the + // stored world) must ride this branch — plugin turns return + // before the proxy-lane taps. The client-facing bytes here ARE + // the upstream bytes (the strip pipes are byte-identical for + // tag-free streams; a response id is never a render tag), so a + // raw tap sees exactly what the client chains from. + let mintTap: MintIdTap | undefined; + let mintDone: Promise | undefined; + if (prepared.stream && prepared.protocol === "responses" && prepared.mintFrontierPos !== undefined && upstreamResult.response.status < 400) { + const [mMain, mTwin] = pluginBody.tee(); + pluginBody = mMain; + mintTap = new MintIdTap(); + mintDone = (async () => { + try { + for await (const c of mTwin) mintTap?.feed(Buffer.from(c).toString("utf8")); + } catch { + /* upstream cut — the client-facing branch decides */ + } + })(); + } if (prepared.stream) { // #2328: what the pipes know about the upstream HTTP response — // first attempt's status/content-type (a resolveFakeCompletion @@ -2376,6 +2442,15 @@ async function forwardCaptured( } else { await pipePluginJson(pluginBody, res, prepared.session, prepared.protocol, targetOrigin, absorbInstructed, wireBodyText, reply); } + if (mintTap !== undefined && mintDone !== undefined && upstreamResult.response.status < 400 && prepared.mintFrontierPos !== undefined) { + // Drain BEFORE the finally's session persist rides markDirty, + // so lastMintedResponse is recorded in the same save. + await mintDone; + const seen = mintTap.completed ?? mintTap.created; + if (seen !== undefined) { + prepared.session.lastMintedResponse = { id: seen, frontierPos: prepared.mintFrontierPos }; + } + } } finally { clearUpstreamTimer(); // #2347: finish the dump before the turn closes — its value is the @@ -2473,6 +2548,30 @@ async function forwardCaptured( prepared !== null && prepared.compressInjected && prepared.processedMessages.length > 0; + // #6181 round 2, Residual 2 hardening: the mint scan above lives in the + // compress-loop (injected) branch — but a non-injected responses turn + // (small context, classifier bypass, ACP_NO_INJECT_TOOL) never enters it, + // so session.lastMintedResponse was never recorded and the first chained + // round re-seeded the whole world. Tee a twin of the raw stream for those + // turns: here the client-facing bytes ARE the upstream bytes (the strip + // pipes are byte-identical for tag-free streams, and a response id is + // never a render tag), so the raw tap sees exactly what the client sees. + let plainMintTap: MintIdTap | undefined; + let plainMintDone: Promise | undefined; + let plainMintFrontier: number | undefined; + if ( + !useRewriter && prepared !== null && prepared.stream + && prepared.protocol === "responses" && prepared.mintFrontierPos !== undefined + && upstreamResult.response.status < 400 + ) { + plainMintFrontier = prepared.mintFrontierPos; + const [mintMain, mintTap] = responseBody.tee(); + responseBody = mintMain; + plainMintTap = new MintIdTap(); + plainMintDone = (async () => { + for await (const c of mintTap) plainMintTap?.feed(Buffer.from(c).toString("utf8")); + })(); + } if (!useRewriter || prepared === null) { // #2328: pipes cite the first upstream response's HTTP identity. const upstreamMeta: UpstreamMeta = { status: upstream.status, contentType: upstream.headers.get("content-type") ?? undefined }; @@ -2604,6 +2703,15 @@ async function forwardCaptured( } // #2347: the raw-SSE dump is finished in the safety-net finally below — // an inline await here would be skipped when a pipe rejects. + // #6181: drain the mint twin BEFORE the finally's markDirty persists + // the session, so lastMintedResponse rides the same save. + if (plainMintTap !== undefined && plainMintDone !== undefined && prepared !== null && plainMintFrontier !== undefined) { + await plainMintDone; + const seen = plainMintTap.completed ?? plainMintTap.created; + if (seen !== undefined) { + prepared.session.lastMintedResponse = { id: seen, frontierPos: plainMintFrontier }; + } + } return; } const ctx: RewriteCtx = { @@ -2774,40 +2882,10 @@ async function forwardCaptured( // proxy's minted one otherwise, inline-minted included: the scan // runs on the exact bytes written to the client). The relay pairs // it with prepared.mintFrontierPos into session.lastMintedResponse. - let mintCreatedId: string | undefined; - let mintCompletedId: string | undefined; - let mintSseTail = ""; + const mintTap = new MintIdTap(); const mintScan = (text: string): void => { if (prepared.protocol !== "responses" || prepared.mintFrontierPos === undefined) return; - mintSseTail += text; - if (mintSseTail.length > 65536) mintSseTail = safeSuffix(mintSseTail, 32768); - if (!mintSseTail.includes("response.completed") && !mintSseTail.includes("response.created")) return; - let cut = mintSseTail.indexOf("\n\n"); - while (cut !== -1) { - const event = mintSseTail.slice(0, cut); - mintSseTail = mintSseTail.slice(cut + 2); - // The event type rides in the data JSON for real upstreams - // ({"type":"response.completed",...}); some clients/tests - // emit it on the `event:` line instead — accept both. - if (!event.includes("response.completed") && !event.includes("response.created")) continue; - const lines = event.split("\n"); - const dataLine = lines.find((l) => l.startsWith("data: ")); - const eventLine = lines.find((l) => l.startsWith("event: ")); - if (dataLine !== undefined) { - try { - const evt = JSON.parse(dataLine.slice(6)) as { type?: string; response?: { id?: unknown } }; - const id = evt.response?.id; - if (typeof id === "string" && id.length > 0) { - const type = evt.type ?? (eventLine !== undefined ? eventLine.slice(7).trim() : ""); - if (type === "response.completed") mintCompletedId = id; - else if (type === "response.created") mintCreatedId = id; - } - } catch { - /* partial/corrupt event — ignore */ - } - } - cut = mintSseTail.indexOf("\n\n"); - } + mintTap.feed(text); }; for await (const chunk of loop) { if (res.destroyed || res.writableEnded) break; @@ -2835,7 +2913,7 @@ async function forwardCaptured( // full world (mintFrontierPos set). The markDirty in the finally // below persists it with the rest of the turn's state. if (prepared.protocol === "responses" && upstreamResult.response.status < 400 && prepared.mintFrontierPos !== undefined) { - const seen = mintCompletedId ?? mintCreatedId; + const seen = mintTap.completed ?? mintTap.created; if (seen !== undefined) { prepared.session.lastMintedResponse = { id: seen, frontierPos: prepared.mintFrontierPos }; } diff --git a/src/storage/store.ts b/src/storage/store.ts index cbe3ea1c69..f5bdd7eb38 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -30,6 +30,7 @@ import { markSnapPlaneDead, readSnapView, snapGcCoveredRows, + snapL1EnvelopeRead, snapL1Record, snapR3LatchedRead, snapR3LatchWrite, @@ -39,6 +40,7 @@ import { type SnapEvent, type SnapView, } from "./snap.js"; +import { parseEnvelope } from "../snapshot/identity.js"; import { FoundationRejectError, stage0Decide, type Stage0Result } from "../snapshot/plane.js"; import { recordIntegrityOutcome, recordSecondProcessOwner, recordBackupOutcome } from "./runtime-status.js"; import { integrityModeFor, writeIntegrityCache, scheduleBackgroundIntegrityCheck, backgroundIntegrityInflightForTests, noteIntegrityCheckRun } from "./integrity-check.js"; @@ -1570,6 +1572,38 @@ export class UnifiedStore { // plane's tagged forms). const view = input.planeMessages; const tagged = Array.isArray(view) && view.length > 0; + // #6181 B4 (stack on the #5901 head with #6326 durability-before-render): + // an UNTAGGED save against a sid that already holds active plane rows + // can only update the plane when its ledger-id view EXTENDS the + // stage-0 world: stage0Decide writes the l1 envelope and declares the + // new positions, but the ROWS for those positions land at the NEXT + // save — and for a pinned cohort that next save is often an untagged + // one (#6326's ingestRowsNow persists before render). So: + // • view ids ⊇ envelope ids (prefix-identical) → ALLOW: the legacy + // diff materializes the declared rows (ids agree, idempotent); + // • anything else is STALE w.r.t. the pinned world — re-derived + // kernel ids (h_…), a pre-turn transcript, a compaction — and the + // legacy diff could only delete/re-home the pinned rows (the R2 + // doctrine: the tagged plane is addressable from tagged views + // only) → SKIP the plane; the disk record still lands (the + // #6326 durability win survives) and the plane is updated by the + // next TAGGED save or a stage-0 resync (one-round convergence). + // MUST run before the transaction — after it the destructive legacy + // diff is already committed. + if (!tagged && (input.planeMessages ?? []).length === 0) { + const anyRow = (snapRowsAll(this.db, sid) ?? []).some((r) => r.active === 1); + if (anyRow) { + const env = snapL1EnvelopeRead(this.db, sessRaw); + const envIds = + env && env.envelopeJson !== null + ? parseEnvelope(env.envelopeJson).envelope.map((e) => e.id) + : []; + const viewIds = input.messages.map((m) => m.msgId); + const extendsEnvelope = + viewIds.length >= envIds.length && envIds.every((id, i) => viewIds[i] === id); + if (!extendsEnvelope) return; + } + } try { const out = this.db.transaction(() => { const l1 = snapL1Record(this.db, sessRaw, sid, witness, now); @@ -1604,21 +1638,6 @@ export class UnifiedStore { ? `sync,fold(n=${out.nEvents})` : `sync(n=${out.nEvents})`; loggerLog("info", `[snaplane] sid=${sid} l1=${out.l1} ${kind} tail=${out.commit.tailSeq}`); - // #6181 B4 (stack on the #5901 head with #6326 durability-before-render): - // an UNTAGGED save against a sid that already holds active plane rows - // is destructive and can never be an update: the tagged plane is only - // addressable from tagged views (the R2 doctrine — re-derived kernel ids - // are unaddressable), so the legacy diff can only delete/re-home the - // pinned rows. #6326's ingestRowsNow persist is such an untagged save: - // it runs before render, when the in-memory transcript holds the pre-turn - // world (the previous response was never ingested), and its diff then - // deleted the just-flushed rows. The disk record above still lands (the - // durability win survives); the plane is updated by the NEXT tagged save - // (the next turn's flush carries the authoritative view). - if (!tagged && (input.planeMessages ?? []).length === 0) { - const anyRow = (snapRowsAll(this.db, sid) ?? []).some((r) => r.active === 1); - if (anyRow) return; - } } catch (err) { if (!(err instanceof Error) || err.name !== "SnapPlaneDeadError") markSnapPlaneDead(sid); // R2/R3 degrade: the round stays on the legacy path. Log the From 838f46d54c0a06d85cf600a0b49ccb1633fe7020 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Fri, 13 Nov 2026 18:41:50 +0800 Subject: [PATCH 8/8] =?UTF-8?q?fix(#6181):=20repair=20the=20merged=20proce?= =?UTF-8?q?ss.yaml=20=E2=80=94=20desc=5Fja=20line=20lost=20its=20newline?= =?UTF-8?q?=20in=20the=20#5901=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 941447a54 merge joined the egress-cohort desc_ja value with the next '- key: sessions' entry onto one line, which tools/gen-config-docs.mjs rejects (quoted value not valid JSON) — docs-sync + docs-coverage red. Split the line back; generator checks pass (coverage ok: 204 env + 63 keys; i18n ok: 254 keys). --- website/config-reference/process.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/config-reference/process.yaml b/website/config-reference/process.yaml index 9e999f9101..d699e50f65 100644 --- a/website/config-reference/process.yaml +++ b/website/config-reference/process.yaml @@ -482,7 +482,8 @@ entries: group: process desc_en: "Protocol cohorts allowed to cut the egress source over (#5874 P2); a subset of openai/anthropic/google/responses. Sessions whose protocol is absent keep legacy bytes unconditionally (the plane still shadow-writes for them). An empty/absent array means all four." desc_zh: "允许切出站源的协议 cohort(#5874 P2),取 openai/anthropic/google/responses 的子集。协议不在名单内的会话无条件保留旧字节(平面照常影子写)。空/缺省 = 四个全开。" - desc_ja: "egress ソースの切替を許可するプロトコル cohort (#5874 P2); openai/anthropic/google/responses の部分集合。プロトコルがリストにないセッションは無条件で legacy バイトを保持する (プレーンはシャドウライト継続)。空/欠如の配列は 4 つすべてを意味する。" - key: sessions + desc_ja: "egress ソースの切替を許可するプロトコル cohort (#5874 P2); openai/anthropic/google/responses の部分集合。プロトコルがリストにないセッションは無条件で legacy バイトを保持する (プレーンはシャドウライト継続)。空/欠如の配列は 4 つすべてを意味する。" + - key: sessions type: object default: "{} (all defaults)" group: process