From b5756bb6fc3f6772c0fec200a1188a3eba0f9cd7 Mon Sep 17 00:00:00 2001 From: awork Date: Fri, 16 Oct 2026 02:03:41 +0800 Subject: [PATCH 1/4] fix: re-anchor folded history across host wire-shape flips (#3301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host metadata-table refresh can re-serialize every past thinking turn of one session in a different wire form (reasoning_content field vs reasoning spliced into content), minting new ids for unchanged content. Fold re-anchor then reports 'covered id(s) missing with no anchor match' on every request: the turn-key pass only understood bare-concat merges, and pre-#2644 anchors lack p/x entirely — while the wire-side backfill can never fire because the flip puts the old id off the wire on the very pass that also rolls the order backbone forward without it. - fold-reconcile: extractThinkingMerge mirrors the kernel's demoted-thinking FORMS so merged-form cores register the same turn keys as split-form anchors (turnContexts + Pass 3 candidate side); ReconcileOptions.storedMessages rebuilds eligible legacy anchors from stored decompositions BEFORE the plan runs, so the flip pass itself claims. - storage: UnifiedStore.messagesForIds (+seq±1 neighbors for rc-pair adjacency), shadow/persist pass-throughs gated on full mode. - new fold-decomposition-lookup: per-pass capped lookup with a tried-set (index absence is stable, index unavailability retries). - import entry: record/session/ingest carry an optional decompositionLog (external converter payload) interned first at ingest; consume-once at save keeps the #3133 delta fast path intact; buildRecord never emits it. No new config surface. Additive optional fields only (ReconcileOptions. storedMessages, LegacySessionLike.decompositionLog, PersistedSession. decompositionLog) plus an internal metadata bookkeeping key foldStoredTried. No kernel/ changes. Fixes #3301 --- src/fold-decomposition-lookup.ts | 125 +++++++ src/fold-reconcile.ts | 109 ++++++- src/persist.ts | 34 +- src/server/prepare-anthropic.ts | 4 +- src/server/prepare-google.ts | 4 +- src/server/prepare-openai.ts | 6 +- src/server/prepare-responses.ts | 4 +- src/session.ts | 5 + src/storage/ingest.ts | 14 + src/storage/shadow.ts | 7 + src/storage/store.ts | 43 +++ tests/issue3301-shape-flip-reanchor.test.ts | 340 ++++++++++++++++++++ 12 files changed, 679 insertions(+), 16 deletions(-) create mode 100644 src/fold-decomposition-lookup.ts create mode 100644 tests/issue3301-shape-flip-reanchor.test.ts diff --git a/src/fold-decomposition-lookup.ts b/src/fold-decomposition-lookup.ts new file mode 100644 index 0000000000..6ba7e7bc85 --- /dev/null +++ b/src/fold-decomposition-lookup.ts @@ -0,0 +1,125 @@ +// #3301: stored-decomposition source for fold-anchor backfill. +// +// Anchors persisted before turn keys existed (pre-#2644) carry no p/x, and the +// wire-side backfill can only fire while the old id is still on the wire — a +// wire-shape flip (host metadata refresh re-serializing every past thinking +// turn) puts it off the wire forever — and the flip pass is the LAST pass in +// which those ids sit in the order backbone (claims only iterate backbone ids). +// This module pulls those ids' original message decompositions out of the +// unified index (full mode only) so reconcileFoldCoverage can rebuild p/x +// BEFORE its plan runs and the flip pass itself claims. Ids absent from the index are +// marked tried once (absence is stable for folded-out history); an UNAVAILABLE +// index is not marked tried (it may appear later). + +import type { CoreMessage } from "acp-kernel"; +import type { Session } from "./session.js"; +import { getStore } from "./persist.js"; +import { log as loggerLog } from "./logger.js"; +import { METADATA_ANCHORS, type FoldAnchor } from "./fold-reconcile.js"; + +export interface StoredRow { + seq: number; + msgId: string; + content: string; +} + +export interface DecompositionSource { + /** null = no index available (legacy/shadow/degraded); [] = index has none of them. */ + rows(sessionId: string, msgIds: readonly string[]): StoredRow[] | null; +} + +const MAX_LOOKUP_PER_PASS = 256; +const TRIED_METADATA_KEY = "foldStoredTried"; +const MAX_TRIED = 8192; + +function defaultSource(): DecompositionSource { + return { + rows(sessionId, msgIds) { + try { + return getStore().unifiedMessageRows(sessionId, msgIds); + } catch (err) { + loggerLog("warn", `[fold-reconcile] stored decomposition lookup failed: ${String(err)}`); + return null; + } + }, + }; +} + +/** Wire-site convenience: build the wire-id set and run the lookup in one + * call (used by all four prepare-* lanes). */ +export function storedMessagesFor(session: Session, sessionId: string, msgs: readonly CoreMessage[]): CoreMessage[] | undefined { + const wireIds = new Set(); + for (const m of msgs) if (m.id !== undefined) wireIds.add(m.id); + return resolveStoredDecompositions(session, sessionId, wireIds); +} + +export function resolveStoredDecompositions( + session: Session, + sessionId: string, + wireIds: ReadonlySet, + source: DecompositionSource = defaultSource(), +): CoreMessage[] | undefined { + const metadata = session.metadata; + if (metadata === undefined) return undefined; + const anchors = (metadata[METADATA_ANCHORS] as Record | undefined) ?? {}; + const anchorIds = Object.keys(anchors); + if (anchorIds.length === 0) return undefined; + const tried = (metadata[TRIED_METADATA_KEY] as Record | undefined) ?? {}; + const needed: string[] = []; + for (const id of anchorIds) { + const a = anchors[id]; + if (a.p !== undefined || a.r !== "assistant" || a.t !== undefined) continue; + if (wireIds.has(id)) continue; + if (tried[id] !== undefined) continue; + needed.push(id); + if (needed.length >= MAX_LOOKUP_PER_PASS) break; + } + if (needed.length === 0) return undefined; + const rows = source.rows(sessionId, needed); + if (rows === null) return undefined; // index unavailable: retry next pass + for (const id of needed) tried[id] = 1; + const pruned: Record = {}; + let n = 0; + for (const [id, v] of Object.entries(tried)) { + if (!anchors[id]) continue; + pruned[id] = v; + if (++n >= MAX_TRIED) break; + } + metadata[TRIED_METADATA_KEY] = pruned; + // Latest seq per id wins (edits append rows); neighbors stay in so rc + // pairs keep adjacency for turn-key derivation. Unparseable rows are + // auxiliary-evidence misses, not errors — count them, don't throw. + let skipped = 0; + const latestSeq = new Map(); + const ordered: { seq: number; m: CoreMessage }[] = []; + const orderedIdx = new Map(); + for (const row of rows) { + let parsed: unknown; + try { + parsed = JSON.parse(row.content); + } catch { + skipped++; + continue; + } + if (typeof parsed !== "object" || parsed === null) { + skipped++; + continue; + } + const m = parsed as CoreMessage; + if (typeof m.id !== "string" || typeof m.role !== "string") { + skipped++; + continue; + } + const prev = latestSeq.get(row.msgId); + if (prev !== undefined && prev >= row.seq) continue; + latestSeq.set(row.msgId, row.seq); + const prevIdx = orderedIdx.get(row.msgId); + if (prevIdx !== undefined) ordered.splice(prevIdx, 1); + ordered.push({ seq: row.seq, m }); + orderedIdx.set(row.msgId, ordered.length - 1); + } + if (skipped > 0) loggerLog("debug", `[fold-reconcile] stored decomposition: ${skipped} unparseable row(s) skipped`); + if (ordered.length === 0) return undefined; + ordered.sort((a, b) => a.seq - b.seq); + return ordered.map((o) => o.m); +} diff --git a/src/fold-reconcile.ts b/src/fold-reconcile.ts index a715f7af54..4296eb8b88 100644 --- a/src/fold-reconcile.ts +++ b/src/fold-reconcile.ts @@ -139,7 +139,7 @@ import { dshNativeCompactionWitness, recordConflict } from "./conflict-watch.js" type FoldReconcileMode = "off" | "warn" | "repair"; -const METADATA_ANCHORS = "foldAnchors"; +export const METADATA_ANCHORS = "foldAnchors"; const METADATA_ORDER = "foldAnchorOrder"; /** #2480: positional identity copy — canonical fingerprint per message, * aligned 1:1 with foldAnchorOrder (same rolling window, same cap, written @@ -475,6 +475,42 @@ function turnKeyOfSeamless(reasoning: string | undefined, text: string | undefin return h.digest("hex").slice(0, 16); } +/** #3301: split tagged thinking-merge text (host metadata refresh re-serializes + * split-form turns as tagged reasoning spliced into content). MIRRORS + * kernel/src/wire/demoted-thinking.ts FORMS + splitDemotedThinking in lockstep + * (not exported from the kernel root) — keep both lists in sync. Offset-0 tag + * start is the only position demotion renderers produce. Auxiliary evidence + * for turn-key derivation ONLY — never identity minting. */ +const THINKING_MERGE_FORMS: readonly { open: string; close: string }[] = [ + { open: "\x3cthink\x3e\n", close: "\n\x3c/think\x3e" }, + { open: "\x3cthinking\x3e\n", close: "\n\x3c/thinking\x3e" }, + { open: "```thinking\n", close: "\n```" }, +]; + +export function extractThinkingMerge(text: string | undefined): { reasoning: string; text: string } | null { + if (text === undefined || text === "") return null; + let rest = text; + const parts: string[] = []; + for (;;) { + let matched = false; + for (const form of THINKING_MERGE_FORMS) { + if (!rest.startsWith(form.open)) continue; + const end = rest.indexOf(form.close, form.open.length); + if (end < 0) continue; + const inner = rest.slice(form.open.length, end); + if (inner.length === 0) continue; + parts.push(inner); + rest = rest.slice(end + form.close.length); + if (rest.startsWith("\n")) rest = rest.slice(1); + matched = true; + break; + } + if (!matched) break; + } + if (parts.length === 0) return null; + return { reasoning: parts.join("\n"), text: rest }; +} + /** Turn context for the anchor seed loop: per assistant-core id, the turn * key(s) and (for split-form turns) the sibling core id. Built lazily — a * steady-state pass reuses stored anchors and never pays this. */ @@ -499,7 +535,18 @@ export function turnContexts(msgs: CoreMessage[]): Map { const prev = msgs[i - 1]; if (prev === undefined || prev.id === undefined || prev.role !== "assistant" || prev.contentType !== "reasoning" || (prev.toolCallId !== undefined && prev.toolCallId !== "")) { - ctx.set(m.id, { p: turnKeyOf(undefined, m.text) }); + // #3301: merged-form core — its text may be tagged reasoning + reply. + // The extracted pair key bridges anchors seeded from the split form + // (same turn, different wire shape); the raw full-text key stays for + // identical resends of this exact form. + const ex = extractThinkingMerge(m.text); + if (ex !== null) { + const key = turnKeyOf(ex.reasoning, ex.text); + const seamless = turnKeyOfSeamless(ex.reasoning, ex.text); + ctx.set(m.id, seamless !== key ? { p: key, p2: seamless } : { p: key }); + } else { + ctx.set(m.id, { p: turnKeyOf(undefined, m.text) }); + } } } } @@ -947,10 +994,20 @@ export function planReconciliation( if (claimedCandidates.has(cand.id)) continue; if (!isBareAssistantCore(cand.message)) continue; if (cand.message.contentType === "text") { - const key = turnKeyOf(undefined, cand.message.text); - const bucket = singleTurnKeys.get(key); - if (bucket === undefined) singleTurnKeys.set(key, [cand.id]); - else bucket.push(cand.id); + // #3301: a merged-form core stands in for the whole turn — + // register the raw full-text key AND the extracted pair key(s) + // so anchors seeded from either wire form match. + const keys = new Set([turnKeyOf(undefined, cand.message.text)]); + const ex = extractThinkingMerge(cand.message.text); + if (ex !== null) { + keys.add(turnKeyOf(ex.reasoning, ex.text)); + keys.add(turnKeyOfSeamless(ex.reasoning, ex.text)); + } + for (const key of keys) { + const bucket = singleTurnKeys.get(key); + if (bucket === undefined) singleTurnKeys.set(key, [cand.id]); + else bucket.push(cand.id); + } } else if (cand.message.contentType === "reasoning") { const next = byPos.get(cand.pos + 1); if (next !== undefined && !claimedCandidates.has(next.id) && isBareAssistantCore(next.message) && next.message.contentType === "text") { @@ -1262,6 +1319,13 @@ export interface ReconcileOptions { * cleans the resend — benign churn that must not feed the #2193 destroy * streak. #2815/#2789. */ markerStripped?: number; + /** #3301: stored message decompositions for anchors that predate turn-key + * backfill (pre-#2644) — after a wire-shape flip the wire-side backfill + * can never fire (the old id is gone from the wire), so eligible anchors + * are rebuilt from these rows BEFORE the reconciliation plan runs and the + * flip pass itself claims. Latest-seq-per-id, neighbors included; + * undefined = no source available (legacy-only mode). */ + storedMessages?: CoreMessage[]; } /** Entry point wired into the four wire sites, immediately BEFORE the @@ -1294,7 +1358,7 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt } if (!session.metadata) return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: 0, unmatched: 0 }; - const anchors: Record = + let anchors: Record = (session.metadata[METADATA_ANCHORS] as Record | undefined) ?? {}; const oldOrder: string[] = (session.metadata[METADATA_ORDER] as string[] | undefined) ?? []; // #2480: self-contained positional copy {ids, canon} (#2487's storage @@ -1310,6 +1374,35 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt positions = { ids: storedRaw.ids as string[], canon: storedRaw.canon as string[] }; } + const byId = new Map(); + for (const m of msgs) if (m.id !== undefined) byId.set(m.id, m); + // #3301: a wire-shape flip puts legacy (pre-#2644, no p/x) ids off the wire + // on the FIRST post-flip pass — and this same pass rolls the order backbone + // forward without them, so any backfill landing later can never claim (the + // claim passes only iterate ids still in the backbone). Rebuild eligible + // anchors from the stored decomposition BEFORE the plan runs so the flip + // pass itself can claim. Wire bytes always win: an id present on this pass + // keeps its live-bytes path untouched. + if (opts.storedMessages !== undefined && opts.storedMessages.length > 0) { + const sById = new Map(); + for (const m of opts.storedMessages) if (m.id !== undefined) sById.set(m.id, m); + const sCtx = turnContexts(opts.storedMessages); + let changed = false; + const upgraded: Record = {}; + for (const [id, a] of Object.entries(anchors)) { + if (a.p === undefined && a.r === "assistant" && a.t === undefined && !byId.has(id)) { + const sm = sById.get(id); + if (sm !== undefined) { + upgraded[id] = anchorFrom(sm, sCtx.get(id)); + changed = true; + continue; + } + } + upgraded[id] = a; + } + if (changed) anchors = upgraded; + } + const plan = planReconciliation(oldOrder, anchors, msgs, covered, positions); if (process.env.FOLD_RECONCILE_DEBUG === "1") { const active = blocks.filter((b) => b.active).length; @@ -1321,8 +1414,6 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt // freshly claimed ones — the next churn must re-anchor from post-churn // bytes), prune anchors for ids no longer covered, and roll the order // backbone forward to this pass. - const byId = new Map(); - for (const m of msgs) if (m.id !== undefined) byId.set(m.id, m); // #2636: turn keys for freshly seeded anchors — built lazily so // steady-state passes (no churn) never pay the per-core normalization // (#2334). Only bare assistant cores can carry a turn key diff --git a/src/persist.ts b/src/persist.ts index ec93ccf4e1..97fe69baaa 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -200,6 +200,9 @@ interface PersistedSession { * history and must still be pruned at export time. */ messagesFolded?: boolean; pluginSnapshot?: CoreMessage[]; + /** #3301: external-converter import payload (see Session.decompositionLog) — + * accepted on read; buildRecord deliberately does not emit it. */ + decompositionLog?: CoreMessage[]; forkContentStore?: MessageContentStore; } @@ -1216,6 +1219,20 @@ export class SessionStore { } } + /** #3301: stored message decompositions (+ seq±1 neighbors) for + * fold-anchor backfill after a wire-shape flip (consumed by + * fold-decomposition-lookup → reconcileFoldCoverage.storedMessages). + * Full mode only (same gate as the other unified reads); null when + * unavailable — callers fall back to wire-side backfill. Never throws. */ + unifiedMessageRows(sessionId: string, msgIds: readonly string[]): { seq: number; msgId: string; content: string }[] | null { + if (!this.shadow?.readsUnified || msgIds.length === 0) return null; + try { + return this.shadow.messagesForIds(sessionId, msgIds, true); + } catch { + return null; + } + } + replicasAvailable(): boolean { return this.shadow !== undefined && !this.shadow.stats.disabled; } @@ -1631,6 +1648,14 @@ export class SessionStore { const record = buildRecord(session, stamp); const disk = this.staleDiskPayload(record); if (disk === null) { + // #3301: external-converter import payload rides exactly ONE + // save into the unified index (first-open imports take the + // importRecord path), then is consumed so steady saves keep + // the delta fast path (#3133) instead of paying full + // re-ingests forever. buildRecord omits it, so the disk file + // never carries it. + const payload = session.decompositionLog; + if (payload !== undefined) session.decompositionLog = undefined; // #2960 ⑤: the table holds the RAW tail (full fidelity); the // envelope keeps its bounded folded view for legacy readers. // ②a caveat: once foldResidentTail has folded the resident @@ -1644,9 +1669,10 @@ export class SessionStore { // folded saves present the envelope render (no override), // which the delta path already routes to the full ingest. const tail = session.lastMessagesFolded !== true ? session.lastMessages : undefined; - this.shadow?.record(record, tail && tail.length > 0 - ? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded } - : undefined, session.contentStore ?? undefined); + this.shadow?.record(payload !== undefined ? { ...record, decompositionLog: payload } : record, + tail && tail.length > 0 + ? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded } + : undefined, session.contentStore ?? undefined); this.noteOwnCommit(session.id, record); return record; } @@ -1711,6 +1737,7 @@ export class SessionStore { session.lastMessages = fresh.lastMessages ?? session.lastMessages; session.lastMessagesFolded = fresh.lastMessages !== undefined ? fresh.lastMessagesFolded : session.lastMessagesFolded; session.pluginSnapshot = fresh.pluginSnapshot; + session.decompositionLog = fresh.decompositionLog ?? session.decompositionLog; session.lastSeen = fresh.lastSeen; if (!session.meta.protocol && fresh.meta.protocol) session.meta.protocol = fresh.meta.protocol; if (!session.meta.upstreamOrigin && fresh.meta.upstreamOrigin) session.meta.upstreamOrigin = fresh.meta.upstreamOrigin; @@ -2117,6 +2144,7 @@ function buildSession(parsed: PersistedSession): Session { lastMessages: Array.isArray(parsed.messages) ? parsed.messages : undefined, lastMessagesFolded: parsed.messagesFolded === true, pluginSnapshot: Array.isArray(parsed.pluginSnapshot) ? parsed.pluginSnapshot : undefined, + decompositionLog: Array.isArray(parsed.decompositionLog) ? parsed.decompositionLog : undefined, contentStore: parsed.forkContentStore, inFlight: 0, persisted: true, diff --git a/src/server/prepare-anthropic.ts b/src/server/prepare-anthropic.ts index b862723ab5..e9b0f57fb7 100644 --- a/src/server/prepare-anthropic.ts +++ b/src/server/prepare-anthropic.ts @@ -9,6 +9,7 @@ import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTo import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { storedMessagesFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js"; @@ -202,7 +203,8 @@ export async function prepareAnthropic( // [#1921] re-anchor fold coverage onto churned-but-same messages // before the #1195 snapshot, so covered ids surviving a client // re-serialization stay covered (src/fold-reconcile.ts). - reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) }); + const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); + reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) }); noteSystemPromptFingerprint(session, clientSystem, { sessionId, log }); // #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside // processTurn may deactivate fully-drifted blocks, erasing them. diff --git a/src/server/prepare-google.ts b/src/server/prepare-google.ts index b26e8714ac..ad7307cc87 100644 --- a/src/server/prepare-google.ts +++ b/src/server/prepare-google.ts @@ -8,6 +8,7 @@ import { diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBilli import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { storedMessagesFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js"; @@ -149,7 +150,8 @@ export async function prepareGoogle( // [#1921] re-anchor fold coverage onto churned-but-same messages // before the #1195 snapshot, so covered ids surviving a client // re-serialization stay covered (src/fold-reconcile.ts). - reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) }); + const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); + reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) }); if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log }); // #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside // processTurn may deactivate fully-drifted blocks, erasing them. diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 94fadf9083..635fcc0977 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -10,6 +10,7 @@ import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTo import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { storedMessagesFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js"; @@ -155,7 +156,10 @@ export async function prepareOpenai( // [#1921] re-anchor fold coverage onto churned-but-same messages // before the #1195 snapshot, so covered ids surviving a client // re-serialization stay covered (src/fold-reconcile.ts). - reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) }); + // #3301: stored decompositions for pre-#2644 anchors whose wire-side + // backfill can never fire after a wire-shape flip. + const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); + reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) }); if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log }); // #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside // processTurn may deactivate fully-drifted blocks, erasing them. diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 0cb8b37fa5..3a972c248d 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -11,6 +11,7 @@ import { excludeDeadRefRanges } from "../acp-status.js"; import { mergeAdjacentConfigurationUpdates, patchResponsesInputWithToolImages as patchResponsesInput, responsesToCoreWithToolImages as responsesToCore } from "../responses-tool-output.js"; import { dropWhitespaceResponsesMessages, normalizeResponsesMessageItems, sanitizeResponsesInputIds } from "../loop/adapter-responses.js"; import { METADATA_LAST_RESPONSES_MODE, noteCanonicalSystemFingerprint, reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { storedMessagesFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed, pluginLaneDegraded } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { coveredRealHistoryIds, foldCoverage, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js"; @@ -256,7 +257,8 @@ export async function prepareResponses( // [#1921] re-anchor fold coverage onto churned-but-same messages // before the #1195 snapshot, so covered ids surviving a client // re-serialization stay covered (src/fold-reconcile.ts). - reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) }); + const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); + reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) }); if (!isCompactionTrigger) { noteSystemPromptFingerprint(session, responsesClientSystem, { sessionId, log }); // #2867: the mode-independent twin — lets the ledger classify a diff --git a/src/session.ts b/src/session.ts index 86396bd820..727adf7d5e 100644 --- a/src/session.ts +++ b/src/session.ts @@ -512,6 +512,11 @@ export type Session = { * request — the client re-sends full raw history, restoring the invariant. */ lastMessagesFolded?: boolean; pluginSnapshot?: CoreMessage[]; + /** #3301: original message decompositions supplied by an external state + * converter (e.g. opencode-acp migration) at import time. Transient + * import payload: consumed once by the unified ingest (rows land in the + * index for fold re-anchor backfill) and never written back to records. */ + decompositionLog?: CoreMessage[]; /** In-memory only (NOT persisted — buildRecord omits it): monotone counter * bumped by markDirty after every mutation. Keys the fork-revision cache * below so status polling skips re-hashing the whole history (#2017). */ diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 429a8a8f57..589bcf19ca 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -71,6 +71,12 @@ export interface LegacySessionLike { lastMessages?: CoreMessage[]; lastMessagesFolded?: boolean; pluginSnapshot?: CoreMessage[]; + /** #3301: external-converter import payload — original message + * decompositions interned BEFORE the snapshot/tail so their rows get the + * lowest seqs (they are the oldest evidence). Consumed by the full + * ingest only; the delta planner falls back to the full path while it is + * present (its rows would otherwise be skipped as "trusted prefix"). */ + decompositionLog?: CoreMessage[]; contentStore?: { version: 1; byHash: Record; byRef: Record }; } @@ -115,6 +121,7 @@ export function toLegacyLike(s: { lastMessages?: LegacySessionLike["lastMessages"]; lastMessagesFolded?: boolean; pluginSnapshot?: LegacySessionLike["pluginSnapshot"]; + decompositionLog?: LegacySessionLike["decompositionLog"]; contentStore?: LegacySessionLike["contentStore"]; }): LegacySessionLike { return { @@ -129,6 +136,7 @@ export function toLegacyLike(s: { lastMessages: s.lastMessages, lastMessagesFolded: s.lastMessagesFolded, pluginSnapshot: s.pluginSnapshot, + decompositionLog: s.decompositionLog, contentStore: s.contentStore, }; } @@ -159,6 +167,9 @@ export function planIngest(session: LegacySessionLike): IngestPlan { log.push({ seq, msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null }); return seq; }; + // #3301: import payload first — its rows must sit below the snapshot/tail + // seqs so the stored log reads oldest-evidence-first. + if (session.decompositionLog && session.decompositionLog.length > 0) session.decompositionLog.forEach(intern); const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null; const tailSeqs = session.lastMessages ? session.lastMessages.map(intern) : null; @@ -193,6 +204,9 @@ export function planIngestDelta( hasRawTail: boolean, ): IngestPlan | null { if (session.pluginSnapshot && session.pluginSnapshot.length > 0) return null; + // #3301: an import payload would be skipped as "trusted prefix" here — force + // the full path so its rows actually land. + if (session.decompositionLog && session.decompositionLog.length > 0) return null; const tail = session.lastMessages; const trustedLen = trustedTailSeqs.length; if ( diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 6a1e1b6cb7..dfe94bb968 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -338,6 +338,13 @@ export class UnifiedShadow { return tip; } + /** #3301: message rows for specific ids (+ seq±1 neighbors) — stored + * decompositions for fold-anchor backfill after a wire-shape flip. */ + messagesForIds(sessionId: string, msgIds: readonly string[], includeNeighbors: boolean): { seq: number; msgId: string; content: string }[] { + this.assertLive(); + return this.store.messagesForIds(sessionId, msgIds, includeNeighbors); + } + /** All sessions row ids present in the index (#2960 R2.1). Used by the * full-mode boot union so rows whose legacy file disappeared remain * enumerable. */ diff --git a/src/storage/store.ts b/src/storage/store.ts index 4fc3c8d3a4..6c3fa79971 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -729,6 +729,49 @@ export class UnifiedStore { return { msgId: row.msg_id, contentHash: row.content_hash, maxSeq: max.m ?? -1 }; } + /** #3301: message rows for the given ids — EVERY seq row matching an id + * (edits append; callers keep the latest per id) plus, when + * includeNeighbors, the seq±1 rows so split-form rc pairs stay adjacent + * for turn-key derivation. Content bytes resolved via CAS; rows whose + * blob is gone are skipped. Read-only. */ + messagesForIds(sessionId: string, msgIds: readonly string[], includeNeighbors: boolean): { seq: number; msgId: string; content: string }[] { + if (msgIds.length === 0) return []; + const CHUNK = 500; + const rows = new Map(); + const neighborSeqs = new Set(); + for (let i = 0; i < msgIds.length; i += CHUNK) { + const chunk = msgIds.slice(i, i + CHUNK); + const ph = chunk.map(() => "?").join(","); + for (const r of this.db + .prepare(`SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ? AND msg_id IN (${ph})`) + .all(sessionId, ...chunk) as { seq: number; msg_id: string; content_hash: string }[]) { + if (!rows.has(r.seq)) rows.set(r.seq, { seq: r.seq, msgId: r.msg_id, contentHash: r.content_hash }); + if (includeNeighbors) { + neighborSeqs.add(r.seq - 1); + neighborSeqs.add(r.seq + 1); + } + } + } + if (neighborSeqs.size > 0) { + const seqs = [...neighborSeqs]; + for (let i = 0; i < seqs.length; i += CHUNK) { + const chunk = seqs.slice(i, i + CHUNK); + const ph = chunk.map(() => "?").join(","); + for (const r of this.db + .prepare(`SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ? AND seq IN (${ph})`) + .all(sessionId, ...chunk) as { seq: number; msg_id: string; content_hash: string }[]) { + if (!rows.has(r.seq)) rows.set(r.seq, { seq: r.seq, msgId: r.msg_id, contentHash: r.content_hash }); + } + } + } + const out: { seq: number; msgId: string; content: string }[] = []; + for (const r of [...rows.values()].sort((a, b) => a.seq - b.seq)) { + const content = this.getContent(r.contentHash); + if (content !== null) out.push({ seq: r.seq, msgId: r.msgId, content }); + } + return out; + } + /** Deletes one session and releases every CAS refcount it holds * (GC parity with the legacy path's file removal). Rows that other * sessions also reference survive via their own refcounts. Checkpoints diff --git a/tests/issue3301-shape-flip-reanchor.test.ts b/tests/issue3301-shape-flip-reanchor.test.ts new file mode 100644 index 0000000000..3fde649924 --- /dev/null +++ b/tests/issue3301-shape-flip-reanchor.test.ts @@ -0,0 +1,340 @@ +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import { + extractThinkingMerge, + planReconciliation, + reconcileFoldCoverage, + turnContexts, + normalizedIdentity, + type FoldAnchor, + type ReconcileOptions, +} from "../src/fold-reconcile.ts"; +import { resolveStoredDecompositions, type StoredRow } from "../src/fold-decomposition-lookup.ts"; +import { planIngest, planIngestDelta, type LegacySessionLike } from "../src/storage/ingest.ts"; +import type { CoreMessage } from "acp-kernel"; +import type { Session } from "../src/session.ts"; + +function msg(id: string, role: string, text: string, extra?: Partial): CoreMessage { + return { id, role, contentType: "text", text, ...extra } as CoreMessage; +} + +const THINK_OPEN = "\x3cthink\x3e\n"; +const THINK_CLOSE = "\n\x3c/think\x3e"; +const THINKING_OPEN = "\x3cthinking\x3e\n"; +const THINKING_CLOSE = "\n\x3c/thinking\x3e"; +const FENCE_OPEN = "```thinking\n"; +const FENCE_CLOSE = "\n```"; + +describe("extractThinkingMerge (#3301)", () => { + test("each kernel-listed form splits at offset 0", () => { + assert.deepEqual(extractThinkingMerge(`${THINK_OPEN}R${THINK_CLOSE}\nT`), { reasoning: "R", text: "T" }); + assert.deepEqual(extractThinkingMerge(`${THINKING_OPEN}R${THINKING_CLOSE}\nT`), { reasoning: "R", text: "T" }); + assert.deepEqual(extractThinkingMerge(`${FENCE_OPEN}R${FENCE_CLOSE}\nT`), { reasoning: "R", text: "T" }); + }); + test("multi-block runs join reasoning with \\n (kernel parity)", () => { + const s = `${THINK_OPEN}A${THINK_CLOSE}\n${THINK_OPEN}B${THINK_CLOSE}\nT`; + assert.deepEqual(extractThinkingMerge(s), { reasoning: "A\nB", text: "T" }); + }); + test("thinking-only merge keeps empty text", () => { + assert.deepEqual(extractThinkingMerge(`${THINK_OPEN}R${THINK_CLOSE}`), { reasoning: "R", text: "" }); + }); + test("tag not at offset 0 never matches (prose safety)", () => { + assert.equal(extractThinkingMerge(`prefix ${THINK_OPEN}R${THINK_CLOSE}\nT`), null); + assert.equal(extractThinkingMerge(`\n${THINK_OPEN}R${THINK_CLOSE}\nT`), null); + }); + test("empty or unterminated blocks do not match", () => { + assert.equal(extractThinkingMerge(`${THINK_OPEN}${THINK_CLOSE}\nT`), null); + assert.equal(extractThinkingMerge(`${THINK_OPEN}R never closed`), null); + }); + test("plain prose, empty and undefined return null", () => { + assert.equal(extractThinkingMerge("just prose"), null); + assert.equal(extractThinkingMerge(""), null); + assert.equal(extractThinkingMerge(undefined), null); + }); +}); + +describe("turnContexts bridges split vs merged forms (#3301)", () => { + test("merged tagged core carries the same turn keys as the rc pair", () => { + const R = "alpha beta gamma delta epsilon zeta eta theta"; + const T = "one two three four five six seven eight nine ten"; + const rcPairCtx = turnContexts([ + msg("r1", "assistant", R, { contentType: "reasoning" }), + msg("t1", "assistant", T), + ]); + const mergedCtx = turnContexts([ + msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`), + ]); + const pair = rcPairCtx.get("r1"); + const merged = mergedCtx.get("m1"); + assert.ok(pair && merged); + assert.equal(merged!.p, pair!.p, "primary seam key matches"); + assert.equal(merged!.p2 ?? merged!.p, pair!.p2 ?? pair!.p, "seamless key matches"); + }); +}); + +describe("planReconciliation shape flips (#3301)", () => { + const R = "alpha beta gamma delta epsilon zeta eta theta"; + const T = "one two three four five six seven eight nine ten"; + const MERGED = `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`; + + function rcAnchors(): Record { + const r1 = msg("r1", "assistant", R, { contentType: "reasoning" }); + const t1 = msg("t1", "assistant", T); + const ctx = turnContexts([r1, t1]); + const a1 = { n: normalizedIdentity(r1), r: "assistant", c: "reasoning", b: R.length, p: ctx.get("r1")?.p, p2: ctx.get("r1")?.p2, x: "t1" }; + const a2 = { n: normalizedIdentity(t1), r: "assistant", c: "text", b: T.length, p: ctx.get("t1")?.p, p2: ctx.get("t1")?.p2, x: "r1" }; + return { r1: a1, t1: a2 }; + } + + test("rc pair -> merged core: both member ids claim the one witness", () => { + const merged = msg("m1", "assistant", MERGED); + const plan = planReconciliation(["r1", "t1"], rcAnchors(), [merged], new Set(["r1", "t1"])); + assert.deepEqual(plan.claims.get("r1"), ["m1"]); + assert.deepEqual(plan.claims.get("t1"), ["m1"]); + assert.equal(plan.unmatched.length, 0); + assert.ok(plan.byTurn >= 1, `turn-key pass did the bridging (counters: ${JSON.stringify({ byTurn: plan.byTurn, bySim: plan.bySim, byOrder: plan.byOrder })})`); + }); + + test("merged core -> rc pair: one old id claims both new ids", () => { + const merged = msg("m1", "assistant", MERGED); + const ctx = turnContexts([merged]); + const anchors = { m1: { n: normalizedIdentity(merged), r: "assistant", c: "text", b: MERGED.length, p: ctx.get("m1")?.p } }; + const plan = planReconciliation(["m1"], anchors, [ + msg("r2", "assistant", R, { contentType: "reasoning" }), + msg("t2", "assistant", T), + ], new Set(["m1"])); + assert.deepEqual(plan.claims.get("m1"), ["r2", "t2"]); + assert.equal(plan.unmatched.length, 0); + assert.ok(plan.byTurn >= 1); + }); + + test("untagged merges stay bridgeable through the bare-concat keys", () => { + for (const seam of ["\n", ""]) { + const mergedUntagged = msg(`mu-${seam === "\n" ? "nl" : "flat"}`, "assistant", R + seam + T); + const plan = planReconciliation(["r1", "t1"], rcAnchors(), [mergedUntagged], new Set(["r1", "t1"])); + assert.equal(plan.unmatched.length, 0, `seam ${JSON.stringify(seam)} bridged`); + assert.deepEqual(plan.claims.get("r1"), [mergedUntagged.id]); + } + }); + + test("ambiguous duplicate turns skip honestly (never fuzzy)", () => { + const mergedA = msg("mA", "assistant", MERGED); + const mergedB = msg("mB", "assistant", MERGED); + const plan = planReconciliation(["r1", "t1"], rcAnchors(), [mergedA, mergedB], new Set(["r1", "t1"])); + assert.equal(plan.claims.size, 0); + assert.deepEqual(new Set(plan.unmatched), new Set(["r1", "t1"])); + }); +}); + +describe("reconcileFoldCoverage end-to-end flips (#3301)", () => { + function fakeSession(blocks: { effectiveMessageIds: string[]; directMessageIds?: string[] }[]): Session { + return { + state: { blocks: blocks.map((b) => ({ active: true, ...b })) }, + metadata: {}, + } as unknown as Session; + } + const opts = (extra?: Partial) => + ({ mode: "repair", sessionId: "s1", log: () => {}, ...extra }) satisfies ReconcileOptions; + + // 10 fillers + one rc thinking turn (ids f0..f8, r1, t1) + function rcHistory() { + const R = "alpha beta gamma delta epsilon zeta eta theta"; + const T = "one two three four five six seven eight nine ten"; + const fillers = Array.from({ length: 9 }, (_, i) => msg(`f${i}`, "user", `filler context words number ${i}`)); + const r1 = msg("r1", "assistant", R, { contentType: "reasoning" }); + const t1 = msg("t1", "assistant", T); + return { R, T, all: [...fillers.slice(0, 5), r1, t1, ...fillers.slice(5)] }; + } + + test("rc -> merged: covered thinking turn survives the flip", () => { + const { R, T, all } = rcHistory(); + const allIds = all.map((m) => m.id!); + const session = fakeSession([{ effectiveMessageIds: allIds, directMessageIds: allIds }]); + reconcileFoldCoverage(session, all, opts()); + const merged = msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`); + const flipped = all.filter((m) => m.id !== "t1").map((m) => (m.id === "r1" ? merged : m)); + const result = reconcileFoldCoverage(session, flipped, opts()); + assert.equal(result.unmatched, 0, `drift residual must be zero (got ${JSON.stringify(result)})`); + const rewritten = allIds.map((id) => (id === "r1" || id === "t1" ? "m1" : id)); + assert.deepEqual(session.state.blocks[0].effectiveMessageIds, rewritten); + assert.deepEqual(session.state.blocks[0].directMessageIds, rewritten); + }); + + test("merged -> rc: the reverse flip survives too", () => { + const { R, T, all } = rcHistory(); + const mergedFirst = all.map((m) => (m.id === "r1" ? msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`) : m)).filter((m) => m.id !== "t1"); + const idsAfterMerge = mergedFirst.map((m) => m.id!); + const session = fakeSession([{ effectiveMessageIds: idsAfterMerge, directMessageIds: idsAfterMerge }]); + reconcileFoldCoverage(session, mergedFirst, opts()); + const backToRc = mergedFirst.flatMap((m) => m.id === "m1" + ? [msg("r2", "assistant", R, { contentType: "reasoning" }), msg("t2", "assistant", T)] + : [m]); + const result = reconcileFoldCoverage(session, backToRc, opts()); + assert.equal(result.unmatched, 0, `reverse flip must fully re-anchor (got ${JSON.stringify(result)})`); + const expected = idsAfterMerge.flatMap((id) => (id === "m1" ? ["r2", "t2"] : [id])); + assert.deepEqual(session.state.blocks[0].effectiveMessageIds, expected); + }); + + test("pre-#2644 anchors heal on the flip pass itself from stored decompositions", () => { + const { R, T, all } = rcHistory(); + const allIds = all.map((m) => m.id!); + const session = fakeSession([{ effectiveMessageIds: allIds, directMessageIds: allIds }]); + reconcileFoldCoverage(session, all, opts()); + // emulate anchors persisted before turn keys existed + const anchors = session.metadata.foldAnchors as Record; + for (const id of ["r1", "t1"]) { + delete anchors[id].p; + delete anchors[id].p2; + delete anchors[id].x; + } + const merged = msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`); + const flipped = all.filter((m) => m.id !== "t1").map((m) => (m.id === "r1" ? merged : m)); + const stored = [ + msg("r1", "assistant", R, { contentType: "reasoning" }), + msg("t1", "assistant", T), + ]; + // The flip pass is the LAST pass in which r1/t1 sit in the order + // backbone (nextOrder rolls forward to the flipped history), so the + // stored-decomposition backfill must land BEFORE the plan runs and + // this very pass must claim — a later backfill can never do it. + const pass1 = reconcileFoldCoverage(session, flipped, opts({ storedMessages: stored })); + assert.equal(pass1.unmatched, 0, `flip pass claims through stored keys (got ${JSON.stringify(pass1)})`); + const rewritten = allIds.map((id) => (id === "r1" || id === "t1" ? "m1" : id)); + assert.deepEqual(session.state.blocks[0].effectiveMessageIds, rewritten); + const healed = session.metadata.foldAnchors as Record; + assert.ok(healed["m1"] !== undefined && healed["m1"].p !== undefined, "merged core anchored with extracted turn key"); + assert.equal(healed["r1"], undefined, "old form ids retired from anchors"); + assert.equal(healed["t1"], undefined); + const pass2 = reconcileFoldCoverage(session, flipped, opts()); + assert.equal(pass2.unmatched, 0, "steady state after recovery is clean"); + }); + + test("legacy anchors without a stored source stay honestly blind", () => { + const { R, T, all } = rcHistory(); + const allIds = all.map((m) => m.id!); + const session = fakeSession([{ effectiveMessageIds: allIds }]); + reconcileFoldCoverage(session, all, opts()); + const anchors = session.metadata.foldAnchors as Record; + for (const id of ["r1", "t1"]) { + delete anchors[id].p; + delete anchors[id].p2; + delete anchors[id].x; + } + const merged = msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`); + const flipped = all.filter((m) => m.id !== "t1").map((m) => (m.id === "r1" ? merged : m)); + const result = reconcileFoldCoverage(session, flipped, opts()); + assert.equal(result.unmatched, 2, "no index, no wire bytes: honest miss, no fuzzy claim"); + }); +}); + +describe("resolveStoredDecompositions (#3301)", () => { + function lookupSession(anchorCount: number, mutate?: (anchors: Record) => void): Session { + const anchors: Record = {}; + for (let i = 0; i < anchorCount; i++) { + anchors[`a${i}`] = { n: `n${i}`, r: "assistant", c: "text", b: 10 }; + } + mutate?.(anchors); + return { state: { blocks: [] }, metadata: { foldAnchors: anchors } } as unknown as Session; + } + const wireIds = new Set(); + + test("filters to legacy bare-assistant anchors off the wire, caps at 256", () => { + const session = lookupSession(300, (a) => { + a["a0"].p = "has-key"; + a["a1"].r = "user"; + a["a2"].t = "tool-call-id"; + }); + const seenCalls: string[][] = []; + const source = { rows: (_sid: string, ids: readonly string[]) => { seenCalls.push([...ids]); return []; } }; + const out = resolveStoredDecompositions(session, "s1", wireIds, source); + assert.equal(out, undefined, "empty rows yield undefined"); + assert.equal(seenCalls.length, 1, "source consulted once"); + const seen = seenCalls[0]; + assert.equal(seen.length, 256, "cap applied"); + assert.ok(!seen.includes("a0") && !seen.includes("a1") && !seen.includes("a2"), "ineligible anchors excluded"); + }); + + test("rows === null (index unavailable) does not burn the retry budget", () => { + const session = lookupSession(3); + let calls = 0; + const source = { rows: () => { calls++; return null; } }; + assert.equal(resolveStoredDecompositions(session, "s1", wireIds, source), undefined); + assert.equal(resolveStoredDecompositions(session, "s1", wireIds, source), undefined); + assert.equal(calls, 2, "still retried on the next pass"); + }); + + test("absent rows mark tried once; later passes skip the lookup", () => { + const session = lookupSession(3); + let calls = 0; + const source = { rows: () => { calls++; return []; } }; + assert.equal(resolveStoredDecompositions(session, "s1", wireIds, source), undefined); + assert.equal(calls, 1); + assert.equal(resolveStoredDecompositions(session, "s1", wireIds, source), undefined); + assert.equal(calls, 1, "tried-set suppresses repeat lookups"); + }); + + test("latest-seq-per-id wins, neighbors kept, output sorted by seq", () => { + const rows: StoredRow[] = [ + { seq: 1, msgId: "r1", content: JSON.stringify(msg("r1", "assistant", "OLD", { contentType: "reasoning" })) }, + { seq: 2, msgId: "t1", content: JSON.stringify(msg("t1", "assistant", "reply")) }, + { seq: 3, msgId: "r1", content: JSON.stringify(msg("r1", "assistant", "NEW", { contentType: "reasoning" })) }, + { seq: 4, msgId: "u2", content: "{not-json" }, + ]; + const session = lookupSession(2); + delete (session.metadata.foldAnchors as Record)["a1"]; + const source = { rows: () => rows }; + const out = resolveStoredDecompositions(session, "s1", wireIds, source); + assert.ok(out !== undefined); + assert.deepEqual(out!.map((m) => m.id), ["t1", "r1"], "sorted by surviving seq (t1=2 < r1=3), unparseable row dropped"); + assert.equal(out![1].text, "NEW", "latest serialization wins"); + }); + + test("wire-present ids are excluded from the lookup", () => { + const session = lookupSession(2); + let seen: readonly string[] = []; + const source = { rows: (_sid: string, ids: readonly string[]) => { seen = ids; return []; } }; + resolveStoredDecompositions(session, "s1", new Set(["a0"]), source); + assert.deepEqual(seen, ["a1"]); + }); +}); + +describe("decompositionLog ingest (#3301)", () => { + function mkLegacy(id: string, lastMessages: CoreMessage[], decompositionLog?: CoreMessage[]): LegacySessionLike { + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "openai" }, + stats: { requests: 1 }, + metadata: {}, + state: { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages, + lastMessagesFolded: false, + decompositionLog, + }; + } + + test("import payload rows sit below tail seqs and dedup against the tail", () => { + const r1 = msg("r1", "assistant", "R words", { contentType: "reasoning" }); + const t1 = msg("t1", "assistant", "T words"); + const u2 = msg("u2", "user", "hello"); + // payload: one older message + a byte-identical copy of a tail message + const legacy = mkLegacy("imp", [r1, t1, u2], [msg("d0", "user", "older evidence"), r1]); + const plan = planIngest(legacy); + assert.deepEqual(plan.messages.map((m) => m.msgId), ["d0", "r1", "t1", "u2"], "payload interned first, duplicate not re-logged"); + assert.deepEqual(plan.tailSeqs, [1, 2, 3]); + }); + + test("delta planner forces the full path while an import payload is present", () => { + const r1 = msg("r1", "assistant", "R words", { contentType: "reasoning" }); + const legacy = mkLegacy("imp2", [r1], [msg("d0", "user", "older evidence")]); + const out = planIngestDelta(legacy, 0, [], { msgId: "x", contentHash: "y" }, true); + assert.equal(out, null, "payload must not be skipped as trusted prefix"); + }); +}); From 4e4e337e4f1205a0164aa24b2447b8d422a3a4b9 Mon Sep 17 00:00:00 2001 From: awork Date: Fri, 16 Oct 2026 02:21:22 +0800 Subject: [PATCH 2/4] test: pin UnifiedStore.messagesForIds row/neighbor behavior (#3301 review supplement) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new DB method carried chunked IN-lists, seq±1 neighbor expansion and CAS content resolution but no direct store-level test — every other facet of #3301 was pinned, this one only exercised through injected fakes. --- tests/issue3301-shape-flip-reanchor.test.ts | 75 +++++++++++++++------ 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/tests/issue3301-shape-flip-reanchor.test.ts b/tests/issue3301-shape-flip-reanchor.test.ts index 3fde649924..acc2182968 100644 --- a/tests/issue3301-shape-flip-reanchor.test.ts +++ b/tests/issue3301-shape-flip-reanchor.test.ts @@ -9,8 +9,12 @@ import { type FoldAnchor, type ReconcileOptions, } from "../src/fold-reconcile.ts"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; import { resolveStoredDecompositions, type StoredRow } from "../src/fold-decomposition-lookup.ts"; import { planIngest, planIngestDelta, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; import type { CoreMessage } from "acp-kernel"; import type { Session } from "../src/session.ts"; @@ -18,6 +22,27 @@ function msg(id: string, role: string, text: string, extra?: Partial { }); describe("decompositionLog ingest (#3301)", () => { - function mkLegacy(id: string, lastMessages: CoreMessage[], decompositionLog?: CoreMessage[]): LegacySessionLike { - return { - id, - createdAt: 1_000, - lastSeen: 2_000, - meta: { protocol: "openai" }, - stats: { requests: 1 }, - metadata: {}, - state: { - blocks: [], - messageRefs: { byRaw: {}, byRef: {} }, - nextBlockId: 1, - nextRunId: 1, - }, - blockContents: new Map(), - lastMessages, - lastMessagesFolded: false, - decompositionLog, - }; - } - test("import payload rows sit below tail seqs and dedup against the tail", () => { const r1 = msg("r1", "assistant", "R words", { contentType: "reasoning" }); const t1 = msg("t1", "assistant", "T words"); @@ -338,3 +342,32 @@ describe("decompositionLog ingest (#3301)", () => { assert.equal(out, null, "payload must not be skipped as trusted prefix"); }); }); + +describe("UnifiedStore.messagesForIds (#3301)", () => { + test("returns id rows plus seq±1 neighbors, sorted by seq, CAS-resolved content", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3301-mfi-")); + let store: UnifiedStore | null = null; + try { + const r1 = msg("r1", "assistant", "R words here", { contentType: "reasoning" }); + const t1 = msg("t1", "assistant", "T reply words"); + const u2 = msg("u2", "user", "hello user"); + store = UnifiedStore.open(path.join(tmp, "index.db")); + store.ingestSession(planIngest(mkLegacy("s-mfi", [r1, t1, u2]))); + // one rc member with neighbors → the adjacent core lands too, so + // turn-key derivation keeps split-form pairs adjacent. + const rows = store.messagesForIds("s-mfi", ["r1"], true); + assert.deepEqual(rows.map((r) => r.msgId), ["r1", "t1"], "rc pair stays adjacent"); + assert.equal(rows[1].seq - rows[0].seq, 1, "neighbor is the very next row"); + assert.equal(rows[0].content, JSON.stringify(r1), "content resolved through the CAS blob"); + // without neighbors: exactly the requested ids, nothing more + const bare = store.messagesForIds("s-mfi", ["u2"], false); + assert.deepEqual(bare.map((r) => r.msgId), ["u2"]); + // ids absent from the index → empty list (null is the persist-layer + // signal for "no index at all"; [] means the index has none) + assert.deepEqual(store.messagesForIds("s-mfi", ["missing"], true), []); + } finally { + store?.close(); + try { fs.rmSync(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handle */ } + } + }); +}); From 3e05cd6605bf5420b85a7b87725437ba2e284b42 Mon Sep 17 00:00:00 2001 From: awork Date: Fri, 16 Oct 2026 02:27:40 +0800 Subject: [PATCH 3/4] fix: drop unused DecompositionSource export (unused-exports gate, #3316 review) --- src/fold-decomposition-lookup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fold-decomposition-lookup.ts b/src/fold-decomposition-lookup.ts index 6ba7e7bc85..543e6b160b 100644 --- a/src/fold-decomposition-lookup.ts +++ b/src/fold-decomposition-lookup.ts @@ -23,7 +23,7 @@ export interface StoredRow { content: string; } -export interface DecompositionSource { +interface DecompositionSource { /** null = no index available (legacy/shadow/degraded); [] = index has none of them. */ rows(sessionId: string, msgIds: readonly string[]): StoredRow[] | null; } From a233c6c818fa111f6ad04eed8470d1fc96893d73 Mon Sep 17 00:00:00 2001 From: awork Date: Fri, 16 Oct 2026 09:25:41 +0800 Subject: [PATCH 4/4] refactor: scope fold re-anchor to deterministic recovery, drop shape guessing (#3301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner review: runtime form extraction in the reconciler chases symptoms — the flip producer is the host serializer choosing the reasoning field from the current metadata table. The root fix pins the wire shape at the plugin transform boundary (separate PRs: opencode-acp for the 1.x lane, opencode-native here for the 2.x lane), after which the wire re-mints the original ids and existing exact-id + backfill mechanisms self-heal. This commit removes that speculative half (THINKING_MERGE_FORMS, extractThinkingMerge, its turnContexts/Pass-3 registrations and their tests) and keeps the deterministic half: pre-plan anchor rebuild from stored decompositions (unified index rows) + the decompositionLog import entry. --- src/fold-reconcile.ts | 67 +------ tests/issue3301-shape-flip-reanchor.test.ts | 185 +++----------------- 2 files changed, 31 insertions(+), 221 deletions(-) diff --git a/src/fold-reconcile.ts b/src/fold-reconcile.ts index 4296eb8b88..0cdda18556 100644 --- a/src/fold-reconcile.ts +++ b/src/fold-reconcile.ts @@ -475,42 +475,6 @@ function turnKeyOfSeamless(reasoning: string | undefined, text: string | undefin return h.digest("hex").slice(0, 16); } -/** #3301: split tagged thinking-merge text (host metadata refresh re-serializes - * split-form turns as tagged reasoning spliced into content). MIRRORS - * kernel/src/wire/demoted-thinking.ts FORMS + splitDemotedThinking in lockstep - * (not exported from the kernel root) — keep both lists in sync. Offset-0 tag - * start is the only position demotion renderers produce. Auxiliary evidence - * for turn-key derivation ONLY — never identity minting. */ -const THINKING_MERGE_FORMS: readonly { open: string; close: string }[] = [ - { open: "\x3cthink\x3e\n", close: "\n\x3c/think\x3e" }, - { open: "\x3cthinking\x3e\n", close: "\n\x3c/thinking\x3e" }, - { open: "```thinking\n", close: "\n```" }, -]; - -export function extractThinkingMerge(text: string | undefined): { reasoning: string; text: string } | null { - if (text === undefined || text === "") return null; - let rest = text; - const parts: string[] = []; - for (;;) { - let matched = false; - for (const form of THINKING_MERGE_FORMS) { - if (!rest.startsWith(form.open)) continue; - const end = rest.indexOf(form.close, form.open.length); - if (end < 0) continue; - const inner = rest.slice(form.open.length, end); - if (inner.length === 0) continue; - parts.push(inner); - rest = rest.slice(end + form.close.length); - if (rest.startsWith("\n")) rest = rest.slice(1); - matched = true; - break; - } - if (!matched) break; - } - if (parts.length === 0) return null; - return { reasoning: parts.join("\n"), text: rest }; -} - /** Turn context for the anchor seed loop: per assistant-core id, the turn * key(s) and (for split-form turns) the sibling core id. Built lazily — a * steady-state pass reuses stored anchors and never pays this. */ @@ -535,18 +499,7 @@ export function turnContexts(msgs: CoreMessage[]): Map { const prev = msgs[i - 1]; if (prev === undefined || prev.id === undefined || prev.role !== "assistant" || prev.contentType !== "reasoning" || (prev.toolCallId !== undefined && prev.toolCallId !== "")) { - // #3301: merged-form core — its text may be tagged reasoning + reply. - // The extracted pair key bridges anchors seeded from the split form - // (same turn, different wire shape); the raw full-text key stays for - // identical resends of this exact form. - const ex = extractThinkingMerge(m.text); - if (ex !== null) { - const key = turnKeyOf(ex.reasoning, ex.text); - const seamless = turnKeyOfSeamless(ex.reasoning, ex.text); - ctx.set(m.id, seamless !== key ? { p: key, p2: seamless } : { p: key }); - } else { - ctx.set(m.id, { p: turnKeyOf(undefined, m.text) }); - } + ctx.set(m.id, { p: turnKeyOf(undefined, m.text) }); } } } @@ -994,20 +947,10 @@ export function planReconciliation( if (claimedCandidates.has(cand.id)) continue; if (!isBareAssistantCore(cand.message)) continue; if (cand.message.contentType === "text") { - // #3301: a merged-form core stands in for the whole turn — - // register the raw full-text key AND the extracted pair key(s) - // so anchors seeded from either wire form match. - const keys = new Set([turnKeyOf(undefined, cand.message.text)]); - const ex = extractThinkingMerge(cand.message.text); - if (ex !== null) { - keys.add(turnKeyOf(ex.reasoning, ex.text)); - keys.add(turnKeyOfSeamless(ex.reasoning, ex.text)); - } - for (const key of keys) { - const bucket = singleTurnKeys.get(key); - if (bucket === undefined) singleTurnKeys.set(key, [cand.id]); - else bucket.push(cand.id); - } + const key = turnKeyOf(undefined, cand.message.text); + const bucket = singleTurnKeys.get(key); + if (bucket === undefined) singleTurnKeys.set(key, [cand.id]); + else bucket.push(cand.id); } else if (cand.message.contentType === "reasoning") { const next = byPos.get(cand.pos + 1); if (next !== undefined && !claimedCandidates.has(next.id) && isBareAssistantCore(next.message) && next.message.contentType === "text") { diff --git a/tests/issue3301-shape-flip-reanchor.test.ts b/tests/issue3301-shape-flip-reanchor.test.ts index acc2182968..da08848312 100644 --- a/tests/issue3301-shape-flip-reanchor.test.ts +++ b/tests/issue3301-shape-flip-reanchor.test.ts @@ -1,17 +1,13 @@ import { test, describe } from "node:test"; import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; import { - extractThinkingMerge, - planReconciliation, reconcileFoldCoverage, - turnContexts, - normalizedIdentity, type FoldAnchor, type ReconcileOptions, } from "../src/fold-reconcile.ts"; -import * as fs from "node:fs"; -import * as os from "node:os"; -import * as path from "node:path"; import { resolveStoredDecompositions, type StoredRow } from "../src/fold-decomposition-lookup.ts"; import { planIngest, planIngestDelta, type LegacySessionLike } from "../src/storage/ingest.ts"; import { UnifiedStore } from "../src/storage/store.ts"; @@ -43,115 +39,13 @@ function mkLegacy(id: string, lastMessages: CoreMessage[], decompositionLog?: Co }; } -const THINK_OPEN = "\x3cthink\x3e\n"; -const THINK_CLOSE = "\n\x3c/think\x3e"; -const THINKING_OPEN = "\x3cthinking\x3e\n"; -const THINKING_CLOSE = "\n\x3c/thinking\x3e"; -const FENCE_OPEN = "```thinking\n"; -const FENCE_CLOSE = "\n```"; - -describe("extractThinkingMerge (#3301)", () => { - test("each kernel-listed form splits at offset 0", () => { - assert.deepEqual(extractThinkingMerge(`${THINK_OPEN}R${THINK_CLOSE}\nT`), { reasoning: "R", text: "T" }); - assert.deepEqual(extractThinkingMerge(`${THINKING_OPEN}R${THINKING_CLOSE}\nT`), { reasoning: "R", text: "T" }); - assert.deepEqual(extractThinkingMerge(`${FENCE_OPEN}R${FENCE_CLOSE}\nT`), { reasoning: "R", text: "T" }); - }); - test("multi-block runs join reasoning with \\n (kernel parity)", () => { - const s = `${THINK_OPEN}A${THINK_CLOSE}\n${THINK_OPEN}B${THINK_CLOSE}\nT`; - assert.deepEqual(extractThinkingMerge(s), { reasoning: "A\nB", text: "T" }); - }); - test("thinking-only merge keeps empty text", () => { - assert.deepEqual(extractThinkingMerge(`${THINK_OPEN}R${THINK_CLOSE}`), { reasoning: "R", text: "" }); - }); - test("tag not at offset 0 never matches (prose safety)", () => { - assert.equal(extractThinkingMerge(`prefix ${THINK_OPEN}R${THINK_CLOSE}\nT`), null); - assert.equal(extractThinkingMerge(`\n${THINK_OPEN}R${THINK_CLOSE}\nT`), null); - }); - test("empty or unterminated blocks do not match", () => { - assert.equal(extractThinkingMerge(`${THINK_OPEN}${THINK_CLOSE}\nT`), null); - assert.equal(extractThinkingMerge(`${THINK_OPEN}R never closed`), null); - }); - test("plain prose, empty and undefined return null", () => { - assert.equal(extractThinkingMerge("just prose"), null); - assert.equal(extractThinkingMerge(""), null); - assert.equal(extractThinkingMerge(undefined), null); - }); -}); - -describe("turnContexts bridges split vs merged forms (#3301)", () => { - test("merged tagged core carries the same turn keys as the rc pair", () => { - const R = "alpha beta gamma delta epsilon zeta eta theta"; - const T = "one two three four five six seven eight nine ten"; - const rcPairCtx = turnContexts([ - msg("r1", "assistant", R, { contentType: "reasoning" }), - msg("t1", "assistant", T), - ]); - const mergedCtx = turnContexts([ - msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`), - ]); - const pair = rcPairCtx.get("r1"); - const merged = mergedCtx.get("m1"); - assert.ok(pair && merged); - assert.equal(merged!.p, pair!.p, "primary seam key matches"); - assert.equal(merged!.p2 ?? merged!.p, pair!.p2 ?? pair!.p, "seamless key matches"); - }); -}); - -describe("planReconciliation shape flips (#3301)", () => { - const R = "alpha beta gamma delta epsilon zeta eta theta"; - const T = "one two three four five six seven eight nine ten"; - const MERGED = `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`; - - function rcAnchors(): Record { - const r1 = msg("r1", "assistant", R, { contentType: "reasoning" }); - const t1 = msg("t1", "assistant", T); - const ctx = turnContexts([r1, t1]); - const a1 = { n: normalizedIdentity(r1), r: "assistant", c: "reasoning", b: R.length, p: ctx.get("r1")?.p, p2: ctx.get("r1")?.p2, x: "t1" }; - const a2 = { n: normalizedIdentity(t1), r: "assistant", c: "text", b: T.length, p: ctx.get("t1")?.p, p2: ctx.get("t1")?.p2, x: "r1" }; - return { r1: a1, t1: a2 }; - } +// #3301: producer-side shape pinning (opencode-acp / opencode-native transform) +// makes the wire form deterministic, so the runtime reconciler no longer +// guesses shapes. What remains here is the DETERMINISTIC recovery half: +// rebuilding pre-#2644 anchors from stored ground truth (index.db rows / +// import payload) — never fuzzy, never speculative. - test("rc pair -> merged core: both member ids claim the one witness", () => { - const merged = msg("m1", "assistant", MERGED); - const plan = planReconciliation(["r1", "t1"], rcAnchors(), [merged], new Set(["r1", "t1"])); - assert.deepEqual(plan.claims.get("r1"), ["m1"]); - assert.deepEqual(plan.claims.get("t1"), ["m1"]); - assert.equal(plan.unmatched.length, 0); - assert.ok(plan.byTurn >= 1, `turn-key pass did the bridging (counters: ${JSON.stringify({ byTurn: plan.byTurn, bySim: plan.bySim, byOrder: plan.byOrder })})`); - }); - - test("merged core -> rc pair: one old id claims both new ids", () => { - const merged = msg("m1", "assistant", MERGED); - const ctx = turnContexts([merged]); - const anchors = { m1: { n: normalizedIdentity(merged), r: "assistant", c: "text", b: MERGED.length, p: ctx.get("m1")?.p } }; - const plan = planReconciliation(["m1"], anchors, [ - msg("r2", "assistant", R, { contentType: "reasoning" }), - msg("t2", "assistant", T), - ], new Set(["m1"])); - assert.deepEqual(plan.claims.get("m1"), ["r2", "t2"]); - assert.equal(plan.unmatched.length, 0); - assert.ok(plan.byTurn >= 1); - }); - - test("untagged merges stay bridgeable through the bare-concat keys", () => { - for (const seam of ["\n", ""]) { - const mergedUntagged = msg(`mu-${seam === "\n" ? "nl" : "flat"}`, "assistant", R + seam + T); - const plan = planReconciliation(["r1", "t1"], rcAnchors(), [mergedUntagged], new Set(["r1", "t1"])); - assert.equal(plan.unmatched.length, 0, `seam ${JSON.stringify(seam)} bridged`); - assert.deepEqual(plan.claims.get("r1"), [mergedUntagged.id]); - } - }); - - test("ambiguous duplicate turns skip honestly (never fuzzy)", () => { - const mergedA = msg("mA", "assistant", MERGED); - const mergedB = msg("mB", "assistant", MERGED); - const plan = planReconciliation(["r1", "t1"], rcAnchors(), [mergedA, mergedB], new Set(["r1", "t1"])); - assert.equal(plan.claims.size, 0); - assert.deepEqual(new Set(plan.unmatched), new Set(["r1", "t1"])); - }); -}); - -describe("reconcileFoldCoverage end-to-end flips (#3301)", () => { +describe("reconcileFoldCoverage stored-decomposition recovery (#3301)", () => { function fakeSession(blocks: { effectiveMessageIds: string[]; directMessageIds?: string[] }[]): Session { return { state: { blocks: blocks.map((b) => ({ active: true, ...b })) }, @@ -171,48 +65,26 @@ describe("reconcileFoldCoverage end-to-end flips (#3301)", () => { return { R, T, all: [...fillers.slice(0, 5), r1, t1, ...fillers.slice(5)] }; } - test("rc -> merged: covered thinking turn survives the flip", () => { - const { R, T, all } = rcHistory(); - const allIds = all.map((m) => m.id!); - const session = fakeSession([{ effectiveMessageIds: allIds, directMessageIds: allIds }]); - reconcileFoldCoverage(session, all, opts()); - const merged = msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`); - const flipped = all.filter((m) => m.id !== "t1").map((m) => (m.id === "r1" ? merged : m)); - const result = reconcileFoldCoverage(session, flipped, opts()); - assert.equal(result.unmatched, 0, `drift residual must be zero (got ${JSON.stringify(result)})`); - const rewritten = allIds.map((id) => (id === "r1" || id === "t1" ? "m1" : id)); - assert.deepEqual(session.state.blocks[0].effectiveMessageIds, rewritten); - assert.deepEqual(session.state.blocks[0].directMessageIds, rewritten); - }); - - test("merged -> rc: the reverse flip survives too", () => { - const { R, T, all } = rcHistory(); - const mergedFirst = all.map((m) => (m.id === "r1" ? msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`) : m)).filter((m) => m.id !== "t1"); - const idsAfterMerge = mergedFirst.map((m) => m.id!); - const session = fakeSession([{ effectiveMessageIds: idsAfterMerge, directMessageIds: idsAfterMerge }]); - reconcileFoldCoverage(session, mergedFirst, opts()); - const backToRc = mergedFirst.flatMap((m) => m.id === "m1" - ? [msg("r2", "assistant", R, { contentType: "reasoning" }), msg("t2", "assistant", T)] - : [m]); - const result = reconcileFoldCoverage(session, backToRc, opts()); - assert.equal(result.unmatched, 0, `reverse flip must fully re-anchor (got ${JSON.stringify(result)})`); - const expected = idsAfterMerge.flatMap((id) => (id === "m1" ? ["r2", "t2"] : [id])); - assert.deepEqual(session.state.blocks[0].effectiveMessageIds, expected); - }); - - test("pre-#2644 anchors heal on the flip pass itself from stored decompositions", () => { - const { R, T, all } = rcHistory(); - const allIds = all.map((m) => m.id!); - const session = fakeSession([{ effectiveMessageIds: allIds, directMessageIds: allIds }]); - reconcileFoldCoverage(session, all, opts()); - // emulate anchors persisted before turn keys existed + function stripTurnKeys(session: Session): void { + // emulate anchors persisted before turn keys existed (#2644) const anchors = session.metadata.foldAnchors as Record; for (const id of ["r1", "t1"]) { delete anchors[id].p; delete anchors[id].p2; delete anchors[id].x; } - const merged = msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`); + } + + test("pre-#2644 anchors heal on the flip pass itself from stored decompositions", () => { + const { R, T, all } = rcHistory(); + const allIds = all.map((m) => m.id!); + const session = fakeSession([{ effectiveMessageIds: allIds, directMessageIds: allIds }]); + reconcileFoldCoverage(session, all, opts()); + stripTurnKeys(session); + // the host re-serialized the turn as ONE untagged core (bytes changed, + // structure 2->1): exact-id cannot match, and the legacy anchors have + // no turn keys to bridge it — only the stored ground truth can. + const merged = msg("m1", "assistant", `${R}\n${T}`); const flipped = all.filter((m) => m.id !== "t1").map((m) => (m.id === "r1" ? merged : m)); const stored = [ msg("r1", "assistant", R, { contentType: "reasoning" }), @@ -227,7 +99,7 @@ describe("reconcileFoldCoverage end-to-end flips (#3301)", () => { const rewritten = allIds.map((id) => (id === "r1" || id === "t1" ? "m1" : id)); assert.deepEqual(session.state.blocks[0].effectiveMessageIds, rewritten); const healed = session.metadata.foldAnchors as Record; - assert.ok(healed["m1"] !== undefined && healed["m1"].p !== undefined, "merged core anchored with extracted turn key"); + assert.ok(healed["m1"] !== undefined && healed["m1"].p !== undefined, "merged core anchored"); assert.equal(healed["r1"], undefined, "old form ids retired from anchors"); assert.equal(healed["t1"], undefined); const pass2 = reconcileFoldCoverage(session, flipped, opts()); @@ -239,13 +111,8 @@ describe("reconcileFoldCoverage end-to-end flips (#3301)", () => { const allIds = all.map((m) => m.id!); const session = fakeSession([{ effectiveMessageIds: allIds }]); reconcileFoldCoverage(session, all, opts()); - const anchors = session.metadata.foldAnchors as Record; - for (const id of ["r1", "t1"]) { - delete anchors[id].p; - delete anchors[id].p2; - delete anchors[id].x; - } - const merged = msg("m1", "assistant", `${THINK_OPEN}${R}${THINK_CLOSE}\n${T}`); + stripTurnKeys(session); + const merged = msg("m1", "assistant", `${R}\n${T}`); const flipped = all.filter((m) => m.id !== "t1").map((m) => (m.id === "r1" ? merged : m)); const result = reconcileFoldCoverage(session, flipped, opts()); assert.equal(result.unmatched, 2, "no index, no wire bytes: honest miss, no fuzzy claim");