diff --git a/src/fold-decomposition-lookup.ts b/src/fold-decomposition-lookup.ts new file mode 100644 index 0000000000..543e6b160b --- /dev/null +++ b/src/fold-decomposition-lookup.ts @@ -0,0 +1,125 @@ +// #3301: stored-decomposition source for fold-anchor backfill. +// +// Anchors persisted before turn keys existed (pre-#2644) carry no p/x, and the +// wire-side backfill can only fire while the old id is still on the wire — a +// wire-shape flip (host metadata refresh re-serializing every past thinking +// turn) puts it off the wire forever — and the flip pass is the LAST pass in +// which those ids sit in the order backbone (claims only iterate backbone ids). +// This module pulls those ids' original message decompositions out of the +// unified index (full mode only) so reconcileFoldCoverage can rebuild p/x +// BEFORE its plan runs and the flip pass itself claims. Ids absent from the index are +// marked tried once (absence is stable for folded-out history); an UNAVAILABLE +// index is not marked tried (it may appear later). + +import type { CoreMessage } from "acp-kernel"; +import type { Session } from "./session.js"; +import { getStore } from "./persist.js"; +import { log as loggerLog } from "./logger.js"; +import { METADATA_ANCHORS, type FoldAnchor } from "./fold-reconcile.js"; + +export interface StoredRow { + seq: number; + msgId: string; + content: string; +} + +interface DecompositionSource { + /** null = no index available (legacy/shadow/degraded); [] = index has none of them. */ + rows(sessionId: string, msgIds: readonly string[]): StoredRow[] | null; +} + +const MAX_LOOKUP_PER_PASS = 256; +const TRIED_METADATA_KEY = "foldStoredTried"; +const MAX_TRIED = 8192; + +function defaultSource(): DecompositionSource { + return { + rows(sessionId, msgIds) { + try { + return getStore().unifiedMessageRows(sessionId, msgIds); + } catch (err) { + loggerLog("warn", `[fold-reconcile] stored decomposition lookup failed: ${String(err)}`); + return null; + } + }, + }; +} + +/** Wire-site convenience: build the wire-id set and run the lookup in one + * call (used by all four prepare-* lanes). */ +export function storedMessagesFor(session: Session, sessionId: string, msgs: readonly CoreMessage[]): CoreMessage[] | undefined { + const wireIds = new Set(); + for (const m of msgs) if (m.id !== undefined) wireIds.add(m.id); + return resolveStoredDecompositions(session, sessionId, wireIds); +} + +export function resolveStoredDecompositions( + session: Session, + sessionId: string, + wireIds: ReadonlySet, + source: DecompositionSource = defaultSource(), +): CoreMessage[] | undefined { + const metadata = session.metadata; + if (metadata === undefined) return undefined; + const anchors = (metadata[METADATA_ANCHORS] as Record | undefined) ?? {}; + const anchorIds = Object.keys(anchors); + if (anchorIds.length === 0) return undefined; + const tried = (metadata[TRIED_METADATA_KEY] as Record | undefined) ?? {}; + const needed: string[] = []; + for (const id of anchorIds) { + const a = anchors[id]; + if (a.p !== undefined || a.r !== "assistant" || a.t !== undefined) continue; + if (wireIds.has(id)) continue; + if (tried[id] !== undefined) continue; + needed.push(id); + if (needed.length >= MAX_LOOKUP_PER_PASS) break; + } + if (needed.length === 0) return undefined; + const rows = source.rows(sessionId, needed); + if (rows === null) return undefined; // index unavailable: retry next pass + for (const id of needed) tried[id] = 1; + const pruned: Record = {}; + let n = 0; + for (const [id, v] of Object.entries(tried)) { + if (!anchors[id]) continue; + pruned[id] = v; + if (++n >= MAX_TRIED) break; + } + metadata[TRIED_METADATA_KEY] = pruned; + // Latest seq per id wins (edits append rows); neighbors stay in so rc + // pairs keep adjacency for turn-key derivation. Unparseable rows are + // auxiliary-evidence misses, not errors — count them, don't throw. + let skipped = 0; + const latestSeq = new Map(); + const ordered: { seq: number; m: CoreMessage }[] = []; + const orderedIdx = new Map(); + for (const row of rows) { + let parsed: unknown; + try { + parsed = JSON.parse(row.content); + } catch { + skipped++; + continue; + } + if (typeof parsed !== "object" || parsed === null) { + skipped++; + continue; + } + const m = parsed as CoreMessage; + if (typeof m.id !== "string" || typeof m.role !== "string") { + skipped++; + continue; + } + const prev = latestSeq.get(row.msgId); + if (prev !== undefined && prev >= row.seq) continue; + latestSeq.set(row.msgId, row.seq); + const prevIdx = orderedIdx.get(row.msgId); + if (prevIdx !== undefined) ordered.splice(prevIdx, 1); + ordered.push({ seq: row.seq, m }); + orderedIdx.set(row.msgId, ordered.length - 1); + } + if (skipped > 0) loggerLog("debug", `[fold-reconcile] stored decomposition: ${skipped} unparseable row(s) skipped`); + if (ordered.length === 0) return undefined; + ordered.sort((a, b) => a.seq - b.seq); + return ordered.map((o) => o.m); +} diff --git a/src/fold-reconcile.ts b/src/fold-reconcile.ts index a715f7af54..0cdda18556 100644 --- a/src/fold-reconcile.ts +++ b/src/fold-reconcile.ts @@ -139,7 +139,7 @@ import { dshNativeCompactionWitness, recordConflict } from "./conflict-watch.js" type FoldReconcileMode = "off" | "warn" | "repair"; -const METADATA_ANCHORS = "foldAnchors"; +export const METADATA_ANCHORS = "foldAnchors"; const METADATA_ORDER = "foldAnchorOrder"; /** #2480: positional identity copy — canonical fingerprint per message, * aligned 1:1 with foldAnchorOrder (same rolling window, same cap, written @@ -1262,6 +1262,13 @@ export interface ReconcileOptions { * cleans the resend — benign churn that must not feed the #2193 destroy * streak. #2815/#2789. */ markerStripped?: number; + /** #3301: stored message decompositions for anchors that predate turn-key + * backfill (pre-#2644) — after a wire-shape flip the wire-side backfill + * can never fire (the old id is gone from the wire), so eligible anchors + * are rebuilt from these rows BEFORE the reconciliation plan runs and the + * flip pass itself claims. Latest-seq-per-id, neighbors included; + * undefined = no source available (legacy-only mode). */ + storedMessages?: CoreMessage[]; } /** Entry point wired into the four wire sites, immediately BEFORE the @@ -1294,7 +1301,7 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt } if (!session.metadata) return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: 0, unmatched: 0 }; - const anchors: Record = + let anchors: Record = (session.metadata[METADATA_ANCHORS] as Record | undefined) ?? {}; const oldOrder: string[] = (session.metadata[METADATA_ORDER] as string[] | undefined) ?? []; // #2480: self-contained positional copy {ids, canon} (#2487's storage @@ -1310,6 +1317,35 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt positions = { ids: storedRaw.ids as string[], canon: storedRaw.canon as string[] }; } + const byId = new Map(); + for (const m of msgs) if (m.id !== undefined) byId.set(m.id, m); + // #3301: a wire-shape flip puts legacy (pre-#2644, no p/x) ids off the wire + // on the FIRST post-flip pass — and this same pass rolls the order backbone + // forward without them, so any backfill landing later can never claim (the + // claim passes only iterate ids still in the backbone). Rebuild eligible + // anchors from the stored decomposition BEFORE the plan runs so the flip + // pass itself can claim. Wire bytes always win: an id present on this pass + // keeps its live-bytes path untouched. + if (opts.storedMessages !== undefined && opts.storedMessages.length > 0) { + const sById = new Map(); + for (const m of opts.storedMessages) if (m.id !== undefined) sById.set(m.id, m); + const sCtx = turnContexts(opts.storedMessages); + let changed = false; + const upgraded: Record = {}; + for (const [id, a] of Object.entries(anchors)) { + if (a.p === undefined && a.r === "assistant" && a.t === undefined && !byId.has(id)) { + const sm = sById.get(id); + if (sm !== undefined) { + upgraded[id] = anchorFrom(sm, sCtx.get(id)); + changed = true; + continue; + } + } + upgraded[id] = a; + } + if (changed) anchors = upgraded; + } + const plan = planReconciliation(oldOrder, anchors, msgs, covered, positions); if (process.env.FOLD_RECONCILE_DEBUG === "1") { const active = blocks.filter((b) => b.active).length; @@ -1321,8 +1357,6 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt // freshly claimed ones — the next churn must re-anchor from post-churn // bytes), prune anchors for ids no longer covered, and roll the order // backbone forward to this pass. - const byId = new Map(); - for (const m of msgs) if (m.id !== undefined) byId.set(m.id, m); // #2636: turn keys for freshly seeded anchors — built lazily so // steady-state passes (no churn) never pay the per-core normalization // (#2334). Only bare assistant cores can carry a turn key diff --git a/src/persist.ts b/src/persist.ts index ec93ccf4e1..97fe69baaa 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -200,6 +200,9 @@ interface PersistedSession { * history and must still be pruned at export time. */ messagesFolded?: boolean; pluginSnapshot?: CoreMessage[]; + /** #3301: external-converter import payload (see Session.decompositionLog) — + * accepted on read; buildRecord deliberately does not emit it. */ + decompositionLog?: CoreMessage[]; forkContentStore?: MessageContentStore; } @@ -1216,6 +1219,20 @@ export class SessionStore { } } + /** #3301: stored message decompositions (+ seq±1 neighbors) for + * fold-anchor backfill after a wire-shape flip (consumed by + * fold-decomposition-lookup → reconcileFoldCoverage.storedMessages). + * Full mode only (same gate as the other unified reads); null when + * unavailable — callers fall back to wire-side backfill. Never throws. */ + unifiedMessageRows(sessionId: string, msgIds: readonly string[]): { seq: number; msgId: string; content: string }[] | null { + if (!this.shadow?.readsUnified || msgIds.length === 0) return null; + try { + return this.shadow.messagesForIds(sessionId, msgIds, true); + } catch { + return null; + } + } + replicasAvailable(): boolean { return this.shadow !== undefined && !this.shadow.stats.disabled; } @@ -1631,6 +1648,14 @@ export class SessionStore { const record = buildRecord(session, stamp); const disk = this.staleDiskPayload(record); if (disk === null) { + // #3301: external-converter import payload rides exactly ONE + // save into the unified index (first-open imports take the + // importRecord path), then is consumed so steady saves keep + // the delta fast path (#3133) instead of paying full + // re-ingests forever. buildRecord omits it, so the disk file + // never carries it. + const payload = session.decompositionLog; + if (payload !== undefined) session.decompositionLog = undefined; // #2960 ⑤: the table holds the RAW tail (full fidelity); the // envelope keeps its bounded folded view for legacy readers. // ②a caveat: once foldResidentTail has folded the resident @@ -1644,9 +1669,10 @@ export class SessionStore { // folded saves present the envelope render (no override), // which the delta path already routes to the full ingest. const tail = session.lastMessagesFolded !== true ? session.lastMessages : undefined; - this.shadow?.record(record, tail && tail.length > 0 - ? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded } - : undefined, session.contentStore ?? undefined); + this.shadow?.record(payload !== undefined ? { ...record, decompositionLog: payload } : record, + tail && tail.length > 0 + ? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded } + : undefined, session.contentStore ?? undefined); this.noteOwnCommit(session.id, record); return record; } @@ -1711,6 +1737,7 @@ export class SessionStore { session.lastMessages = fresh.lastMessages ?? session.lastMessages; session.lastMessagesFolded = fresh.lastMessages !== undefined ? fresh.lastMessagesFolded : session.lastMessagesFolded; session.pluginSnapshot = fresh.pluginSnapshot; + session.decompositionLog = fresh.decompositionLog ?? session.decompositionLog; session.lastSeen = fresh.lastSeen; if (!session.meta.protocol && fresh.meta.protocol) session.meta.protocol = fresh.meta.protocol; if (!session.meta.upstreamOrigin && fresh.meta.upstreamOrigin) session.meta.upstreamOrigin = fresh.meta.upstreamOrigin; @@ -2117,6 +2144,7 @@ function buildSession(parsed: PersistedSession): Session { lastMessages: Array.isArray(parsed.messages) ? parsed.messages : undefined, lastMessagesFolded: parsed.messagesFolded === true, pluginSnapshot: Array.isArray(parsed.pluginSnapshot) ? parsed.pluginSnapshot : undefined, + decompositionLog: Array.isArray(parsed.decompositionLog) ? parsed.decompositionLog : undefined, contentStore: parsed.forkContentStore, inFlight: 0, persisted: true, diff --git a/src/server/prepare-anthropic.ts b/src/server/prepare-anthropic.ts index b862723ab5..e9b0f57fb7 100644 --- a/src/server/prepare-anthropic.ts +++ b/src/server/prepare-anthropic.ts @@ -9,6 +9,7 @@ import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTo import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { storedMessagesFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js"; @@ -202,7 +203,8 @@ export async function prepareAnthropic( // [#1921] re-anchor fold coverage onto churned-but-same messages // before the #1195 snapshot, so covered ids surviving a client // re-serialization stay covered (src/fold-reconcile.ts). - reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) }); + const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); + reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) }); noteSystemPromptFingerprint(session, clientSystem, { sessionId, log }); // #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside // processTurn may deactivate fully-drifted blocks, erasing them. diff --git a/src/server/prepare-google.ts b/src/server/prepare-google.ts index b26e8714ac..ad7307cc87 100644 --- a/src/server/prepare-google.ts +++ b/src/server/prepare-google.ts @@ -8,6 +8,7 @@ import { diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBilli import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { storedMessagesFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js"; @@ -149,7 +150,8 @@ export async function prepareGoogle( // [#1921] re-anchor fold coverage onto churned-but-same messages // before the #1195 snapshot, so covered ids surviving a client // re-serialization stay covered (src/fold-reconcile.ts). - reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) }); + const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); + reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) }); if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log }); // #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside // processTurn may deactivate fully-drifted blocks, erasing them. diff --git a/src/server/prepare-openai.ts b/src/server/prepare-openai.ts index 94fadf9083..635fcc0977 100644 --- a/src/server/prepare-openai.ts +++ b/src/server/prepare-openai.ts @@ -10,6 +10,7 @@ import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTo import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { storedMessagesFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js"; @@ -155,7 +156,10 @@ export async function prepareOpenai( // [#1921] re-anchor fold coverage onto churned-but-same messages // before the #1195 snapshot, so covered ids surviving a client // re-serialization stay covered (src/fold-reconcile.ts). - reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) }); + // #3301: stored decompositions for pre-#2644 anchors whose wire-side + // backfill can never fire after a wire-shape flip. + const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); + reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) }); if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log }); // #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside // processTurn may deactivate fully-drifted blocks, erasing them. diff --git a/src/server/prepare-responses.ts b/src/server/prepare-responses.ts index 0cb8b37fa5..3a972c248d 100644 --- a/src/server/prepare-responses.ts +++ b/src/server/prepare-responses.ts @@ -11,6 +11,7 @@ import { excludeDeadRefRanges } from "../acp-status.js"; import { mergeAdjacentConfigurationUpdates, patchResponsesInputWithToolImages as patchResponsesInput, responsesToCoreWithToolImages as responsesToCore } from "../responses-tool-output.js"; import { dropWhitespaceResponsesMessages, normalizeResponsesMessageItems, sanitizeResponsesInputIds } from "../loop/adapter-responses.js"; import { METADATA_LAST_RESPONSES_MODE, noteCanonicalSystemFingerprint, reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js"; +import { storedMessagesFor } from "../fold-decomposition-lookup.js"; import { nudgeSuppressed, pluginLaneDegraded } from "../session-self-heal.js"; import { compressBreakerArmed } from "../stream.js"; import { coveredRealHistoryIds, foldCoverage, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js"; @@ -256,7 +257,8 @@ export async function prepareResponses( // [#1921] re-anchor fold coverage onto churned-but-same messages // before the #1195 snapshot, so covered ids surviving a client // re-serialization stay covered (src/fold-reconcile.ts). - reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) }); + const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile); + reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) }); if (!isCompactionTrigger) { noteSystemPromptFingerprint(session, responsesClientSystem, { sessionId, log }); // #2867: the mode-independent twin — lets the ledger classify a diff --git a/src/session.ts b/src/session.ts index 86396bd820..727adf7d5e 100644 --- a/src/session.ts +++ b/src/session.ts @@ -512,6 +512,11 @@ export type Session = { * request — the client re-sends full raw history, restoring the invariant. */ lastMessagesFolded?: boolean; pluginSnapshot?: CoreMessage[]; + /** #3301: original message decompositions supplied by an external state + * converter (e.g. opencode-acp migration) at import time. Transient + * import payload: consumed once by the unified ingest (rows land in the + * index for fold re-anchor backfill) and never written back to records. */ + decompositionLog?: CoreMessage[]; /** In-memory only (NOT persisted — buildRecord omits it): monotone counter * bumped by markDirty after every mutation. Keys the fork-revision cache * below so status polling skips re-hashing the whole history (#2017). */ diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 429a8a8f57..589bcf19ca 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -71,6 +71,12 @@ export interface LegacySessionLike { lastMessages?: CoreMessage[]; lastMessagesFolded?: boolean; pluginSnapshot?: CoreMessage[]; + /** #3301: external-converter import payload — original message + * decompositions interned BEFORE the snapshot/tail so their rows get the + * lowest seqs (they are the oldest evidence). Consumed by the full + * ingest only; the delta planner falls back to the full path while it is + * present (its rows would otherwise be skipped as "trusted prefix"). */ + decompositionLog?: CoreMessage[]; contentStore?: { version: 1; byHash: Record; byRef: Record }; } @@ -115,6 +121,7 @@ export function toLegacyLike(s: { lastMessages?: LegacySessionLike["lastMessages"]; lastMessagesFolded?: boolean; pluginSnapshot?: LegacySessionLike["pluginSnapshot"]; + decompositionLog?: LegacySessionLike["decompositionLog"]; contentStore?: LegacySessionLike["contentStore"]; }): LegacySessionLike { return { @@ -129,6 +136,7 @@ export function toLegacyLike(s: { lastMessages: s.lastMessages, lastMessagesFolded: s.lastMessagesFolded, pluginSnapshot: s.pluginSnapshot, + decompositionLog: s.decompositionLog, contentStore: s.contentStore, }; } @@ -159,6 +167,9 @@ export function planIngest(session: LegacySessionLike): IngestPlan { log.push({ seq, msgId: m.id, content: entry.content, hash: entry.hash, role: entry.role, kind: entry.kind, ref: byRawRef[m.id] ?? null }); return seq; }; + // #3301: import payload first — its rows must sit below the snapshot/tail + // seqs so the stored log reads oldest-evidence-first. + if (session.decompositionLog && session.decompositionLog.length > 0) session.decompositionLog.forEach(intern); const pluginSnapshotSeqs = session.pluginSnapshot ? session.pluginSnapshot.map(intern) : null; const tailSeqs = session.lastMessages ? session.lastMessages.map(intern) : null; @@ -193,6 +204,9 @@ export function planIngestDelta( hasRawTail: boolean, ): IngestPlan | null { if (session.pluginSnapshot && session.pluginSnapshot.length > 0) return null; + // #3301: an import payload would be skipped as "trusted prefix" here — force + // the full path so its rows actually land. + if (session.decompositionLog && session.decompositionLog.length > 0) return null; const tail = session.lastMessages; const trustedLen = trustedTailSeqs.length; if ( diff --git a/src/storage/shadow.ts b/src/storage/shadow.ts index 6a1e1b6cb7..dfe94bb968 100644 --- a/src/storage/shadow.ts +++ b/src/storage/shadow.ts @@ -338,6 +338,13 @@ export class UnifiedShadow { return tip; } + /** #3301: message rows for specific ids (+ seq±1 neighbors) — stored + * decompositions for fold-anchor backfill after a wire-shape flip. */ + messagesForIds(sessionId: string, msgIds: readonly string[], includeNeighbors: boolean): { seq: number; msgId: string; content: string }[] { + this.assertLive(); + return this.store.messagesForIds(sessionId, msgIds, includeNeighbors); + } + /** All sessions row ids present in the index (#2960 R2.1). Used by the * full-mode boot union so rows whose legacy file disappeared remain * enumerable. */ diff --git a/src/storage/store.ts b/src/storage/store.ts index 4fc3c8d3a4..6c3fa79971 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -729,6 +729,49 @@ export class UnifiedStore { return { msgId: row.msg_id, contentHash: row.content_hash, maxSeq: max.m ?? -1 }; } + /** #3301: message rows for the given ids — EVERY seq row matching an id + * (edits append; callers keep the latest per id) plus, when + * includeNeighbors, the seq±1 rows so split-form rc pairs stay adjacent + * for turn-key derivation. Content bytes resolved via CAS; rows whose + * blob is gone are skipped. Read-only. */ + messagesForIds(sessionId: string, msgIds: readonly string[], includeNeighbors: boolean): { seq: number; msgId: string; content: string }[] { + if (msgIds.length === 0) return []; + const CHUNK = 500; + const rows = new Map(); + const neighborSeqs = new Set(); + for (let i = 0; i < msgIds.length; i += CHUNK) { + const chunk = msgIds.slice(i, i + CHUNK); + const ph = chunk.map(() => "?").join(","); + for (const r of this.db + .prepare(`SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ? AND msg_id IN (${ph})`) + .all(sessionId, ...chunk) as { seq: number; msg_id: string; content_hash: string }[]) { + if (!rows.has(r.seq)) rows.set(r.seq, { seq: r.seq, msgId: r.msg_id, contentHash: r.content_hash }); + if (includeNeighbors) { + neighborSeqs.add(r.seq - 1); + neighborSeqs.add(r.seq + 1); + } + } + } + if (neighborSeqs.size > 0) { + const seqs = [...neighborSeqs]; + for (let i = 0; i < seqs.length; i += CHUNK) { + const chunk = seqs.slice(i, i + CHUNK); + const ph = chunk.map(() => "?").join(","); + for (const r of this.db + .prepare(`SELECT seq, msg_id, content_hash FROM messages WHERE session_id = ? AND seq IN (${ph})`) + .all(sessionId, ...chunk) as { seq: number; msg_id: string; content_hash: string }[]) { + if (!rows.has(r.seq)) rows.set(r.seq, { seq: r.seq, msgId: r.msg_id, contentHash: r.content_hash }); + } + } + } + const out: { seq: number; msgId: string; content: string }[] = []; + for (const r of [...rows.values()].sort((a, b) => a.seq - b.seq)) { + const content = this.getContent(r.contentHash); + if (content !== null) out.push({ seq: r.seq, msgId: r.msgId, content }); + } + return out; + } + /** Deletes one session and releases every CAS refcount it holds * (GC parity with the legacy path's file removal). Rows that other * sessions also reference survive via their own refcounts. Checkpoints diff --git a/tests/issue3301-shape-flip-reanchor.test.ts b/tests/issue3301-shape-flip-reanchor.test.ts new file mode 100644 index 0000000000..da08848312 --- /dev/null +++ b/tests/issue3301-shape-flip-reanchor.test.ts @@ -0,0 +1,240 @@ +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { + reconcileFoldCoverage, + type FoldAnchor, + type ReconcileOptions, +} from "../src/fold-reconcile.ts"; +import { resolveStoredDecompositions, type StoredRow } from "../src/fold-decomposition-lookup.ts"; +import { planIngest, planIngestDelta, type LegacySessionLike } from "../src/storage/ingest.ts"; +import { UnifiedStore } from "../src/storage/store.ts"; +import type { CoreMessage } from "acp-kernel"; +import type { Session } from "../src/session.ts"; + +function msg(id: string, role: string, text: string, extra?: Partial): CoreMessage { + return { id, role, contentType: "text", text, ...extra } as CoreMessage; +} + +function mkLegacy(id: string, lastMessages: CoreMessage[], decompositionLog?: CoreMessage[]): LegacySessionLike { + return { + id, + createdAt: 1_000, + lastSeen: 2_000, + meta: { protocol: "openai" }, + stats: { requests: 1 }, + metadata: {}, + state: { + blocks: [], + messageRefs: { byRaw: {}, byRef: {} }, + nextBlockId: 1, + nextRunId: 1, + }, + blockContents: new Map(), + lastMessages, + lastMessagesFolded: false, + decompositionLog, + }; +} + +// #3301: producer-side shape pinning (opencode-acp / opencode-native transform) +// makes the wire form deterministic, so the runtime reconciler no longer +// guesses shapes. What remains here is the DETERMINISTIC recovery half: +// rebuilding pre-#2644 anchors from stored ground truth (index.db rows / +// import payload) — never fuzzy, never speculative. + +describe("reconcileFoldCoverage stored-decomposition recovery (#3301)", () => { + function fakeSession(blocks: { effectiveMessageIds: string[]; directMessageIds?: string[] }[]): Session { + return { + state: { blocks: blocks.map((b) => ({ active: true, ...b })) }, + metadata: {}, + } as unknown as Session; + } + const opts = (extra?: Partial) => + ({ mode: "repair", sessionId: "s1", log: () => {}, ...extra }) satisfies ReconcileOptions; + + // 10 fillers + one rc thinking turn (ids f0..f8, r1, t1) + function rcHistory() { + const R = "alpha beta gamma delta epsilon zeta eta theta"; + const T = "one two three four five six seven eight nine ten"; + const fillers = Array.from({ length: 9 }, (_, i) => msg(`f${i}`, "user", `filler context words number ${i}`)); + const r1 = msg("r1", "assistant", R, { contentType: "reasoning" }); + const t1 = msg("t1", "assistant", T); + return { R, T, all: [...fillers.slice(0, 5), r1, t1, ...fillers.slice(5)] }; + } + + function stripTurnKeys(session: Session): void { + // emulate anchors persisted before turn keys existed (#2644) + const anchors = session.metadata.foldAnchors as Record; + for (const id of ["r1", "t1"]) { + delete anchors[id].p; + delete anchors[id].p2; + delete anchors[id].x; + } + } + + test("pre-#2644 anchors heal on the flip pass itself from stored decompositions", () => { + const { R, T, all } = rcHistory(); + const allIds = all.map((m) => m.id!); + const session = fakeSession([{ effectiveMessageIds: allIds, directMessageIds: allIds }]); + reconcileFoldCoverage(session, all, opts()); + stripTurnKeys(session); + // the host re-serialized the turn as ONE untagged core (bytes changed, + // structure 2->1): exact-id cannot match, and the legacy anchors have + // no turn keys to bridge it — only the stored ground truth can. + const merged = msg("m1", "assistant", `${R}\n${T}`); + const flipped = all.filter((m) => m.id !== "t1").map((m) => (m.id === "r1" ? merged : m)); + const stored = [ + msg("r1", "assistant", R, { contentType: "reasoning" }), + msg("t1", "assistant", T), + ]; + // The flip pass is the LAST pass in which r1/t1 sit in the order + // backbone (nextOrder rolls forward to the flipped history), so the + // stored-decomposition backfill must land BEFORE the plan runs and + // this very pass must claim — a later backfill can never do it. + const pass1 = reconcileFoldCoverage(session, flipped, opts({ storedMessages: stored })); + assert.equal(pass1.unmatched, 0, `flip pass claims through stored keys (got ${JSON.stringify(pass1)})`); + const rewritten = allIds.map((id) => (id === "r1" || id === "t1" ? "m1" : id)); + assert.deepEqual(session.state.blocks[0].effectiveMessageIds, rewritten); + const healed = session.metadata.foldAnchors as Record; + assert.ok(healed["m1"] !== undefined && healed["m1"].p !== undefined, "merged core anchored"); + assert.equal(healed["r1"], undefined, "old form ids retired from anchors"); + assert.equal(healed["t1"], undefined); + const pass2 = reconcileFoldCoverage(session, flipped, opts()); + assert.equal(pass2.unmatched, 0, "steady state after recovery is clean"); + }); + + test("legacy anchors without a stored source stay honestly blind", () => { + const { R, T, all } = rcHistory(); + const allIds = all.map((m) => m.id!); + const session = fakeSession([{ effectiveMessageIds: allIds }]); + reconcileFoldCoverage(session, all, opts()); + stripTurnKeys(session); + const merged = msg("m1", "assistant", `${R}\n${T}`); + const flipped = all.filter((m) => m.id !== "t1").map((m) => (m.id === "r1" ? merged : m)); + const result = reconcileFoldCoverage(session, flipped, opts()); + assert.equal(result.unmatched, 2, "no index, no wire bytes: honest miss, no fuzzy claim"); + }); +}); + +describe("resolveStoredDecompositions (#3301)", () => { + function lookupSession(anchorCount: number, mutate?: (anchors: Record) => void): Session { + const anchors: Record = {}; + for (let i = 0; i < anchorCount; i++) { + anchors[`a${i}`] = { n: `n${i}`, r: "assistant", c: "text", b: 10 }; + } + mutate?.(anchors); + return { state: { blocks: [] }, metadata: { foldAnchors: anchors } } as unknown as Session; + } + const wireIds = new Set(); + + test("filters to legacy bare-assistant anchors off the wire, caps at 256", () => { + const session = lookupSession(300, (a) => { + a["a0"].p = "has-key"; + a["a1"].r = "user"; + a["a2"].t = "tool-call-id"; + }); + const seenCalls: string[][] = []; + const source = { rows: (_sid: string, ids: readonly string[]) => { seenCalls.push([...ids]); return []; } }; + const out = resolveStoredDecompositions(session, "s1", wireIds, source); + assert.equal(out, undefined, "empty rows yield undefined"); + assert.equal(seenCalls.length, 1, "source consulted once"); + const seen = seenCalls[0]; + assert.equal(seen.length, 256, "cap applied"); + assert.ok(!seen.includes("a0") && !seen.includes("a1") && !seen.includes("a2"), "ineligible anchors excluded"); + }); + + test("rows === null (index unavailable) does not burn the retry budget", () => { + const session = lookupSession(3); + let calls = 0; + const source = { rows: () => { calls++; return null; } }; + assert.equal(resolveStoredDecompositions(session, "s1", wireIds, source), undefined); + assert.equal(resolveStoredDecompositions(session, "s1", wireIds, source), undefined); + assert.equal(calls, 2, "still retried on the next pass"); + }); + + test("absent rows mark tried once; later passes skip the lookup", () => { + const session = lookupSession(3); + let calls = 0; + const source = { rows: () => { calls++; return []; } }; + assert.equal(resolveStoredDecompositions(session, "s1", wireIds, source), undefined); + assert.equal(calls, 1); + assert.equal(resolveStoredDecompositions(session, "s1", wireIds, source), undefined); + assert.equal(calls, 1, "tried-set suppresses repeat lookups"); + }); + + test("latest-seq-per-id wins, neighbors kept, output sorted by seq", () => { + const rows: StoredRow[] = [ + { seq: 1, msgId: "r1", content: JSON.stringify(msg("r1", "assistant", "OLD", { contentType: "reasoning" })) }, + { seq: 2, msgId: "t1", content: JSON.stringify(msg("t1", "assistant", "reply")) }, + { seq: 3, msgId: "r1", content: JSON.stringify(msg("r1", "assistant", "NEW", { contentType: "reasoning" })) }, + { seq: 4, msgId: "u2", content: "{not-json" }, + ]; + const session = lookupSession(2); + delete (session.metadata.foldAnchors as Record)["a1"]; + const source = { rows: () => rows }; + const out = resolveStoredDecompositions(session, "s1", wireIds, source); + assert.ok(out !== undefined); + assert.deepEqual(out!.map((m) => m.id), ["t1", "r1"], "sorted by surviving seq (t1=2 < r1=3), unparseable row dropped"); + assert.equal(out![1].text, "NEW", "latest serialization wins"); + }); + + test("wire-present ids are excluded from the lookup", () => { + const session = lookupSession(2); + let seen: readonly string[] = []; + const source = { rows: (_sid: string, ids: readonly string[]) => { seen = ids; return []; } }; + resolveStoredDecompositions(session, "s1", new Set(["a0"]), source); + assert.deepEqual(seen, ["a1"]); + }); +}); + +describe("decompositionLog ingest (#3301)", () => { + test("import payload rows sit below tail seqs and dedup against the tail", () => { + const r1 = msg("r1", "assistant", "R words", { contentType: "reasoning" }); + const t1 = msg("t1", "assistant", "T words"); + const u2 = msg("u2", "user", "hello"); + // payload: one older message + a byte-identical copy of a tail message + const legacy = mkLegacy("imp", [r1, t1, u2], [msg("d0", "user", "older evidence"), r1]); + const plan = planIngest(legacy); + assert.deepEqual(plan.messages.map((m) => m.msgId), ["d0", "r1", "t1", "u2"], "payload interned first, duplicate not re-logged"); + assert.deepEqual(plan.tailSeqs, [1, 2, 3]); + }); + + test("delta planner forces the full path while an import payload is present", () => { + const r1 = msg("r1", "assistant", "R words", { contentType: "reasoning" }); + const legacy = mkLegacy("imp2", [r1], [msg("d0", "user", "older evidence")]); + const out = planIngestDelta(legacy, 0, [], { msgId: "x", contentHash: "y" }, true); + assert.equal(out, null, "payload must not be skipped as trusted prefix"); + }); +}); + +describe("UnifiedStore.messagesForIds (#3301)", () => { + test("returns id rows plus seq±1 neighbors, sorted by seq, CAS-resolved content", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "bili-3301-mfi-")); + let store: UnifiedStore | null = null; + try { + const r1 = msg("r1", "assistant", "R words here", { contentType: "reasoning" }); + const t1 = msg("t1", "assistant", "T reply words"); + const u2 = msg("u2", "user", "hello user"); + store = UnifiedStore.open(path.join(tmp, "index.db")); + store.ingestSession(planIngest(mkLegacy("s-mfi", [r1, t1, u2]))); + // one rc member with neighbors → the adjacent core lands too, so + // turn-key derivation keeps split-form pairs adjacent. + const rows = store.messagesForIds("s-mfi", ["r1"], true); + assert.deepEqual(rows.map((r) => r.msgId), ["r1", "t1"], "rc pair stays adjacent"); + assert.equal(rows[1].seq - rows[0].seq, 1, "neighbor is the very next row"); + assert.equal(rows[0].content, JSON.stringify(r1), "content resolved through the CAS blob"); + // without neighbors: exactly the requested ids, nothing more + const bare = store.messagesForIds("s-mfi", ["u2"], false); + assert.deepEqual(bare.map((r) => r.msgId), ["u2"]); + // ids absent from the index → empty list (null is the persist-layer + // signal for "no index at all"; [] means the index has none) + assert.deepEqual(store.messagesForIds("s-mfi", ["missing"], true), []); + } finally { + store?.close(); + try { fs.rmSync(tmp, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handle */ } + } + }); +});