Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 125 additions & 0 deletions src/fold-decomposition-lookup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
// #3301: stored-decomposition source for fold-anchor backfill.
//
// Anchors persisted before turn keys existed (pre-#2644) carry no p/x, and the
// wire-side backfill can only fire while the old id is still on the wire — a
// wire-shape flip (host metadata refresh re-serializing every past thinking
// turn) puts it off the wire forever — and the flip pass is the LAST pass in
// which those ids sit in the order backbone (claims only iterate backbone ids).
// This module pulls those ids' original message decompositions out of the
// unified index (full mode only) so reconcileFoldCoverage can rebuild p/x
// BEFORE its plan runs and the flip pass itself claims. Ids absent from the index are
// marked tried once (absence is stable for folded-out history); an UNAVAILABLE
// index is not marked tried (it may appear later).

import type { CoreMessage } from "acp-kernel";
import type { Session } from "./session.js";
import { getStore } from "./persist.js";
import { log as loggerLog } from "./logger.js";
import { METADATA_ANCHORS, type FoldAnchor } from "./fold-reconcile.js";

export interface StoredRow {
seq: number;
msgId: string;
content: string;
}

interface DecompositionSource {
/** null = no index available (legacy/shadow/degraded); [] = index has none of them. */
rows(sessionId: string, msgIds: readonly string[]): StoredRow[] | null;
}

const MAX_LOOKUP_PER_PASS = 256;
const TRIED_METADATA_KEY = "foldStoredTried";
const MAX_TRIED = 8192;

function defaultSource(): DecompositionSource {
return {
rows(sessionId, msgIds) {
try {
return getStore().unifiedMessageRows(sessionId, msgIds);
} catch (err) {
loggerLog("warn", `[fold-reconcile] stored decomposition lookup failed: ${String(err)}`);
return null;
}
},
};
}

/** Wire-site convenience: build the wire-id set and run the lookup in one
* call (used by all four prepare-* lanes). */
export function storedMessagesFor(session: Session, sessionId: string, msgs: readonly CoreMessage[]): CoreMessage[] | undefined {
const wireIds = new Set<string>();
for (const m of msgs) if (m.id !== undefined) wireIds.add(m.id);
return resolveStoredDecompositions(session, sessionId, wireIds);
}

export function resolveStoredDecompositions(
session: Session,
sessionId: string,
wireIds: ReadonlySet<string>,
source: DecompositionSource = defaultSource(),
): CoreMessage[] | undefined {
const metadata = session.metadata;
if (metadata === undefined) return undefined;
const anchors = (metadata[METADATA_ANCHORS] as Record<string, FoldAnchor> | undefined) ?? {};
const anchorIds = Object.keys(anchors);
if (anchorIds.length === 0) return undefined;
const tried = (metadata[TRIED_METADATA_KEY] as Record<string, number> | undefined) ?? {};
const needed: string[] = [];
for (const id of anchorIds) {
const a = anchors[id];
if (a.p !== undefined || a.r !== "assistant" || a.t !== undefined) continue;
if (wireIds.has(id)) continue;
if (tried[id] !== undefined) continue;
needed.push(id);
if (needed.length >= MAX_LOOKUP_PER_PASS) break;
}
if (needed.length === 0) return undefined;
const rows = source.rows(sessionId, needed);
if (rows === null) return undefined; // index unavailable: retry next pass
for (const id of needed) tried[id] = 1;
const pruned: Record<string, number> = {};
let n = 0;
for (const [id, v] of Object.entries(tried)) {
if (!anchors[id]) continue;
pruned[id] = v;
if (++n >= MAX_TRIED) break;
}
metadata[TRIED_METADATA_KEY] = pruned;
// Latest seq per id wins (edits append rows); neighbors stay in so rc
// pairs keep adjacency for turn-key derivation. Unparseable rows are
// auxiliary-evidence misses, not errors — count them, don't throw.
let skipped = 0;
const latestSeq = new Map<string, number>();
const ordered: { seq: number; m: CoreMessage }[] = [];
const orderedIdx = new Map<string, number>();
for (const row of rows) {
let parsed: unknown;
try {
parsed = JSON.parse(row.content);
} catch {
skipped++;
continue;
}
if (typeof parsed !== "object" || parsed === null) {
skipped++;
continue;
}
const m = parsed as CoreMessage;
if (typeof m.id !== "string" || typeof m.role !== "string") {
skipped++;
continue;
}
const prev = latestSeq.get(row.msgId);
if (prev !== undefined && prev >= row.seq) continue;
latestSeq.set(row.msgId, row.seq);
const prevIdx = orderedIdx.get(row.msgId);
if (prevIdx !== undefined) ordered.splice(prevIdx, 1);
ordered.push({ seq: row.seq, m });
orderedIdx.set(row.msgId, ordered.length - 1);
}
if (skipped > 0) loggerLog("debug", `[fold-reconcile] stored decomposition: ${skipped} unparseable row(s) skipped`);
if (ordered.length === 0) return undefined;
ordered.sort((a, b) => a.seq - b.seq);
return ordered.map((o) => o.m);
}
42 changes: 38 additions & 4 deletions src/fold-reconcile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ import { dshNativeCompactionWitness, recordConflict } from "./conflict-watch.js"

type FoldReconcileMode = "off" | "warn" | "repair";

const METADATA_ANCHORS = "foldAnchors";
export const METADATA_ANCHORS = "foldAnchors";
const METADATA_ORDER = "foldAnchorOrder";
/** #2480: positional identity copy — canonical fingerprint per message,
* aligned 1:1 with foldAnchorOrder (same rolling window, same cap, written
Expand Down Expand Up @@ -1262,6 +1262,13 @@ export interface ReconcileOptions {
* cleans the resend — benign churn that must not feed the #2193 destroy
* streak. #2815/#2789. */
markerStripped?: number;
/** #3301: stored message decompositions for anchors that predate turn-key
* backfill (pre-#2644) — after a wire-shape flip the wire-side backfill
* can never fire (the old id is gone from the wire), so eligible anchors
* are rebuilt from these rows BEFORE the reconciliation plan runs and the
* flip pass itself claims. Latest-seq-per-id, neighbors included;
* undefined = no source available (legacy-only mode). */
storedMessages?: CoreMessage[];
}

/** Entry point wired into the four wire sites, immediately BEFORE the
Expand Down Expand Up @@ -1294,7 +1301,7 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt
}
if (!session.metadata) return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: 0, unmatched: 0 };

const anchors: Record<string, FoldAnchor> =
let anchors: Record<string, FoldAnchor> =
(session.metadata[METADATA_ANCHORS] as Record<string, FoldAnchor> | undefined) ?? {};
const oldOrder: string[] = (session.metadata[METADATA_ORDER] as string[] | undefined) ?? [];
// #2480: self-contained positional copy {ids, canon} (#2487's storage
Expand All @@ -1310,6 +1317,35 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt
positions = { ids: storedRaw.ids as string[], canon: storedRaw.canon as string[] };
}

const byId = new Map<string, CoreMessage>();
for (const m of msgs) if (m.id !== undefined) byId.set(m.id, m);
// #3301: a wire-shape flip puts legacy (pre-#2644, no p/x) ids off the wire
// on the FIRST post-flip pass — and this same pass rolls the order backbone
// forward without them, so any backfill landing later can never claim (the
// claim passes only iterate ids still in the backbone). Rebuild eligible
// anchors from the stored decomposition BEFORE the plan runs so the flip
// pass itself can claim. Wire bytes always win: an id present on this pass
// keeps its live-bytes path untouched.
if (opts.storedMessages !== undefined && opts.storedMessages.length > 0) {
const sById = new Map<string, CoreMessage>();
for (const m of opts.storedMessages) if (m.id !== undefined) sById.set(m.id, m);
const sCtx = turnContexts(opts.storedMessages);
let changed = false;
const upgraded: Record<string, FoldAnchor> = {};
for (const [id, a] of Object.entries(anchors)) {
if (a.p === undefined && a.r === "assistant" && a.t === undefined && !byId.has(id)) {
const sm = sById.get(id);
if (sm !== undefined) {
upgraded[id] = anchorFrom(sm, sCtx.get(id));
changed = true;
continue;
}
}
upgraded[id] = a;
}
if (changed) anchors = upgraded;
}

const plan = planReconciliation(oldOrder, anchors, msgs, covered, positions);
if (process.env.FOLD_RECONCILE_DEBUG === "1") {
const active = blocks.filter((b) => b.active).length;
Expand All @@ -1321,8 +1357,6 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt
// freshly claimed ones — the next churn must re-anchor from post-churn
// bytes), prune anchors for ids no longer covered, and roll the order
// backbone forward to this pass.
const byId = new Map<string, CoreMessage>();
for (const m of msgs) if (m.id !== undefined) byId.set(m.id, m);
// #2636: turn keys for freshly seeded anchors — built lazily so
// steady-state passes (no churn) never pay the per-core normalization
// (#2334). Only bare assistant cores can carry a turn key
Expand Down
34 changes: 31 additions & 3 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,9 @@ interface PersistedSession {
* history and must still be pruned at export time. */
messagesFolded?: boolean;
pluginSnapshot?: CoreMessage[];
/** #3301: external-converter import payload (see Session.decompositionLog) —
* accepted on read; buildRecord deliberately does not emit it. */
decompositionLog?: CoreMessage[];
forkContentStore?: MessageContentStore;
}

Expand Down Expand Up @@ -1216,6 +1219,20 @@ export class SessionStore {
}
}

/** #3301: stored message decompositions (+ seq±1 neighbors) for
* fold-anchor backfill after a wire-shape flip (consumed by
* fold-decomposition-lookup → reconcileFoldCoverage.storedMessages).
* Full mode only (same gate as the other unified reads); null when
* unavailable — callers fall back to wire-side backfill. Never throws. */
unifiedMessageRows(sessionId: string, msgIds: readonly string[]): { seq: number; msgId: string; content: string }[] | null {
if (!this.shadow?.readsUnified || msgIds.length === 0) return null;
try {
return this.shadow.messagesForIds(sessionId, msgIds, true);
} catch {
return null;
}
}

replicasAvailable(): boolean {
return this.shadow !== undefined && !this.shadow.stats.disabled;
}
Expand Down Expand Up @@ -1631,6 +1648,14 @@ export class SessionStore {
const record = buildRecord(session, stamp);
const disk = this.staleDiskPayload(record);
if (disk === null) {
// #3301: external-converter import payload rides exactly ONE
// save into the unified index (first-open imports take the
// importRecord path), then is consumed so steady saves keep
// the delta fast path (#3133) instead of paying full
// re-ingests forever. buildRecord omits it, so the disk file
// never carries it.
const payload = session.decompositionLog;
if (payload !== undefined) session.decompositionLog = undefined;
// #2960 ⑤: the table holds the RAW tail (full fidelity); the
// envelope keeps its bounded folded view for legacy readers.
// ②a caveat: once foldResidentTail has folded the resident
Expand All @@ -1644,9 +1669,10 @@ export class SessionStore {
// folded saves present the envelope render (no override),
// which the delta path already routes to the full ingest.
const tail = session.lastMessagesFolded !== true ? session.lastMessages : undefined;
this.shadow?.record(record, tail && tail.length > 0
? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded }
: undefined, session.contentStore ?? undefined);
this.shadow?.record(payload !== undefined ? { ...record, decompositionLog: payload } : record,
tail && tail.length > 0
? { lastMessages: tail, lastMessagesFolded: session.lastMessagesFolded }
: undefined, session.contentStore ?? undefined);
this.noteOwnCommit(session.id, record);
return record;
}
Expand Down Expand Up @@ -1711,6 +1737,7 @@ export class SessionStore {
session.lastMessages = fresh.lastMessages ?? session.lastMessages;
session.lastMessagesFolded = fresh.lastMessages !== undefined ? fresh.lastMessagesFolded : session.lastMessagesFolded;
session.pluginSnapshot = fresh.pluginSnapshot;
session.decompositionLog = fresh.decompositionLog ?? session.decompositionLog;
session.lastSeen = fresh.lastSeen;
if (!session.meta.protocol && fresh.meta.protocol) session.meta.protocol = fresh.meta.protocol;
if (!session.meta.upstreamOrigin && fresh.meta.upstreamOrigin) session.meta.upstreamOrigin = fresh.meta.upstreamOrigin;
Expand Down Expand Up @@ -2117,6 +2144,7 @@ function buildSession(parsed: PersistedSession): Session {
lastMessages: Array.isArray(parsed.messages) ? parsed.messages : undefined,
lastMessagesFolded: parsed.messagesFolded === true,
pluginSnapshot: Array.isArray(parsed.pluginSnapshot) ? parsed.pluginSnapshot : undefined,
decompositionLog: Array.isArray(parsed.decompositionLog) ? parsed.decompositionLog : undefined,
contentStore: parsed.forkContentStore,
inFlight: 0,
persisted: true,
Expand Down
4 changes: 3 additions & 1 deletion src/server/prepare-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTo
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { excludeDeadRefRanges } from "../acp-status.js";
import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js";
import { storedMessagesFor } from "../fold-decomposition-lookup.js";
import { nudgeSuppressed } from "../session-self-heal.js";
import { compressBreakerArmed } from "../stream.js";
import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js";
Expand Down Expand Up @@ -202,7 +203,8 @@ export async function prepareAnthropic(
// [#1921] re-anchor fold coverage onto churned-but-same messages
// before the #1195 snapshot, so covered ids surviving a client
// re-serialization stay covered (src/fold-reconcile.ts).
reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) });
const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile);
reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) });
noteSystemPromptFingerprint(session, clientSystem, { sessionId, log });
// #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside
// processTurn may deactivate fully-drifted blocks, erasing them.
Expand Down
4 changes: 3 additions & 1 deletion src/server/prepare-google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBilli
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { excludeDeadRefRanges } from "../acp-status.js";
import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js";
import { storedMessagesFor } from "../fold-decomposition-lookup.js";
import { nudgeSuppressed } from "../session-self-heal.js";
import { compressBreakerArmed } from "../stream.js";
import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type Session } from "../session.js";
Expand Down Expand Up @@ -149,7 +150,8 @@ export async function prepareGoogle(
// [#1921] re-anchor fold coverage onto churned-but-same messages
// before the #1195 snapshot, so covered ids surviving a client
// re-serialization stay covered (src/fold-reconcile.ts).
reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) });
const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile);
reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) });
if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log });
// #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside
// processTurn may deactivate fully-drifted blocks, erasing them.
Expand Down
6 changes: 5 additions & 1 deletion src/server/prepare-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { applyReasoningDrop, diagNudge, diagTagSummary, deriveTitle, effectiveTo
import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js";
import { excludeDeadRefRanges } from "../acp-status.js";
import { reconcileFoldCoverage, noteSystemPromptFingerprint, resolveFoldReconcileMode } from "../fold-reconcile.js";
import { storedMessagesFor } from "../fold-decomposition-lookup.js";
import { nudgeSuppressed } from "../session-self-heal.js";
import { compressBreakerArmed } from "../stream.js";
import { applyCompactionArchive, coveredRealHistoryIds, detectUnannouncedHistoryRewrite, foldCoverage, markCompactionBoundary, markDirty, markNativeCompactionBoundary, reconcileNativeCompactionBoundary, REWRITE_MIN_INCOMING_TOTAL, snapshotMessages, type PendingRetrieval, type Session } from "../session.js";
Expand Down Expand Up @@ -155,7 +156,10 @@ export async function prepareOpenai(
// [#1921] re-anchor fold coverage onto churned-but-same messages
// before the #1195 snapshot, so covered ids surviving a client
// re-serialization stay covered (src/fold-reconcile.ts).
reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts) });
// #3301: stored decompositions for pre-#2644 anchors whose wire-side
// backfill can never fire after a wire-shape flip.
const foldMode = resolveFoldReconcileMode(process.env, opts.compress.reconcile);
reconcileFoldCoverage(session, msgs, { mode: foldMode, sessionId, log, markerStripped: markerArtifactCount(markerCounts), storedMessages: foldMode === "off" ? undefined : storedMessagesFor(session, sessionId, msgs) });
if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log });
// #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside
// processTurn may deactivate fully-drifted blocks, erasing them.
Expand Down
Loading
Loading