Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 40 additions & 1 deletion src/fold-reconcile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -711,6 +711,8 @@ interface BlockLike {
active?: boolean;
effectiveMessageIds?: string[];
directMessageIds?: string[];
/** Host-decompressed block — the kernel keeps it inactive by design. */
expanded?: boolean;
}

/** #2297: only LIVE blocks count as covered. The kernel deactivates a block
Expand All @@ -719,11 +721,44 @@ interface BlockLike {
* those dead-lineage ids can never re-anchor and would sit in `missing`
* permanently, inflating the drift warn ~2x (#2293). Delegates to
* coveredRealHistoryIds so the single real-history caliber (#2627) is
* implemented exactly once. */
* implemented exactly once. reconcileFoldCoverage additionally feeds the
* retained view-deactivated lineage back in via recoveryCandidateIds
* (#4050) — different question, same file; this caliber stays active-only. */
function coveredIdsOf(blocks: BlockLike[]): Set<string> {
return coveredRealHistoryIds(blocks);
}

/** #4050: coverage of RETAINED folds that are inactive only because the last
* view skipped their span (kernel syncBlocks deactivation after a short
* request, #3844/#3845). That lineage is RECOVERABLE, not dead: the span's
* bytes may come back under churned ids (client re-serialization), and only
* the claim ladder can re-key them — but it never runs if they stay out of
* `covered` (coveredRealHistoryIds skips inactive blocks → covered.size===0
* → noop → stale ids never re-anchor → syncBlocks can never find them again
* → permanent coverage loss, folded originals re-sent every turn). The claim
* ladder decides PER CELL with multi-channel corroboration, so nothing is
* restored blindly: corroborated cells get re-keyed (syncBlocks then
* re-activates the block on the next pipeline run), non-corroborated cells
* stay honestly unmatched and complete the existing lifecycle (z-streak reap
* for verified substrates #2695, truncation suppression, epoch verdict
* #3641). Excluded: `expanded` blocks — the host decompressed them ON PURPOSE
* (the originals ride the wire verbatim and a fresh compress creates a NEW
* block, kernel/src/sync.ts), so there is nothing to restore and including
* their ids would feed #2293-style missing inflation when the whole wire
* re-serializes. Consumed (merged-away) blocks need no exclusion: their ids
* duplicate the surviving parent's ids, which the Set dedupes, and they stay
* inactive regardless via the consumed guard in kernel/src/sync.ts. */
function recoveryCandidateIds(blocks: ReadonlyArray<BlockLike>): Set<string> {
const ids = new Set<string>();
for (const block of blocks) {
if (block.active || block.expanded === true) continue;
for (const id of block.effectiveMessageIds ?? []) {
if (!isResponsesTurnSeparatorId(id)) ids.add(id);
}
}
return ids;
}

/** Pure core: plan the reconciliation between the previous pass order and the
* incoming messages. Exposed for unit tests. `positions` (#2480) is the
* stored positional fingerprint copy {ids, canon} (#2487's self-contained
Expand Down Expand Up @@ -1514,7 +1549,11 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt
return { kind: "off", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, byShape: 0, reaped: 0, unmatched: 0 };
}
const blocks = (session.state?.blocks ?? []) as BlockLike[];
// Active coverage first (the anchor cap's survivor order, #2334), then the
// retained view-deactivated lineage re-fed into the claim ladder (#4050);
// see recoveryCandidateIds for what is excluded and why.
const covered = coveredIdsOf(blocks);
for (const id of recoveryCandidateIds(blocks)) covered.add(id);
if (covered.size === 0) {
resetFoldDriftState(session);
return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, byShape: 0, reaped: 0, unmatched: 0 };
Expand Down
188 changes: 182 additions & 6 deletions tests/fold-reconcile.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -760,20 +760,22 @@ describe("reconcileFoldCoverage owner-deletion drift (#2297)", () => {
Array.from({ length: n }, (_, i) => msg(`${tag}${i}`, "assistant", `${tag} fresh tail turn ${i} padding words`));

test("missing counts only ACTIVE block coverage; dead-lineage ids drop out", () => {
// #2293 shape: the host truncated its own history; syncBlocks has
// already deactivated the old blocks (active=false) but their
// effectiveMessageIds persist in state — they can never re-anchor and
// #2293 shape: the host decompressed these blocks and they died —
// syncBlocks keeps them inactive (expanded=true) while their
// effectiveMessageIds persist in state; they can never re-anchor and
// must not sit in `missing` forever. Pre-seeded anchors simulate the
// persisted state from when those blocks were still live.
// persisted state from when those blocks were still live. (#4050
// refined the split: VIEW-deactivated inactive blocks are recovery
// candidates again — see the #4050 describe below.)
const liveIds = Array.from({ length: 12 }, (_, i) => `live${i}`);
const deadIds = Array.from({ length: 50 }, (_, i) => `dead${i}`);
const seedAnchors: Record<string, FoldAnchor> = {};
for (const id of [...deadIds, ...liveIds]) seedAnchors[id] = anchorOf(msg(id, "user", `text ${id}`));
const session = {
state: { blocks: [
{ active: true, effectiveMessageIds: liveIds },
{ active: false, effectiveMessageIds: deadIds.slice(0, 40) },
{ active: false, effectiveMessageIds: deadIds.slice(40) },
{ active: false, expanded: true, effectiveMessageIds: deadIds.slice(0, 40) },
{ active: false, expanded: true, effectiveMessageIds: deadIds.slice(40) },
]},
metadata: { foldAnchors: seedAnchors, foldAnchorOrder: [...deadIds, ...liveIds] },
} as unknown as Session;
Expand Down Expand Up @@ -846,6 +848,180 @@ describe("reconcileFoldCoverage owner-deletion drift (#2297)", () => {
assert.equal(session.metadata["foldDriftStreak"], 8, "the streak keeps counting across the restart");
});
});

describe("reconcileFoldCoverage view-deactivated recovery (#4050)", () => {
type LogLine = { level: string; msg: string };
const makeOpts = (logs: LogLine[]): ReconcileOptions & { mode: "repair" } =>
({ sessionId: "s4050", mode: "repair", log: (level: string, m: string) => logs.push({ level, msg: m }) });
const errorLines = (logs: LogLine[]) => logs.filter((l) => l.level === "error");
// A live fold plus a fold deactivated by the LAST view skipping its span
// (kernel syncBlocks after a short request) — inactive, NOT expanded.
function dualSession(): Session {
const liveIds = Array.from({ length: 12 }, (_, i) => `l${i}`);
const viewIds = Array.from({ length: 5 }, (_, i) => `v${i}`);
const seedAnchors: Record<string, FoldAnchor> = {};
for (const id of [...liveIds, ...viewIds]) seedAnchors[id] = anchorOf(msg(id, "user", `seed text ${id} words`));
return {
state: {
blocks: [
{ blockId: "live-block", active: true, effectiveMessageIds: liveIds },
{ blockId: "view-block", active: false, effectiveMessageIds: viewIds },
],
},
metadata: { foldAnchors: seedAnchors, foldAnchorOrder: [...liveIds, ...viewIds] },
} as unknown as Session;
}
const liveMsgs = Array.from({ length: 12 }, (_, i) => msg(`l${i}`, "user", `live turn ${i} payload words`));
const viewMsgs = Array.from({ length: 5 }, (_, i) => msg(`v${i}`, "user", `view span ${i} content words`));
const tailMsgs = Array.from({ length: 10 }, (_, i) => msg(`t${i}`, "assistant", `tail turn ${i} padding words`));

test("canonical-equivalent id churn re-keys the view-deactivated fold", () => {
// The reported shape: a short request deactivates the fold, then the
// main history returns with every id re-minted over canonical-equivalent
// bytes (whitespace churn). Pre-fix covered.size saw only active blocks
// and the stale v-ids could never re-anchor → permanent coverage loss.
const session = dualSession();
const opts = makeOpts([]);
assert.equal(reconcileFoldCoverage(session, [...liveMsgs, ...viewMsgs, ...tailMsgs], opts).kind, "resend");
const churned: CoreMessage[] = [
...liveMsgs.map((m, i) => msg(`L${i}`, m.role, m.text!.replaceAll(" ", " "))),
...viewMsgs.map((m, i) => msg(`V${i}`, m.role, m.text!.replaceAll(" ", " "))),
...tailMsgs.map((m, i) => msg(`T${i}`, m.role, m.text!.replaceAll(" ", " "))),
];
const result = reconcileFoldCoverage(session, churned, opts);
assert.equal(result.kind, "reanchored");
assert.equal(result.claims, 17, "every churned cell corroborates and claims");
assert.equal(result.unmatched, 0);
assert.deepEqual(
(session.state.blocks[0] as { effectiveMessageIds: string[] }).effectiveMessageIds,
Array.from({ length: 12 }, (_, i) => `L${i}`),
"the live fold re-keys under the churned ids",
);
assert.deepEqual(
(session.state.blocks[1] as { effectiveMessageIds: string[] }).effectiveMessageIds,
Array.from({ length: 5 }, (_, i) => `V${i}`),
"the view-deactivated fold re-keys too — syncBlocks re-activates it on the next pipeline run",
);
assert.equal((session.state.blocks[1] as { active: boolean }).active, false,
"reconcile only re-keys; reactivation stays the kernel sync's job (no blind flag flip)");
});

test("sanctioned head-truncation of a verified span is never reaped", () => {
// The client truncated its own head history (the #2627/#3641/#3724
// sanctioned-deletion class): every absent id has zero word evidence
// and the wire still continues from the witness tail → structurally
// absent, so the #2695 streak must stay at zero even for a verified
// (e===1) view-deactivated fold. The snapshot IS the session's memory
// of that span and must survive indefinitely.
const session = dualSession();
const logs: LogLine[] = [];
const opts = makeOpts(logs);
assert.equal(reconcileFoldCoverage(session, [...liveMsgs, ...viewMsgs, ...tailMsgs], opts).kind, "resend");
session.metadata[METADATA_FOLD_COVERAGE] = { "view-block": { p: 5, r: 0, t: 5, e: 1 } };
// Head truncation cuts the OLDEST ids first: everything up to and
// including both folded spans is gone, the wire keeps its clean tail
// (same ids, re-serialized bytes) plus a fresh continuation — the
// common set is a contiguous suffix of the witness, which is what
// makes this the sanctioned class (a mid-span cut would stay honest
// drift instead). Alien vocabulary keeps the word-evidence census at
// zero deterministically.
const alienTail = Array.from({ length: 10 }, (_, i) => msg(`t${i}`, "assistant", `tail message ${String.fromCharCode(97 + i)} continuation context`));
const appended = [msg("n0", "user", "fresh continuation question here"), msg("n1", "assistant", "fresh continuation answer now")];
const truncated = [...alienTail, ...appended]; // l0..l11 + v0..v4 gone
for (let pass = 0; pass < 3; pass++) {
const result = reconcileFoldCoverage(session, truncated, opts);
assert.equal(result.reaped, 0, `pass ${pass + 1}: truncation-suppressed span is never reaped`);
assert.equal((session.metadata[METADATA_FOLD_COVERAGE] as Record<string, FoldBlockCoverage>)["view-block"].z, 0,
`pass ${pass + 1}: structural absence keeps the streak at zero`);
}
assert.deepEqual(
(session.state.blocks[1] as { effectiveMessageIds: string[] }).effectiveMessageIds,
viewMsgs.map((m) => m.id!),
"no corroboration → no re-key, the stale ids are kept for honest loss",
);
assert.equal(session.state.blocks.length, 2, "the snapshot survives the truncation");
assert.equal(errorLines(logs).length, 0, "continuity explains the absence — no drift escalation");
assert.equal(session.metadata["foldDriftEscalated"], undefined, "no escalation latch set");
});

test("a mid-history edit of a verified span accrues honest drift (same #2695 lifecycle as active folds)", () => {
// A genuine in-session rewrite of folded content breaks wire contiguity
// BY DESIGN (mid-history edits are not the sanctioned-truncation class),
// so the verified span accrues its zombie streak exactly like an active
// fold would: z climbs per pass and the third pass completes the #2695
// reap. Two passes here pin the accrual without triggering the removal
// (the reap itself is pinned by the #2695 tests against active folds).
const session = dualSession();
const logs: LogLine[] = [];
const opts = makeOpts(logs);
assert.equal(reconcileFoldCoverage(session, [...liveMsgs, ...viewMsgs, ...tailMsgs], opts).kind, "resend");
session.metadata[METADATA_FOLD_COVERAGE] = { "view-block": { p: 5, r: 0, t: 5, e: 1 } };
const edited: CoreMessage[] = [
...liveMsgs,
...Array.from({ length: 5 }, (_, i) => msg(`V${i}`, "user", `alien rewrite ${["alpha", "beta", "gamma", "delta", "epsilon"][i]} omega`)),
...tailMsgs,
];
for (let pass = 0; pass < 2; pass++) {
const result = reconcileFoldCoverage(session, edited, opts);
assert.equal(result.unmatched, 5, `pass ${pass + 1}: the rewritten cells stay honestly unmatched`);
assert.equal(result.reaped, 0, `pass ${pass + 1}: not yet ripe`);
assert.equal((session.metadata[METADATA_FOLD_COVERAGE] as Record<string, FoldBlockCoverage>)["view-block"].z, pass + 1,
`pass ${pass + 1}: honest streak accrual for the genuinely-gone verified span`);
}
assert.deepEqual(
(session.state.blocks[1] as { effectiveMessageIds: string[] }).effectiveMessageIds,
viewMsgs.map((m) => m.id!),
"no corroboration → no re-key while the span is still accounted for",
);
});

test("consumed child ids dedupe against the surviving parent — no exclusion needed", () => {
// A merged-away (consumed) block keeps its pre-merge ids, a subset of
// the parent's ids. Both blocks ride the same claim: the Set dedupes,
// the claim applies once, and BOTH id lists get re-keyed. If someone
// later adds a consumed-exclusion here, this test catches the lost
// child re-key.
const parentIds = Array.from({ length: 5 }, (_, i) => `x${i}`);
const childIds = parentIds.slice(2);
const seedAnchors: Record<string, FoldAnchor> = {};
for (const id of parentIds) seedAnchors[id] = anchorOf(msg(id, "user", `merged turn ${id} words`));
const session = {
state: {
blocks: [
{ blockId: "parent", active: true, effectiveMessageIds: parentIds },
{ blockId: "child", active: false, effectiveMessageIds: childIds },
],
},
metadata: { foldAnchors: seedAnchors, foldAnchorOrder: [...parentIds] },
} as unknown as Session;
const opts = makeOpts([]);
const tail = Array.from({ length: 6 }, (_, i) => msg(`z${i}`, "assistant", `dedupe tail ${i} words`));
const originals = [
...Array.from({ length: 5 }, (_, i) => msg(`x${i}`, "user", `merged turn x${i} words`)),
...tail,
];
assert.equal(reconcileFoldCoverage(session, originals, opts).kind, "resend");
const churned: CoreMessage[] = [
msg("x0", "user", "merged turn x0 words"),
msg("x1", "user", "merged turn x1 words"),
...Array.from({ length: 3 }, (_, i) => msg(`X${i + 2}`, "user", `merged turn x${i + 2} words\r\n`)),
...tail,
];
const result = reconcileFoldCoverage(session, churned, opts);
assert.equal(result.claims, 3, "each unique id claimed once despite two blocks listing it");
assert.equal(result.unmatched, 0);
assert.deepEqual(
(session.state.blocks[0] as { effectiveMessageIds: string[] }).effectiveMessageIds,
["x0", "x1", "X2", "X3", "X4"],
);
assert.deepEqual(
(session.state.blocks[1] as { effectiveMessageIds: string[] }).effectiveMessageIds,
["X2", "X3", "X4"],
"the consumed child re-keys alongside its parent",
);
});
});

describe("reconcileFoldCoverage anchor cap boundary (#2334)", () => {
// MAX_ANCHORS (src/fold-reconcile.ts) pinned at 16384 — hardcoded here on
// purpose so a constant move breaks the pin loudly instead of sliding it.
Expand Down
Loading
Loading