diff --git a/src/fold-reconcile.ts b/src/fold-reconcile.ts index 0bfd631eae..5b04864d94 100644 --- a/src/fold-reconcile.ts +++ b/src/fold-reconcile.ts @@ -1727,6 +1727,91 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt && witness.slice(0, commonPrefix.length).every((id) => inboundSet.has(id)) && plan.newOrder.slice(0, commonPrefix.length).every((id) => witnessSet.has(id)); } + // #2815: id churn explained by raw-stage marker stripping. Covered ids + // minted BEFORE the marker engine (or by any client that keeps resending + // the artifact-laden bytes) hash marker-laden content; once stripRawStage + // cleans the resent history every marker-bearing covered message arrives + // under a fresh id (clean bytes), every pass, forever, on a stateless + // client. The covered CONTENT is still on the wire (benign churn that + // re-folds self-heal — probe-verified: user ids verbatim, positional + // anchors hold, nothing reaped), so feeding the #2193 destroy streak + // here would misdiagnose exactly the #2789 victim population as + // "substrate destroyed" while compression is healthy. Same shape as the + // reasoning-only-drop sanction above: per-pass evidence, and a pass + // without stripped markers still escalates on its own streak. + const markerChurnExplained = (opts.markerStripped ?? 0) > 0; + // Per-wire model-switch audit (#2193 follow-up): a PURE reasoning drop is + // sanctioned host behavior, not substrate destruction. Cross-model replay + // discards structured thinking (anthropic thinking blocks, responses + // reasoning items, google thought parts) while every text/tool core keeps + // its id — kernel ids are seeded blind to thinking siblings. When the + // ONLY missing covered ids are reasoning anchors and nothing needed + // reanchoring (every other covered id is present verbatim), the fold is + // fully alive: do not accrue the destroy streak, do not latch the + // escalation. + const reasoningOnlyDrop = plan.claims.size === 0 && plan.unmatched.length > 0 + && plan.unmatched.every((id) => anchors[id]?.c === "reasoning"); + // Dead reasoning ids after a sanctioned drop are permanent residents of + // the unmatched set; they must not count toward the destroy threshold + // either — otherwise "model switch + one honest user edit" (10 dead + // reasoning ids + 1 dead user id) would cry destruction three passes + // later while the fold is healthy. + // #2903: also exclude LEGACY anchors — persisted by ≤0.1.191 builds before + // FoldAnchor.c existed (no content-type class). Their keys sit outside the + // current id space, so they read as missing every pass and can never be + // backfilled once off the wire (the #2732 backfill needs live bytes). The + // old `!== "reasoning"` classified that permanent bili self-debt as + // non-reasoning and latched foldDriftEscalated on healthy long sessions + // after 3 passes. Only a KNOWN non-reasoning class counts: a genuine + // client-side rewrite/deletion still carries a concrete `c`, so real loss + // still escalates — pure-legacy debt alone never does. + const nonReasoningUnmatched = plan.unmatched.filter((id) => { + const cls = anchors[id]?.c; + return cls !== undefined && cls !== "reasoning"; + }).length; + // #3641 verdict classifier — the census evidence names the phenomenon + // before any streak accrues: + // shapeExplained: word evidence survived on the wire under churned ids + // (protocol switch #3640, image offload #3588) — the shape pass + // re-anchored some cells (byShape > 0) or the residual mass still + // carries near-duplicate word evidence (≥60% of class-carrying + // residual). Transform, not destruction: no streak, one info. + // truncationExplained: EVERY class-carrying residual is absent with + // zero word evidence anywhere, while some covered substrate is still + // present (resent or re-anchored) — the client dropped old history + // (sanctioned truncation #2695/#2297). Absence, not destruction: the + // snapshot is kept (z suppressed below), no streak, one info. + // Everything else (genuine rewrite with real content change, wholesale + // replacement by foreign history) keeps the #2193 streak — and its + // terminal verdict is now an epoch confirmation, not "destroyed". + const classCarryingResidual = plan.unmatched.filter((id) => { + const cls = anchors[id]?.c; + return cls !== undefined && cls !== "reasoning"; + }); + const shapeExplained = plan.byShape > 0 + || (classCarryingResidual.length > 0 && plan.nearDupResidual.length >= 0.6 * classCarryingResidual.length); + const truncationExplained = classCarryingResidual.length > 0 + && plan.nearDupResidual.length === 0 + && classCarryingResidual.every((id) => absentSet.has(id)) + && wireContinuity; + // #3724: MIRROR — every class-carrying residual absent with zero word + // evidence AND the wire is a clean prefix-cut = host rewound/undid the + // newest turns (deleting a freshly-folded span). Same sanctioned-deletion + // class as head truncation: snapshot retained (z suppressed above), no + // destroy streak, one distinct info line. Without this the all-covered- + // rewound case read as total-loss drift → false "epoch confirmation" + // escalation (+ false proxy-mode reap), or went silent when re-anchoring + // kept claims > 0. + const tailTruncationExplained = classCarryingResidual.length > 0 + && plan.nearDupResidual.length === 0 + && classCarryingResidual.every((id) => absentSet.has(id)) + && tailTruncated; + const totalDrift = plan.claims.size === 0 && plan.unmatched.length > 0 && !reasoningOnlyDrop && !markerChurnExplained + && !shapeExplained && !truncationExplained && !tailTruncationExplained; + // Classify before reaping: the epoch-confirm gate promises to retain the + // snapshot, even on the pass that first reaches the escalation threshold. + const epochProtected = session.metadata[METADATA_EPOCH_CONFIRM] === true + || (totalDrift && nonReasoningUnmatched >= FOLD_DRIFT_ESCALATE_MIN_UNMATCHED); const prevCov = (session.metadata[METADATA_FOLD_COVERAGE] as Record | undefined) ?? {}; const nextCov: Record = {}; let covCount = 0; @@ -1792,7 +1877,7 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt // only (warn observes). Runs BEFORE the backbone persistence below so // the reaped ids are stripped from the same objects that get stored. let reapedCount = 0; - if (mode === "repair" && session.metadata.pluginAgent === undefined) { + if (mode === "repair" && session.metadata.pluginAgent === undefined && !epochProtected) { const ripe = new Set(); for (const block of blocks) { const bid = block.blockId; @@ -1862,87 +1947,6 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt // side history rewrite. One error line per episode names it instead of // letting the identical warn print hundreds of times. const tag = opts.sessionId === undefined ? "" : `[${opts.sessionId}] `; - // #2815: id churn explained by raw-stage marker stripping. Covered ids - // minted BEFORE the marker engine (or by any client that keeps resending - // the artifact-laden bytes) hash marker-laden content; once stripRawStage - // cleans the resent history every marker-bearing covered message arrives - // under a fresh id (clean bytes), every pass, forever, on a stateless - // client. The covered CONTENT is still on the wire (benign churn that - // re-folds self-heal — probe-verified: user ids verbatim, positional - // anchors hold, nothing reaped), so feeding the #2193 destroy streak - // here would misdiagnose exactly the #2789 victim population as - // "substrate destroyed" while compression is healthy. Same shape as the - // reasoning-only-drop sanction above: per-pass evidence, and a pass - // without stripped markers still escalates on its own streak. - const markerChurnExplained = (opts.markerStripped ?? 0) > 0; - // Per-wire model-switch audit (#2193 follow-up): a PURE reasoning drop is - // sanctioned host behavior, not substrate destruction. Cross-model replay - // discards structured thinking (anthropic thinking blocks, responses - // reasoning items, google thought parts) while every text/tool core keeps - // its id — kernel ids are seeded blind to thinking siblings. When the - // ONLY missing covered ids are reasoning anchors and nothing needed - // reanchoring (every other covered id is present verbatim), the fold is - // fully alive: do not accrue the destroy streak, do not latch the - // escalation. - const reasoningOnlyDrop = plan.claims.size === 0 && plan.unmatched.length > 0 - && plan.unmatched.every((id) => anchors[id]?.c === "reasoning"); - // Dead reasoning ids after a sanctioned drop are permanent residents of - // the unmatched set; they must not count toward the destroy threshold - // either — otherwise "model switch + one honest user edit" (10 dead - // reasoning ids + 1 dead user id) would cry destruction three passes - // later while the fold is healthy. - // #2903: also exclude LEGACY anchors — persisted by ≤0.1.191 builds before - // FoldAnchor.c existed (no content-type class). Their keys sit outside the - // current id space, so they read as missing every pass and can never be - // backfilled once off the wire (the #2732 backfill needs live bytes). The - // old `!== "reasoning"` classified that permanent bili self-debt as - // non-reasoning and latched foldDriftEscalated on healthy long sessions - // after 3 passes. Only a KNOWN non-reasoning class counts: a genuine - // client-side rewrite/deletion still carries a concrete `c`, so real loss - // still escalates — pure-legacy debt alone never does. - const nonReasoningUnmatched = plan.unmatched.filter((id) => { - const cls = anchors[id]?.c; - return cls !== undefined && cls !== "reasoning"; - }).length; - // #3641 verdict classifier — the census evidence names the phenomenon - // before any streak accrues: - // shapeExplained: word evidence survived on the wire under churned ids - // (protocol switch #3640, image offload #3588) — the shape pass - // re-anchored some cells (byShape > 0) or the residual mass still - // carries near-duplicate word evidence (≥60% of class-carrying - // residual). Transform, not destruction: no streak, one info. - // truncationExplained: EVERY class-carrying residual is absent with - // zero word evidence anywhere, while some covered substrate is still - // present (resent or re-anchored) — the client dropped old history - // (sanctioned truncation #2695/#2297). Absence, not destruction: the - // snapshot is kept (z suppressed below), no streak, one info. - // Everything else (genuine rewrite with real content change, wholesale - // replacement by foreign history) keeps the #2193 streak — and its - // terminal verdict is now an epoch confirmation, not "destroyed". - const classCarryingResidual = plan.unmatched.filter((id) => { - const cls = anchors[id]?.c; - return cls !== undefined && cls !== "reasoning"; - }); - const shapeExplained = plan.byShape > 0 - || (classCarryingResidual.length > 0 && plan.nearDupResidual.length >= 0.6 * classCarryingResidual.length); - const truncationExplained = classCarryingResidual.length > 0 - && plan.nearDupResidual.length === 0 - && classCarryingResidual.every((id) => absentSet.has(id)) - && wireContinuity; - // #3724: MIRROR — every class-carrying residual absent with zero word - // evidence AND the wire is a clean prefix-cut = host rewound/undid the - // newest turns (deleting a freshly-folded span). Same sanctioned-deletion - // class as head truncation: snapshot retained (z suppressed above), no - // destroy streak, one distinct info line. Without this the all-covered- - // rewound case read as total-loss drift → false "epoch confirmation" - // escalation (+ false proxy-mode reap), or went silent when re-anchoring - // kept claims > 0. - const tailTruncationExplained = classCarryingResidual.length > 0 - && plan.nearDupResidual.length === 0 - && classCarryingResidual.every((id) => absentSet.has(id)) - && tailTruncated; - const totalDrift = plan.claims.size === 0 && plan.unmatched.length > 0 && !reasoningOnlyDrop && !markerChurnExplained - && !shapeExplained && !truncationExplained && !tailTruncationExplained; const unfoldedDrift = mode === "repair" ? Math.max(0, (plan.churnTokens ?? 0) - (plan.claimedChurnTokens ?? 0)) : (plan.churnTokens ?? 0); diff --git a/tests/epoch-confirm-lifecycle.test.ts b/tests/epoch-confirm-lifecycle.test.ts new file mode 100644 index 0000000000..071571beba --- /dev/null +++ b/tests/epoch-confirm-lifecycle.test.ts @@ -0,0 +1,81 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { createCore, defaultConfig, type CoreMessage } from "acp-kernel"; +import { parseCompressInput } from "../src/compress-tool.ts"; +import { resolveDecompress } from "../src/decompress-shared.ts"; +import { reconcileFoldCoverage } from "../src/fold-reconcile.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { createSession } from "../src/session.ts"; +import { applyRanges } from "../src/stream.ts"; + +_setStoreForTest(new SessionStore({ enabled: false })); + +for (const shape of ["one-interior-survivor", "all-missing", "small-zombie"] as const) { + test(`#4260 epoch confirmation retains real kernel folds: ${shape}`, () => { + const dir = mkdtempSync(join(tmpdir(), "bili-epoch-confirm-")); + try { + const core = createCore(); + const config = defaultConfig(200_000); + const session = createSession(`epoch-confirm-${shape}`); + const messages: CoreMessage[] = Array.from({ length: 40 }, (_, i) => ({ + id: `h_original_${i}`, role: i % 2 ? "assistant" : "user", contentType: "text", + text: `Historical detail ${i}: ${String.fromCharCode(65 + i % 26).repeat(2000)}`, + })); + const tick = (incoming: CoreMessage[]) => { + const turn = core.processTurn({ messages: incoming, state: session.state, config, tokenCount: 9999, renderTags: "text-only" }); + session.state = turn.state; + return turn; + }; + const initial = tick(messages); + const summary = "The early history establishes independent details for an epoch-confirmation recovery regression, with original content retained for later retrieval."; + applyRanges(parseCompressInput({ content: [{ + startId: "m00001", endId: shape === "small-zombie" ? "m00007" : "m00020", summary, + }] }), { core, config, messages: initial.messages, session, log: () => {} }); + assert.equal(session.state.blocks.length, 1); + const blockId = session.state.blocks[0].blockId; + const original = session.blockContents.get(blockId)?.full.text; + assert.ok(original); + const opts = { mode: "repair" as const, inboundMessageCount: 40, log: () => {} }; + reconcileFoldCoverage(session, messages, opts); + tick(messages); + const foreign: CoreMessage[] = Array.from({ length: 40 }, (_, i) => ({ + id: `h_foreign_${i}`, role: i % 2 ? "assistant" : "user", contentType: "text", + text: `Unrelated narrative ${i} with different evidence ${"qzjy".repeat(500)}`, + })); + if (shape !== "all-missing") foreign[15] = messages[shape === "small-zombie" ? 3 : 10]; + for (let pass = 1; pass <= 5; pass++) { + const rec = reconcileFoldCoverage(session, foreign, opts); + tick(foreign); + assert.equal(rec.reaped, shape === "small-zombie" && pass === 3 ? 1 : 0); + if (shape === "one-interior-survivor") { + assert.equal(session.state.blocks.length, 1); + assert.equal(session.state.blocks[0].active, true); + assert.equal(session.metadata.foldEpochConfirmPending === true, pass >= 3); + } + if (shape === "all-missing") { + assert.equal(session.state.blocks.length, 1); + assert.equal(session.state.blocks[0].active, false); + assert.equal(session.metadata.foldEpochConfirmPending === true, false); + } + } + assert.equal(session.blockContents.get(blockId)?.full.text, original); + const receipt = resolveDecompress({ blockId, full: true, toFile: join(dir, "original.txt") }, + { core, config, messages: [], session, log: () => {} }).text; + assert.equal(receipt.startsWith("[decompress FAILED:"), shape === "small-zombie"); + reconcileFoldCoverage(session, messages, opts); + const returned = tick(messages); + assert.equal(session.metadata.foldEpochConfirmPending === true, false); + assert.equal(session.state.blocks.length, shape === "small-zombie" ? 0 : 1); + assert.equal(returned.messages.some(m => m.text?.includes(summary)), shape !== "small-zombie"); + if (shape !== "small-zombie") { + assert.equal(session.state.blocks[0].active, true); + assert.equal(returned.messages.some(m => m.text?.includes("Historical detail 5:")), false); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +} diff --git a/tests/fold-reconcile.test.ts b/tests/fold-reconcile.test.ts index ab8d3dcb7c..3efe234b69 100644 --- a/tests/fold-reconcile.test.ts +++ b/tests/fold-reconcile.test.ts @@ -602,18 +602,23 @@ describe("reconcileFoldCoverage coverage evidence + side-request guard (#2202)", assert.equal(JSON.stringify(session.metadata[METADATA_FOLD_COVERAGE]), covBefore, "coverage records untouched"); assert.equal(JSON.stringify(session.metadata.foldAnchors), anchorsBefore, "anchors untouched"); assert.equal(JSON.stringify(session.metadata.foldAnchorOrder), orderBefore, "backbone untouched"); - // The next conversation-sized total-loss pass escalates on its own merit - // AND completes the #2695 zombie streak (z was 2 before the side pass; - // h3 makes it 3): the block is reaped — removed from state.blocks, the - // loss recorded as a conflict event, blockContents semantics preserved - // by the reap helper (kept, so derived decompress still works). + // The next full pass reaches the epoch-confirm gate. That gate must + // retain the fold snapshot, including on later foreign-history passes. const h3 = reconcileFoldCoverage(session, filler("h3"), opts); assert.equal(logs.filter((l) => l.level === "error").length, 1); - assert.equal(h3.reaped, 1, "third consecutive majority-loss pass reaps the zombie block"); - assert.equal(session.state.blocks.length, 0, "reaped block removed from state.blocks"); - assert.ok(!covOf(session)["blk3"], "coverage record for the reaped block dropped"); - const conflict = (session.metadata.conflictEvents as Array<{ kind: string }>)[0]; - assert.equal(conflict?.kind, "orphan-reap"); + assert.equal(h3.reaped, 0, "epoch confirmation takes priority over zombie reaping"); + assert.equal(session.metadata.foldEpochConfirmPending, true); + for (let i = 4; i <= 6; i++) { + assert.equal(reconcileFoldCoverage(session, filler(`h${i}`), opts).reaped, 0); + } + assert.equal(session.state.blocks.length, 1, "pending confirmation retains the original fold"); + assert.ok(covOf(session)["blk3"], "coverage evidence survives for recovery"); + assert.equal(conflictEventsOf(session).filter((e) => e.kind === "orphan-reap").length, 0); + assert.equal(logs.filter((l) => l.level === "error").length, 1, "one epoch warning per episode"); + const recovered = reconcileFoldCoverage(session, originals, opts); + assert.equal(recovered.missing, 0); + assert.notEqual(session.metadata.foldEpochConfirmPending, true); + assert.equal(session.state.blocks[0].blockId, "blk3"); }); test("majority-reasoning span survives a sanctioned thinking drop (no false reap, #2732)", () => { @@ -694,10 +699,10 @@ describe("reconcileFoldCoverage coverage evidence + side-request guard (#2202)", ]; for (const w of wires) { assert.ok(w.msgs.length >= 10, `${w.name}: fixture must defeat the core-count guard, got ${w.msgs.length}`); - const ctrlIds = Array.from({ length: 12 }, (_, i) => `c${i}`); + const ctrlIds = Array.from({ length: 5 }, (_, i) => `c${i}`); const ctrl = blockSession("main-fold", ctrlIds); const ctrlOriginals = ctrlIds.map((id, i) => msg(id, "user", `control evidence ${i}`)); - reconcileFoldCoverage(ctrl, ctrlOriginals, makeOpts()); + reconcileFoldCoverage(ctrl, [...ctrlOriginals, ...filler("control")], makeOpts()); for (let i = 0; i < 4; i++) reconcileFoldCoverage(ctrl, w.msgs, makeOpts()); assert.equal(ctrl.state.blocks.length, 0, `${w.name}: uncounted expanded short passes feed deletion evidence and reap the main fold (pre-fix behavior)`); const ids = Array.from({ length: 12 }, (_, i) => `main${i}`);