Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 86 additions & 82 deletions src/fold-reconcile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1727,6 +1727,91 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt
&& witness.slice(0, commonPrefix.length).every((id) => inboundSet.has(id))
&& plan.newOrder.slice(0, commonPrefix.length).every((id) => witnessSet.has(id));
}
// #2815: id churn explained by raw-stage marker stripping. Covered ids
// minted BEFORE the marker engine (or by any client that keeps resending
// the artifact-laden bytes) hash marker-laden content; once stripRawStage
// cleans the resent history every marker-bearing covered message arrives
// under a fresh id (clean bytes), every pass, forever, on a stateless
// client. The covered CONTENT is still on the wire (benign churn that
// re-folds self-heal — probe-verified: user ids verbatim, positional
// anchors hold, nothing reaped), so feeding the #2193 destroy streak
// here would misdiagnose exactly the #2789 victim population as
// "substrate destroyed" while compression is healthy. Same shape as the
// reasoning-only-drop sanction above: per-pass evidence, and a pass
// without stripped markers still escalates on its own streak.
const markerChurnExplained = (opts.markerStripped ?? 0) > 0;
// Per-wire model-switch audit (#2193 follow-up): a PURE reasoning drop is
// sanctioned host behavior, not substrate destruction. Cross-model replay
// discards structured thinking (anthropic thinking blocks, responses
// reasoning items, google thought parts) while every text/tool core keeps
// its id — kernel ids are seeded blind to thinking siblings. When the
// ONLY missing covered ids are reasoning anchors and nothing needed
// reanchoring (every other covered id is present verbatim), the fold is
// fully alive: do not accrue the destroy streak, do not latch the
// escalation.
const reasoningOnlyDrop = plan.claims.size === 0 && plan.unmatched.length > 0
&& plan.unmatched.every((id) => anchors[id]?.c === "reasoning");
// Dead reasoning ids after a sanctioned drop are permanent residents of
// the unmatched set; they must not count toward the destroy threshold
// either — otherwise "model switch + one honest user edit" (10 dead
// reasoning ids + 1 dead user id) would cry destruction three passes
// later while the fold is healthy.
// #2903: also exclude LEGACY anchors — persisted by ≤0.1.191 builds before
// FoldAnchor.c existed (no content-type class). Their keys sit outside the
// current id space, so they read as missing every pass and can never be
// backfilled once off the wire (the #2732 backfill needs live bytes). The
// old `!== "reasoning"` classified that permanent bili self-debt as
// non-reasoning and latched foldDriftEscalated on healthy long sessions
// after 3 passes. Only a KNOWN non-reasoning class counts: a genuine
// client-side rewrite/deletion still carries a concrete `c`, so real loss
// still escalates — pure-legacy debt alone never does.
const nonReasoningUnmatched = plan.unmatched.filter((id) => {
const cls = anchors[id]?.c;
return cls !== undefined && cls !== "reasoning";
}).length;
// #3641 verdict classifier — the census evidence names the phenomenon
// before any streak accrues:
// shapeExplained: word evidence survived on the wire under churned ids
// (protocol switch #3640, image offload #3588) — the shape pass
// re-anchored some cells (byShape > 0) or the residual mass still
// carries near-duplicate word evidence (≥60% of class-carrying
// residual). Transform, not destruction: no streak, one info.
// truncationExplained: EVERY class-carrying residual is absent with
// zero word evidence anywhere, while some covered substrate is still
// present (resent or re-anchored) — the client dropped old history
// (sanctioned truncation #2695/#2297). Absence, not destruction: the
// snapshot is kept (z suppressed below), no streak, one info.
// Everything else (genuine rewrite with real content change, wholesale
// replacement by foreign history) keeps the #2193 streak — and its
// terminal verdict is now an epoch confirmation, not "destroyed".
const classCarryingResidual = plan.unmatched.filter((id) => {
const cls = anchors[id]?.c;
return cls !== undefined && cls !== "reasoning";
});
const shapeExplained = plan.byShape > 0
|| (classCarryingResidual.length > 0 && plan.nearDupResidual.length >= 0.6 * classCarryingResidual.length);
const truncationExplained = classCarryingResidual.length > 0
&& plan.nearDupResidual.length === 0
&& classCarryingResidual.every((id) => absentSet.has(id))
&& wireContinuity;
// #3724: MIRROR — every class-carrying residual absent with zero word
// evidence AND the wire is a clean prefix-cut = host rewound/undid the
// newest turns (deleting a freshly-folded span). Same sanctioned-deletion
// class as head truncation: snapshot retained (z suppressed above), no
// destroy streak, one distinct info line. Without this the all-covered-
// rewound case read as total-loss drift → false "epoch confirmation"
// escalation (+ false proxy-mode reap), or went silent when re-anchoring
// kept claims > 0.
const tailTruncationExplained = classCarryingResidual.length > 0
&& plan.nearDupResidual.length === 0
&& classCarryingResidual.every((id) => absentSet.has(id))
&& tailTruncated;
const totalDrift = plan.claims.size === 0 && plan.unmatched.length > 0 && !reasoningOnlyDrop && !markerChurnExplained
&& !shapeExplained && !truncationExplained && !tailTruncationExplained;
// Classify before reaping: the epoch-confirm gate promises to retain the
// snapshot, even on the pass that first reaches the escalation threshold.
const epochProtected = session.metadata[METADATA_EPOCH_CONFIRM] === true
|| (totalDrift && nonReasoningUnmatched >= FOLD_DRIFT_ESCALATE_MIN_UNMATCHED);
const prevCov = (session.metadata[METADATA_FOLD_COVERAGE] as Record<string, FoldBlockCoverage> | undefined) ?? {};
const nextCov: Record<string, FoldBlockCoverage> = {};
let covCount = 0;
Expand Down Expand Up @@ -1792,7 +1877,7 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt
// only (warn observes). Runs BEFORE the backbone persistence below so
// the reaped ids are stripped from the same objects that get stored.
let reapedCount = 0;
if (mode === "repair" && session.metadata.pluginAgent === undefined) {
if (mode === "repair" && session.metadata.pluginAgent === undefined && !epochProtected) {
const ripe = new Set<string>();
for (const block of blocks) {
const bid = block.blockId;
Expand Down Expand Up @@ -1862,87 +1947,6 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt
// side history rewrite. One error line per episode names it instead of
// letting the identical warn print hundreds of times.
const tag = opts.sessionId === undefined ? "" : `[${opts.sessionId}] `;
// #2815: id churn explained by raw-stage marker stripping. Covered ids
// minted BEFORE the marker engine (or by any client that keeps resending
// the artifact-laden bytes) hash marker-laden content; once stripRawStage
// cleans the resent history every marker-bearing covered message arrives
// under a fresh id (clean bytes), every pass, forever, on a stateless
// client. The covered CONTENT is still on the wire (benign churn that
// re-folds self-heal — probe-verified: user ids verbatim, positional
// anchors hold, nothing reaped), so feeding the #2193 destroy streak
// here would misdiagnose exactly the #2789 victim population as
// "substrate destroyed" while compression is healthy. Same shape as the
// reasoning-only-drop sanction above: per-pass evidence, and a pass
// without stripped markers still escalates on its own streak.
const markerChurnExplained = (opts.markerStripped ?? 0) > 0;
// Per-wire model-switch audit (#2193 follow-up): a PURE reasoning drop is
// sanctioned host behavior, not substrate destruction. Cross-model replay
// discards structured thinking (anthropic thinking blocks, responses
// reasoning items, google thought parts) while every text/tool core keeps
// its id — kernel ids are seeded blind to thinking siblings. When the
// ONLY missing covered ids are reasoning anchors and nothing needed
// reanchoring (every other covered id is present verbatim), the fold is
// fully alive: do not accrue the destroy streak, do not latch the
// escalation.
const reasoningOnlyDrop = plan.claims.size === 0 && plan.unmatched.length > 0
&& plan.unmatched.every((id) => anchors[id]?.c === "reasoning");
// Dead reasoning ids after a sanctioned drop are permanent residents of
// the unmatched set; they must not count toward the destroy threshold
// either — otherwise "model switch + one honest user edit" (10 dead
// reasoning ids + 1 dead user id) would cry destruction three passes
// later while the fold is healthy.
// #2903: also exclude LEGACY anchors — persisted by ≤0.1.191 builds before
// FoldAnchor.c existed (no content-type class). Their keys sit outside the
// current id space, so they read as missing every pass and can never be
// backfilled once off the wire (the #2732 backfill needs live bytes). The
// old `!== "reasoning"` classified that permanent bili self-debt as
// non-reasoning and latched foldDriftEscalated on healthy long sessions
// after 3 passes. Only a KNOWN non-reasoning class counts: a genuine
// client-side rewrite/deletion still carries a concrete `c`, so real loss
// still escalates — pure-legacy debt alone never does.
const nonReasoningUnmatched = plan.unmatched.filter((id) => {
const cls = anchors[id]?.c;
return cls !== undefined && cls !== "reasoning";
}).length;
// #3641 verdict classifier — the census evidence names the phenomenon
// before any streak accrues:
// shapeExplained: word evidence survived on the wire under churned ids
// (protocol switch #3640, image offload #3588) — the shape pass
// re-anchored some cells (byShape > 0) or the residual mass still
// carries near-duplicate word evidence (≥60% of class-carrying
// residual). Transform, not destruction: no streak, one info.
// truncationExplained: EVERY class-carrying residual is absent with
// zero word evidence anywhere, while some covered substrate is still
// present (resent or re-anchored) — the client dropped old history
// (sanctioned truncation #2695/#2297). Absence, not destruction: the
// snapshot is kept (z suppressed below), no streak, one info.
// Everything else (genuine rewrite with real content change, wholesale
// replacement by foreign history) keeps the #2193 streak — and its
// terminal verdict is now an epoch confirmation, not "destroyed".
const classCarryingResidual = plan.unmatched.filter((id) => {
const cls = anchors[id]?.c;
return cls !== undefined && cls !== "reasoning";
});
const shapeExplained = plan.byShape > 0
|| (classCarryingResidual.length > 0 && plan.nearDupResidual.length >= 0.6 * classCarryingResidual.length);
const truncationExplained = classCarryingResidual.length > 0
&& plan.nearDupResidual.length === 0
&& classCarryingResidual.every((id) => absentSet.has(id))
&& wireContinuity;
// #3724: MIRROR — every class-carrying residual absent with zero word
// evidence AND the wire is a clean prefix-cut = host rewound/undid the
// newest turns (deleting a freshly-folded span). Same sanctioned-deletion
// class as head truncation: snapshot retained (z suppressed above), no
// destroy streak, one distinct info line. Without this the all-covered-
// rewound case read as total-loss drift → false "epoch confirmation"
// escalation (+ false proxy-mode reap), or went silent when re-anchoring
// kept claims > 0.
const tailTruncationExplained = classCarryingResidual.length > 0
&& plan.nearDupResidual.length === 0
&& classCarryingResidual.every((id) => absentSet.has(id))
&& tailTruncated;
const totalDrift = plan.claims.size === 0 && plan.unmatched.length > 0 && !reasoningOnlyDrop && !markerChurnExplained
&& !shapeExplained && !truncationExplained && !tailTruncationExplained;
const unfoldedDrift = mode === "repair"
? Math.max(0, (plan.churnTokens ?? 0) - (plan.claimedChurnTokens ?? 0))
: (plan.churnTokens ?? 0);
Expand Down
81 changes: 81 additions & 0 deletions tests/epoch-confirm-lifecycle.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "node:test";
import { createCore, defaultConfig, type CoreMessage } from "acp-kernel";
import { parseCompressInput } from "../src/compress-tool.ts";
import { resolveDecompress } from "../src/decompress-shared.ts";
import { reconcileFoldCoverage } from "../src/fold-reconcile.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { createSession } from "../src/session.ts";
import { applyRanges } from "../src/stream.ts";

_setStoreForTest(new SessionStore({ enabled: false }));

for (const shape of ["one-interior-survivor", "all-missing", "small-zombie"] as const) {
test(`#4260 epoch confirmation retains real kernel folds: ${shape}`, () => {
const dir = mkdtempSync(join(tmpdir(), "bili-epoch-confirm-"));
try {
const core = createCore();
const config = defaultConfig(200_000);
const session = createSession(`epoch-confirm-${shape}`);
const messages: CoreMessage[] = Array.from({ length: 40 }, (_, i) => ({
id: `h_original_${i}`, role: i % 2 ? "assistant" : "user", contentType: "text",
text: `Historical detail ${i}: ${String.fromCharCode(65 + i % 26).repeat(2000)}`,
}));
const tick = (incoming: CoreMessage[]) => {
const turn = core.processTurn({ messages: incoming, state: session.state, config, tokenCount: 9999, renderTags: "text-only" });
session.state = turn.state;
return turn;
};
const initial = tick(messages);
const summary = "The early history establishes independent details for an epoch-confirmation recovery regression, with original content retained for later retrieval.";
applyRanges(parseCompressInput({ content: [{
startId: "m00001", endId: shape === "small-zombie" ? "m00007" : "m00020", summary,
}] }), { core, config, messages: initial.messages, session, log: () => {} });
assert.equal(session.state.blocks.length, 1);
const blockId = session.state.blocks[0].blockId;
const original = session.blockContents.get(blockId)?.full.text;
assert.ok(original);
const opts = { mode: "repair" as const, inboundMessageCount: 40, log: () => {} };
reconcileFoldCoverage(session, messages, opts);
tick(messages);
const foreign: CoreMessage[] = Array.from({ length: 40 }, (_, i) => ({
id: `h_foreign_${i}`, role: i % 2 ? "assistant" : "user", contentType: "text",
text: `Unrelated narrative ${i} with different evidence ${"qzjy".repeat(500)}`,
}));
if (shape !== "all-missing") foreign[15] = messages[shape === "small-zombie" ? 3 : 10];
for (let pass = 1; pass <= 5; pass++) {
const rec = reconcileFoldCoverage(session, foreign, opts);
tick(foreign);
assert.equal(rec.reaped, shape === "small-zombie" && pass === 3 ? 1 : 0);
if (shape === "one-interior-survivor") {
assert.equal(session.state.blocks.length, 1);
assert.equal(session.state.blocks[0].active, true);
assert.equal(session.metadata.foldEpochConfirmPending === true, pass >= 3);
}
if (shape === "all-missing") {
assert.equal(session.state.blocks.length, 1);
assert.equal(session.state.blocks[0].active, false);
assert.equal(session.metadata.foldEpochConfirmPending === true, false);
}
}
assert.equal(session.blockContents.get(blockId)?.full.text, original);
const receipt = resolveDecompress({ blockId, full: true, toFile: join(dir, "original.txt") },
{ core, config, messages: [], session, log: () => {} }).text;
assert.equal(receipt.startsWith("[decompress FAILED:"), shape === "small-zombie");
reconcileFoldCoverage(session, messages, opts);
const returned = tick(messages);
assert.equal(session.metadata.foldEpochConfirmPending === true, false);
assert.equal(session.state.blocks.length, shape === "small-zombie" ? 0 : 1);
assert.equal(returned.messages.some(m => m.text?.includes(summary)), shape !== "small-zombie");
if (shape !== "small-zombie") {
assert.equal(session.state.blocks[0].active, true);
assert.equal(returned.messages.some(m => m.text?.includes("Historical detail 5:")), false);
}
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
}
Loading
Loading