Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions src/conflict-watch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { markDirty, type Session } from "./session.js";
import { isCodexClient } from "./codex-compact.js";
import { isDisplayOnlyConflictDetail, isSiblingConflictDetail } from "./thirdparty-scan.js";

type ConflictKind = "third-party-plugin" | "unannounced-rewrite" | "orphan-reap" | "native-compaction" | "native-compaction-inferred" | "identity-orphan";
type ConflictKind = "third-party-plugin" | "unannounced-rewrite" | "orphan-reap" | "native-compaction" | "native-compaction-inferred" | "identity-orphan" | "id-churn-adoption";

export interface ConflictEvent {
at: number;
Expand Down Expand Up @@ -203,10 +203,17 @@ export function formatConflictSection(events: ConflictEvent[], now: number = Dat
// landing was fully rebased onto (#2372/#2432) — handled, no ongoing risk. Gated to the
// pure case so mixed ledgers keep their existing tier verbatim.
const handledNativeOnly = meaningful.length > 0 && meaningful.every((e) => e.kind === "native-compaction");
// #4131 review: an adoption-only ledger is bili's OWN fully-handled cross-build
// id re-mint (#3642 D1) — not external interference; the [suspected]-plugin
// tiers below would misframe it (same class as #2802 orphan-only / #2912
// handled-native-only).
const adoptionOnly = meaningful.length > 0 && meaningful.every((e) => e.kind === "id-churn-adoption");
const activeConfirmed = confirmed.some((e) => now - e.at <= CONFLICT_ACTIVE_WINDOW_MS);
lines.push(`COMPRESSION CONFLICTS — ${events.length} event(s) in this session (${active.length} active · ${historical.length} historical; active = within ${CONFLICT_ACTIVE_WINDOW_MS / 86_400_000} days). ${foreignConfirmed || nativePresent
? "Two compressors on one conversation (bili + another compression plugin — third-party or bili's own sibling — or client native compaction) double-compress and corrupt message refs:"
: "Diagnostic evidence that something outside bili touched this conversation — how strong that evidence is is tiered below:"}`);
: adoptionOnly
? "Bili's own cross-build id re-mints (#3642/#4131), adopted with full fold re-anchor — nothing outside bili touched this conversation:"
: "Diagnostic evidence that something outside bili touched this conversation — how strong that evidence is is tiered below:"}`);
for (const e of events.slice(-10)) {
lines.push(` [${fmtTime(e.at)}] ${e.kind} — ${e.detail}`);
}
Expand All @@ -222,6 +229,7 @@ export function formatConflictSection(events: ConflictEvent[], now: number = Dat
// foreign confirmed -> strong one-compressor command (live) or verify-then-clear (stock)
// native landing (no foreign)-> host-side explanation, no plugin hunt (#2432)
// identity-orphan only -> host-lane identity loss, no second compressor (#2802 review)
// id-churn-adoption only -> bili's own re-mint adopted, no external cause (#4131 review)
// pure [suspected] -> #1736 soft wording
// unconfirmed signals -> "cause not identified" framing, never imperative
let advice: string;
Expand All @@ -247,6 +255,14 @@ export function formatConflictSection(events: ConflictEvent[], now: number = Dat
advice = active.length > 0
? "The events above are client-native compaction landings bili ALREADY rebased onto (#2372/#2432): each detected checkpoint rebuilt the fold base, so there is no ongoing double-compression risk and nothing to disable. If you did not expect these sessions to auto-compact, review the client's own auto-compaction setting; otherwise clear this ledger — Web UI conflict banner / session page, or POST /__bili/conflicts/clear?session=<id>."
: "All events above are client-native compaction landings older than 7 days (historical stock) that bili already rebased onto (#2372/#2432): handled at the time, no ongoing risk. Clear this ledger — Web UI conflict banner / session page, or POST /__bili/conflicts/clear?session=<id>.";
} else if (adoptionOnly) {
// #4131 review: bili ADOPTED its own cross-build id re-mint with full fold
// re-anchor — no external cause exists, so neither the [suspected]-plugin
// wording nor any remediation command applies (same misframing class as
// #2802/#2912).
advice = active.length > 0
? "The events above are bili's OWN cross-build id re-mints (#3642/#4131): after an upgrade restart bili re-minted non-text message ids and fold-reconcile re-anchored every covered id, so it ADOPTED the re-minted history instead of marking a destructive compaction boundary — there is NO second compressor to hunt for and nothing to disable. If you did not expect this upgrade churn, check your auto-update settings; otherwise clear this ledger — Web UI conflict banner / session page, or POST /__bili/conflicts/clear."
: "All events above are bili's own cross-build id re-mints older than 7 days (historical stock): each was adopted with full fold re-anchor at the time, no ongoing risk. Clear this ledger — Web UI conflict banner / session page, or POST /__bili/conflicts/clear.";
} else if (orphanOnly) {
// #2802 review: identity-loss takeover (#2791 family) — bili KNOWS what
// happened (its own resolver detected the headerless replay of this
Expand Down
2 changes: 1 addition & 1 deletion src/fold-reconcile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ interface ReconciliationPlan {
claimedChurnTokens?: number;
}

interface FoldReconcileResult {
export interface FoldReconcileResult {
kind: "off" | "noop" | "resend" | "reanchored" | "unmatched";
missing: number;
claims: number;
Expand Down
11 changes: 8 additions & 3 deletions src/server/prepare-anthropic.ts
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ export async function prepareAnthropic(
// [#1921] re-anchor fold coverage onto churned-but-same messages
// before the #1195 snapshot, so covered ids surviving a client
// re-serialization stay covered (src/fold-reconcile.ts).
reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts), inboundMessageCount });
const reconcileOutcome = reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts), inboundMessageCount });
noteSystemPromptFingerprint(session, clientSystem, { sessionId, log });
// #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside
// processTurn may deactivate fully-drifted blocks, erasing them.
Expand Down Expand Up @@ -373,8 +373,13 @@ export async function prepareAnthropic(
// rebased — a second "unannounced-rewrite" entry beside it would present
// the substrate destruction as "another compressor fighting you".
if (!dshRebased) {
const rewrite = detectUnannouncedHistoryRewrite(session, knownRefsBefore, msgs.map((m) => m.id));
if (rewrite.detected) {
const rewrite = detectUnannouncedHistoryRewrite(session, knownRefsBefore, msgs.map((m) => m.id), reconcileOutcome);
if (rewrite.churnExplained) {
// #4131: cross-build id re-mint (#3642) — fold-reconcile found every
// covered id on the wire; adopt instead of destroying.
recordConflict(session, "id-churn-adoption", `${rewrite.knownIncoming}/${rewrite.incomingTotal} incoming message(s) carry pre-turn refs of ${rewrite.knownBefore} known; fold-reconcile re-anchored every covered id (0 unmatched) — bili-side id re-mint across builds (#3642), not a client rewrite — no compaction boundary (#4131)`);
log("info", `[${sessionId}] cross-build id churn adopted: hash-identity collapse (${rewrite.knownIncoming}/${rewrite.incomingTotal} pre-turn refs of ${rewrite.knownBefore} known) explained by fold-reconcile full re-anchor (0 unmatched) — skipping compaction boundary (#4131/#3642)`);
} else if (rewrite.detected) {
// #2709: attribute a framing-absent rewrite to client-native compaction when bili witnessed the client's own compaction being refused — stops the phantom-second-compressor errand.
const dshWitness = dshNativeCompactionWitness(session, { incomingTotal: rewrite.incomingTotal, knownBefore: rewrite.knownBefore });
log("warn", `[${sessionId}] ${dshWitness !== undefined ? "client-native compaction landing (framing absent; attributed from refusal ledger)" : "unannounced client history rewrite"} detected (${rewrite.knownIncoming}/${rewrite.incomingTotal} incoming message(s) carry pre-turn refs of ${rewrite.knownBefore} known) — marking compaction boundary (#1001${dshWitness !== undefined ? "/#2709" : ""})`);
Expand Down
11 changes: 8 additions & 3 deletions src/server/prepare-google.ts
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ export async function prepareGoogle(
// [#1921] re-anchor fold coverage onto churned-but-same messages
// before the #1195 snapshot, so covered ids surviving a client
// re-serialization stay covered (src/fold-reconcile.ts).
reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts), inboundMessageCount });
const reconcileOutcome = reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts), inboundMessageCount });
if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log });
// #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside
// processTurn may deactivate fully-drifted blocks, erasing them.
Expand Down Expand Up @@ -247,8 +247,13 @@ export async function prepareGoogle(
// as an announced /compact boundary — syncBlocks above has already
// deactivated the blocks whose sources left the context.
{
const rewrite = detectUnannouncedHistoryRewrite(session, knownRefsBefore, msgs.map((m) => m.id));
if (rewrite.detected) {
const rewrite = detectUnannouncedHistoryRewrite(session, knownRefsBefore, msgs.map((m) => m.id), reconcileOutcome);
if (rewrite.churnExplained) {
// #4131: cross-build id re-mint (#3642) — fold-reconcile found every
// covered id on the wire; adopt instead of destroying.
recordConflict(session, "id-churn-adoption", `${rewrite.knownIncoming}/${rewrite.incomingTotal} incoming message(s) carry pre-turn refs of ${rewrite.knownBefore} known; fold-reconcile re-anchored every covered id (0 unmatched) — bili-side id re-mint across builds (#3642), not a client rewrite — no compaction boundary (#4131)`);
log("info", `[${sessionId}] cross-build id churn adopted: hash-identity collapse (${rewrite.knownIncoming}/${rewrite.incomingTotal} pre-turn refs of ${rewrite.knownBefore} known) explained by fold-reconcile full re-anchor (0 unmatched) — skipping compaction boundary (#4131/#3642)`);
} else if (rewrite.detected) {
log("warn", `[${sessionId}] unannounced client history rewrite detected (${rewrite.knownIncoming}/${rewrite.incomingTotal} incoming message(s) carry pre-turn refs of ${rewrite.knownBefore} known) — marking compaction boundary (#1001)`);
recordConflict(session, "unannounced-rewrite", `${rewrite.knownIncoming}/${rewrite.incomingTotal} incoming message(s) carry pre-turn refs of ${rewrite.knownBefore} known`);
markCompactionBoundary(session);
Expand Down
14 changes: 10 additions & 4 deletions src/server/prepare-openai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,9 @@ export async function prepareOpenai(
}
// [#1921] re-anchor fold coverage onto churned-but-same messages
// before the #1195 snapshot, so covered ids surviving a client
// re-serialization stay covered (src/fold-reconcile.ts).
reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts), inboundMessageCount });
// re-serialization stay covered (src/fold-reconcile.ts). The verdict
// feeds the #4131 rewrite gate below (explained churn ≠ destruction).
const reconcileOutcome = reconcileFoldCoverage(session, msgs, { mode: resolveFoldReconcileMode(process.env, opts.compress.reconcile), sessionId, log, markerStripped: markerArtifactCount(markerCounts), inboundMessageCount });
if (!isTitleGen) noteSystemPromptFingerprint(session, systemText, { sessionId, log });
// #1195: pre-turn snapshot of the fold's covered ids — syncBlocks inside
// processTurn may deactivate fully-drifted blocks, erasing them.
Expand Down Expand Up @@ -270,8 +271,13 @@ export async function prepareOpenai(
// a second "unannounced-rewrite" entry beside it would present the
// substrate destruction as "another compressor fighting you".
if (!dshRebased) {
const rewrite = detectUnannouncedHistoryRewrite(session, knownRefsBefore, msgs.map((m) => m.id));
if (rewrite.detected) {
const rewrite = detectUnannouncedHistoryRewrite(session, knownRefsBefore, msgs.map((m) => m.id), reconcileOutcome);
if (rewrite.churnExplained) {
// #4131: cross-build id re-mint (#3642) — fold-reconcile found every
// covered id on the wire; adopt instead of destroying.
recordConflict(session, "id-churn-adoption", `${rewrite.knownIncoming}/${rewrite.incomingTotal} incoming message(s) carry pre-turn refs of ${rewrite.knownBefore} known; fold-reconcile re-anchored every covered id (0 unmatched) — bili-side id re-mint across builds (#3642), not a client rewrite — no compaction boundary (#4131)`);
log("info", `[${sessionId}] cross-build id churn adopted: hash-identity collapse (${rewrite.knownIncoming}/${rewrite.incomingTotal} pre-turn refs of ${rewrite.knownBefore} known) explained by fold-reconcile full re-anchor (0 unmatched) — skipping compaction boundary (#4131/#3642)`);
} else if (rewrite.detected) {
// #2709: attribute a framing-absent rewrite to client-native compaction when bili witnessed the client's own compaction being refused — stops the phantom-second-compressor errand.
const dshWitness = dshNativeCompactionWitness(session, { incomingTotal: rewrite.incomingTotal, knownBefore: rewrite.knownBefore });
log("warn", `[${sessionId}] ${dshWitness !== undefined ? "client-native compaction landing (framing absent; attributed from refusal ledger)" : "unannounced client history rewrite"} detected (${rewrite.knownIncoming}/${rewrite.incomingTotal} incoming message(s) carry pre-turn refs of ${rewrite.knownBefore} known) — marking compaction boundary (#1001${dshWitness !== undefined ? "/#2709" : ""})`);
Expand Down
36 changes: 33 additions & 3 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { durableMessageToolPaths, withDurableProtection } from "./durable-messag
import type { AnonymousProvenance } from "./anon-provenance.js";
import type { WireProtocol } from "./util.js";
import type { InboundSlot, SeamSlot } from "./cache-ledger.js";
import type { FoldReconcileResult } from "./fold-reconcile.js";

export type BlockView = { text: string; count: number };

Expand Down Expand Up @@ -1453,6 +1454,14 @@ export function applyCompactionArchive(
}

const { byRaw, byRef } = session.state.messageRefs;
// #4131: name what the prune costs — ref(s) about to be dropped that still
// sit under some block's coverage, so silent coverage loss is auditable.
const droppedIds = new Set(Object.keys(byRaw).filter((id) => id !== REF_FLOOR_RAW_ID && !liveRawIds.has(id)));
let coverageNote = "";
if (droppedIds.size > 0) {
const affectedBlocks = session.state.blocks.filter((b) => (b.effectiveMessageIds ?? []).some((id) => droppedIds.has(id))).map((b) => b.blockId);
coverageNote = `; ${droppedIds.size} ref(s) dropped${affectedBlocks.length > 0 ? `, covering block(s) ${affectedBlocks.join(", ")}` : ""}`;
}
pruneRefsToLiveIds(session, liveRawIds, byRaw, byRef);

session.metadata.compactionBoundary = {
Expand All @@ -1462,7 +1471,7 @@ export function applyCompactionArchive(
archivedBlocks: deactivated,
};
markDirty(session);
log("info", `[${session.id}] native compaction boundary: archived ${deactivated.length} pre-compaction block(s)${deactivated.length > 0 ? ` (${deactivated.join(", ")})` : ""}; pruned ref maps to ${liveRawIds.size} live raw id(s)`);
log("info", `[${session.id}] native compaction boundary: archived ${deactivated.length} pre-compaction block(s)${deactivated.length > 0 ? ` (${deactivated.join(", ")})` : ""}; pruned ref maps to ${liveRawIds.size} live raw id(s)${coverageNote}`);
}

/** Prune the ref maps to `liveRawIds` in place (same maps, not new objects:
Expand Down Expand Up @@ -1574,12 +1583,19 @@ interface RewriteDetection {
knownBefore: number;
incomingTotal: number;
knownIncoming: number;
/** #4131: the ratio collapse was explained by this pass's fold-reconcile
* finding EVERY covered id on the wire (present or re-anchored, zero
* unmatched) — bili's own sanctioned id churn across builds (#3642 D1),
* not a client rewrite. The destructive boundary is suppressed and
* callers record an adoption event instead of marking one. */
churnExplained: boolean;
}

export function detectUnannouncedHistoryRewrite(
session: Session,
knownRefsBefore: ReadonlySet<string>,
liveRawIds: Iterable<string>,
reconcile?: Pick<FoldReconcileResult, "kind" | "unmatched"> | null,
): RewriteDetection {
const knownBefore = knownRefsBefore.size - (knownRefsBefore.has(REF_FLOOR_RAW_ID) ? 1 : 0);
let incomingTotal = 0;
Expand All @@ -1588,14 +1604,28 @@ export function detectUnannouncedHistoryRewrite(
incomingTotal++;
if (knownRefsBefore.has(id)) knownIncoming++;
}
const detected =
const rawDetected =
knownBefore >= REWRITE_MIN_KNOWN_REFS &&
session.state.blocks.length > 0 &&
incomingTotal >= REWRITE_MIN_INCOMING_TOTAL &&
knownIncoming > 0 &&
knownIncoming / incomingTotal < REWRITE_MAX_KNOWN_RATIO &&
knownIncoming / knownBefore < REWRITE_MAX_KNOWN_RATIO;
return { detected, knownBefore, incomingTotal, knownIncoming };
// #4131: order-centric doctrine — no single dimension may veto the
// multi-evidence verdict. When this pass's fold-reconcile found EVERY
// covered id on the wire (kind reanchored/resend, unmatched === 0), the
// hash-identity collapse is explained churn: bili's own cross-build id
// re-mint (#3642 D1 accepted exactly this once-per-upgrade mass churn,
// promising the reconcile ladder would absorb it). Marking the destructive
// compaction boundary here instead tombstones hundreds of still-cited
// refs (#4131 incident). Off/noop verdicts (reconcile disabled, side-request
// skip) carry no evidence either way → legacy behavior unchanged; partial
// claims (unmatched > 0) keep the legacy escalation path.
const churnExplained = rawDetected
&& reconcile !== undefined && reconcile !== null
&& (reconcile.kind === "reanchored" || reconcile.kind === "resend")
&& reconcile.unmatched === 0;
return { detected: rawDetected && !churnExplained, knownBefore, incomingTotal, knownIncoming, churnExplained };
}

/** #1195: message ids are content hashes — if a client resends history whose
Expand Down
Loading
Loading