diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 1956cd54ec..1d88f406cb 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.143", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.143", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index ed0ff46830..0457970312 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.143", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/absorb.ts b/kernel/src/absorb.ts index b1365b3972..4f3906ac1f 100644 --- a/kernel/src/absorb.ts +++ b/kernel/src/absorb.ts @@ -373,6 +373,11 @@ export function applyAbsorb(input: AbsorbInput): AbsorbOutcome { ...input.state.stats, absorbedTokens: (input.state.stats.absorbedTokens ?? 0) + tokens, }, + // #3939 ⑤: the absorb call is a deliberate history-rewriting action (a + // fold-class boundary, same family as applyCompression) — release the + // view latch here so the hide lands THIS turn, not at some later fold. + // Keeps absorb's instant-reclaim contract; quiet turns stay latched. + viewLatches: undefined, }; const bloat = diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 7257814f02..67ff2df893 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -1,4 +1,4 @@ -import { assignRefs, highestUsedIndex, indexToRef } from "./refs.js"; +import { assignRefs, BLOCKED_REF, highestUsedIndex, indexToRef } from "./refs.js"; import { remintCoveredLiveIds } from "./instance-reid.js"; import { prune } from "./prune.js"; import { syncBlocks } from "./sync.js"; @@ -14,11 +14,13 @@ import { import type { ResolvedRange } from "./boundaries.js"; import { truncateLargeToolOutputs } from "./truncate-tools.js"; import { scrubLoneSurrogates } from "./truncate.js"; -import { hideConsumedCompressCalls } from "./hide-consumed.js"; +import { hideConsumedCompressCalls, refNum } from "./hide-consumed.js"; import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyCrushToMessages } from "./crush.js"; import { applyRetrieve, ccrStoreNode, RETRIEVED_ID_PREFIX } from "./ccr.js"; +import { preTransformSnapshotNode, viewLatchNode } from "./view-latch.js"; import type { ApplyRetrieveResult, CcrEffect } from "./ccr.js"; +import type { EmergencyTruncateLatchEntry } from "./types.js"; import { createContentStore } from "./content-store.js"; import type { MessageContentStore } from "./content-store.js"; import { applyMessageFilters, listMessageFilters } from "./filter/index.js"; @@ -145,6 +147,11 @@ type RangeResolution = | { status: "unknown"; error: BoundaryNotFoundError } | { status: "invalid"; error: Error }; +/** #3939 ③: bound on persisted emergency-truncate latch entries. Each entry + * carries a clipped text (≤ keepPrefix+marker+keepSuffix ≈ 4KiB); a fold + * clears the latch, so normal sessions hold a handful at most. Oldest drop. */ +const MAX_TRUNCATE_LATCH = 64; + function rangeError( spec: { startRef: string; endRef: string }, message: string, @@ -600,6 +607,76 @@ export function createCore(ports: Ports = {}): CompressionCore { } } + // #3003: all-or-nothing summary-LENGTH pre-check. Length-class failures are + // pure functions of (spec.summary, spec.summaryMaxChars, cfg) — no state and + // no cross-range coverage — so they can be screened BEFORE any block commits. + // Screening them here turns the common PARTIAL outcome (one oversized summary + // rejects while sibling ranges land and open a seam) into a clean 0-commit + // reject: nothing lands, so the model's single retry pays ONE seam instead of + // two. Content-class failures (protected-zone / turn-integrity / no-new- + // messages) DO depend on state + cross-range coverage and stay per-range below. + // Scope is EXACTLY the fold-candidate set that mutates state in the loop — + // never every requested range, else a consumed range whose summary happens to + // be long would wrongly kill a batch whose real candidates all validate. + { + const lengthFailures: string[] = []; + for (const spec of input.ranges) { + if (skipSpecs.has(spec)) continue; + const resolution = classifications.get(spec); + if (resolution === undefined) continue; + const foldsFresh = resolution.status === "ok"; + const foldsRefold = + resolution.status === "consumed" && + refoldDecisions.get(spec)?.kind === "refold"; + if (!foldsFresh && !foldsRefold) continue; + try { + validateSummaryLength(spec, input.config.compress); + } catch (error) { + lengthFailures.push( + rangeError( + spec, + error instanceof Error ? error.message : String(error), + ), + ); + } + } + if (lengthFailures.length > 0) { + // Rejected batches carry their dead-ref tombstone side effect (#2362): + // the clone differs from input.state only in deadRefs before this point, + // so returning it is safe (same contract as the minCompressRange gate). + return { + state, + result: { + blocksCreated: 0, + tokensCompressed: 0, + errors: [ + `Batch rejected before folding any range — ${lengthFailures.length} of ${input.ranges.length} requested range(s) failed summary-length validation: ${lengthFailures.join(" | ")}. Nothing was folded, so no prefix was rewritten. Fix it one way or the other and resend: shorten or split each offending range's summary to fit its cap, then resend the whole batch; or drop the offending range(s) and resend only the rest that validate.`, + ...classificationErrors, + ], + warnings: [], + }, + }; + } + } + + // #3939 (PR-B item ④, extends #3670/#3003 to content class): the length + // pre-check above cannot screen content-class fatals (protected-zone / + // turn-integrity / livelock / empty-range) because they depend on state + // plus cross-range coverage — range N+1's eligibility may hinge on range N + // having landed. So apply the loop and keep a pristine pre-loop snapshot; + // any apply-level throw rejects the WHOLE batch with zero commits instead + // of leaving earlier siblings landed (the #3003 double-seam shape: a fold + // seam PLUS the retry seam on the model's next attempt). Classification- + // level outcomes (unknown / invalid / consumed-skip) never mutate state, + // so they stay per-range and do NOT poison the batch. + const preLoopState = cloneState(state); + const applyErrors: string[] = []; + const warningsBase = warnings.length; + // #3769: capture which blocks already exist so, after the loop, we can tell + // which were created by THIS fold and where it starts (for the orphan sweep). + const preFoldBlockIds = new Set(state.blocks.map((b) => b.blockId)); + + for (const spec of input.ranges) { if (skipSpecs.has(spec)) continue; const resolution = classifications.get(spec); @@ -617,7 +694,7 @@ export function createCore(ports: Ports = {}): CompressionCore { }); blocksCreated += decision.blocks.length; } catch (error) { - errors.push( + applyErrors.push( rangeError( spec, error instanceof Error ? error.message : String(error), @@ -657,7 +734,7 @@ export function createCore(ports: Ports = {}): CompressionCore { tokensCompressed += outcome.tokens; warnings.push(...outcome.warnings); } catch (error) { - errors.push( + applyErrors.push( rangeError( spec, error instanceof Error ? error.message : String(error), @@ -666,10 +743,54 @@ export function createCore(ports: Ports = {}): CompressionCore { } } + if (applyErrors.length > 0) { + // Same contract as the gate paths above: the returned state differs + // from input.state only in the #2362 dead-ref tombstones recorded during + // classification. Nothing else landed, so no prefix was rewritten. + return { + state: preLoopState, + result: { + blocksCreated: 0, + tokensCompressed: 0, + errors: [ + `Batch rejected before folding any range — ${applyErrors.length} of ${input.ranges.length} requested range(s) failed during application: ${applyErrors.join(" | ")}. Nothing was folded, so no prefix was rewritten. Fix or drop the offending range(s), then resend the whole batch.`, + ...classificationErrors, + ], + warnings: warnings.slice(0, warningsBase), + }, + }; + } + state.stats.compressionCount += blocksCreated; state.stats.tokensCompressed += tokensCompressed; if (blocksCreated > 0) { + // #3769: record where THIS fold starts so the next processTurn sweeps + // orphan failures at/after it cache-neutrally, leaving orphans before it + // in the still-cached prefix. Kept as the running min across folds that + // commit before a sweep consumes it. The position comes from the covered + // message refs of each new block, NOT its startRef string: a block-boundary + // spec stores startRef="bN", whose digits are a BLOCK number, not a message + // position — parsing it would mark the sweep far too early and rewrite the + // still-cached prefix (#3769 again). refNum returns null for unmapped/blocked + // ids, so a fully-pruned coverage simply sets no marker (under-sweep is safe; + // over-sweep is not). + let foldStartRef: number | undefined; + for (const b of state.blocks) { + if (preFoldBlockIds.has(b.blockId)) continue; + for (const id of b.effectiveMessageIds) { + const num = refNum(state.messageRefs.byRaw[id]); + if (num !== null && (foldStartRef === undefined || num < foldStartRef)) { + foldStartRef = num; + } + } + } + if (foldStartRef !== undefined) { + state.pendingOrphanSweepMinRef = + state.pendingOrphanSweepMinRef === undefined + ? foldStartRef + : Math.min(state.pendingOrphanSweepMinRef, foldStartRef); + } // Compress succeeded: clear the growth baseline so the next turn // re-establishes it at the new (lower) token count. Without this the // nudge re-fires in a feedback loop (the §5.7 baseline-reset bug). @@ -683,6 +804,20 @@ export function createCore(ports: Ports = {}): CompressionCore { // the terminal-floor streak so the escape signal can only fire again // after N fresh stuck events (#300). state.terminalStreak = 0; + // #3939 ③/#3691: the fold reset the truncation landscape — clipped + // originals under the fold are gone from the view and the watermark + // re-evaluates from the new (lower) count, so the latch retires with it. + // Holding it would replay stale clips onto a view the fold rewrote. + state.emergencyTruncateLatch = undefined; + // #3939 ②/#3805: the fold rewrote history — the migration deferral + // rides it. Any surviving carrier may render its tag from here on; the + // rewrite is attributed to the fold, not to a quiet turn. + state.renderDeferredRefs = undefined; + // #3939 ⑤: the latch refresh rides the fold's own history rewrite — + // the next pass re-latches every visible message's transform output + // fresh (absorb hides apply, crush re-evaluates under the new pressure, + // keep-last-only demotions land). + state.viewLatches = undefined; } return { @@ -814,12 +949,14 @@ export function createCore(ports: Ports = {}): CompressionCore { assignRefsNode, syncBlocksNode, pruneNode, + preTransformSnapshotNode, ccrStoreNode, absorbHideNode, crushNode, absorbPromptNode, filterNode, hideCompressCallsNode, + viewLatchNode, recommendNode, nudgeNode, emergencyTruncateNode, @@ -896,7 +1033,23 @@ const assignRefsNode: PipelineNode = { // Ephemeral retrieval injections never consume a ref slot. shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), }); - return { ...io, state: { ...io.state, messageRefs: refResult.map } }; + // #3939 ②/#3805: a BLOCKED→numeric migration mints the ref NOW but the + // carrier's tag render is DEFERRED to the next fold boundary — + // stamping it this turn would rewrite an old message mid-history on an + // otherwise quiet turn (prefix-cache breaker). Cleared on fold success. + const migratedRefs = refResult.migratedIds + .map((id) => refResult.map.byRaw[id]) + .filter((r): r is string => typeof r === "string" && r !== BLOCKED_REF); + const state = migratedRefs.length + ? { + ...io.state, + messageRefs: refResult.map, + renderDeferredRefs: [ + ...new Set([...(io.state.renderDeferredRefs ?? []), ...migratedRefs]), + ], + } + : { ...io.state, messageRefs: refResult.map }; + return { ...io, state }; }, }; @@ -996,10 +1149,26 @@ const hideCompressCallsNode: PipelineNode = { io.messages, ctx.config.isCompressTool, ); + // #3769: fold the newly-swept orphans into the persistent monotonic set so + // they stay hidden across turns/restarts/upgrades, then consume the fold + // marker (one-shot) — a quiet turn leaves both untouched except clearing a + // stale marker. + const patch: Partial = { + hiddenOrphanRefs: hidden.hiddenOrphanRefs, + pendingOrphanSweepMinRef: undefined, + }; + if (hidden.newlySweptCallIds.length > 0) { + patch.sweptOrphanCallIds = [ + ...new Set([ + ...(io.state.sweptOrphanCallIds ?? []), + ...hidden.newlySweptCallIds, + ]), + ]; + } return { ...io, messages: hidden.messages, - state: { ...io.state, hiddenOrphanRefs: hidden.hiddenOrphanRefs }, + state: { ...io.state, ...patch }, }; }, }; @@ -1151,14 +1320,53 @@ const emergencyTruncateNode: PipelineNode = { ? ctx.tokenCount / ctx.config.modelContextLimit : 0; const prevStreak = io.state.terminalStreak ?? 0; + // #3939 ③/#3691: replay latched clips BEFORE the watermark check. The + // decision to clip was made once at the watermark; re-evaluating it every + // turn against a fluctuating estimate flips the same bytes full↔clipped — + // a mid-history rewrite on every request (prefix-cache breaker). Refs are + // content-bound, so an edited message misses its entry and re-judges from + // scratch; entries whose messages left the view retire; the whole latch + // resets on the next successful fold (applyCompression). + const latch = io.state.emergencyTruncateLatch ?? []; + const liveRefs = new Set(); + for (const m of io.messages) { + const ref = io.state.messageRefs.byRaw[m.id]; + if (ref !== undefined) liveRefs.add(ref); + } + const latchLive = latch.filter((e) => liveRefs.has(e.ref)); + let messages = io.messages; + let replaySaved = 0; + if (latch.length > 0) { + const byRef = new Map(latch.map((e) => [e.ref, e.text])); + messages = io.messages.map((m) => { + const ref = io.state.messageRefs.byRaw[m.id]; + const clipped = ref !== undefined ? byRef.get(ref) : undefined; + if (clipped === undefined || clipped === m.text) return m; + replaySaved += Math.max( + 0, + ctx.countTokens(m.text ?? "") - ctx.countTokens(clipped), + ); + return { ...m, text: clipped }; + }); + } if (usage < ctx.config.truncate.threshold) { - return prevStreak > 0 - ? { ...io, state: { ...io.state, terminalStreak: 0 } } - : io; + if (latch.length === 0 && prevStreak === 0) return io; + return { + ...io, + messages, + state: { + ...io.state, + terminalStreak: 0, + emergencyTruncateLatch: + latchLive.length > 0 ? latchLive : undefined, + }, + }; } const trunc = truncateLargeToolOutputs( - io.messages, - ctx.tokenCount, + messages, + // Account for replayed clips: they already shrank the real request; + // charging their tokens again would over-truncate fresh candidates. + Math.max(0, ctx.tokenCount - replaySaved), ctx.config, ctx.countTokens, { @@ -1203,10 +1411,35 @@ const emergencyTruncateNode: PipelineNode = { stuckEvents: streak, }; } + // #3939 ③: fold this pass's clips into the latch (replacing any same-ref + // entry) so later turns replay them byte-identically. Bounded so a long + // session cannot grow the persisted state without limit; the oldest + // entries drop first. + const clips: EmergencyTruncateLatchEntry[] = []; + for (const id of trunc.truncatedIds ?? []) { + const ref = io.state.messageRefs.byRaw[id]; + const clippedMsg = trunc.messages.find((m) => m.id === id); + if (ref !== undefined && clippedMsg?.text !== undefined) { + clips.push({ ref, text: clippedMsg.text }); + } + } + const clipRefs = new Set(clips.map((c) => c.ref)); + const nextLatch = [ + ...latchLive.filter((e) => !clipRefs.has(e.ref)), + ...clips, + ]; + if (nextLatch.length > MAX_TRUNCATE_LATCH) { + nextLatch.splice(0, nextLatch.length - MAX_TRUNCATE_LATCH); + } return { ...io, messages: trunc.messages, - state: { ...io.state, terminalStreak: streak }, + state: { + ...io.state, + terminalStreak: streak, + emergencyTruncateLatch: + nextLatch.length > 0 ? nextLatch : undefined, + }, effects, }; }, @@ -1794,6 +2027,10 @@ function cloneState(state: CompressionState): CompressionState { hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined, + sweptOrphanCallIds: state.sweptOrphanCallIds + ? [...state.sweptOrphanCallIds] + : undefined, + pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef, deadRefs: state.deadRefs ? [...state.deadRefs] : undefined, lastPassIds: state.lastPassIds ? [...state.lastPassIds] : undefined, }; diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index 25e674605d..fd62ef7585 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -4,28 +4,99 @@ import { COMPRESS_TOOL_NAME } from "./compress-tools.js"; import type { CompressionState, CoreMessage } from "./types.js"; // Orphaned compress calls (no matching block — failed attempts, or historical -// calls whose blocks predate compressCallId recording) keep their NEWEST two -// call+result pairs visible: failures must stay observable or a deterministic -// model re-issues the same no-op compress forever, pinned at a fixed point -// (billion-context-pi issue #9: 3,849 identical calls over 5h13m under -// KEEP_LAST_ORPHANED=0). Older orphans are hidden, so the residue is bounded -// at two pairs regardless of session length — PR #18's unbounded accumulation -// does not return (its own live check showed the cap: 10 in → 6 out). -const KEEP_LAST_ORPHANED = 2; +// calls whose blocks predate compressCallId recording) stay visible until a REAL +// fold gives a cache-safe place to remove them (#3769). Failures must remain +// observable or a deterministic model re-issues the same no-op compress forever +// (billion-context-pi #9: 3,849 identical calls over 5h13m under keep=0), AND a +// quiet turn must never move the cached prefix breakpoint forward just to delete +// a few hundred chars of old failure residue — the old keep-last-two rotation did +// exactly that (hid the oldest failure on EVERY turn, seam cost far exceeding the +// bytes removed). Now: between folds nothing is swept; on a fold the node sweeps +// orphans at/after the fold's earliest start ref onto the persistent +// sweptOrphanCallIds set (they stay hidden forever), leaving orphans before it +// visible. Residue is bounded by fold cadence, not a fixed pair count. export interface HideConsumedResult { messages: CoreMessage[]; hidden: number; - /** Padded refs of the orphan compress call/result messages hidden by this - * pass (orphans beyond the keep window). Consumed by boundary resolution to - * snap a range endpoint onto a hidden-but-intact orphan (#396). */ + /** Padded refs of ALL currently-hidden orphan compress call/result messages + * (the persistent swept set, not just this pass). Consumed by boundary + * resolution to snap a range endpoint onto a hidden-but-intact orphan (#396). */ hiddenOrphanRefs: string[]; + /** Orphan call ids newly swept by THIS pass (only when a fold was pending); + * the node unions these into state.sweptOrphanCallIds. Empty on quiet turns. */ + newlySweptCallIds: string[]; } function rangeKey(startRef: string, endRef: string): string { return `${startRef}::${endRef}`; } +// #3769: does this call's args quote the folded range of an ACTIVE block? Such a +// re-sent pair IS that block's summary carrier — the host strips the in-place +// anchor only while it rides the history (stripKernelSummaries, #1567/#2042) — +// so the fold-gated sweep must leave it alone. Both arg shapes are accepted to +// mirror the host index ({content:[…]} AND the flat {startId,endId} models emit +// natively); parseCallText alone misses the flat form. Unparseable args fail +// safe as NOT a carrier (under-sweep is safe, over-sweep is not). +function quotesActiveBlock( + text: string | undefined, + activeRangeKeys: Set, +): boolean { + const raw = text ?? ""; + const start = raw.indexOf("{"); + if (start < 0) return false; + let parsed: unknown; + try { + parsed = JSON.parse(raw.slice(start)); + } catch { + return false; + } + if (!parsed || typeof parsed !== "object") return false; + const obj = parsed as Record; + let entries: unknown[]; + if (Array.isArray(obj.content)) { + entries = obj.content; + } else if (typeof obj.content === "string") { + try { + const inner: unknown = JSON.parse(obj.content); + entries = Array.isArray(inner) ? inner : []; + } catch { + return false; + } + } else { + entries = [obj]; + } + for (const entry of entries) { + if (!entry || typeof entry !== "object") continue; + const e = entry as Record; + const s = + typeof e.startId === "string" + ? e.startId + : typeof e.messageId === "string" + ? e.messageId + : ""; + const end = + typeof e.endId === "string" + ? e.endId + : typeof e.messageId === "string" + ? e.messageId + : ""; + if (s && end && activeRangeKeys.has(rangeKey(s, end))) return true; + } + return false; +} + +// Numeric position of a padded ref (mNNNNN -> NNNNN); null when unmapped/blocked +// so the caller treats it as "position unknown" and never sweeps on a guess. +// Exported: applyCompression derives the fold sweep marker from covered message +// refs through the same helper so both sides agree on "unknown" (#3769). +export function refNum(ref: string | undefined): number | null { + if (!ref || ref === BLOCKED_REF) return null; + const n = Number(ref.replace(/\D/g, "")); + return Number.isFinite(n) ? n : null; +} + // Named return type (not inline): with the object literal in the signature, // prettier's layout heuristic was not idempotent here (#416). interface ParsedCallText { @@ -240,18 +311,17 @@ export function hideConsumedCompressCalls( }, ); - const lastOrphaned = new Set(); - for ( - let i = messages.length - 1; - i >= 0 && lastOrphaned.size < KEEP_LAST_ORPHANED; - i-- - ) { - const info = infos[i]!; - const callId = messages[i]!.toolCallId; - if (info.isCompressCall && callId && info.fate === "orphan") { - lastOrphaned.add(callId); - } - } + // #3769: orphans are removed only where a real fold already moved the cached + // breakpoint. sweptSet holds orphans removed on earlier folds (persistent + + // monotonic across turns/restarts/upgrades); pendingOrphanSweepMinRef carries + // this fold's earliest start ref so we additionally sweep orphans at/after it. + // Orphans before it sit in the still-cached prefix and stay visible; with no + // fold pending nothing new is swept, so a quiet turn never breaks the prefix. + // Orphans quoting an ACTIVE block's folded range are that block's summary + // carrier (#1567) and are exempt from the sweep (quotesActiveBlock). + const sweptSet = new Set(state.sweptOrphanCallIds ?? []); + const foldMinRef = state.pendingOrphanSweepMinRef; + const newlySweptCallIds: string[] = []; const keepCallIds = new Set(); const hiddenCallIds = new Set(); @@ -259,14 +329,31 @@ export function hideConsumedCompressCalls( const info = infos[i]!; const callId = messages[i]!.toolCallId; if (!info.isCompressCall || !callId) continue; - if ( - info.fate === "inactive" || - (info.fate === "orphan" && !lastOrphaned.has(callId)) - ) { - hiddenCallIds.add(callId); - } else { - keepCallIds.add(callId); + let hide = false; + if (info.fate === "inactive") { + hide = true; + } else if (info.fate === "orphan") { + const ref = state.messageRefs.byRaw[messages[i]!.id]; + const rememberedHidden: string[] = state.hiddenOrphanRefs ?? []; + if (sweptSet.has(callId)) { + hide = true; + } else if (ref && rememberedHidden.includes(ref)) { + // Pre-#3769 states persisted the rotation's hidden set as message refs + // (hiddenOrphanRefs). Honor it on upgrade: resurrecting a pair the old + // kernel already removed would re-insert it mid-history and break the + // cached prefix again. The set self-perpetuates — rememberHiddenRef + // re-records these refs on every pass they stay hidden. + hide = true; + } else if (foldMinRef !== undefined && !quotesActiveBlock(messages[i]!.text, activeRangeKeys)) { + const num = refNum(ref); + if (num !== null && num >= foldMinRef) { + hide = true; + newlySweptCallIds.push(callId); + } + } } + if (hide) hiddenCallIds.add(callId); + else keepCallIds.add(callId); } const fateByCallId = new Map(); @@ -349,5 +436,5 @@ export function hideConsumedCompressCalls( result.push(message); } - return { messages: result, hidden, hiddenOrphanRefs }; + return { messages: result, hidden, hiddenOrphanRefs, newlySweptCallIds }; } diff --git a/kernel/src/pipeline.ts b/kernel/src/pipeline.ts index 96d063f9d9..24dfa976ba 100644 --- a/kernel/src/pipeline.ts +++ b/kernel/src/pipeline.ts @@ -25,6 +25,10 @@ export interface NodeEffects { * foldability judgment — on lanes whose processTurn input is itself a * pruned/render view, even the pre-prune input is not the apply class. */ applyView?: CoreMessage[]; + /** #3939 ⑤: the post-prune, pre-transform resent view, stashed by the + * pre-transform-snapshot node so the view-latch node can replay latched + * bytes for messages the transform nodes dropped or rewrote. */ + preTransformMessages?: CoreMessage[]; readonly [key: string]: unknown; } diff --git a/kernel/src/refs.ts b/kernel/src/refs.ts index 106bcdde13..21606cda23 100644 --- a/kernel/src/refs.ts +++ b/kernel/src/refs.ts @@ -63,6 +63,9 @@ export interface AssignRefsResult { map: MessageRefMap; nextIndex: number; newlyAssigned: number; + /** Ids whose stale BLOCKED entry migrated to a fresh numeric ref this pass + * (#2607). The caller defers their carrier re-render (#3939 ②/#3805). */ + migratedIds: string[]; } export interface AssignRefsOptions { @@ -85,6 +88,7 @@ export function assignRefs( ? options.nextIndex : MIN_INDEX; let newlyAssigned = 0; + const migratedIds: string[] = []; for (const message of messages) { if (!message.id || options.shouldSkip?.(message)) continue; @@ -96,8 +100,10 @@ export function assignRefs( // is lifted (#2607: #1188 pinned all media; archivable images now fold // and must be citable via decompress({ imageRef })). Fresh number only — // the id-never-reused contract holds. - if (prev === BLOCKED_REF && !protectedNow) delete map.byRaw[message.id]; - else continue; + if (prev === BLOCKED_REF && !protectedNow) { + delete map.byRaw[message.id]; + migratedIds.push(message.id); + } else continue; } if (protectedNow) { @@ -112,7 +118,7 @@ export function assignRefs( newlyAssigned++; } - return { map, nextIndex: cursor, newlyAssigned }; + return { map, nextIndex: cursor, newlyAssigned, migratedIds }; } function allocateFreeRef( diff --git a/kernel/src/render-refs.ts b/kernel/src/render-refs.ts index 7338c4632f..6590d25843 100644 --- a/kernel/src/render-refs.ts +++ b/kernel/src/render-refs.ts @@ -62,10 +62,16 @@ function renderMessage( countTokens: (text: string) => number, strategy: RenderStrategy, snapshot: Record | null = null, + deferred: ReadonlySet | null = null, ): CoreMessage { const ref = refForRaw(map, message.id); if (!ref || ref === BLOCKED_REF) return message; + // #3939 ②/#3805: refs minted by the BLOCKED→numeric migration keep their + // pre-migration UNTAGGED bytes until the next fold boundary — stamping the + // tag now would rewrite an old message mid-history on a quiet turn. + if (deferred?.has(ref)) return message; + // "none": host reads the ref map directly — never pollute text. if (strategy === "none") return message; @@ -110,8 +116,11 @@ export function renderVisibleRefs( ): CoreMessage[] { // Legacy behavior: recompute tokens every render (snapshot = null). const map = state.messageRefs; + const deferred = state.renderDeferredRefs + ? new Set(state.renderDeferredRefs) + : null; return messages.map((message) => - renderMessage(message, map, countTokens, strategy), + renderMessage(message, map, countTokens, strategy, null, deferred), ); } @@ -132,8 +141,11 @@ export function renderWithSnapshot( ): RenderWithSnapshotResult { const map = state.messageRefs; const snapshot = { ...(state.tokenSnapshot ?? {}) }; + const deferred = state.renderDeferredRefs + ? new Set(state.renderDeferredRefs) + : null; const rendered = messages.map((message) => - renderMessage(message, map, countTokens, strategy, snapshot), + renderMessage(message, map, countTokens, strategy, snapshot, deferred), ); return { messages: rendered, tokenSnapshot: snapshot }; } diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index 76dc42c259..cb9564e82a 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -46,6 +46,32 @@ export function syncBlocks( nextBlockId: state.nextBlockId, nextRunId: state.nextRunId, deadRefs: state.deadRefs ? [...state.deadRefs] : undefined, + // #3769 orphan-cleanup ledgers — same carry set as cloneState: dropping any + // of these here would wipe them before the hide-compress-calls node reads + // them (this node runs first), silently disabling both the fold-gated sweep + // and the legacy hiddenOrphanRefs upgrade-compat clause every turn. + hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined, + sweptOrphanCallIds: state.sweptOrphanCallIds ? [...state.sweptOrphanCallIds] : undefined, + pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef, + // #3939 ③: the truncation latch must survive the sync node — dropping it + // here would re-expose clipped bytes mid-history after a sync carried the + // rest of the state (#3691 oscillation via the sync path). + emergencyTruncateLatch: state.emergencyTruncateLatch + ? state.emergencyTruncateLatch.map((e) => ({ ref: e.ref, text: e.text })) + : undefined, + // #3939 ②: the render deferral must survive sync — losing it would stamp + // migration tags mid-history after a sync carried the ref map (#3805). + renderDeferredRefs: state.renderDeferredRefs + ? [...state.renderDeferredRefs] + : undefined, + // #3939 ⑤: the view latch must survive sync — losing it re-derives + // transform output mid-history after a sync carried the state (#3805). + viewLatches: state.viewLatches + ? Object.fromEntries( + Object.entries(state.viewLatches).map(([k, v]) => [k, { ...v }]), + ) + : undefined, + viewLatchConfigDigest: state.viewLatchConfigDigest, }; // Refs are additive (assignRefs never removes them from messageRefs), so diff --git a/kernel/src/truncate-tools.ts b/kernel/src/truncate-tools.ts index 964bfc1b0d..25cebb433b 100644 --- a/kernel/src/truncate-tools.ts +++ b/kernel/src/truncate-tools.ts @@ -21,6 +21,9 @@ export interface TruncateResult { /** Size-qualified candidates found (pre-truncation). 0 explains a silent * no-op at/above threshold — hosts should warn-log (#300). */ candidatesFound: number; + /** Ids of the messages actually clipped this pass (#3939 ③ latch). Empty + * when truncatedCount is 0. */ + truncatedIds?: string[]; } const TRUNCATION_MARKER = "[truncated for context space]"; @@ -42,7 +45,7 @@ export function truncateLargeToolOutputs( const opts = { ...DEFAULTS, ...options }; const limit = config.modelContextLimit; if (limit <= 0 || tokenCount < config.truncate.threshold * limit) { - return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound: 0 }; + return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound: 0, truncatedIds: [] }; } const protectedIndex = messages.length - opts.protectRecentMessages; @@ -87,6 +90,7 @@ export function truncateLargeToolOutputs( let remaining = tokenCount; const targetTokens = config.truncate.threshold * limit * 0.9; const replacements = new Map(); + const truncatedIds: string[] = []; const applyTo = (candidates: CoreMessage[]) => { for (const candidate of candidates) { @@ -104,6 +108,7 @@ export function truncateLargeToolOutputs( ); const replacement = `${prefix}\n\n...${TRUNCATION_MARKER} — original ~${tokens} tokens]...\n\n${suffix}`; replacements.set(candidate.id, replacement); + truncatedIds.push(candidate.id); truncatedCount++; remaining -= tokens - countTokens(replacement); savedTokens += tokens - countTokens(replacement); @@ -114,7 +119,7 @@ export function truncateLargeToolOutputs( applyTo(textMessages); if (replacements.size === 0) { - return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound }; + return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound, truncatedIds: [] }; } return { messages: messages.map((m) => @@ -123,5 +128,6 @@ export function truncateLargeToolOutputs( truncatedCount, savedTokens, candidatesFound, + truncatedIds, }; } diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 22523eeea7..68e729582e 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -36,6 +36,13 @@ export type CompressionTier = 1 | 2 | 3; export type BlockGeneration = "young" | "old"; +/** One #3939 ③ emergency-truncation latch entry: the post-clip text of a + * message (keyed by its padded ref). See CompressionState.emergencyTruncateLatch. */ +export interface EmergencyTruncateLatchEntry { + ref: string; + text: string; +} + export interface CompressionBlock { blockId: string; runId: string; @@ -107,6 +114,12 @@ export interface CompressionStats { retrievalCount?: number; } +/** View-latch entry shape (#3939 ⑤) — see CompressionState.viewLatches. */ +export interface ViewLatchEntry { + text: string | null; + hidden?: boolean; +} + /** One instant tool-result absorption: the original tool-call + tool-result * pair is hidden from view once recorded; the model's absorb call (carrying * the distilled summary) remains as the durable record. */ @@ -241,6 +254,71 @@ export interface CompressionState { * this set and keeps the clear "cannot be anchored" error. Optional for * pre-feature persisted states. */ hiddenOrphanRefs?: string[]; + /** Persistent, MONOTONIC set of orphan compress-call ids permanently swept + * (#3769). An orphan (a failed compress call/result with no matching block) + * is only removed from the visible view when a REAL fold gives a cache-safe + * place to do it: the hide node sweeps orphans at/after the fold's earliest + * start ref onto this set, and they stay hidden forever — across the next + * turn, repeated processTurn calls, restarts and upgrades. Monotonic (never + * shrinks) so a previously-hidden record can never reappear once the client + * resends full history. Replaces the old keep-last-two rotation, which hid + * the oldest failure on EVERY quiet turn and moved the cached prefix + * breakpoint forward for no new fold. Optional: pre-feature persisted states + * lack it (nothing was permanently hidden under the old per-pass rule). */ + sweptOrphanCallIds?: string[]; + /** Numeric part (mNNNNN -> NNNNN) of the EARLIEST start ref of the most + * recent fold that has not yet been consumed by an orphan sweep (#3769). + * Written by applyCompression when it creates >=1 block (min over the newly + * created blocks' startRef); consumed + cleared by the hide-compress-calls + * node on the next processTurn, which then sweeps orphans with ref >= this + * value (orphans before it sit in the still-cached prefix and stay visible). + * Carries the position so the sweep is cache-neutral: it only removes records + * the fold already invalidated. Optional: absent until the first post-fix fold. */ + pendingOrphanSweepMinRef?: number; + /** Emergency-truncation latch (#3939 ③/#3691): the clipped texts (post-clip + * bytes, keyed by padded ref) of messages clipped by the emergency-truncate + * node. The watermark decision is made ONCE — replaying these clips on every + * later turn keeps the outbound bytes stable when the token estimate + * fluctuates across the threshold (the #3691 full↔clipped oscillation was a + * mid-history rewrite on every request). Entries retire when their message + * leaves the resent view (folded/edited-away) and the whole latch resets on + * the next successful fold (applyCompression) — after a fold the watermark + * re-evaluates from scratch. Optional: pre-feature states have none (the + * pre-latch node clipped from scratch each turn). */ + emergencyTruncateLatch?: EmergencyTruncateLatchEntry[]; + /** Ref-migration render deferral (#3939 ②/#3805): padded refs minted by the + * BLOCKED→numeric migration (assignRefs, #2607). The number exists (citable + * via decompress({ imageRef }) bookkeeping) but the carrier message keeps + * its pre-migration, UNTAGGED bytes until the next successful fold — + * stamping the tag on the migration turn would rewrite an old + * message mid-history on an otherwise quiet turn (a prefix-cache breaker: + * #3805 durable carriers flipping bytes with no fold in sight). Cleared by + * applyCompression on fold success; entries are inert once their message + * leaves the view (ids are never re-issued). Optional: absent until the + * first migration. */ + renderDeferredRefs?: string[]; + /** Fold-gated view latch (#3939 ⑤): post-transform view bytes per padded + * ref, replayed VERBATIM on quiet turns. The opt-in mid-history rewriters + * (absorb-hide, crush, filter keepLastOnly) re-derive their output every + * pass from state/context (absorb records growing, crush responding to + * token pressure, keep-last-only demotions as new occurrences arrive) — + * without a latch each change flips OLD message bytes on an otherwise + * quiet turn (the #3805 prefix-cache breaker class). Structural, not + * per-feature: every transform output is latched at first application and + * replays until the next successful fold (applyCompression resets the + * whole map — the refresh rides the fold's own history rewrite); new or + * edited messages (new ref) latch fresh at their arrival boundary. ccr + * needs no latch (replace-once-at-arrival, byte-stable by design #1460). + * Entry: text=null = passthrough (source bytes stand); text=string = + * transformed bytes; hidden=true = dropped from the view. Optional: + * absent until the first latched transform. */ + viewLatches?: Record; + /** #3939 ⑤: digest of the transform-relevant config slices (absorb/crush/ + * message-filters). A config edit is a user action boundary — matching + * ①'s fold/refold/user-action rule, the latch releases when the digest + * changes so enabling a transform applies on the next turn (one rewrite + * riding the config change), never on a quiet turn. */ + viewLatchConfigDigest?: string; /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk diff --git a/kernel/src/view-latch.ts b/kernel/src/view-latch.ts new file mode 100644 index 0000000000..2ca2d2100e --- /dev/null +++ b/kernel/src/view-latch.ts @@ -0,0 +1,163 @@ +import { BLOCKED_REF } from "./refs.js"; +import { RETRIEVED_ID_PREFIX } from "./ccr.js"; +import { resolveAbsorbConfig } from "./absorb.js"; +import { resolveCcrConfig } from "./ccr.js"; +import { resolveCrushConfig } from "./crush.js"; +import type { CompressionState, CoreMessage, ViewLatchEntry } from "./types.js"; +import type { NodeIO, PipelineNode, PipelineContext } from "./pipeline.js"; + +/** Stash the post-prune, pre-transform view (#3939 ⑤). The view-latch node + * replays latched bytes from this array — the transform nodes have already + * dropped or rewritten their targets by the time the latch runs, so the + * source-of-truth resent bytes live here. */ +export const preTransformSnapshotNode: PipelineNode = { + name: "pre-transform-snapshot", + run(io) { + return { + ...io, + effects: { ...io.effects, preTransformMessages: io.messages }, + }; + }, +}; + +/** Fold-gated view latch (#3939 ⑤/#3805): replay post-transform bytes + * verbatim on quiet turns; refresh latches only at fold boundaries. Runs + * after every opt-in transform node (ccr-store, absorb-hide, crush, + * absorb-prompt, filter) and after hide-compress-calls (its own #3784 fold + * gate stays authoritative — the latch only records its output), before + * recommend/nudge/emergency-truncate/render so downstream nodes judge the + * same view the model actually sees. */ +export const viewLatchNode: PipelineNode = { + name: "view-latch", + run(io, ctx) { + return applyViewLatch(io, ctx); + }, +}; + +function latchKeyEqual( + a: Record | undefined, + b: Record | undefined, +): boolean { + if (!a || !b) return !a && !b; + if (Object.keys(a).length !== Object.keys(b).length) return false; + for (const [k, v] of Object.entries(a)) { + const o = b[k]; + if (!o || o.text !== v.text || o.hidden !== v.hidden) return false; + } + return true; +} + +/** Transform-relevant config slices — a change here is a user action + * boundary (①): the latch releases so enable/disable applies immediately + * instead of waiting for a fold. tokenCount is deliberately NOT included + * (pressure fluctuation is what the latch exists to gate). */ +export function viewLatchConfigDigest(ctx: PipelineContext): string { + return JSON.stringify({ + absorb: resolveAbsorbConfig(ctx.config), + crush: resolveCrushConfig(ctx.config), + // CCR rides the same boundary: a session that disarms CCR must never + // keep projecting latched [acp-stored] placeholders — the retrieve + // channel they advertise would be gone (dangling placeholder = silent + // loss, the #1273 plugin-governance contract). A digest change releases + // the latch and re-judges every message under the CURRENT config. + ccr: resolveCcrConfig(ctx.config), + filters: { + enabled: ctx.config.messageFilters?.enabled ?? false, + filters: ctx.config.messageFilters?.filters ?? null, + }, + }); +} + +export function applyViewLatch(io: NodeIO, ctx?: PipelineContext): NodeIO { + const pre = io.effects.preTransformMessages; + if (!pre) return io; + // Config boundary (#3939 ①/⑤): digest change releases the latch — the + // transform enable/disable applies this turn (one rewrite riding the user's + // config edit), never silently later. + const digest = ctx ? viewLatchConfigDigest(ctx) : undefined; + const configChanged = + digest !== undefined && digest !== io.state.viewLatchConfigDigest; + const prev = configChanged ? undefined : io.state.viewLatches; + const post = new Map(); + for (const m of io.messages) if (m.id) post.set(m.id, m); + + const out: CoreMessage[] = []; + const latches: Record = {}; + let latchedTransform = false; + + for (const m of pre) { + if (!m.id || m.id.startsWith(RETRIEVED_ID_PREFIX)) { + const t = post.get(m.id!); + if (t) out.push(t); + continue; + } + const ref = io.state.messageRefs.byRaw[m.id]; + if (!ref || ref === BLOCKED_REF) { + const t = post.get(m.id); + if (t) out.push(t); + continue; + } + const transformed = post.get(m.id); + const prevEntry = prev?.[ref]; + if (prevEntry) { + // REPLAY the latched bytes — a transform's fresh output for an + // already-latched ref is deferred to the next fold boundary. + if (prevEntry.hidden) { + latches[ref] = prevEntry; + continue; + } + if (prevEntry.text === null) { + if (transformed) out.push(m); + latches[ref] = prevEntry; + continue; + } + latchedTransform = true; + if (transformed && transformed.text === prevEntry.text) out.push(transformed); + else out.push({ ...m, text: prevEntry.text }); + latches[ref] = prevEntry; + } else { + // FRESH: latch whatever the transforms produced at this arrival + // boundary (arrival/first-enable IS a history change; the flip rides it). + if (!transformed) { + latches[ref] = { text: null, hidden: true }; + continue; + } + if (transformed.text !== m.text) { + latches[ref] = { text: transformed.text ?? null }; + latchedTransform = true; + out.push(transformed); + } else { + latches[ref] = { text: null }; + out.push(m); + } + } + } + + // Steady state (identical latch map) must not churn the state object and + // force an adapter save every turn (render-refs snapshot precedent). + if ( + !configChanged && + digest === io.state.viewLatchConfigDigest && + latchKeyEqual(io.state.viewLatches, latches) + ) { + return { ...io, messages: out }; + } + // No prior latches and every entry passthrough → nothing worth pinning + // (no opt-in transform active); skip the write entirely. + if (!prev && !latchedTransform) return io; + return { + ...io, + messages: out, + state: { + ...io.state, + viewLatches: latches, + ...(digest !== undefined ? { viewLatchConfigDigest: digest } : {}), + }, + }; +} + +/** #3939 ⑤: a successful fold rewrote history — the latch refresh rides it. */ +export function resetViewLatches(state: CompressionState): CompressionState { + if (state.viewLatches === undefined) return state; + return { ...state, viewLatches: undefined }; +} diff --git a/kernel/tests/compress.test.ts b/kernel/tests/compress.test.ts index 7f65c0bae5..0d8383d033 100644 --- a/kernel/tests/compress.test.ts +++ b/kernel/tests/compress.test.ts @@ -486,7 +486,7 @@ test("applyCompression reports error for unknown boundary ref", () => { assert.equal(result.result.errors.length, 1); }); -test("batch compress attributes per-range errors and keeps partial success", () => { +test("batch compress rejects the whole batch when a summary fails length validation (#3003)", () => { const core = createCore(); const state = createInitialState(); const messages = [ @@ -526,11 +526,16 @@ test("batch compress attributes per-range errors and keeps partial success", () }), }); - assert.equal(result.result.blocksCreated, 1, "valid range still compresses"); + // #3003: a length-class failure in ANY candidate rejects the WHOLE batch — + // the valid sibling no longer lands (that partial commit opened a seam that + // the retry then paid twice). All-or-nothing means zero blocks here. + assert.equal(result.result.blocksCreated, 0, "no sibling may land while one summary fails length"); + assert.equal(result.state.blocks.length, 0); assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); assert.match( result.result.errors[0]!, - /^range m00003\.\.m00004: Summary too short \(22 chars, min 50\)/, + /range m00003\.\.m00004: Summary too short \(22 chars, min 50\)/, ); }); @@ -1024,9 +1029,11 @@ test("empty summary is attributed to its range", () => { assert.equal(result.result.blocksCreated, 0); assert.equal(result.result.errors.length, 1); + // #3003: single-range length failures now surface through the all-or-nothing + // batch gate; the per-range detail is embedded verbatim in that message. assert.match( result.result.errors[0]!, - /^range m00001\.\.m00002: Summary is empty/, + /range m00001\.\.m00002: Summary is empty/, ); }); diff --git a/kernel/tests/hide-consumed-mcp.test.ts b/kernel/tests/hide-consumed-mcp.test.ts index fdeb460487..0f618de67c 100644 --- a/kernel/tests/hide-consumed-mcp.test.ts +++ b/kernel/tests/hide-consumed-mcp.test.ts @@ -89,8 +89,11 @@ test("#2970 L2: active synthetic-id carrier survives beyond the orphan window", const r = hideConsumedCompressCalls(state, msgs, isMcp); const kept = new Set(r.messages.map((m) => m.toolCallId)); assert.ok(kept.has("tool_a"), "active synthetic-id carrier must survive"); - assert.ok(!kept.has("tool_b"), "oldest true orphan still capped at two"); - assert.ok(kept.has("tool_c") && kept.has("tool_d"), "newest two orphans kept"); + assert.equal(r.hidden, 0, "no fold pending -> nothing swept (#3769)"); + assert.ok( + kept.has("tool_b") && kept.has("tool_c") && kept.has("tool_d"), + "orphans remain visible until a real fold", + ); }); test("#2970 L2: inactive synthetic-id carrier is hidden (stale receipt cleanup)", () => { @@ -112,7 +115,7 @@ test("#2970 L2: inactive synthetic-id carrier is hidden (stale receipt cleanup)" assert.equal(r.hidden, 2, "call + result removed"); }); -test("#2970/#2362: mcp__bili__compress orphans keep only the newest two", () => { +test("#2970/#2362: mcp__bili__compress orphans stay visible between folds (#3769)", () => { const state = createInitialState(); const msgs: CoreMessage[] = []; for (const id of ["o1", "o2", "o3", "o4"]) { @@ -121,8 +124,11 @@ test("#2970/#2362: mcp__bili__compress orphans keep only the newest two", () => const r = hideConsumedCompressCalls(state, msgs, isMcp); const kept = new Set(r.messages.map((m) => m.toolCallId)); - assert.ok(!kept.has("o1") && !kept.has("o2"), "older orphans hidden"); - assert.ok(kept.has("o3") && kept.has("o4"), "newest two kept"); + assert.equal(r.hidden, 0, "quiet turn sweeps nothing (#3769)"); + assert.ok( + ["o1", "o2", "o3", "o4"].every((id) => kept.has(id)), + "all orphans remain visible until a real fold (#3769)", + ); }); test("#2970: active synthetic-id carrier's long summary is stubbed", () => { diff --git a/kernel/tests/hide-merge-rebuild.test.ts b/kernel/tests/hide-merge-rebuild.test.ts index 96d5e9b4a9..f0a464dedc 100644 --- a/kernel/tests/hide-merge-rebuild.test.ts +++ b/kernel/tests/hide-merge-rebuild.test.ts @@ -27,7 +27,7 @@ function block(overrides: Partial): CompressionBlock { }; } -test("hideConsumedCompressCalls keeps active-block calls, hides consumed, keeps newest two orphans", () => { +test("hideConsumedCompressCalls keeps active-block calls and orphans until a fold (#3769)", () => { const state: CompressionState = { ...createInitialState(), blocks: [ @@ -70,20 +70,21 @@ test("hideConsumedCompressCalls keeps active-block calls, hides consumed, keeps { id: "m5", role: "user", contentType: "text", text: "hello" }, ]; const result = hideConsumedCompressCalls(state, messages); - // active-block call kept; consumed hidden; newest KEEP_LAST_ORPHANED=2 - // orphans kept visible (failure observability — billion-context-pi #9). - assert.equal(result.hidden, 1); + assert.equal(result.hidden, 0, "no fold pending -> nothing swept (#3769)"); const remainingCallIds = result.messages .filter((m) => m.toolName === "compress") .map((m) => m.toolCallId); - assert.ok(remainingCallIds.includes("call-active")); - assert.ok(!remainingCallIds.includes("call-consumed")); + assert.ok(remainingCallIds.includes("call-active"), "active carrier kept"); + assert.ok( + remainingCallIds.includes("call-consumed"), + "orphan stays visible between folds", + ); assert.ok(remainingCallIds.includes("call-orphan1")); assert.ok(remainingCallIds.includes("call-orphan2")); assert.ok(result.messages.some((m) => m.text === "hello")); }); -test("orphaned compress residue is bounded: only the newest two pairs survive any session length", () => { +test("orphan residue is swept onto a committed fold, not rotated on quiet turns (#3769)", () => { const state: CompressionState = { ...createInitialState(), blocks: [ @@ -125,22 +126,37 @@ test("orphaned compress residue is bounded: only the newest two pairs survive an text: "tail", }); + messages.forEach((m, i) => { + state.messageRefs.byRaw[m.id] = `m${String(i + 1).padStart(5, "0")}`; + }); + state.pendingOrphanSweepMinRef = 10; + const result = hideConsumedCompressCalls(state, messages); - // 12 failed pairs in → only the newest 2 pairs + the active call remain: - // PR #18's unbounded accumulation cannot recur under KEEP_LAST_ORPHANED=2. - assert.equal(result.hidden, 20); + assert.equal( + result.hidden, + 16, + "orphans at/after fold start (ref 10) swept; before it kept (#3769)", + ); + assert.deepEqual(result.newlySweptCallIds, [ + "call-fail5", + "call-fail6", + "call-fail7", + "call-fail8", + "call-fail9", + "call-fail10", + "call-fail11", + "call-fail12", + ]); const remainingCallIds = result.messages .filter((m) => m.toolName === "compress" && m.contentType === "tool-call") .map((m) => m.toolCallId); assert.deepEqual(remainingCallIds, [ "call-active", - "call-fail11", - "call-fail12", + "call-fail1", + "call-fail2", + "call-fail3", + "call-fail4", ]); - const remainingResultIds = result.messages - .filter((m) => m.contentType === "tool-result") - .map((m) => m.toolCallId); - assert.deepEqual(remainingResultIds, ["call-fail11", "call-fail12"]); }); test("a failed compress call and its result stay visible (issue #9 fixed-point precondition removed)", () => { diff --git a/kernel/tests/issue3003-all-or-nothing-length.test.ts b/kernel/tests/issue3003-all-or-nothing-length.test.ts new file mode 100644 index 0000000000..6e8b942945 --- /dev/null +++ b/kernel/tests/issue3003-all-or-nothing-length.test.ts @@ -0,0 +1,253 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { createInitialState } from "../src/state.js"; +import { prune } from "../src/prune.js"; +import { assignRefs } from "../src/refs.js"; +import type { Config, CoreMessage } from "../src/types.js"; + +// #3003: PARTIAL fold double-seam cost. When a multi-range compress batch has +// one length-class failure (summary over/under cap) among otherwise-valid +// folds, the old per-range commit landed the valid siblings FIRST (opening a +// seam) and the model then retried the bad range into a SECOND seam. The kernel +// now screens summary LENGTH all-or-nothing BEFORE any block commits: any +// length-class failure rejects the WHOLE batch (0 commits, 0 seams), so one +// retry pays once. Content-class failures (protected-zone / turn-integrity / +// no-new-messages) stay per-range. These tests pin both directions of the gate +// (#3617): it fires on a candidate's length failure, and it does NOT fire for +// a non-candidate range whose summary merely happens to be out of range. + +function msg( + id: string, + text: string, + role: CoreMessage["role"] = "user", +): CoreMessage { + return { id, role, contentType: "text", text }; +} + +function config(overrides: Partial = {}): Config { + return { + tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 }, + nudge: { + maxContextLimitPct: 0.55, + minContextLimitPct: 0.45, + frequency: 5, + iterationThreshold: 15, + force: "soft", + growthRatio: 0.05, + growthFloor: 6000, + growthCap: 50000, + minGrowthFloor: 5000, + minGrowthRatio: 0.45, + emergencyThresholdPct: 0.98, + }, + promotionThreshold: 5, + truncate: { threshold: 1 }, + merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 }, + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + protectedTools: [], + preserveRecentMessages: 0, + preserveRecentTokens: 0, + modelContextLimit: 100000, + ...overrides, + }; +} + +test("a too-long summary in a multi-range batch rejects the WHOLE batch (#3003)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // Range A's summary exceeds maxSummaryLength (20); range B is within range. + // Without the pre-check, B would land (seam #1) and A would retry (seam #2). + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "x".repeat(40), topic: "too long" }, + { startRef: "m00003", endRef: "m00004", summary: "ok", topic: "fine" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 0, "nothing may commit"); + assert.equal(result.state.blocks.length, 0, "no block may exist after rejection"); + assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + result.result.errors[0]!, + /range m00001\.\.m00002: Summary too long \(40 chars, max 20\)/, + ); + assert.match(result.result.errors[0]!, /Nothing was folded, so no prefix was rewritten/); + // Both actionable exits must be offered. + assert.match(result.result.errors[0]!, /shorten or split/); + assert.match(result.result.errors[0]!, /drop the offending range\(s\) and resend only the rest/); +}); + +test("a too-short summary in a multi-range batch rejects the WHOLE batch (#3003)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "y".repeat(60), topic: "fine" }, + { startRef: "m00003", endRef: "m00004", summary: "x".repeat(5), topic: "too short" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 50 }, + }), + }); + + assert.equal(result.result.blocksCreated, 0, "the valid sibling must NOT land either"); + assert.equal(result.state.blocks.length, 0); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + result.result.errors[0]!, + /range m00003\.\.m00004: Summary too short \(5 chars, min 50\)/, + ); +}); + +test("a multi-range batch whose summaries all validate still commits every range (#3003 control)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "ok-a", topic: "a" }, + { startRef: "m00003", endRef: "m00004", summary: "ok-b", topic: "b" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 2, "both valid ranges commit"); + assert.equal(result.state.blocks.length, 2); + assert.equal(result.result.errors.length, 0); +}); + +test("a non-candidate range with an oversized summary does NOT reject a valid fresh fold (#3003 scoping)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [msg("a", "alpha"), msg("b", "beta")]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // m00099..m00100 are valid-format but never allocated -> status "unknown", + // never a fold candidate. Its oversized summary must be ignored by the + // all-or-nothing length screen (checking it would wrongly kill the batch). + const result = core.applyCompression({ + ranges: [ + { startRef: "m00099", endRef: "m00100", summary: "z".repeat(500) }, + { startRef: "m00001", endRef: "m00002", summary: "ok", topic: "fine" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 1, "the valid candidate still commits"); + assert.equal(result.state.blocks.length, 1); + assert.doesNotMatch( + result.result.errors.join("\n"), + /Batch rejected before folding any range/, + ); + // The unknown ref is still reported per-range (content-class, unchanged). + assert.match(result.result.errors.join("\n"), /m00099/); +}); + +test("a too-long refold summary rejects the WHOLE batch incl. a fresh sibling (#3003 refold direction)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // First fold m00001..m00002 into a block. + const first = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "first fold", topic: "t" }, + ], + messages, + state, + config: config(), + }); + assert.equal(first.result.blocksCreated, 1); + const blockId = first.state.blocks[0]!.blockId; + + // Restore it inline so re-requesting its span classifies as a refold candidate. + first.state.blocks.find((b) => b.blockId === blockId)!.restoredInline = true; + + // Second call: refold with an OVERSIZED new summary + a fresh valid sibling. + // Without the pre-check the fresh sibling would land first (seam #1) and the + // refold retry would pay seam #2; the gate must reject before either commits. + const second = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "z".repeat(400) }, + { startRef: "m00003", endRef: "m00004", summary: "fresh ok", topic: "sib" }, + ], + messages, + state: first.state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(second.result.blocksCreated, 0, "nothing may commit"); + assert.equal(second.state.blocks.length, 1, "only the pre-existing block remains"); + assert.match(second.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + second.result.errors[0]!, + /range m00001\.\.m00002: Summary too long \(400 chars, max 20\)/, + ); + // Refold atomicity: the restored block keeps its old summary (no partial + // in-place update may precede the batch rejection). + const b1 = second.state.blocks.find((b) => b.blockId === blockId)!; + assert.equal(b1.summary, "first fold"); +}); diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts new file mode 100644 index 0000000000..f5765195c4 --- /dev/null +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -0,0 +1,374 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { assignRefs } from "../src/refs.js"; +import { hideConsumedCompressCalls } from "../src/hide-consumed.js"; +import { createInitialState } from "../src/state.js"; +import type { CompressionState, Config, CoreMessage } from "../src/types.js"; + +// #3769 regression: orphaned failed compress calls are hidden only where a committed +// fold already broke the cached prefix (at/after its earliest start ref), tracked in a +// persistent monotonic set — never rotated away on a quiet turn. These cases pin that timing. + +const isMcp = (n: string) => n === "compress" || n === "mcp__bili__compress"; + +function msg(id: string, text: string): CoreMessage { + return { id, role: "user", contentType: "text", text }; +} + +function failPair(callId: string): [CoreMessage, CoreMessage] { + return [ + { + id: `c-${callId}`, + role: "assistant", + contentType: "tool-call", + toolName: "mcp__bili__compress", + toolCallId: callId, + text: JSON.stringify({ content: [{ startId: "mA", endId: "mB", summary: "s" }] }), + }, + { + id: `r-${callId}`, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text: "nothing to compress", + }, + ]; +} + +function seqRefs(state: CompressionState, messages: CoreMessage[]): void { + messages.forEach((m, i) => { + state.messageRefs.byRaw[m.id] = `m${String(i + 1).padStart(5, "0")}`; + }); +} + +function config(): Config { + return { + tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 }, + nudge: { + maxContextLimitPct: 0.55, + minContextLimitPct: 0.45, + frequency: 5, + iterationThreshold: 15, + force: "soft", + growthRatio: 0.05, + growthFloor: 6000, + growthCap: 50000, + minGrowthFloor: 5000, + minGrowthRatio: 0.45, + emergencyThresholdPct: 0.98, + }, + promotionThreshold: 5, + truncate: { threshold: 1 }, + merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 }, + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + protectedTools: [], + preserveRecentMessages: 0, + preserveRecentTokens: 0, + modelContextLimit: 100000, + }; +} + +test("a third failed compress appended to history does NOT rewrite mid-history on a quiet turn (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early context")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid context")); + messages.push(...failPair("f2")); + messages.push(msg("p3", "newer context")); + messages.push(...failPair("f3")); + seqRefs(state, messages); + + const inbound = messages.map((m) => m.id); + const r = hideConsumedCompressCalls(state, messages, isMcp); + + assert.equal(r.hidden, 0, "quiet turn sweeps nothing"); + assert.equal(r.newlySweptCallIds.length, 0, "no fold -> nothing newly swept"); + assert.deepEqual( + r.messages.map((m) => m.id), + inbound, + "outbound ids identical to inbound: oldest failure NOT rotated away mid-history", + ); +}); + +test("a fold sweeps orphans at/after its start but keeps earlier ones in the cached prefix (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid")); + messages.push(...failPair("f2")); + messages.push(msg("p3", "late")); + messages.push(...failPair("f3")); + seqRefs(state, messages); + // refs: p1=1 f1c=2 f1r=3 p2=4 f2c=5 f2r=6 p3=7 f3c=8 f3r=9 + state.pendingOrphanSweepMinRef = 5; + + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f2", "f3"], "only orphans at/after fold start swept"); + const ids = r.messages.map((m) => m.id); + assert.ok(ids.includes("c-f1"), "orphan before fold start stays visible"); + assert.ok(!ids.includes("c-f2") && !ids.includes("c-f3"), "orphans at/after fold start removed"); +}); + +test("once swept, an orphan stays hidden on later quiet turns (persistent, restart-safe) (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid")); + messages.push(...failPair("f2")); + seqRefs(state, messages); + + state.pendingOrphanSweepMinRef = 5; + const first = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(first.newlySweptCallIds, ["f2"], "the fold sweep removes only f2"); + + state.sweptOrphanCallIds = [...first.newlySweptCallIds]; + state.pendingOrphanSweepMinRef = undefined; + + const second = hideConsumedCompressCalls(state, messages, isMcp); + assert.equal(second.newlySweptCallIds.length, 0, "quiet turn adds nothing"); + const ids = second.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f2"), "previously-swept orphan stays hidden"); + assert.ok(ids.includes("c-f1"), "unswept orphan remains visible"); +}); + +test("failures do not roll-clean per attempt: N quiet turns sweep nothing, one fold sweeps once (#3769)", () => { + const base: CoreMessage[] = [msg("p1", "early")]; + for (let t = 1; t <= 3; t++) { + const state = createInitialState(); + const messages: CoreMessage[] = [...base]; + for (let k = 1; k <= t; k++) messages.push(...failPair(`f${k}`)); + seqRefs(state, messages); + + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.equal(r.hidden, 0, `turn ${t}: no fold -> nothing swept despite ${t} failures`); + assert.equal(r.newlySweptCallIds.length, 0, `turn ${t}: no new sweep`); + } + + const state = createInitialState(); + const messages: CoreMessage[] = [...base]; + for (let k = 1; k <= 3; k++) messages.push(...failPair(`f${k}`)); + seqRefs(state, messages); + state.pendingOrphanSweepMinRef = 1; + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f1", "f2", "f3"], "one fold sweeps all eligible orphans at once"); +}); + +test("a real fold writes the sweep marker and the next pass removes the orphan behind it (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("h1", "first detailed work item alpha content"), + msg("h2", "second detailed work item beta content"), + msg("h3", "third detailed work item gamma content"), + ...failPair("f1"), + msg("h4", "fourth detailed work item delta content"), + msg("last", "latest user intent theta"), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const out = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00003", summary: "fold the spent head", topic: "head" }], + messages, + state, + config: config(), + }); + assert.equal(out.result.errors.length, 0, `unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, "the head genuinely folds"); + assert.equal(out.state.pendingOrphanSweepMinRef, 1, "marker points at the fold's earliest start ref"); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f1"], "orphan behind the fold is swept"); + assert.ok(!r.messages.some((m) => m.toolCallId === "f1"), "swept orphan removed from outbound"); +}); + +test("a block-boundary fold marks the sweep by covered message refs, not the block number (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("p1", "early cached prefix alpha"), + ...failPair("f1"), + msg("p2", "mid detailed work item beta content"), + msg("p3", "mid detailed work item gamma content"), + msg("p4", "mid detailed work item delta content"), + msg("p5", "mid detailed work item epsilon content"), + msg("p6", "latest user intent theta"), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + // refs: p1=1 c-f1=2 r-f1=3 p2=4 p3=5 p4=6 p5=7 p6=8 + + const first = core.applyCompression({ + ranges: [{ startRef: "m00004", endRef: "m00005", summary: "fold the mid pair", topic: "mid" }], + messages, + state, + config: config(), + }); + assert.equal(first.result.errors.length, 0, `unexpected errors: ${first.result.errors.join("; ")}`); + assert.ok(first.result.blocksCreated >= 1, "the mid pair folds"); + const blockId = first.state.blocks[first.state.blocks.length - 1].blockId; + + const out = core.applyCompression({ + ranges: [{ startRef: blockId, endRef: "m00006", summary: "distill the block plus its tail", topic: "distill" }], + messages, + state: first.state, + config: config(), + }); + assert.equal(out.result.errors.length, 0, `unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, "the block-boundary range folds"); + // Parsing the new block's startRef string ("b1") yields the BLOCK number 1, which + // would sweep the cached-prefix orphan f1 (ref 2). Covered-ref derivation must say 4. + assert.equal(out.state.pendingOrphanSweepMinRef, 4, "marker sits at the earliest covered message ref"); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, [], "cached-prefix orphan before the fold is untouched"); + assert.ok(r.messages.some((m) => m.toolCallId === "f1"), "f1 remains visible"); +}); + +test("an upgraded pre-fold-gate state keeps its rotation-hidden orphan hidden (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = [ + msg("p1", "early"), + ...failPair("f1"), + msg("p2", "mid"), + ...failPair("f2"), + msg("p3", "late"), + ...failPair("f3"), + ]; + seqRefs(state, messages); + // refs: p1=1 c-f1=2 r-f1=3 p2=4 c-f2=5 r-f2=6 p3=7 c-f3=8 r-f3=9 + + // Exact shape of a state the old keep-two kernel persisted: with three failures + // the rotation had already removed f1 from mid-history and remembered its refs. + // sweptOrphanCallIds / pendingOrphanSweepMinRef do not exist yet. + state.hiddenOrphanRefs = ["m00002", "m00003"]; + + const first = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(first.newlySweptCallIds, [], "nothing newly swept without a fold"); + const ids = first.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f1") && !ids.includes("r-f1"), "previously-hidden pair does not resurface on upgrade"); + assert.ok(ids.includes("c-f2") && ids.includes("c-f3"), "recent failures stay visible"); + + // Pipeline write-back (hide-compress-calls node) then another quiet turn: the + // legacy set self-perpetuates through rememberHiddenRef, no new field required. + state.hiddenOrphanRefs = first.hiddenOrphanRefs; + const second = hideConsumedCompressCalls(state, messages, isMcp); + const ids2 = second.messages.map((m) => m.id); + assert.ok(!ids2.includes("c-f1") && !ids2.includes("r-f1"), "still hidden on the next quiet turn"); +}); + +test("a re-sent pair quoting an active block's range is its carrier: the fold sweep leaves it visible (#3769/#1567)", () => { + // Plugin-style fold: the block is minted a server-side callId the client can + // never echo, so the client's re-sent pair for that exact fold is fate=orphan + // by id — yet it IS the block's summary carrier (the host strips the in-place + // anchor only while the pair rides the history). Sweeping it would strand the + // fold with zero wire carriers. Both arg shapes must be recognized, mirroring + // the host carrier index (flat {startId,endId} AND {content:[…]}). + const core = createCore(); + const build = (callId: string, callText: string): CoreMessage[] => [ + msg("h1", "first detailed work item alpha content"), + msg("h2", "second detailed work item beta content"), + msg("h3", "third detailed work item gamma content"), + { + id: `c-${callId}`, + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: callId, + text: callText, + }, + { + id: `r-${callId}`, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text: "compressed the head", + }, + msg("last", "latest user intent theta"), + ]; + const fold = (messages: CoreMessage[], state: CompressionState) => + core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00003", summary: "fold the spent head", topic: "head", compressCallId: "plugin_1234" }, + ], + messages, + state, + config: config(), + }); + + const carriers: Array<[string, string]> = [ + ["flat {startId,endId}", JSON.stringify({ startId: "m00001", endId: "m00003", topic: "head", summary: "fold the spent head" })], + ["{content:[…]} array", JSON.stringify({ content: [{ startId: "m00001", endId: "m00003", topic: "head", summary: "fold the spent head" }] })], + ]; + for (const [label, callText] of carriers) { + const messages = build("call_echo_1", callText); + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; + const out = fold(messages, state); + assert.equal(out.result.errors.length, 0, `${label}: unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, `${label}: the head genuinely folds`); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, [], `${label}: carrier pair exempt from the fold sweep`); + assert.ok(r.messages.some((m) => m.toolCallId === "call_echo_1"), `${label}: carrier pair stays visible (#1567)`); + } + + // Negative control: a failure whose quoted range matches NO block is still swept. + const messages = build( + "call_fail_9", + JSON.stringify({ content: [{ startId: "mX", endId: "mY", summary: "s" }] }), + ); + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; + const out = fold(messages, state); + assert.equal(out.result.errors.length, 0, `negative: unexpected errors: ${out.result.errors.join("; ")}`); + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["call_fail_9"], "true failure residue at/after the fold is swept"); +}); + +test("the fold marker survives the full pipeline: sync-blocks carries it to the hide node (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("p1", "early cached prefix alpha"), + ...failPair("f2"), + msg("p2", "mid detailed work item beta content"), + ]; + const state = createInitialState(); + seqRefs(state, messages); + // refs: p1=1 c-f2=2 r-f2=3 p2=4 — a fold starting at ref 2 committed last turn. + // Before the sync-blocks carry fix this marker died on the FIRST pipeline node + // (syncBlocks rebuilt state without the #3769 fields), so the sweep never fired. + state.pendingOrphanSweepMinRef = 2; + + const cfg = { ...config(), isCompressTool: isMcp }; + const turn = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 1000, + renderTags: "none", + }); + + const ids = turn.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f2") && !ids.includes("r-f2"), "orphan at/after fold start swept inside the pipeline"); + assert.deepEqual(turn.state.sweptOrphanCallIds, ["f2"], "swept id folded into the persistent set"); + assert.equal(turn.state.pendingOrphanSweepMinRef, undefined, "one-shot marker consumed"); + + const quiet = core.processTurn({ + messages: [...messages, msg("p3", "latest user intent theta")], + state: turn.state, + config: cfg, + tokenCount: 1000, + renderTags: "none", + }); + const ids2 = quiet.messages.map((m) => m.id); + assert.ok(!ids2.includes("c-f2"), "previously-swept orphan stays hidden on the next pipeline pass"); +}); diff --git a/kernel/tests/issue3939-fold-boundary-pins.test.ts b/kernel/tests/issue3939-fold-boundary-pins.test.ts index 2ed635b6d7..2b0dda2aa1 100644 --- a/kernel/tests/issue3939-fold-boundary-pins.test.ts +++ b/kernel/tests/issue3939-fold-boundary-pins.test.ts @@ -93,33 +93,39 @@ function liveBlockState(): { state: ReturnType } { return { state }; } -test("pin: hide-consumed keeps only the newest 2 orphaned compress calls (KEEP_LAST_ORPHANED=2)", () => { +test("pin: between folds no orphan is swept — quiet turns keep every failed compress call visible (#3769 fold gate)", () => { const messages = orphanHistory(); const res = hideConsumedCompressCalls(createInitialState(), messages); const keptCalls = res.messages .filter((m) => m.contentType === "tool-call") .map((m) => m.toolCallId); - assert.deepEqual(keptCalls, ["c3", "c4"]); - assert.equal(res.messages.length, 4); + // The pre-#3769 keep-last-two rotation hid the oldest failures on EVERY + // turn, rewriting the still-cached prefix for a few hundred chars. Now the + // window only sweeps once a committed fold stamps pendingOrphanSweepMinRef. + assert.deepEqual(keptCalls, ["c1", "c2", "c3", "c4"]); + assert.equal(res.messages.length, 8); + assert.equal(res.hidden, 0); + assert.deepEqual(res.newlySweptCallIds, []); }); -test("pin: hidden orphan calls report their refs when state has them assigned", () => { +test("pin: a pending fold sweep hides orphans at/after its min ref and reports their refs (#3769)", () => { const messages = orphanHistory(); const state = createInitialState(); state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1, }).map; + state.pendingOrphanSweepMinRef = 1; const res = hideConsumedCompressCalls(state, messages); - assert.deepEqual(res.hiddenOrphanRefs, [ - "m00001", - "m00002", - "m00003", - "m00004", - ]); + assert.equal(res.hidden, 8); + assert.deepEqual( + res.hiddenOrphanRefs, + ["m00001", "m00002", "m00003", "m00004", "m00005", "m00006", "m00007", "m00008"], + ); + assert.deepEqual(res.newlySweptCallIds, ["c1", "c2", "c3", "c4"]); }); test("pin: active compress call keeps only block-live ranges (dead entries stripped from args)", () => { @@ -295,7 +301,7 @@ test("pin: already-truncated text is skipped on the next pass (marker idempotenc assert.equal(second.truncatedCount, 0); }); -test("pin: applyCompression lands front ranges even when a later range errors (non-atomic partial landing, #3003 shape)", () => { +test("pin: unknown-ref sibling errors do not poison valid ranges (classification-level partial report)", () => { const core = createCore(); const state = createInitialState(); const messages: CoreMessage[] = [ @@ -326,3 +332,36 @@ test("pin: applyCompression lands front ranges even when a later range errors (n /m00009.*above this session's highest allocated ref/, ); }); + +test("pin: a range that dies DURING application rejects the whole batch all-or-nothing (#3939 ④, closes the #3003 double-seam)", () => { + const core = createCore(); + const state = createInitialState(); + const messages: CoreMessage[] = [ + { id: "a", role: "user", contentType: "text", text: "alpha" }, + { id: "b", role: "user", contentType: "text", text: "beta" }, + { id: "c", role: "user", contentType: "text", text: "gamma" }, + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "s1", topic: "t1" }, + { startRef: "m00003", endRef: "m00003", summary: "s2", topic: "t2" }, + ], + messages, + state, + config: config(), + protectedMessageIds: new Set(["m00003"]), + }); + + assert.equal(result.result.blocksCreated, 0); + assert.equal(result.state.blocks.length, 0); + assert.equal(result.state.stats.compressionCount, 0); + assert.match( + result.result.errors[0]!, + /Batch rejected before folding any range.*entirely within the protected zone/s, + ); +}); diff --git a/kernel/tests/issue3939-ref-migration-defer.test.ts b/kernel/tests/issue3939-ref-migration-defer.test.ts new file mode 100644 index 0000000000..c6a3bfb7c9 --- /dev/null +++ b/kernel/tests/issue3939-ref-migration-defer.test.ts @@ -0,0 +1,89 @@ +// #3939 ② / #3805: when a BLOCKED ref migrates to a fresh numeric ref +// (protection lifted, #2607), the NUMBER is minted immediately but the +// tag on the old carrier must NOT appear until a fold boundary rewrites +// history anyway. Stamping it on the migration turn silently rewrites an old +// message mid-history on an otherwise quiet turn — a prefix-cache breaker +// (#3805: durable carriers flipping bytes with no fold in sight). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { createInitialState } from "../src/state.js"; +import { BLOCKED_REF } from "../src/refs.js"; +import type { CoreMessage } from "../src/types.js"; + +const W = 100_000; +const cfg = () => ({ ...defaultConfig(W), preserveRecentTokens: 100 }); + +function hist(): CoreMessage[] { + const filler = "filler ".repeat(340); // push the range over the 5000-char min-compress gate + return [ + { id: "raw-a", role: "user", contentType: "text", text: "first question " + filler }, + { id: "raw-img", role: "user", contentType: "text", text: "look at this " + filler }, + { id: "raw-b", role: "assistant", contentType: "text", text: "answer " + filler }, + { id: "raw-c", role: "user", contentType: "text", text: "second question" }, + { id: "raw-d", role: "assistant", contentType: "text", text: "second answer" }, + { id: "raw-e", role: "user", contentType: "text", text: "third question" }, + { id: "raw-f", role: "assistant", contentType: "text", text: "third answer" }, + ]; +} + +test("pin: BLOCKED→numeric migration mints the ref but defers the carrier tag (#3805)", () => { + const core = createCore(cfg()); + // Simulate a pre-existing BLOCKED assignment on raw-img (protection lifted + // since — the message is plain text now). + const state = createInitialState(); + state.messageRefs.byRaw["raw-img"] = BLOCKED_REF; + + const t1 = core.processTurn({ + messages: hist(), + state, + config: cfg(), + tokenCount: 1_000, + renderTags: "all", + }); + + const imgRef = t1.state.messageRefs.byRaw["raw-img"]!; + assert.notEqual(imgRef, BLOCKED_REF, "migration mints a numeric ref"); + const imgText = t1.messages.find((m) => m.id === "raw-img")!.text!; + assert.ok(!imgText.includes("\x3cacp "), "carrier text stays untagged on the migration turn"); + assert.deepEqual(t1.state.renderDeferredRefs, [imgRef], "deferral recorded in state"); + // Sanity: other messages render normally. + assert.ok(t1.messages.find((m) => m.id === "raw-a")!.text!.includes("\x3cacp "), "untagged only for the migrated carrier"); + + // Quiet turn 2: deferral persists — still no tag, still byte-stable. + const t2 = core.processTurn({ + messages: hist(), + state: t1.state, + config: cfg(), + tokenCount: 1_000, + renderTags: "all", + }); + assert.ok(!t2.messages.find((m) => m.id === "raw-img")!.text!.includes("\x3cacp "), "deferral self-perpetuates on quiet turns"); +}); + +test("pin: a fold boundary releases the deferral and the tag rides the fold's rewrite", () => { + const core = createCore(cfg()); + const state = createInitialState(); + state.messageRefs.byRaw["raw-img"] = BLOCKED_REF; + const t1 = core.processTurn({ messages: hist(), state, config: cfg(), tokenCount: 1_000, renderTags: "all" }); + assert.ok(t1.state.renderDeferredRefs?.length, "deferral active"); + + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const fold = core.applyCompression({ + messages: t1.messages, + state: t1.state, + config: cfg(), + ranges: [{ + startRef: refOf(t1.state, "raw-a")!, + endRef: refOf(t1.state, "raw-b")!, + summary: "a long enough summary for the fold gate to accept this range, over fifty chars", + topic: "t", + }], + }); + assert.equal(fold.result.blocksCreated, 1, `fold lands: ${JSON.stringify(fold.result.errors ?? [])} ${JSON.stringify(fold.result.warnings ?? [])}`); + assert.equal(fold.state.renderDeferredRefs, undefined, "fold releases the deferral"); + + const t3 = core.processTurn({ messages: hist(), state: fold.state, config: cfg(), tokenCount: 1_000, renderTags: "all" }); + const imgText = t3.messages.find((m) => m.id === "raw-img")?.text ?? ""; + assert.ok(!imgText.includes("\x3cacp ") || t3.messages.every((m) => m.id !== "raw-img"), "raw-img rode the fold away or renders tagged at the fold boundary"); +}); diff --git a/kernel/tests/issue3939-truncate-latch.test.ts b/kernel/tests/issue3939-truncate-latch.test.ts new file mode 100644 index 0000000000..bd44ee9b15 --- /dev/null +++ b/kernel/tests/issue3939-truncate-latch.test.ts @@ -0,0 +1,139 @@ +// #3939 ③ / #3691: emergency truncation is a one-shot decision that must stay +// byte-stable across turns. The pre-latch node re-evaluated the watermark on +// every turn against the resent FULL history, so a fluctuating token estimate +// (95%+ one turn, below the next) flipped the same user text between its full +// and clipped forms every turn — a mid-history rewrite on every request and a +// prefix-cache breaker. The latch pins the clipped bytes until a fold resets +// them (#3939 contract: 水位触发 clip 后保持 clip 直到折叠事件重置). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { createInitialState } from "../src/state.js"; +import type { CoreMessage } from "../src/types.js"; + +const BIG = "alpha-beta ".repeat(4000); // ~48k chars, well past prefix+suffix +// Roomy filler so a fold spanning the whole history clears the 5000-char +// min-range gate even after the big text is clipped (~4.1K). +const filler = (t: string) => `filler ${t} ` + "x".repeat(800); +const tiny = (t: string): CoreMessage[] => [ + { id: `raw-${t}`, role: "user", contentType: "text", text: "hi " + t + " " + filler(t) }, + { id: `raw-${t}-a`, role: "assistant", contentType: "text", text: "ok " + t }, +]; + +function hist(): CoreMessage[] { + return [ + ...tiny("warm"), + { id: "raw-big", role: "user", contentType: "text", text: BIG }, + ...tiny("tail1"), + ...tiny("tail2"), + ...tiny("tail3"), + ...tiny("tail4"), + ]; +} + +const WINDOW = 100_000; +// Narrow the recent-token protection so a unit-scale history still has a +// foldable head (default preserveRecentTokens=5000 swallows this whole fixture +// and every range dies in the protected zone). The latch logic under test is +// orthogonal to the protected-zone math. +const cfg = () => ({ ...defaultConfig(WINDOW), preserveRecentTokens: 100 }); + +test("pin: a clip fires at 96% and the latch replays it on the next under-watermark turn (#3691)", () => { + const core = createCore(defaultConfig(WINDOW)); + const messages = hist(); + + const t1 = core.processTurn({ + messages, + state: createInitialState(), + config: defaultConfig(WINDOW), + tokenCount: 96_000, + renderTags: "none", + }); + const clipped1 = t1.messages.find((m) => m.id === "raw-big")!.text!; + assert.ok(clipped1.includes("[truncated for context space]"), "turn 1 clips the oversized text"); + assert.ok(t1.state.emergencyTruncateLatch?.some((e) => e.text === clipped1), "clip is latched"); + + // Turn 2: SAME full history resent, estimate now well under the watermark — + // pre-latch this returned the full text (the #3691 oscillation). + const t2 = core.processTurn({ + messages: hist(), + state: t1.state, + config: defaultConfig(WINDOW), + tokenCount: 60_000, + renderTags: "none", + }); + const clipped2 = t2.messages.find((m) => m.id === "raw-big")!.text!; + assert.equal(clipped2, clipped1, "latch replays the byte-identical clip under the watermark"); + assert.ok(t2.state.emergencyTruncateLatch?.some((e) => e.text === clipped1), "latch self-perpetuates while the message stays live"); +}); + +test("pin: a successful fold resets the latch and the clip retires with the fold", () => { + const core = createCore(cfg()); + const t1 = core.processTurn({ + messages: hist(), + state: createInitialState(), + config: cfg(), + tokenCount: 96_000, + renderTags: "none", + }); + assert.ok(t1.state.emergencyTruncateLatch?.length, "clip latched"); + + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const startRef = refOf(t1.state, "raw-warm"); + const endRef = refOf(t1.state, "raw-tail1-a"); + assert.ok(startRef && endRef, "refs assigned"); + + const fold = core.applyCompression({ + messages: t1.messages, + state: t1.state, + config: cfg(), + ranges: [{ startRef, endRef, summary: "a long enough summary for the fold gate to accept this range, over fifty chars", topic: "t" }], + }); + assert.equal(fold.result.blocksCreated, 1, "fold lands"); + assert.equal(fold.state.emergencyTruncateLatch, undefined, "fold resets the latch"); + + // Post-fold: a NEW oversized message under the watermark must NOT be + // clipped — the latch retired with the fold and the watermark re-evaluates + // from scratch (the original raw-big left the view inside the fold). + const post = [ + ...hist().filter((m) => m.id !== "raw-big"), + { id: "raw-big2", role: "user" as const, contentType: "text" as const, text: BIG }, + ]; + const t3 = core.processTurn({ + messages: post, + state: fold.state, + config: cfg(), + tokenCount: 60_000, + renderTags: "none", + }); + const big2 = t3.messages.find((m) => m.id === "raw-big2"); + assert.ok(big2, "new oversized message in view"); + assert.equal(big2!.text, BIG, "no clip after the fold reset (under watermark)"); + assert.equal(t3.state.emergencyTruncateLatch, undefined, "latch stays retired"); +}); + +test("pin: an edited message (new content id) is not replay-clipped from a stale latch entry", () => { + const core = createCore(defaultConfig(WINDOW)); + const t1 = core.processTurn({ + messages: hist(), + state: createInitialState(), + config: defaultConfig(WINDOW), + tokenCount: 96_000, + renderTags: "none", + }); + assert.ok(t1.state.emergencyTruncateLatch?.length, "clip latched"); + + const edited = hist(); + const bigIdx = edited.findIndex((m) => m.id === "raw-big"); + edited[bigIdx] = { ...edited[bigIdx]!, id: "raw-big-v2", text: "completely rewritten, short" }; + + const t2 = core.processTurn({ + messages: edited, + state: t1.state, + config: defaultConfig(WINDOW), + tokenCount: 60_000, + renderTags: "none", + }); + assert.equal(t2.messages.find((m) => m.id === "raw-big-v2")!.text, "completely rewritten, short"); + assert.equal(t2.state.emergencyTruncateLatch?.length ?? 0, 0, "stale entry retired with its message"); +}); diff --git a/kernel/tests/issue3939-view-latch.test.ts b/kernel/tests/issue3939-view-latch.test.ts new file mode 100644 index 0000000000..d17ac0522e --- /dev/null +++ b/kernel/tests/issue3939-view-latch.test.ts @@ -0,0 +1,184 @@ +// #3939 ⑤: the opt-in mid-history rewriters (absorb-hide, crush, filter +// keepLastOnly) re-derive their output every pass; without a latch each state/ +// context change flips OLD message bytes on an otherwise quiet turn (#3805 +// class). The view latch replays post-transform bytes verbatim; refreshes ride +// fold boundaries, absorb actions, and config edits (user action boundaries). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { applyAbsorb } from "../src/absorb.js"; +import { createInitialState } from "../src/state.js"; +import { + registerMessageFilter, + clearMessageFilters, +} from "../src/filter/registry.js"; +import type { CoreMessage } from "../src/types.js"; + +const W = 100_000; +const cfg = () => ({ ...defaultConfig(W), preserveRecentTokens: 100 }); + +function hist(): CoreMessage[] { + const filler = "filler ".repeat(500); + return [ + { id: "raw-a", role: "user", contentType: "text", text: "first question " + filler }, + { id: "raw-b", role: "assistant", contentType: "text", text: "answer " + filler }, + { id: "raw-c", role: "user", contentType: "text", text: "second question" }, + { id: "raw-d", role: "assistant", contentType: "text", text: "second answer" }, + { id: "raw-e", role: "user", contentType: "text", text: "third question" }, + { id: "raw-f", role: "assistant", contentType: "text", text: "third answer" }, + ]; +} + +test("pin: filter keepLastOnly demotion is deferred to the fold boundary", () => { + clearMessageFilters(); + registerMessageFilter({ + name: "pin-marker", + version: "1", + description: "test filter", + keepLastOnly: true, + filter: (ctx) => + ctx.text.includes("SECRETTOKEN") + ? { action: "modify", text: ctx.text.replace(/SECRETTOKEN/g, "[redacted]") } + : { action: "keep" }, + }); + try { + const core = createCore(cfg()); + const config = { + ...cfg(), + messageFilters: { enabled: true, filters: { "pin-marker": { enabled: true } } }, + }; + const m1 = [ + ...hist(), + { id: "raw-occ1", role: "user", contentType: "text" as const, text: "here is SECRETTOKEN one" }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config, tokenCount: 1_000, renderTags: "none" }); + // occ1 is the LAST occurrence → the filter's decision applies at arrival + // (modify → [redacted]) and the result is latched. + const occ1a = t1.messages.find((m) => m.id === "raw-occ1")!; + assert.ok(occ1a.text!.includes("[redacted]"), "last occurrence filtered on arrival"); + assert.ok(!occ1a.text!.includes("SECRETTOKEN"), "pattern stripped at arrival"); + + // A NEWER occurrence arrives — the fresh filter would demote occ1 (blank + // it). The latch replays occ1's bytes unchanged: the demotion waits for a + // fold boundary. + const m2 = [ + ...m1, + { id: "raw-occ2", role: "user", contentType: "text" as const, text: "again SECRETTOKEN two" }, + ]; + const t2 = core.processTurn({ messages: m2, state: t1.state, config, tokenCount: 1_000, renderTags: "none" }); + const occ1b = t2.messages.find((m) => m.id === "raw-occ1")!; + assert.equal(occ1b.text, occ1a.text, "demotion deferred — old bytes replay verbatim"); + assert.ok(t2.messages.find((m) => m.id === "raw-occ2")!.text!.includes("[redacted]"), "new last occurrence filtered fresh (arrival boundary)"); + + // Fold covering occ1 releases the latch: post-fold, occ1 is folded away + // and the new state carries no latches. + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const fold = core.applyCompression({ + messages: t2.messages, + state: t2.state, + config, + ranges: [{ + startRef: refOf(t2.state, "raw-a")!, + endRef: refOf(t2.state, "raw-b")!, + summary: "a long enough summary for the fold gate to accept this range, over fifty chars", + topic: "t", + }], + }); + assert.equal(fold.result.blocksCreated, 1, `fold lands: ${JSON.stringify(fold.result.errors ?? [])}`); + assert.equal(fold.state.viewLatches, undefined, "fold releases the view latch"); + } finally { + clearMessageFilters(); + } +}); + +test("pin: absorb-hide defers to the absorb action boundary, not a later fold", () => { + const core = createCore(cfg()); + const config = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 1, contextThresholdPct: 0, excludeTools: [] }, + }); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "read the file" }, + { id: "c1", role: "tool", contentType: "tool-call" as const, toolName: "read", toolCallId: "call1", text: "{}" }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "read", toolCallId: "call1", text: "file body line one\n" + "x".repeat(400) }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config, tokenCount: 3_000, renderTags: "none" }); + assert.ok(t1.messages.find((m) => m.id === "r1"), "pair visible before absorb"); + + // Absorb r1: the absorb call is an action boundary — applyAbsorb releases + // the latch, so the hide lands THIS turn (instant-reclaim contract). + const ref = t1.state.messageRefs.byRaw["r1"]!; + const outcome = applyAbsorb({ + ref, + summary: "file defines foo(a,b) at src/foo.ts:12", + messages: m1, + state: t1.state, + config, + }); + assert.ok(outcome.ok, "absorb ok"); + const t2 = core.processTurn({ messages: m1, state: outcome.state, config, tokenCount: 3_200, renderTags: "none" }); + const ids = t2.messages.map((m) => m.id); + assert.ok(!ids.includes("r1"), "absorbed result hidden on the action boundary turn"); + assert.ok(!ids.includes("c1"), "absorbed call hidden on the action boundary turn"); +}); + +test("pin: config enable of crush applies on the edit turn (config boundary), token-pressure flips do not", () => { + const core = createCore(cfg()); + const off = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 100, contextThresholdPct: 0, excludeTools: [] }, + }); + const rows = JSON.stringify( + Array.from({ length: 40 }, (_, i) => ({ id: i * 7, ts: "2026-09-21T12:00:00Z", level: "INFO", name: `svc-${i}` })), + ); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "read the log" }, + { id: "c1", role: "tool", contentType: "tool-call" as const, toolName: "read", toolCallId: "call1", text: "{}" }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "read", toolCallId: "call1", text: rows }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config: off, tokenCount: 0, renderTags: "none" }); + const fresh = t1.messages.find((m) => m.id === "r1")!.text!; + assert.ok(!fresh.includes("__acp_crush"), "crush off on turn 1"); + + // Enable crush: config edit = user action boundary — the crush applies now. + const onCfg = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 100, contextThresholdPct: 0, excludeTools: [] }, + crush: { enabled: true, minReduction: 0.1 }, + }); + const t2 = core.processTurn({ messages: m1, state: t1.state, config: onCfg, tokenCount: 0, renderTags: "none" }); + assert.ok(t2.messages.find((m) => m.id === "r1")!.text!.includes("__acp_crush"), "enable applies on the config-edit turn"); + + // Token pressure alone never flips bytes: same config, wildly different + // tokenCount — the crushed bytes replay verbatim. + const t3 = core.processTurn({ messages: m1, state: t2.state, config: onCfg, tokenCount: 99_000, renderTags: "none" }); + assert.equal(t3.messages.find((m) => m.id === "r1")!.text, t2.messages.find((m) => m.id === "r1")!.text, "pressure does not re-derive latched bytes"); +}); + +test("pin: CCR disarm releases the latch — a disarmed session never re-projects a dangling placeholder", () => { + // #1273 governance, restated for the latch: placeholders are legal only + // while the retrieve channel they advertise exists. CCR rides the config + // digest, so arming/disarming mid-session is a user-action boundary that + // releases the latch and re-judges under the CURRENT config. + const big = "line of build output ".repeat(400); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "run a big build" }, + { id: "c1", role: "assistant", contentType: "tool-call" as const, toolName: "bash", toolCallId: "call1", text: JSON.stringify({ command: "npm run build" }) }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "bash", toolCallId: "call1", text: big }, + ]; + const armed = { ...cfg(), ccr: { enabled: true, toolName: "acp_retrieve", minToolTokens: 50, excludeTools: [], maxHeadChars: 96 } }; + const core = createCore(cfg()); + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config: armed, tokenCount: 0, renderTags: "none" }); + const t1r = t1.messages.find((m) => m.id === "r1")!.text!; + assert.ok(t1r.includes("[acp-stored"), "armed CCR substitutes at arrival"); + assert.ok(!t1r.includes("line of build output"), "raw bytes leave the wire while armed"); + + // Same session, next turn, CCR still armed: the placeholder replays + // byte-stable (the latch holds; #1460 reprojection agrees with it). + const t2 = core.processTurn({ messages: m1, state: t1.state, config: armed, contentStore: t1.contentStore, tokenCount: 0, renderTags: "none" }); + assert.equal(t2.messages.find((m) => m.id === "r1")!.text, t1r, "latched placeholder replays verbatim while armed"); + + // Disarm (route/base governance flip): the digest changes → the latch + // releases → the raw bytes return. No dangling placeholder, no silent loss. + const t3 = core.processTurn({ messages: m1, state: t2.state, config: cfg(), contentStore: t2.contentStore, tokenCount: 0, renderTags: "none" }); + const t3r = t3.messages.find((m) => m.id === "r1")!.text!; + assert.ok(t3r.includes("line of build output"), "disarmed CCR returns the raw bytes"); + assert.ok(!t3r.includes("[acp-stored"), "no dangling placeholder after disarm"); +}); diff --git a/kernel/tests/orphan-endpoint-snap.test.ts b/kernel/tests/orphan-endpoint-snap.test.ts index 33aa658d20..3bbb72c8d4 100644 --- a/kernel/tests/orphan-endpoint-snap.test.ts +++ b/kernel/tests/orphan-endpoint-snap.test.ts @@ -79,10 +79,15 @@ test("hideConsumedCompressCalls records refs of hidden orphan calls only", () => existing: state.messageRefs, nextIndex: 1, }).map; + state.sweptOrphanCallIds = ["c1"]; const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 2, "oldest orphan pair is hidden"); + assert.equal( + result.hidden, + 2, + "previously-swept orphan pair stays hidden (#3769)", + ); assert.deepEqual( result.hiddenOrphanRefs, ["m00001", "m00002"], diff --git a/kernel/tests/pipeline.test.ts b/kernel/tests/pipeline.test.ts index ec1c877412..f5493f17f4 100644 --- a/kernel/tests/pipeline.test.ts +++ b/kernel/tests/pipeline.test.ts @@ -29,12 +29,14 @@ test("defaultNodes exposes the canonical ordered pipeline", () => { "assign-refs", "sync-blocks", "prune", + "pre-transform-snapshot", "ccr-store", "absorb-hide", "crush", "absorb-prompt", "filter", "hide-compress-calls", + "view-latch", "recommend", "nudge-inject", "emergency-truncate", diff --git a/kernel/tests/protect-recent.test.ts b/kernel/tests/protect-recent.test.ts index 62319940b7..3e0ebd8d4a 100644 --- a/kernel/tests/protect-recent.test.ts +++ b/kernel/tests/protect-recent.test.ts @@ -346,8 +346,12 @@ test("warnings accumulate across multiple ranges in one batch", () => { const state = seededState(messages); const cfg = config({ preserveRecentMessages: 3 }); - // Two ranges: each partially overlaps the recent zone (m00008..m00010). - // Both should produce warnings, and both unprotected heads should compress. + // Two ranges: the second now ends inside the recent zone, which under the + // #3939 all-or-nothing contract would reject the whole batch (a fully- + // protected range is a fatal apply error, pinned in issue3939-fold-boundary- + // pins.test.ts), so it instead OVERLAPS the first range's span — overlap + // warn+skip keeps the batch valid while still exercising multi-range + // warning accumulation. const result = core.applyCompression({ ranges: [ { @@ -357,9 +361,9 @@ test("warnings accumulate across multiple ranges in one batch", () => { topic: "a", }, { - startRef: "m00010", + startRef: "m00008", endRef: "m00010", - summary: "second range fully protected tail", + summary: "second range overlaps the first", topic: "b", }, ], @@ -372,7 +376,7 @@ test("warnings accumulate across multiple ranges in one batch", () => { result.result.blocksCreated >= 1, "at least the unprotected head compresses", ); - assert.ok(result.result.warnings.length >= 1, "warnings surfaced"); + assert.ok(result.result.warnings.length >= 2, "warnings surfaced"); }); // --- neverPreserveRecentTools is configurable (billion-context#1277) --- diff --git a/kernel/tests/validation.test.ts b/kernel/tests/validation.test.ts index 09286377cd..c32cb46c5a 100644 --- a/kernel/tests/validation.test.ts +++ b/kernel/tests/validation.test.ts @@ -259,7 +259,7 @@ test("validation: all checks disabled (0s) allows any input", () => { assert.equal(result.result.errors.length, 0); }); -test("validation: errors are collected per-range in batch compress", () => { +test("validation: an empty-summary range rejects the whole batch (#3003)", () => { const core = createCore(); const messages = [ msg("a", longText), @@ -282,7 +282,12 @@ test("validation: errors are collected per-range in batch compress", () => { state, config, }); - assert.equal(result.result.blocksCreated, 1); + // #3003: a length-class failure (empty summary) in ANY candidate rejects the + // WHOLE batch — the valid sibling no longer lands. Content-class failures + // (unknown/invalid refs) remain per-range; see the invalid-refs batch test. + assert.equal(result.result.blocksCreated, 0); + assert.equal(result.state.blocks.length, 0); assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); assert.match(result.result.errors[0]!, /empty/i); }); diff --git a/package-lock.json b/package-lock.json index e8860f80fc..ef1f24a5d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,7 +50,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.143", "dev": true, "license": "MIT", "devDependencies": { diff --git a/scripts/update-storage-render-goldens.ts b/scripts/update-storage-render-goldens.ts index 46c1f21f5c..63948692d8 100644 --- a/scripts/update-storage-render-goldens.ts +++ b/scripts/update-storage-render-goldens.ts @@ -11,6 +11,11 @@ import { driveWire, WIRES } from "../tests/golden-storage-drive.ts"; // same condition — without it every per-turn wire digest drifts by the image- // rescale delta and the suite reds on the next CI run. process.env.BILI_IMAGE_RESCALE = "0"; +// Parity #2: the test suite also disarms the shadow observability lane +// (#3756) before driving — its metadata blob would otherwise diverge from +// what the committed test run hashes, and every freshly regenerated golden +// would red on the next CI run. +process.env.BILI_SNAPSHOT_SHADOW = "0"; const GOLDEN_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tests", "golden", "storage-render"); fs.mkdirSync(GOLDEN_DIR, { recursive: true }); diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index 8d9486886b..609b87b905 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -119,7 +119,7 @@ interface LedgerLine { * of host churn). Sparse: omitted unless booked. The seam arm sets `seam` * above instead. Lets the web trajectory chart paint the same bucket * membership the economics card aggregates. */ - prov?: "ttl" | "side" | "rewind" | "client"; + prov?: "ttl" | "side" | "rewind" | "client" | "orphan-sweep"; /** #1535: 1 iff `model` differs from the previous sample's KNOWN model * (unknown sides never flag); marks the re-billed stable prefix on the * first request after a model switch. Sparse: omitted unless set. */ @@ -206,6 +206,13 @@ interface LedgerLine { interface CacheLedger { v: 1; lastBlockId: number; + /** #3769: high-water of session.state.sweptOrphanCallIds length — detects a + * proactive failed-compress cleanup between samples so detectSeam can + * attribute a mid-history break to the cleanup instead of "unexplained". */ + lastSweptCount?: number; + /** #3769: orphans swept on the CURRENT sample (0 = none); read by detectSeam's + * orphan-sweep arm, refreshed each recordCacheSample. Transient. */ + sweepThisSample?: number; consumedFoldSeq: number; sampleSeq: number; foldSeqCounter: number; @@ -278,6 +285,11 @@ interface CacheLedger { * bili rebuild seam. Charged only when the client arm wins. */ clientInserts: number; clientInsertMissed: number; + /** #3769: mid-history breaks attributed to a proactive failed-compress + * cleanup (orphan rotation) landing in-window without a fold claiming + * the break — kept out of the unexplained seam tally. */ + orphanSweepSeams: number; + orphanSweepMissed: number; /** #2929: the provider-side arm split by idle gap — samples settled more * than PROVIDER_TTL_GAP_MS after the previous sample are normal TTL * expiry, not an abnormal provider failure. */ @@ -943,6 +955,21 @@ function detectSeam(session: Session, led: CacheLedger): void { // Inbound prefix-stable or a clean tail boundary: the outbound // mid-list break is bili-side — fall through to the seam bucket. } + // #3769: a proactive failed-compress cleanup (orphan rotation) landed in this + // window. After the fold-gated fix such a cleanup only rides along a committed + // fold, which claims its own break above via foldSeq — so reaching here means a + // mid-history break coincided with a cleanup WITHOUT any fold/switch/restart/ + // client arm claiming it. Attribute it to the cleanup rather than leave it + // "unexplained". Additive: none of the earlier arms is touched. + if ((led.sweepThisSample ?? 0) > 0) { + line.prov = "orphan-sweep"; + agg.orphanSweepSeams += 1; + agg.orphanSweepMissed += line.tr; + if (agg.orphanSweepSeams === 1) { + loggerLog("warn", `[${session.id}] [cache-seam] mid-history break attributed to proactive failed-compress cleanup (${led.sweepThisSample} record(s) rotated out, #3769): hit ${line.hitPct}% (unexplained=${Math.round(line.tr)} tok) — no fold claimed this break`); + } + return; + } agg.seamSuspects += 1; agg.seamMissed += line.tr; line.seam = 1; @@ -974,7 +1001,7 @@ export function getCacheLedger(session: Session): CacheLedger { "restartDropMissed", "attributedMissed", "unknownSamples", "unknownInput", "nbSamples", "nbInput", "seamSuspects", "seamMissed", - "providerSideMisses", "providerSideMissed", "providerTtlMisses", "providerTtlMissed", "rewinds", "rewindMissed", "abortCorrelated", "clientInserts", "clientInsertMissed", + "providerSideMisses", "providerSideMissed", "providerTtlMisses", "providerTtlMissed", "rewinds", "rewindMissed", "abortCorrelated", "clientInserts", "clientInsertMissed", "orphanSweepSeams", "orphanSweepMissed", ] as const) { if (typeof g[key] !== "number") g[key] = 0; } @@ -991,7 +1018,7 @@ export function getCacheLedger(session: Session): CacheLedger { foldSeqCounter: 0, folds: [], lines: [], - agg: { requests: 0, input: 0, cached: 0, output: 0, nc: 0, cr: 0, tr: 0, switches: 0, switchMissed: 0, effSwitches: 0, effSwitchMissed: 0, keySwitches: 0, keySwitchMissed: 0, promptSwitches: 0, promptSwitchMissed: 0, projSwitches: 0, projSwitchMissed: 0, wireSwitches: 0, wireSwitchMissed: 0, upstreamSwitches: 0, upstreamSwitchMissed: 0, restartDrops: 0, restartDropMissed: 0, attributedMissed: 0, unknownSamples: 0, unknownInput: 0, nbSamples: 0, nbInput: 0, seamSuspects: 0, seamMissed: 0, providerSideMisses: 0, providerSideMissed: 0, providerTtlMisses: 0, providerTtlMissed: 0, rewinds: 0, rewindMissed: 0, abortCorrelated: 0, clientInserts: 0, clientInsertMissed: 0 }, + agg: { requests: 0, input: 0, cached: 0, output: 0, nc: 0, cr: 0, tr: 0, switches: 0, switchMissed: 0, effSwitches: 0, effSwitchMissed: 0, keySwitches: 0, keySwitchMissed: 0, promptSwitches: 0, promptSwitchMissed: 0, projSwitches: 0, projSwitchMissed: 0, wireSwitches: 0, wireSwitchMissed: 0, upstreamSwitches: 0, upstreamSwitchMissed: 0, restartDrops: 0, restartDropMissed: 0, attributedMissed: 0, unknownSamples: 0, unknownInput: 0, nbSamples: 0, nbInput: 0, seamSuspects: 0, seamMissed: 0, providerSideMisses: 0, providerSideMissed: 0, providerTtlMisses: 0, providerTtlMissed: 0, rewinds: 0, rewindMissed: 0, abortCorrelated: 0, clientInserts: 0, clientInsertMissed: 0, orphanSweepSeams: 0, orphanSweepMissed: 0 }, }; meta[LEDGER_KEY] = led; return led; @@ -1040,6 +1067,22 @@ function detectNewFolds(session: Session, led: CacheLedger): void { } } +// #3769: sweptOrphanCallIds is monotonic, so its length high-water makes growth +// exact. detectSeam's orphan-sweep arm reads sweepThisSample set here. +function detectOrphanSweeps(session: Session, led: CacheLedger): void { + const n = session.state?.sweptOrphanCallIds?.length ?? 0; + // First observation only baselines: a resumed session's pre-existing sweeps + // must not be charged to the current sample. + if (led.lastSweptCount === undefined) { + led.lastSweptCount = n; + led.sweepThisSample = 0; + return; + } + const delta = n - led.lastSweptCount; + led.lastSweptCount = n; + led.sweepThisSample = delta > 0 ? delta : 0; +} + /** Record one provider usage report into the session ledger. `input` must be * NORMALIZED (cached included — promptInputTotal semantics). `cached === null` * means the provider reported NO cache-hit tokens (unmeasurable): such samples @@ -1051,6 +1094,7 @@ export function recordCacheSample( ): void { const led = getCacheLedger(session); detectNewFolds(session, led); + detectOrphanSweeps(session, led); const prevLine = led.lines[led.lines.length - 1]; // #1536: unknown-cache samples still need a prev/cur pair so the fold chain // stays consistent, but decompose with cached=0 then QUARANTINE every derived @@ -1707,7 +1751,7 @@ interface BodyStability { interface BiliCacheReport extends CacheReport { stability: BodyStability; /** #2131: line set widened with the per-call body-stability fields. */ - lines: Array; + lines: Array; modelSwitches: ModelSwitchStats; /** #3803: host generation-parameter switches (reasoning effort / thinking config) — a provider cache-namespace flip that re-bills the whole stable prefix once, distinct from any byte change. */ effSwitches: ModelSwitchStats; @@ -1731,7 +1775,7 @@ interface BiliCacheReport extends CacheReport { * is booked as new content, never as a prefix re-pay. */ initialBills: { samples: number; inputTokens: number }; invalidation: InvalidationTokenBreakdown; - seam: { suspects: number; missed: number; events: SeamEvent[]; providerSide: { count: number; missed: number }; providerTtl: { count: number; missed: number }; rewinds: { count: number; missed: number }; abortCorrelated: number; client: { count: number; missed: number } }; + seam: { suspects: number; missed: number; events: SeamEvent[]; providerSide: { count: number; missed: number }; providerTtl: { count: number; missed: number }; rewinds: { count: number; missed: number }; abortCorrelated: number; client: { count: number; missed: number }; orphanSweep: { count: number; missed: number } }; } export function buildSessionCacheReport(session: Session): BiliCacheReport { @@ -1918,7 +1962,7 @@ export function buildSessionCacheReport(session: Session): BiliCacheReport { initialBills: { samples: a.nbSamples, inputTokens: a.nbInput }, invalidation, stability, - seam: { suspects: a.seamSuspects, missed: a.seamMissed, events: led.seamEvents ?? [], providerSide: { count: a.providerSideMisses, missed: a.providerSideMissed }, providerTtl: { count: a.providerTtlMisses, missed: a.providerTtlMissed }, rewinds: { count: a.rewinds, missed: a.rewindMissed }, abortCorrelated: a.abortCorrelated, client: { count: a.clientInserts, missed: a.clientInsertMissed } }, + seam: { suspects: a.seamSuspects, missed: a.seamMissed, events: led.seamEvents ?? [], providerSide: { count: a.providerSideMisses, missed: a.providerSideMissed }, providerTtl: { count: a.providerTtlMisses, missed: a.providerTtlMissed }, rewinds: { count: a.rewinds, missed: a.rewindMissed }, abortCorrelated: a.abortCorrelated, client: { count: a.clientInserts, missed: a.clientInsertMissed }, orphanSweep: { count: a.orphanSweepSeams, missed: a.orphanSweepMissed } }, }; } @@ -2092,6 +2136,10 @@ function formatSeam(r: BiliCacheReport): string { if (e.client) out.push(` #${e.seq} ${fmtTime(e.at)} hit ${e.hitPct.toFixed(1)}% · input ${fmtTok(e.input)} · inbound message[${e.client.idx}] ${e.client.role}/${e.client.cls} ${e.client.len}B JSON · outbound ${fmtTok(e.lcpBytes)}B @ message[${e.msgIndex ?? "?"}] of ${e.prevMsgs}→${e.curMsgs}`); } } + if (r.seam.orphanSweep.count > 0) { + out.push("🧹 ORPHAN CLEANUP SEAM (proactive failed-compress rotation — #3769)"); + out.push(` ${r.seam.orphanSweep.count} sample(s) · ${fmtTok(r.seam.orphanSweep.missed)} tok re-billed when failed-compress records were rotated out of history without a fold claiming the break — bili's own cleanup, not an upstream or host rewrite`); + } if (r.seam.abortCorrelated > 0) { out.push("⏻ ABORT-CORRELATED"); out.push(` ${r.seam.abortCorrelated} missed sample(s) within 30s of a client mid-stream abort — abort/retry churn rewrites the resent tail; correlation, not causation (see bili.log 'client aborted mid-stream')`); diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 5de73f8a2b..3d15fffdf1 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -262,6 +262,37 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare // dropped (its failed tail, a rewound branch) — and the model's older // text may still cite those. Reserve the parent's whole ref space. reserveRefsThrough(session, highestUsedIndex(parent.state.messageRefs)); + // #3769: carry the orphan-cleanup ledgers too — without this, the failed + // compress pairs the pre-upgrade keep-two rotation hid (hiddenOrphanRefs) + // or the fold-gated sweep already swept (sweptOrphanCallIds) would + // RESURFACE on resume/fork replay, rewriting mid-history at full cache + // cost. Both are monotonic id/ref sets; the hide pass re-records them + // every turn, so the carry self-perpetuates afterwards. + if (parent.state.hiddenOrphanRefs?.length) session.state.hiddenOrphanRefs = [...parent.state.hiddenOrphanRefs]; + if (parent.state.sweptOrphanCallIds?.length) session.state.sweptOrphanCallIds = [...parent.state.sweptOrphanCallIds]; + // #3939 ③: same carry for the truncation latch — resume/fork replay + // re-sends the clipped messages; without the latch the first replayed + // turn under a lower estimate re-exposes the full bytes (oscillation). + if (parent.state.emergencyTruncateLatch?.length) { + session.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch.map((e) => ({ ...e })); + } + // #3939 ②: same carry for the migration render deferral — a resumed/forked + // session re-sends the same history; without the deferral the first replay + // stamps migration tags mid-history (the #3805 byte flip, resurrected). + if (parent.state.renderDeferredRefs?.length) { + session.state.renderDeferredRefs = [...parent.state.renderDeferredRefs]; + } + // #3939 ⑤: same carry for the view latch — a resumed session re-sends the + // same history; a dropped latch would flip transform bytes on the first + // replayed turn (crush re-derived, absorb hides landing late). + if (parent.state.viewLatches) { + session.state.viewLatches = Object.fromEntries( + Object.entries(parent.state.viewLatches).map(([k, v]) => [k, { ...v }]), + ); + } + if (parent.state.viewLatchConfigDigest) { + session.state.viewLatchConfigDigest = parent.state.viewLatchConfigDigest; + } markDirty(session); } diff --git a/src/persist.ts b/src/persist.ts index 608e5a8650..77abbc9386 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -241,6 +241,25 @@ function mergeState(parsed: CompressionState): CompressionState { // resurrects with absorbed=[] and hideAbsorbedMessages has nothing to hide. absorbed: parsed.absorbed ?? fresh.absorbed, rules: parsed.rules ?? fresh.rules, + // #3769: fold-gated orphan-sweep bookkeeping must survive hydration — + // dropping sweptOrphanCallIds resurrects already-cleaned pairs after a + // restart (mid-history rewrite = cache break); the pending marker is + // transient, losing it only defers the sweep to the next fold. + sweptOrphanCallIds: parsed.sweptOrphanCallIds ?? fresh.sweptOrphanCallIds, + pendingOrphanSweepMinRef: parsed.pendingOrphanSweepMinRef ?? fresh.pendingOrphanSweepMinRef, + // #3939 ③/#3691: the emergency-truncate latch must survive hydration — + // dropping it re-exposes clipped bytes mid-history after a restart, + // rewriting the still-cached prefix on the first post-restart turn. + emergencyTruncateLatch: parsed.emergencyTruncateLatch ?? fresh.emergencyTruncateLatch, + // #3939 ②/#3805: the render deferral must survive hydration — losing + // it stamps migration tags onto still-cached carriers right after a + // restart (a silent mid-history rewrite with no fold in sight). + renderDeferredRefs: parsed.renderDeferredRefs ?? fresh.renderDeferredRefs, + // #3939 ⑤: the view latch must survive hydration — losing it + // re-derives transform output on still-cached carriers right after a + // restart (silent mid-history rewrite, no fold in sight). + viewLatches: parsed.viewLatches ?? fresh.viewLatches, + viewLatchConfigDigest: parsed.viewLatchConfigDigest ?? fresh.viewLatchConfigDigest, }; } diff --git a/src/plugin.ts b/src/plugin.ts index 404624285c..e04a08b76c 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1389,6 +1389,24 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr child.state.rules = structuredClone(parent.state.rules); child.state.nextRuleId = parent.state.nextRuleId; child.state.hiddenOrphanRefs = parent.state.hiddenOrphanRefs?.filter((ref) => prefix.some((m) => m.ref === ref)); + // #3939 ③: only latch entries whose messages rode the forked + // prefix — the child never sees the others, so their entries + // would be dead weight (and retire on the next latch pass anyway). + child.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch?.filter((e) => prefix.some((m) => m.ref === e.ref)); + // #3939 ②: deferred refs are content-keyed (byRaw ids) — keep only + // the ones whose carrier rode the forked prefix. + child.state.renderDeferredRefs = parent.state.renderDeferredRefs?.filter((r) => prefix.some((m) => m.ref === r)); + // #3939 ⑤: view-latch entries are ref-keyed — keep only the + // carriers that rode the forked prefix. + const forkRefs = new Set(prefix.map((m) => m.ref)); + if (parent.state.viewLatches) { + child.state.viewLatches = Object.fromEntries( + Object.entries(parent.state.viewLatches).filter(([r]) => forkRefs.has(r)), + ); + } + if (parent.state.viewLatchConfigDigest) { + child.state.viewLatchConfigDigest = parent.state.viewLatchConfigDigest; + } // #2362: inverse of the orphan filter — a dead ref whose message IS // in the forked prefix is visible again in the child; refs outside // it stay dead (their messages are absent from the child too). diff --git a/src/preflight.ts b/src/preflight.ts index 49941a9706..cffe74d8f1 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -2046,6 +2046,15 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // remains (or nothing above minCompressRange, or the whole relaxed // zone already folded) — so no retry against a healthier relay can // change the verdict. + // #2952 Design A: "nothing compressible" can still leave a rescue — + // every raw message already inside a block (view = carriers only) + // is exactly the distillable state. Defer to the post-walk + // recovery funnel instead of failing here, same as the futility + // gate above; tiers=false / emergency digest never enter this way. + if (eligibleForTierRecovery()) { + deps.log("info", `[preflight] no compressible ranges remain${relaxed ? " even with the soft zone relaxed" : ""}; deferring to bounded tier-distillation recovery (#2952)`); + break; + } failure = { kind: "exhausted", deterministic: true, detail: relaxed ? relaxedExhaustedDetail : subMinNoted ? `no foldable compressible ranges remain: all ${viable.length} viable range(s) are below minCompressRange (${minChars} chars)` : "no compressible ranges remain in the conversation" }; diff --git a/src/server/relay.ts b/src/server/relay.ts index b31bc9cbd0..790937be51 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1157,7 +1157,21 @@ export async function forward( const overflowInfo = inspectContextOverflow(upstreamResult.response.status, overflowText); if (overflowInfo.isOverflow) { armOverflowShrink(overflowInfo); - const refolded = await overflowRefold(overflowInfo.window).catch(() => null); + // #3787 follow-up: a NUMBERLESS overflow states no window; the + // armed baseline (min of declared and the rejected payload's + // own estimate) is the only ceiling evidence there is. Judging + // the refold against the DECLARED window made preflight see + // "already fits" and return an unchanged body — the rescue + // then only worked when re-render churn happened to change + // bytes, which byte-stable rebuilds (#3939 ③) eliminate. + // The arm is BY CONSTRUCTION the rejected payload's own size, + // so a refold targeting the arm itself leaves nothing to fold + // (payload == ceiling on every estimate): fold with a 10% + // margin under it, the same headroom discipline as the arm's + // own min(declared, est) clamp — the rejection proved the + // ceiling is below the payload, give the fold real room. + const armedWindow = overflowInfo.window ?? (prepared.session.stats.lastInputTokensSource === "overflow-arm" ? Math.floor(prepared.session.stats.lastInputTokens * 0.9) : undefined); + const refolded = await overflowRefold(armedWindow).catch(() => null); if (refolded) { try { // #816 family: the refold rebuilds the body from state diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 9b27270502..9c741dae79 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -42,6 +42,10 @@ const MERGE_STATE_FIELDS = [ "tokenSnapshot", "lastPassIds", "hiddenOrphanRefs", + "emergencyTruncateLatch", + "renderDeferredRefs", + "viewLatches", + "viewLatchConfigDigest", "deadRefs", "terminalStreak", "nextRuleId", diff --git a/src/stream.ts b/src/stream.ts index 74a8c6bf21..079170b960 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -913,6 +913,7 @@ export function applyRanges(parsed: ReturnType, ctx: : /already compressed/.test(errs) ? "gate:already-compressed" : /requested range\(s\) resolved/.test(errs) ? "gate:none-resolved" : /too small/.test(errs) ? "gate:too-small" + : /Batch rejected before folding any range/.test(errs) ? "gate:summary-length" : "gate:other"; const revNote = revs.length > 0 ? ` Note: startId > endId in range(s) ${revs.map((rg) => `${rg.startRef}→${rg.endRef}`).join(", ")} — your refs were reversed; they were normalized to ascending order before evaluation, so check your ref order.` diff --git a/tests/cache-friendly-plugin.test.ts b/tests/cache-friendly-plugin.test.ts index 11d9927ebe..7362367629 100644 --- a/tests/cache-friendly-plugin.test.ts +++ b/tests/cache-friendly-plugin.test.ts @@ -7,13 +7,18 @@ // Plugin-mode carrier semantics: the ACP-native agent OWNS compression — it // executes `compress` locally through the plugin tool API and re-sends the // call + result in its own persisted history on every later request (pi -// semantics: pairs stay at their chronological position forever). The proxy -// therefore re-injects the in-place acp_summary carrier from block state on -// every request. Since the #1567 fix the carrier is STRIPPED per-block, but -// self-verified: only while that exact fold's compress pair actually rides -// the (post-prepare) inbound history. A pair pruned client-side or hidden by -// the kernel (KEEP_LAST_ORPHANED) keeps the anchor, so an active fold never -// has zero carriers. This suite pins that behavior byte-for-byte. +// semantics: pairs stay at their chronological position forever). That pair +// IS the wire summary carrier — the agent's view never renders the kernel +// acp_summary (KDD#7). Since the #1567 fix the kernel-rendered anchor is +// STRIPPED per-block, self-verified: only while that exact fold's compress +// pair rides the (post-prepare) inbound history quoting an ACTIVE block's +// range (#3769 carrier exemption). The moment a pair stops riding (client +// drop), its anchor re-carries the summary from that turn on (#1567 +// fail-safe); a pair whose block died (swallowed by a wider fold) loses its +// carrier status and is swept at the swallowing fold's boundary, together +// with genuinely failed compress records sitting at/after the fold's start +// ref (fold-gated sweep, #3769). This suite pins that behavior +// byte-for-byte. // // Invariants pinned per wire (bodies = consecutive upstream-bound request // payloads of one session, compared as flat unit lists — text blocks and @@ -31,21 +36,22 @@ // that move); every other byte stays an exact prefix; // P2 fold transitions (client re-sends its compress pair after a // /__bili/plugin/tool fold): -// P2a the pre-fold span is replaced by exactly the expected carrier -// units and nothing else moves or mutates (exact reconstruction); -// P2b the in-place carrier is stripped from every post-fold turn -// while the client's pair rides inbound — occurrence counts -// pinned: exactly 2 summary-text hits per post-fold turn (the -// fold spec echoed in the re-sent tool_call args + the tail -// quote inside the persistently re-sent tool result) and ZERO -// carrier markers (#1567 fix); -// P2c fail-safe (#1567 hardening): the anchor re-carries the summary -// whenever the pair does NOT ride inbound — the client stops -// re-sending it, or the kernel hid it (KEEP_LAST_ORPHANED) — -// pinned by the two hardening subtests at the bottom of this file; -// P3 multi-fold: two folds in one session; both pairs stay visible -// (≤ KEEP_LAST_ORPHANED) and the pair-quote counts stay constant across -// all later turns; +// P2a the pre-fold span is replaced by exactly the expected units and +// nothing else moves or mutates (exact reconstruction; when the +// span touches m00001 the session head survives as boundary, +// mid-history spans are consumed whole); +// P2b the pair RIDES inbound from its fold turn on and IS the summary +// carrier — occurrence counts pinned: ZERO kernel anchors +// (SUM_MARKER) while any pair rides, and exactly TWO summary-text +// hits per riding pair (re-sent call args + tool-result receipt); +// P2c fail-safe (#1567 hardening): the anchor carries the summary +// whenever the pair does NOT ride inbound — the drop-pair subtest +// pins that a client-side pair drop surfaces exactly its anchor +// once, then stays append-stable (no flapping); +// P3 multi-fold: two folds in one session; each pair rides alongside its +// siblings as its own carrier, anchors stay stripped for the life of +// the pairs, per-fold summary counts constant, quiet turns +// append-stable (keep-two rotation is gone, #3769); // P4 no transient acp_loop_* artifact leaks into any forwarded body // (plugin mode has none by construction — pin it); // P5 marker/tag hygiene: every content text unit carries exactly one @@ -68,7 +74,8 @@ // // Scenario (main, all wires): t1=[q1,a1,q2,a2], t2=+q3,a3, t3=+q4,a4, // foldA(m00001..m00002)+pairA@t4, t5=+q5,a5, foldB(m00003..m00005)+pairB@t6, -// t7=+q7,a7, t8=+q8,a8 — every historical pair rides every later request. +// t7=+q7,a7, t8=+q8,a8 — each pair rides as its fold's carrier from its +// fold turn on; anchors stay stripped while their pairs ride (#1567/#3769). // // Scope notes: tool protocol only (the default). The swallow/back-to-back // fold geometries get a dedicated anthropic subtest (kernel-side behavior, @@ -613,28 +620,32 @@ for (const wire of ["anthropic", "openai", "responses", "google"] as Wire[]) { assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // head boundary survives; anchor stripped (#1567) - assertPrefix(cans[3]!.core, cans[4]!.core, L(4), wire === "google" ? ["a5"] : ["q5", "a5"], wire === "google"); // google: #3159 — q5 leads the fold-pair result - assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], [], L(5)); // anchor stripped (#1567) - assertPrefix(cans[5]!.core, cans[6]!.core, L(6), wire === "google" ? ["a7"] : ["q7", "a7"], wire === "google"); // google: #3159 — q7 leads the fold-pair result + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // session-head boundary survives; span consumed, no anchor — pair A rides as carrier (#1567/#3769) + // #3159 (google lane only): the riding pair's functionCall/ + // functionResponse split swallows the turn's leading text part into + // the sanctioned shift, so only the model reply lands in `added` + assertPrefix(cans[3]!.core, cans[4]!.core, L(4), wire === "google" ? ["a5"] : ["q5", "a5"], wire === "google"); + assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], [], L(5)); // mid-history span consumed whole; no anchor — pair B rides (#1567/#3769) + assertPrefix(cans[5]!.core, cans[6]!.core, L(6), wire === "google" ? ["a7"] : ["q7", "a7"], wire === "google"); assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); for (let i = 3; i < cans.length; i++) { const raw = cans[i]!.raw; - // #1567 (fixed): the in-place carrier is stripped per-block while - // the client's re-sent pair rides inbound — only the two pair - // quotes (tool_call args + tool result) keep the summary text. - const expA = 2; - const expB = i < 5 ? 0 : 2; - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: in-place carrier must be stripped while the client's pair rides inbound (got ${countOcc(raw, SUM_MARKER)})`); - assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA} (pair quotes only, #1567)`); - assert.equal(countOcc(raw, SUM_B), expB, `${L(i)}: SUM_B occurrences ${countOcc(raw, SUM_B)} != ${expB} (pair quotes only, #1567)`); - // google part-shapes carry no call id; pin pair presence through - // the fold topic embedded in the re-sent functionCall args - const pairA = wire === "google" ? "CFP-TOPIC-A" : "\"cfp_a\""; - const pairB = wire === "google" ? "CFP-TOPIC-B" : "\"cfp_b\""; - assert.ok(raw.includes(pairA), `${L(i)}: pairA lost from re-sent history`); - if (i >= 5) assert.ok(raw.includes(pairB), `${L(i)}: pairB lost from re-sent history`); + // Carrier regime (#1567/#3769/KDD#7): each fold's echoed pair + // rides from its fold turn on and IS the summary carrier — the + // kernel anchor stays stripped for the pair's life, and the + // summary text appears exactly twice per riding pair (call args + // + result receipt). + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while their pairs ride (#1567)`); + assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A rides pair A args+receipt (got ${countOcc(raw, SUM_A)})`); + assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 2, `${L(i)}: SUM_B rides pair B from its fold turn on (got ${countOcc(raw, SUM_B)})`); + // pin pair PRESENCE — the carriers; google part-shapes carry no + // call id, so pin through the fold-spec refs in the re-sent + // functionCall args + const pairA = wire === "google" ? "\"startId\":\"m00001\"" : "\"cfp_a\""; + const pairB = wire === "google" ? "\"startId\":\"m00003\"" : "\"cfp_b\""; + assert.ok(raw.includes(pairA), `${L(i)}: pair A rides as its fold's carrier (#1567)`); + assert.equal(raw.includes(pairB), i >= 5, `${L(i)}: pair B rides from t6 on`); } }, { timeout: 120_000 }); @@ -665,53 +676,60 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // head boundary survives; anchor stripped (#1567) + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // session-head boundary survives; no anchor — geo_a rides as carrier (#1567/#3769) assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); const wide = FOLD_WIDE.summary; const gamma = FOLD_E.summary; + // The wide fold (m00001..m00004) swallows the alpha fold's range: the + // alpha summary merges into the wide fold, and the geo_a pair — whose + // block died — loses carrier status and is swept at the swallow + // boundary (#3769). geo_wide rides as the merged region's carrier. + // FOLD_E's tail sits in the protected zone, so only its first message + // folds; geo_e rides alongside. for (const i of [5, 6, 7]) { const raw = cans[i]!.raw; - // carrier is superseded on every turn; the two head/tail quotes ride - // the re-sent geo_a tool result until that pair is pruned at t7 - const expA = i === 5 ? 2 : 0; - assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA}`); - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: carriers must be stripped while their pairs ride inbound (got ${countOcc(raw, SUM_MARKER)})`); assert.ok(!raw.includes("q2-marker"), `${L(i)}: swallowed content q2 must stay hidden`); assert.ok(!raw.includes("a2-marker"), `${L(i)}: swallowed content a2 must stay hidden`); + assert.ok(!raw.includes("\"geo_a\""), `${L(i)}: dead-block pair swept at the swallow boundary (#3769)`); } - const surviveKeys = new Set(["q1", "a1", "q2", "a2", "SA"]); - const keepT5 = cans[4]!.core.filter((u) => !surviveKeys.has(keyOf(u))); + assert.equal(countOcc(cans[5]!.raw, SUM_A), 0, `${L(5)}: alpha summary absorbed into the wide fold`); + assert.equal(countOcc(cans[5]!.raw, wide), 2, `${L(5)}: wide summary rides geo_wide args+receipt`); + assert.equal(countOcc(cans[5]!.raw, gamma), 0, `${L(5)}: E not folded yet`); + assert.equal(countOcc(cans[5]!.raw, SUM_MARKER), 0, `${L(5)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(cans[6]!.raw, wide), 2, `${L(6)}: wide carrier persists`); + assert.equal(countOcc(cans[6]!.raw, gamma), 2, `${L(6)}: gamma rides geo_e from its fold turn on`); + assert.equal(countOcc(cans[6]!.raw, SUM_MARKER), 0, `${L(6)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(cans[7]!.raw, wide), 2, `${L(7)}: wide carrier persists`); + assert.equal(countOcc(cans[7]!.raw, gamma), 2, `${L(7)}: gamma carrier persists`); + assert.equal(countOcc(cans[7]!.raw, SUM_MARKER), 0, `${L(7)}: carriers stable across quiet turns`); + // t5→t6 rewrite: the wide span (minus the session-head boundary) is + // consumed and the dead-block pair swept — everything else keeps order + const goneT6 = new Set(["a1", "q2", "a2", "TU:geo_a", "TR:geo_a"]); + const keptT5 = cans[4]!.core.filter((u) => !goneT6.has(keyOf(u))); let cursor = -1; - for (const u of keepT5) { + for (const u of keptT5) { const found = cans[5]!.core.findIndex((v, j) => j > cursor && eqUnit(u, v)); assert.ok(found > cursor, `cfp-geo t6: unit ${JSON.stringify(keyOf(u))} lost or reordered by the wide fold`); cursor = found; } - assert.equal(countOcc(cans[5]!.raw, wide), 2, `${L(5)}: wide fold turn occurrences (pair quotes; anchor stripped)`); - assert.equal(countOcc(cans[6]!.raw, gamma), 2, `${L(6)}: gamma fold turn occurrences (pair quotes; anchor stripped)`); - assert.equal(countOcc(cans[7]!.raw, wide), 2, `${L(7)}: resent wide quotes must survive`); - assert.equal(countOcc(cans[7]!.raw, gamma), 2, `${L(7)}: resent gamma quotes must survive`); - for (const i of [6, 7]) { - const raw = cans[i]!.raw; - assert.ok(raw.includes("\"geo_wide\"") && raw.includes("\"geo_e\""), `${L(i)}: latest two pairs must stay visible`); - assert.ok(!raw.includes("\"geo_a\""), `${L(i)}: oldest pair must be pruned (KEEP_LAST_ORPHANED=2)`); + assertFoldTransition(cans[5]!.core, cans[6]!.core, ["q3"], [], L(6)); // protected-zone carve: only E's first message folds; no anchor — geo_e rides + // #1567: while the pairs ride, their summaries live in TOOL units only — + // never a text-unit anchor + for (const idx of [5, 6, 7]) { + const textCarriers = cans[idx]!.core.filter((u) => u.k === "text" && (u.x.includes(wide) || u.x.includes(gamma))); + assert.equal(textCarriers.length, 0, `${L(idx)}: no text-unit anchor while the riding pairs carry the summaries (#1567)`); } - // #1567 (fixed): no text unit carries the wide summary anymore (stripped - // anchor); the quotes ride structured tool units, not text - const wideT6 = cans[6]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); - const wideT7 = cans[7]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); - assert.equal(wideT6.length, 0, `${L(6)}: wide carrier must be stripped while the geo_wide pair rides inbound`); - assert.equal(wideT7.length, 0, `${L(7)}: wide carrier must be stripped while the geo_wide pair rides inbound`); assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); -// #1567 hardening scenario A: a contract-violating (or pruning) client stops -// re-sending an older compress pair. While pairA rides, its anchor is -// stripped (P2b); the moment the pair vanishes the anchor MUST re-carry the -// summary (zero-carrier fail-safe) and the body must stay append-stable on -// later turns (no flapping). -testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → anchor re-carries (#1567 hardening)`, async () => { +// #1567 hardening scenario A, re-pinned under #3769: a contract-violating +// (or pruning) client stops re-sending an older compress pair at t7. While +// the pairs ride they ARE the carriers (anchors stripped); the moment pairA +// stops riding, its anchor re-carries the summary exactly once at the +// covered span — then the body stays append-stable (no flapping). PairB +// still rides, so anchor B never surfaces. +testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → anchor re-carries once, no flapping (#1567/#3769)`, async () => { const steps: Step[] = [ { send: [T(1), A(1), T(2), A(2)] }, { send: [T(3), A(3)] }, @@ -730,32 +748,43 @@ testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pai assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-drop"); - // t4..t6: both pairs ride → both anchors stripped + // t4..t6: both pairs ride as carriers — anchors stripped for (const i of [3, 4, 5]) { const raw = cans[i]!.raw; - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: carrier must be stripped while pair rides`); - assert.ok(raw.includes('"cfp_a"'), `${L(i)}: pairA still re-sent here`); + assert.ok(raw.includes('"cfp_a"'), `${L(i)}: pair A rides as its fold's carrier (#1567)`); + assert.ok(i >= 5 || !raw.includes('"cfp_b"') , `${L(i)}: pair B rides from t6 on`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A rides pair A args+receipt`); + assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 2, `${L(i)}: SUM_B rides pair B from its fold turn on`); } - assert.ok(cans[5]!.raw.includes('"cfp_b"'), `${L(5)}: pairB rides`); - // t7/t8: pairA gone → anchor A is the ONLY carrier again - for (const i of [6, 7]) { - const raw = cans[i]!.raw; - assert.ok(!raw.includes('"cfp_a"'), `${L(i)}: client no longer re-sends pairA`); - assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(i)}: anchor A must re-carry (exactly one carrier)`); - assert.equal(countOcc(raw, SUM_A), 1, `${L(i)}: SUM_A = carrier only (pair quotes gone with the pair)`); - assert.equal(countOcc(raw, SUM_B), 2, `${L(i)}: SUM_B quotes only, anchor B still stripped (pairB rides)`); - assert.ok(raw.includes('"cfp_b"'), `${L(i)}: pairB rides`); + // t7: pairA dropped → its anchor re-carries exactly once at the covered + // span; pairB still rides so anchor B stays stripped (#1567 fail-safe) + { + const raw = cans[6]!.raw; + assert.ok(!raw.includes('"cfp_a"'), `${L(6)}: dropped pair stays dropped`); + assert.ok(raw.includes('"cfp_b"'), `${L(6)}: pair B unaffected by A's drop`); + assert.equal(countOcc(raw, SUM_A), 1, `${L(6)}: anchor A re-carries the sole copy`); + assert.equal(countOcc(raw, SUM_B), 2, `${L(6)}: pair B still rides`); + assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(6)}: exactly one anchor rendered (A's)`); + assert.ok(raw.includes("q7-marker"), `${L(6)}: new turn visible`); + } + // t8: once re-carried, the body stays append-stable (no flapping) + { + const raw = cans[7]!.raw; + assert.equal(countOcc(raw, SUM_A), 1, `${L(7)}: anchor A stable`); + assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(7)}: no flapping after the drop`); } - assert.ok(cans[6]!.raw.includes("q7-marker"), `${L(6)}: new turn visible`); - // once re-carried, the body stays append-stable (no flapping) assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); -// #1567 hardening scenario B: three concurrent ACTIVE folds. The kernel hides -// consumed compress pairs beyond the newest KEEP_LAST_ORPHANED=2, so the -// oldest pair leaves the wire while its block is still active — the strip -// guard must see it gone and KEEP that anchor (kernel-side fail-safe). -testCompat(`cache-friendly plugin matrix: anthropic — 3rd fold prunes oldest pair → oldest anchor kept (#1567 hardening)`, async () => { +// #3769 contract pin (plugin lane): three back-to-back folds, each with a +// client-visible compress pair. The old KEEP_LAST_ORPHANED=2 rotation hid +// the oldest pair on EVERY turn — a mid-history rewrite with no fold behind +// it, exactly #3769's cache seam. Under the carrier regime every pair rides +// as its fold's summary carrier for good: no per-turn hiding, no churn, +// anchors stay stripped while the pairs ride, and quiet turns stay +// append-stable. +testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: every pair rides as its carrier, quiet turns append-stable (#3769)`, async () => { const SUM_P1 = "Prune-safe fold one summary covering m00001..m00002"; const SUM_P2 = "Prune-safe fold two summary covering m00003..m00004"; const SUM_P3 = "Prune-safe fold three summary covering m00005..m00006"; @@ -779,25 +808,27 @@ testCompat(`cache-friendly plugin matrix: anthropic — 3rd fold prunes oldest p assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-prune"); - // t3/t4: pairs ride (≤ 2) → anchors stripped. From t5 on, three pairs - // coexist and the kernel hides the oldest (KEEP_LAST_ORPHANED=2) — the - // guard sees it gone and keeps that anchor (kernel-side fail-safe) - for (const i of [2, 3]) { - assert.equal(countOcc(cans[i]!.raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while pairs ride`); + // t3..t5: each fold's pair rides from its fold turn on (args+receipt = + // 2 occurrences); anchors stay stripped for the life of the pairs + for (const i of [2, 3, 4]) { + const raw = cans[i]!.raw; + assert.ok(raw.includes('"fs1"'), `${L(i)}: pair 1 rides as its fold's carrier (#1567)`); + assert.ok(i >= 3 || !raw.includes('"fs2"'), `${L(i)}: pair 2 rides from t4 on`); + assert.ok(i >= 4 || !raw.includes('"fs3"'), `${L(i)}: pair 3 rides from t5 on`); + assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: P1 rides pair-1 args+receipt`); + assert.equal(countOcc(raw, SUM_P2), i < 3 ? 0 : 2, `${L(i)}: P2 rides from its fold turn on`); + assert.equal(countOcc(raw, SUM_P3), i < 4 ? 0 : 2, `${L(i)}: P3 rides from its fold turn on`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while the pairs ride`); } - assert.ok(cans[3]!.raw.includes('"fs1"') && cans[3]!.raw.includes('"fs2"'), `${L(3)}: both pairs visible`); - // t5..t7: fs1 hidden by KEEP_LAST_ORPHANED → anchor F1 re-carries; - // F2/F3 anchors stay stripped while their pairs ride - for (const i of [4, 5, 6]) { + // t6/t7: all three pairs persist, stable across quiet turns + for (const i of [5, 6]) { const raw = cans[i]!.raw; - assert.ok(!raw.includes('"fs1"'), `${L(i)}: oldest pair pruned (KEEP_LAST_ORPHANED=2)`); - assert.ok(raw.includes('"fs2"') && raw.includes('"fs3"'), `${L(i)}: newest two pairs stay visible`); - assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(i)}: exactly one carrier (F1 anchor re-carried)`); - assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: SUM_P1 = carrier only (its pair quotes pruned with the pair)`); - assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: SUM_P2 quotes only`); - assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: SUM_P3 quotes only`); + assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: P1 carrier persists`); + assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: P2 carrier persists`); + assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: P3 carrier persists`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: three riding pairs, zero anchors, stable`); } - // once the oldest anchor re-carries, later turns are append-stable + // mid-history never mutates between quiet turns assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q8", "a8"]); }, { timeout: 120_000 }); diff --git a/tests/compress-observability.test.ts b/tests/compress-observability.test.ts index 68b811493b..a31985d82c 100644 --- a/tests/compress-observability.test.ts +++ b/tests/compress-observability.test.ts @@ -88,7 +88,7 @@ test("#189: fold point picks the EARLIEST range start across multiple ranges", ( const ctx = makeCompressibleCtx(); ctx.session.stats.lastInputTokens = 100000; const args = { content: [ - { startId: "m00005", endId: "m00006", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, + { startId: "m00003", endId: "m00004", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, { startId: "m00001", endId: "m00002", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, ] }; runApply(ctx, args); diff --git a/tests/e2e-overflow-selfheal.test.ts b/tests/e2e-overflow-selfheal.test.ts index 08170ef714..45528eefbb 100644 --- a/tests/e2e-overflow-selfheal.test.ts +++ b/tests/e2e-overflow-selfheal.test.ts @@ -193,7 +193,7 @@ test("e2e #987/#1195 T1: a stated-window overflow arms the one-shot shrink, lear } }); -test("e2e #987 T2: an overflow WITHOUT a window number arms at the declared window and learns nothing", async () => { +test("e2e #987 T2: an overflow WITHOUT a window number arms at the payload's own size, rescues in-request (#3787), and learns nothing", async () => { let streamingCall = 0; let summaryCalls = 0; const upstream = http.createServer((req, res) => { @@ -238,24 +238,27 @@ test("e2e #987 T2: an overflow WITHOUT a window number arms at the declared wind const headers = { "content-type": "application/json", "x-acp-session": "t2-sess" }; const r1 = await fetch(url, { method: "POST", headers, body }); - assert.equal(r1.status, 400); + // #3787: a NUMBERLESS overflow used to pass through verbatim (the arm + // fired but the refold judged against the 400k declaration and found + // "already fits" — nothing to fold, 400 to the client). The arm is + // the only ceiling evidence: refold under arm*0.9, fold, re-send. + assert.equal(r1.status, 200, "#3787: numberless overflow rescued in-request under the armed window"); + assert.equal(r1.headers.get("content-type"), "text/event-stream", "the retry's SSE is what the client sees"); await r1.text(); + assert.ok(summaryCalls >= 1, "the rescue ran a preflight fold under the armed window"); const s = listSessions().find((x) => x.id === "t2-sess"); assert.ok(s, "session exists"); assert.equal(s!.metadata.confirmedContextLimits, undefined, "nothing learned"); - assert.ok(s!.stats.lastInputTokens > 10_000 && s!.stats.lastInputTokens < 400_000, `armed at the payload's own size (~16k, not the 400k declaration): ${s!.stats.lastInputTokens}`); - // #1839 ask 3: an arm is NOT a billing report — it carries its own - // provenance grade "overflow-arm" (still rescue-grade: effectiveTokenCount's - // fast path, the #496 evidence gate and the preflight floors accept it; - // billing-grade consumers keep excluding it). - assert.equal(s!.stats.lastInputTokensSource, "overflow-arm", "the arm carries its own provenance grade, never billing-grade \"usage\""); + assert.equal(s!.stats.lastInputTokens, 5000, "the retry's own usage report replaced the armed baseline"); + assert.equal(s!.stats.lastInputTokensSource, "usage", "the successful retry reports usage-grade"); + assert.equal(s!.stats.overflowArmTokens, undefined, "#1129: the retry's usage report retired the one-shot arm"); - // r2: forwarded as-is (no arm, nothing learned) — the pass-through - // shape the old #969 test asserted is unchanged. + // r2: same payload — the fold is sticky, forwarded compacted, no new fold. + const summaryCallsBefore = summaryCalls; const r2 = await fetch(url, { method: "POST", headers, body }); assert.equal(r2.status, 200); await r2.text(); - assert.equal(summaryCalls, 0, "no fold"); + assert.equal(summaryCalls, summaryCallsBefore, "r2 forwards the sticky fold without folding again"); } finally { proxy.close(); proxy.closeIdleConnections?.(); diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index c565999118..91c052b92d 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "1c7b7922230de1bc61ce9b909a9fbfd4a9a28e478d12178568d21e97946760b5", + "stateDigest": "5839fb455554a276e973fc8e2a6039971d430df48c5e64970728cc66f3005ae7", "sectionDigests": { "state": "a4b83f873d16bfc4", "blockContents": "dc7d13d250c94bb4", "stats": "cff3c2546898aeb3", - "metadata": "b6fe321d2ad536b0" + "metadata": "95795174db0e1e50" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", @@ -101,7 +101,13 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 49, diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 6ed76a04ec..ebe66d6ae6 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "995db4a4701baf4b72504dedc8842a79fc17dc9bcc535e4367c8a87e732616dd", + "stateDigest": "38d6459bd83802b5c7ba79b10d6e12a82ba99086377ae2ec8676c41558e08b3d", "sectionDigests": { "state": "463b3dc26a100c2a", "blockContents": "94a58f96d7c5cfc3", "stats": "59842ce5415d3a4d", - "metadata": "348ced08c027b22b" + "metadata": "d16fb5934a3e8779" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", @@ -101,7 +101,13 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 19ddaf90da..13d78598b0 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "32b187839fe05ec69aeebade3bb8aad18587ec7018a1816df7c50471dc8a3187", + "stateDigest": "f4bb18cf96a64525a323d3db05cd160a375464fdfdf5c788f823acdf96a95825", "sectionDigests": { "state": "a9b9531d952b48d8", "blockContents": "473264d9df8ba338", "stats": "e358dc992395fc7e", - "metadata": "124f318e161ebf7f" + "metadata": "9e864ac22f86950a" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", @@ -101,7 +101,13 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 2dd5340f2e..36ae5150bd 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "96f5faf2e54f7d9257f4f423e4abfc089bd90140a1970f862061685707c6b6fa", + "stateDigest": "2f2c59befaa51d0bc726d44821f8c8ddf12d6237ff84dfbe213af7c74ca885cf", "sectionDigests": { "state": "d7951fa36f9e6fe9", "blockContents": "b6701268a0bdcd8a", "stats": "67e33bcc612fdcc1", - "metadata": "ac1c297bf3b603eb" + "metadata": "71bc14af02391230" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", @@ -101,7 +101,13 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 55, diff --git a/tests/issue3769-orphan-sweep-ledger.test.ts b/tests/issue3769-orphan-sweep-ledger.test.ts new file mode 100644 index 0000000000..d6182a4c21 --- /dev/null +++ b/tests/issue3769-orphan-sweep-ledger.test.ts @@ -0,0 +1,77 @@ +// #3769: attribute a proactive failed-compress cleanup (orphan rotation) in the +// cache ledger. detectOrphanSweeps watches the monotonic length of +// session.state.sweptOrphanCallIds between samples; when a mid-history break +// otherwise lands in the unexplained seam bucket AND a cleanup grew this turn, +// the residual books into its own bucket (prov:"orphan-sweep") instead of +// "no fold/switch/restart attribution". Gated on actual growth: no growth keeps +// the legacy seam bucket, and a resumed session's pre-existing sweeps baseline +// without being charged to the first sample. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { noteForwardedBody, settleUsageReport, getCacheLedger, buildSessionCacheReport, handleAcpCache } from "../src/cache-ledger.ts"; +import type { Session } from "../src/session.ts"; + +let seq = 0; +function makeSession(): Session { + seq += 1; + return { + id: `orphan-sweep-${seq}`, + metadata: {}, + stats: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, tokenSnapshot: {} }, + } as unknown as Session; +} + +const body = (msgs: string[]): string => JSON.stringify({ model: "m", messages: msgs.map((c) => ({ role: "user", content: c })) }); +const T0 = Date.parse("2026-10-19T10:49:00Z"); + +function settle(session: Session, at: number, input: number, cached: number): void { + settleUsageReport(session, { total: input, reportedCached: cached, output: 0, protocol: "openai", upstream: "http://u" }); +} + +test("cleanup growth + unattributed break books the orphan-sweep bucket, not the seam bucket", () => { + const s = makeSession(); + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); // baseline: swept empty -> high-water 0 + // A proactive cleanup rotated two failed-compress records out of history this turn. + s.state!.sweptOrphanCallIds = ["c1", "c2"]; + noteForwardedBody(s, body(["a", "B2", "c"])); // outbound mid-history break, no inbound capture + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + const line = led.lines[led.lines.length - 1]!; + assert.equal(line.prov, "orphan-sweep", "residual booked to the cleanup"); + assert.equal(line.seam, undefined, "NOT a seam suspect"); + assert.equal(led.agg.orphanSweepSeams, 1); + assert.ok(led.agg.orphanSweepMissed > 0); + assert.equal(led.agg.seamSuspects, 0, "kept out of the unexplained tally"); + const report = buildSessionCacheReport(s); + assert.equal(report.seam.orphanSweep.count, 1); + assert.ok(report.seam.orphanSweep.missed > 0); + assert.match(handleAcpCache(s).text, /ORPHAN CLEANUP SEAM/); +}); + +test("same break WITHOUT cleanup growth keeps the legacy seam bucket (arm is gated on growth)", () => { + const s = makeSession(); + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); + noteForwardedBody(s, body(["a", "B2", "c"])); // identical break, but nothing was swept + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + const line = led.lines[led.lines.length - 1]!; + assert.equal(line.seam, 1, "legacy seam bucket preserved"); + assert.equal(line.prov, undefined); + assert.equal(led.agg.orphanSweepSeams, 0); + assert.equal(led.agg.seamSuspects, 1); +}); + +test("a resumed session's pre-existing sweeps baseline without charging the first sample", () => { + const s = makeSession(); + s.state!.sweptOrphanCallIds = ["c1"]; // already swept before this process started + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); // first observation baselines high-water at 1 + noteForwardedBody(s, body(["a", "B2", "c"])); // no NEW growth since baseline + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + assert.equal(led.agg.orphanSweepSeams, 0, "pre-existing sweeps not charged to this sample"); + assert.equal(led.agg.seamSuspects, 1, "falls back to the legacy seam bucket"); +}); diff --git a/tests/issue3769-upgrade-compat.test.ts b/tests/issue3769-upgrade-compat.test.ts new file mode 100644 index 0000000000..3b68e542ae --- /dev/null +++ b/tests/issue3769-upgrade-compat.test.ts @@ -0,0 +1,229 @@ +// #3769 upgrade compatibility (owner merge condition): sessions persisted by a +// pre-fold-gate kernel carry their rotation-hidden orphan pairs ONLY in +// `hiddenOrphanRefs` (no sweptOrphanCallIds / pendingOrphanSweepMinRef). After +// upgrading, those pairs must stay hidden on the outbound history — resurrecting +// them re-inserts mid-history bytes and breaks the cached prefix again. +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; +import { mkdtempSync, readdirSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +process.env.NODE_ENV = "test"; +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_CONFIG_FILE = path.join(mkdtempSync(path.join(tmpdir(), "bili-upg-hermetic-")), "no-config.json"); + +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions, _resetSessionsForTest } from "../src/session.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const LABEL = "upg3769"; +const ARG_JSON = JSON.stringify({ content: [{ startId: "m00001", endId: "m00002", topic: "upg", summary: "UPG_SUM_MARKER" }] }); +const RESULT_TEXT = "UPG_RESULT_MARKER"; + +type M = Record; + +const u1: M = { role: "user", content: "q one alpha" }; +const a1: M = { role: "assistant", content: "a one alpha" }; +const u2: M = { role: "user", content: "q two beta" }; +const a2: M = { role: "assistant", content: "a two beta" }; +const pairCall: M = { role: "assistant", content: null, tool_calls: [{ id: "call_upg1", type: "function", function: { name: "compress", arguments: ARG_JSON } }] }; +const pairResult: M = { role: "tool", tool_call_id: "call_upg1", content: RESULT_TEXT }; +const u3: M = { role: "user", content: "q three gamma" }; +const u4: M = { role: "user", content: "q four delta UPG_TAIL_MARKER" }; + +function sse(obj: unknown): string { + return `data: ${JSON.stringify(obj)}\n\n`; +} + +async function post(url: string, messages: M[], bodies: string[]): Promise { + const r = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": LABEL }, + body: JSON.stringify({ model: "gpt-test", messages, stream: true }), + }); + void bodies; + return r.status ?? 0; +} + +type PersistRecord = { + version: number; + savedAt: number; + id: string; + payload: { + meta: { label?: string }; + messages: { id: string; contentType?: string; toolName?: string; toolCallId?: string }[]; + state: { messageRefs: { byRaw: Record }; hiddenOrphanRefs?: string[]; sweptOrphanCallIds?: string[]; pendingOrphanSweepMinRef?: number }; + }; +}; + +function findSessionFile(root: string, label: string): string { + const stack = [root]; + while (stack.length > 0) { + const d = stack.pop()!; + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) { + stack.push(p); + } else if (e.name.endsWith(".json") && !e.name.endsWith(".content-store.json")) { + try { + const rec = JSON.parse(readFileSync(p, "utf8")) as PersistRecord; + if (rec.id === label || rec.payload?.meta?.label === label) return p; + } catch { /* not our file */ } + } + } + } + throw new Error(`no session file for ${label} under ${root}`); +} + +test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-end (#3769)", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "bili-upg-store-")); + const bodies: string[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + bodies.push(Buffer.concat(chunks).toString("utf8")); + const n = bodies.length; + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: `ok ${n}` } }] })); + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })); + res.write("data: [DONE]\n\n"); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upPort = (upstream.address() as { port: number }).port; + + const stores: SessionStore[] = []; + const proxies: http.Server[] = []; + const closedProxies = new Set(); + async function closeProxy(p: http.Server): Promise { + if (closedProxies.has(p)) return; + closedProxies.add(p); + p.close(); + p.closeIdleConnections?.(); + await once(p, "close").catch(() => undefined); + } + try { + // Phase 1: current code runs two turns; the failed pair rides visibly + // (fold-gated contract: no fold -> no sweep) and the record lands on disk. + const store1 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + stores.push(store1); + _setStoreForTest(store1); + _resetSessionsForTest(); + setRegistryForTest({}); + const proxy1 = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upPort}`]: { models: { "gpt-test": { context: 15_000 } } } }, + modelContextLimit: 15_000, + kernelConfig: defaultConfig(15_000), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + passthrough: false, + passthroughSource: null, + autoUpdate: false, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: true, + releaseNotesCheck: true, + compat: { roles: {} }, + streamErrorShape: "protocol", + mitm: { enabled: false, domains: [] }, + } as unknown as ProxyOptions); + await once(proxy1, "listening"); + proxies.push(proxy1); + const port1 = (proxy1.address() as { port: number }).port; + const url1 = `http://127.0.0.1:${port1}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; + + assert.equal(await post(url1, [u1, a1, u2, a2], bodies), 200); + assert.equal(await post(url1, [u1, a1, u2, a2, { role: "assistant", content: "ok 1" }, pairCall, pairResult, u3], bodies), 200); + // New-code baseline: without a fold the pair stays visible outbound. + assert.ok(bodies[bodies.length - 1]!.includes("UPG_SUM_MARKER"), "pre-upgrade baseline: failed pair rides visibly"); + + const s1 = listSessions().find((x) => x.meta.label === LABEL); + assert.ok(s1, "session exists"); + assert.ok(store1.flushSync(s1!)); + await closeProxy(proxy1); + + // Phase 2: a fresh store hydrates the record, then shapes it into + // what a pre-fold-gate kernel left behind — rotation had hidden the + // pair (remembered by ref), new sweep fields absent. Refs are read + // off the dual-written envelope; the mutation itself goes THROUGH THE + // STORE: since the #3818 full-mode default the index row IS the load + // path, so patching only the envelope would leave a stale row the + // loader serves instead. + _resetSessionsForTest(); + const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + stores.push(store2); + _setStoreForTest(store2); + + const file = findSessionFile(dir, LABEL); + const rec = JSON.parse(readFileSync(file, "utf8")) as PersistRecord; + const callMsg = rec.payload.messages.find((m) => m.contentType === "tool-call" && m.toolName === "compress"); + const resMsg = rec.payload.messages.find((m) => m.contentType === "tool-result" && m.toolCallId === "call_upg1"); + assert.ok(callMsg && resMsg, "record snapshot carries the failed pair"); + const callRef = rec.payload.state.messageRefs.byRaw[callMsg!.id]; + const resRef = rec.payload.state.messageRefs.byRaw[resMsg!.id]; + assert.ok(callRef && resRef, "refs resolvable from the persisted state"); + const sPre = store2.loadSync(LABEL); + assert.ok(sPre, "record hydrates before the upgrade turn"); + sPre.state.hiddenOrphanRefs = [callRef, resRef]; + delete sPre.state.sweptOrphanCallIds; + delete sPre.state.pendingOrphanSweepMinRef; + assert.ok(store2.flushSync(sPre), "patched record commits through the store"); + const proxy2 = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upPort}`]: { models: { "gpt-test": { context: 15_000 } } } }, + modelContextLimit: 15_000, + kernelConfig: defaultConfig(15_000), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + passthrough: false, + passthroughSource: null, + autoUpdate: false, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: true, + releaseNotesCheck: true, + compat: { roles: {} }, + streamErrorShape: "protocol", + mitm: { enabled: false, domains: [] }, + } as unknown as ProxyOptions); + await once(proxy2, "listening"); + proxies.push(proxy2); + const port2 = (proxy2.address() as { port: number }).port; + const url2 = `http://127.0.0.1:${port2}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; + + const hist3: M[] = [u1, a1, u2, a2, { role: "assistant", content: "ok 1" }, pairCall, pairResult, u3, { role: "assistant", content: "ok 2" }, u4]; + assert.equal(await post(url2, hist3, bodies), 200); + const out = bodies[bodies.length - 1]!; + assert.ok(!out.includes("UPG_SUM_MARKER"), "upgrade: hidden compress args do not resurface"); + assert.ok(!out.includes("UPG_RESULT_MARKER"), "upgrade: hidden result does not resurface"); + assert.ok(out.includes("UPG_TAIL_MARKER"), "live tail rides normally"); + + const s2 = listSessions().find((x) => x.meta.label === LABEL); + assert.ok(s2, "hydrated session exists"); + assert.ok(s2.state.hiddenOrphanRefs?.includes(callRef!), "legacy hidden set survived hydration and self-perpetuates"); + await closeProxy(proxy2); + } finally { + for (const s of stores) s.closeQuietly(); + for (const p of proxies) await closeProxy(p); + upstream.close(); + upstream.closeIdleConnections?.(); + await once(upstream, "close").catch(() => undefined); + _setStoreForTest(new SessionStore({ enabled: false })); + void rmrf(dir); + } +}); diff --git a/tests/issue3787-overflow-refold.test.ts b/tests/issue3787-overflow-refold.test.ts index ee4058d126..e84651adb5 100644 --- a/tests/issue3787-overflow-refold.test.ts +++ b/tests/issue3787-overflow-refold.test.ts @@ -105,16 +105,24 @@ test("e2e #3787: the agnes/cch numberless context_limit 400 is recognized and th _setStoreForTest(new SessionStore({ enabled: false })); setRegistryForTest({}); - // Declared window below the payload's true size: the first oversized turn - // rides out (the optimistic trigger cannot see CJK-density mass), the - // gateway 400s, and the rescue must come from the recognized error. + // Declared window far above the payload's true size: preflight skips the + // first turn entirely (nothing over 100K), the gateway 400s the RAW body + // anyway (its real ceiling is below the payload — agnes-3.0-flash through + // a mis-declared cch), and the rescue must come from the recognized error: + // arm = min(declared, payload's own estimate), refold under arm*0.9 folds + // real ranges, the smaller body is re-sent within the same request. + // (A declared window BELOW the payload cannot express this contract: the + // calibrated trigger now folds everything compressible BEFORE the send, + // so a numberless rejection arrives with zero foldable mass left — the + // rescue's historical "success" there rode on re-render truncation + // asymmetry, which #3939 ③ eliminated.) const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: "http://127.0.0.1", - routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-test": { context: 10_000 } } } }, - modelContextLimit: 10_000, - kernelConfig: defaultConfig(10_000), + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-test": { context: 100_000 } } } }, + modelContextLimit: 100_000, + kernelConfig: defaultConfig(100_000), compress: { injectTool: true, injectNudge: true }, compat: { roles: {} }, streamErrorShape: "protocol", diff --git a/tests/stub-resubmission-guard.test.ts b/tests/stub-resubmission-guard.test.ts index a1864162d6..42c5e2a097 100644 --- a/tests/stub-resubmission-guard.test.ts +++ b/tests/stub-resubmission-guard.test.ts @@ -124,7 +124,10 @@ test("#2749 canonical integrity pin: full summary survives save → history redu ["user", "Follow-up questions about the configuration."], ["assistant", "y".repeat(3000)], ]); - const out = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00004", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + // The fold is linked to its tool-call id exactly as production does + // (src/loop/core.ts passes callId); the resent "call-b1" below then + // classifies active instead of orphan. + const out = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00004", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(out.startsWith("[Compressed m00001–m00004 → 1 block(s)"), out.split("\n")[0]); // (a) saved canonically in full — no trimming, no truncation. assert.equal(storedSummary(ctx.session.state, "b1"), LONG_SUMMARY); @@ -155,7 +158,7 @@ test("#2749 rejects a byte-identical stub re-submission; state unchanged, refusa ["user", "late part two"], ["assistant", "d".repeat(1000)], ["user", "late part three"], ["assistant", "e".repeat(1000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); // Next request: the model sees its own call reduced to the stub. ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); @@ -181,7 +184,7 @@ test("#2749 self-refold under the block's own stub is blocked (silent-shrink wor ["user", "setup steps discussed early"], ["assistant", "a".repeat(4000)], ["user", "more detail on the setup"], ["assistant", "b".repeat(4000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); // Re-fold the SAME span submitting the block's own stub as the new summary. @@ -204,7 +207,7 @@ test("#2749 legitimate summaries still pass: short, ellipsis-prose, and ±1-char ["user", "late four"], ["assistant", "f".repeat(4000)], ["user", "late five"], ["assistant", "g".repeat(4000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); @@ -234,7 +237,7 @@ test("#2749 host harvest tracks kernel-emitted stubs (format-drift tripwire)", ( seedTurn(ctx, [ ["user", "early part"], ["assistant", "a".repeat(1000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed"), ok.split("\n")[0]); const view = nextRequestView(ctx, "call-b1", "m00001", "m00002"); const visCall = view.find((m) => m.toolCallId === "call-b1")!; diff --git a/tests/verif-bug3769-orphan-window.test.ts b/tests/verif-bug3769-orphan-window.test.ts index 7d2e0eab9b..4bd2888b85 100644 --- a/tests/verif-bug3769-orphan-window.test.ts +++ b/tests/verif-bug3769-orphan-window.test.ts @@ -1,12 +1,18 @@ /** * Bug #3769 — hide-consumed orphan window breaks append-only stability. * - * kernel/src/hide-consumed.ts keeps only the NEWEST KEEP_LAST_ORPHANED=2 - * orphan compress-call pairs visible. When a THIRD failed-compression pair - * is appended, the OLDEST pair (mid-history) is silently dropped — the - * outbound prefix changes with no fold event, so the provider prompt cache - * is invalidated mid-conversation (issue #3769: 184,423 tokens re-billed, - * hit rate 97.7% → 19.2%). + * ORIGINAL bug (pre-#3784): kernel/src/hide-consumed.ts kept only the NEWEST + * KEEP_LAST_ORPHANED=2 orphan compress-call pairs visible; a THIRD failed + * pair silently dropped the OLDEST mid-history (issue #3769: 184,423 tokens + * re-billed, hit rate 97.7% → 19.2%). + * + * FIXED contract (#3784, absorbed into #3939 ①): orphan sweeping is + * FOLD-GATED — nothing hides between folds; the sweep rides the fold's own + * sanctioned rewrite (state.pendingOrphanSweepMinRef). This harness was + * re-baselined when the fix landed: the append-only flip must now be + * UNSAT (Q1), the mechanism must remain derivable THROUGH the fold marker + * only (Q1b sat) — proving the judge still sees the gated mechanism, not a + * vacuous truth — and non-compress appends stay stable (Q2 unsat). * * No-cheat contract (#3848 / user directive m00852): * - we execute the REAL hideConsumedCompressCalls (translateFunction); @@ -78,19 +84,30 @@ test("bug3769: solver proves append-only growth hides a previously-visible pair" } return { kind: "arr", elems: list, len: ctx.Real.val(list.length) }; }; - const stateObj = (): import("../verif/symbolic.js").SymVal => { + const stateObj = (foldGate: boolean): import("../verif/symbolic.js").SymVal => { const fields = new Map(); fields.set("blocks", { kind: "arr", elems: [], len: ctx.Real.val(0) }); // real state shape: messageRefs = { byRaw: Record } const byRawInner = new Map(); + if (foldGate) { + // A fold just committed: refs are minted and the sweep is pending from + // the fold's own min ref (m00001) — the sanctioned rewrite window. + byRawInner.set("t1", { kind: "str", t: ctx.String.val("m00001") }); + byRawInner.set("r1", { kind: "str", t: ctx.String.val("m00002") }); + byRawInner.set("t2", { kind: "str", t: ctx.String.val("m00003") }); + byRawInner.set("r2", { kind: "str", t: ctx.String.val("m00004") }); + byRawInner.set("t3", { kind: "str", t: ctx.String.val("m00005") }); + byRawInner.set("r3", { kind: "str", t: ctx.String.val("m00006") }); + fields.set("pendingOrphanSweepMinRef", { kind: "num", t: ctx.Real.val(1) }); + } fields.set("messageRefs", { kind: "obj", fields: new Map([["byRaw", { kind: "obj", fields: byRawInner }]]) }); return { kind: "obj", fields }; }; - const run = (grown: boolean) => { + const run = (grown: boolean, foldGate = false) => { const { exec } = translateFunction({ file, function: "hideConsumedCompressCalls", - params: { state: stateObj(), messages: msgs(grown), isCompressTool: { kind: "undef" } }, + params: { state: stateObj(foldGate), messages: msgs(grown), isCompressTool: { kind: "undef" } }, ctx, options: { portImpls: { @@ -107,6 +124,7 @@ test("bug3769: solver proves append-only growth hides a previously-visible pair" }; const outA = run(false); const outB = run(true); + const outGate = run(true, true); const arrIds = (o: import("../verif/symbolic.js").ExecOut) => { const ret = o.ret as Extract; @@ -129,12 +147,9 @@ test("bug3769: solver proves append-only growth hides a previously-visible pair" const visibleBefore = hasR1(arrIds(outA)); const visibleAfter = hasR1(arrIds(outB)); - // Q1 (bug exists): with the third pair a genuine compress pair (fresh id), - // the previously-visible r1 disappears — append-only stability violated. - const assumptions: string[] = [ - "(declare-fun __tn3 () String)", - "(declare-fun __tc3 () String)", - ]; + // Q1 (fix holds): with the third pair a genuine compress pair (fresh id), + // append-only growth alone can NO LONGER hide the previously-visible r1 — + // the sweep is fold-gated (#3784). No counterexample exists. const q1 = smt2Of(outB.ctx, [ tn3.eq(ctx.String.val("compress")), tc3.eq(ctx.String.val("c1")).not(), @@ -144,7 +159,23 @@ test("bug3769: solver proves append-only growth hides a previously-visible pair" ...outA.assumptions, ...outB.assumptions, ]); const r1 = await solveSmt2(q1, ["tn3", "tc3"]); - assert.equal(r1.status, "sat", "solver must derive the #3769 flip from the general invariant"); + assert.equal(r1.status, "unsat", "append-only growth must not hide anything after the #3784 fold gate"); + + // Q1b (mechanism still reachable, only through the fold): the same grown + // history under a pending fold sweep (pendingOrphanSweepMinRef minted refs) + // DOES hide r1 — the rewrite is sanctioned to ride the fold boundary. + // SAT here proves the judge is not vacuous: the gated path is derivable. + const gateHidden = hasR1(arrIds(outGate)); + const q1b = smt2Of(outGate.ctx, [ + tn3.eq(ctx.String.val("compress")), + tc3.eq(ctx.String.val("c1")).not(), + tc3.eq(ctx.String.val("c2")).not(), + visibleBefore, + gateHidden.not(), + ...outA.assumptions, ...outGate.assumptions, + ]); + const r1b = await solveSmt2(q1b, ["tn3", "tc3"]); + assert.equal(r1b.status, "sat", "the fold-gated sweep must remain derivable through the fold marker"); // Q2 (negative control): when the appended pair is NOT a compress call, // the same invariant holds — no counterexample exists. @@ -167,10 +198,25 @@ test("bug3769: solver proves append-only growth hides a previously-visible pair" asMsg({ id: "t3", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c3", text: "{}" }), asMsg({ id: "r3", role: "tool", contentType: "tool-result", toolCallId: "c3", text: "fail" }), ]); - assert.equal(b.hidden, 2, "the oldest orphan pair (t1+r1) is hidden — mid-history removal"); - assert.ok(!b.messages.some((m) => m.id === "r1"), "r1 GONE after appending an unrelated third pair — #3769 flip"); - assert.ok(b.messages.some((m) => m.id === "r2") && b.messages.some((m) => m.id === "r3"), - "window keeps only the newest two pairs"); + assert.equal(b.hidden, 0, "#3784: appending an orphan pair no longer sweeps anything between folds"); + assert.ok(b.messages.some((m) => m.id === "r1"), "r1 STAYS visible after the append — the #3769 flip is gone"); + assert.equal(b.newlySweptCallIds.length, 0, "nothing newly swept without a pending fold"); + + // Replay the sanctioned side: a pending fold sweep (refs minted, min ref 1) + // hides the orphan pairs in one pass — the rewrite rides the fold. + const gateState = { + blocks: [], + messageRefs: { byRaw: { t1: "m00001", r1: "m00002", t2: "m00003", r2: "m00004", t3: "m00005", r3: "m00006" } }, + pendingOrphanSweepMinRef: 1, + } as unknown as CompressionState; + const c = hideConsumedCompressCalls(gateState, [ + ...histA(), + asMsg({ id: "t3", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c3", text: "{}" }), + asMsg({ id: "r3", role: "tool", contentType: "tool-result", toolCallId: "c3", text: "fail" }), + ]); + assert.ok(!c.messages.some((m) => m.id === "r1"), "r1 hidden AT the fold boundary — the gated sweep fires"); + assert.deepEqual(c.newlySweptCallIds, ["c1", "c2", "c3"], "all three orphan pairs ride the fold sweep"); + void solverBool; }); // helper: print a z3 Bool in smt2 form via its solver-companion — @@ -178,10 +224,8 @@ test("bug3769: solver proves append-only growth hides a previously-visible pair" const solverBool = (b: import("../verif/terms.js").Bool): string => b.toString(); test("bug3769 counter-test: with a larger window the same append is stable (shows the invariant is the judge, not the shape)", { timeout: 240_000 }, async (t) => { - // KEEP_LAST_ORPHANED=2 is a module constant; we cannot widen it here - // without code changes. Instead we verify the DIRECT sibling: only TWO - // orphan pairs exist and the appended traffic is user text (not a compress - // call) — visibility must be stable. + // Only TWO orphan pairs exist and the appended traffic is user text (not a + // compress call) — visibility must be stable under the fold gate too. const state = { blocks: [], messageRefs: { byRaw: {} } } as unknown as CompressionState; const before = hideConsumedCompressCalls(state, histA()); const after = hideConsumedCompressCalls(state, [ diff --git a/verif/symbolic.ts b/verif/symbolic.ts index 1741051911..5d6cf66fbe 100644 --- a/verif/symbolic.ts +++ b/verif/symbolic.ts @@ -1701,7 +1701,17 @@ export function translateFunction(input: TranslateInput): { exec: () => ExecOut if (isNum(a0)) return numToStringStrFor(a0.t); return { kind: "any" }; } - return isNum(a0) ? a0 : { kind: "any" }; + if (isNum(a0)) return a0; + // Number(constString) folds faithfully — refNum() and friends parse + // digit strings; without this fold Number.isFinite(any) collapses to + // constant false and gated paths silently disappear from the model. + if (isStr(a0)) { + const cs = (a0.t as unknown as { c?: string }).c; + if (typeof cs === "string" && cs.trim() !== "" && !Number.isNaN(Number(cs))) { + return { kind: "num", t: num(ctx, Number(cs)) }; + } + } + return { kind: "any" }; } if (name === "isNaN") { const a0 = argVals[0];