Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
3759b53
kernel: reject whole compress batch when any summary fails length val…
Oct 20, 2026
69b2ad1
fix: classify batch summary-length rejections in gateReason + review …
Oct 20, 2026
fd0b29e
kernel: all-or-nothing batch when a range dies during application (#3…
Oct 20, 2026
5d6b22a
kernel: gate failed-compress orphan rotation on committed folds (#3769)
Oct 19, 2026
4bc5076
fix: attribute proactive orphan-cleanup breaks in the cache ledger (#…
Oct 19, 2026
df3b259
kernel: normalize indentation of #3769 edits (whitespace only)
Oct 19, 2026
ce6c244
kernel: derive orphan-sweep marker from covered message refs (#3769 r…
Oct 19, 2026
6544bf3
fix: hydrate orphan-sweep state fields across restart (#3769 review fix)
Oct 19, 2026
4aaf6ee
test: align #2749/#1567 fixtures and storage-render goldens with fold…
Oct 19, 2026
9488e48
kernel: honor legacy hiddenOrphanRefs on upgrade (#3769 P0)
Oct 19, 2026
3eaf0cd
kernel: carry orphan-cleanup fields through sync-blocks (#3769 P0)
Oct 19, 2026
a8d24aa
kernel: regression tests for upgrade compat and pipeline carry (#3769…
Oct 19, 2026
0be491f
fix: inherit orphan-cleanup ledgers on resume/fork adoption (#3769 P0)
Oct 19, 2026
fe2c46e
test: end-to-end upgrade compat for rotation-hidden orphans (#3769 P0)
Oct 19, 2026
8e8590e
test: pin fold-gated sweep timing through the live server (#3769)
Oct 19, 2026
77ee6c7
test: regen storage-render goldens after rebase onto #3803 ledger fie…
Oct 19, 2026
7d3578c
fix: exempt active-block carriers from the fold-gated orphan sweep (#…
Oct 19, 2026
8289c10
test: align cache-friendly-plugin matrix and upgrade-compat with the …
Oct 19, 2026
b9bae89
kernel: latch emergency truncation bytes until the next fold (#3939 ③…
Oct 20, 2026
88ecbbf
kernel: defer BLOCKED→numeric migration carrier tags to the fold boun…
Oct 20, 2026
4344ff2
kernel: fold-gate the opt-in view rewriters via a structural view lat…
Oct 20, 2026
3057f22
chore: sync package-lock kernel version to 0.0.141
Oct 20, 2026
cf49050
chore: sync kernel/package-lock.json version to 0.0.141
Oct 20, 2026
6aed427
kernel: CCR rides the view-latch config digest — disarm releases latc…
Oct 20, 2026
a61a1f9
fix: rescue a numberless overflow in-request under arm*0.9; exhausted…
Oct 20, 2026
06b7d23
test: keep the #189 fold-point pin outside the protected zone under t…
Oct 20, 2026
74eca4e
kernel: keep the fold-gate files inside the verifiable subset; re-bas…
Oct 20, 2026
54bb967
kernel: every applied truncation clip comes out strictly shorter (#3935)
Oct 20, 2026
ad90fea
kernel: BLOCKED binding migration requires no same-id sibling protect…
Oct 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.137",
"version": "0.0.144",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
5 changes: 5 additions & 0 deletions kernel/src/absorb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,11 @@ export function applyAbsorb(input: AbsorbInput): AbsorbOutcome {
...input.state.stats,
absorbedTokens: (input.state.stats.absorbedTokens ?? 0) + tokens,
},
// #3939 ⑤: the absorb call is a deliberate history-rewriting action (a
// fold-class boundary, same family as applyCompression) — release the
// view latch here so the hide lands THIS turn, not at some later fold.
// Keeps absorb's instant-reclaim contract; quiet turns stay latched.
viewLatches: undefined,
};

const bloat =
Expand Down
261 changes: 249 additions & 12 deletions kernel/src/compress.ts

Large diffs are not rendered by default.

149 changes: 118 additions & 31 deletions kernel/src/hide-consumed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,99 @@ import { COMPRESS_TOOL_NAME } from "./compress-tools.js";
import type { CompressionState, CoreMessage } from "./types.js";

// Orphaned compress calls (no matching block — failed attempts, or historical
// calls whose blocks predate compressCallId recording) keep their NEWEST two
// call+result pairs visible: failures must stay observable or a deterministic
// model re-issues the same no-op compress forever, pinned at a fixed point
// (billion-context-pi issue #9: 3,849 identical calls over 5h13m under
// KEEP_LAST_ORPHANED=0). Older orphans are hidden, so the residue is bounded
// at two pairs regardless of session length — PR #18's unbounded accumulation
// does not return (its own live check showed the cap: 10 in → 6 out).
const KEEP_LAST_ORPHANED = 2;
// calls whose blocks predate compressCallId recording) stay visible until a REAL
// fold gives a cache-safe place to remove them (#3769). Failures must remain
// observable or a deterministic model re-issues the same no-op compress forever
// (billion-context-pi #9: 3,849 identical calls over 5h13m under keep=0), AND a
// quiet turn must never move the cached prefix breakpoint forward just to delete
// a few hundred chars of old failure residue — the old keep-last-two rotation did
// exactly that (hid the oldest failure on EVERY turn, seam cost far exceeding the
// bytes removed). Now: between folds nothing is swept; on a fold the node sweeps
// orphans at/after the fold's earliest start ref onto the persistent
// sweptOrphanCallIds set (they stay hidden forever), leaving orphans before it
// visible. Residue is bounded by fold cadence, not a fixed pair count.

export interface HideConsumedResult {
messages: CoreMessage[];
hidden: number;
/** Padded refs of the orphan compress call/result messages hidden by this
* pass (orphans beyond the keep window). Consumed by boundary resolution to
* snap a range endpoint onto a hidden-but-intact orphan (#396). */
/** Padded refs of ALL currently-hidden orphan compress call/result messages
* (the persistent swept set, not just this pass). Consumed by boundary
* resolution to snap a range endpoint onto a hidden-but-intact orphan (#396). */
hiddenOrphanRefs: string[];
/** Orphan call ids newly swept by THIS pass (only when a fold was pending);
* the node unions these into state.sweptOrphanCallIds. Empty on quiet turns. */
newlySweptCallIds: string[];
}

function rangeKey(startRef: string, endRef: string): string {
return `${startRef}::${endRef}`;
}

// #3769: does this call's args quote the folded range of an ACTIVE block? Such a
// re-sent pair IS that block's summary carrier — the host strips the in-place
// anchor only while it rides the history (stripKernelSummaries, #1567/#2042) —
// so the fold-gated sweep must leave it alone. Both arg shapes are accepted to
// mirror the host index ({content:[…]} AND the flat {startId,endId} models emit
// natively); parseCallText alone misses the flat form. Unparseable args fail
// safe as NOT a carrier (under-sweep is safe, over-sweep is not).
function quotesActiveBlock(
text: string | undefined,
activeRangeKeys: Set<string>,
): boolean {
const raw = text ?? "";
const start = raw.indexOf("{");
if (start < 0) return false;
let parsed: unknown;
try {
parsed = JSON.parse(raw.slice(start));
} catch {
return false;
}
if (!parsed || typeof parsed !== "object") return false;
const obj = parsed as Record<string, unknown>;
let entries: unknown[];
if (Array.isArray(obj.content)) {
entries = obj.content;
} else if (typeof obj.content === "string") {
try {
const inner: unknown = JSON.parse(obj.content);
entries = Array.isArray(inner) ? inner : [];
} catch {
return false;
}
} else {
entries = [obj];
}
for (const entry of entries) {
if (!entry || typeof entry !== "object") continue;
const e = entry as Record<string, unknown>;
const s =
typeof e.startId === "string"
? e.startId
: typeof e.messageId === "string"
? e.messageId
: "";
const end =
typeof e.endId === "string"
? e.endId
: typeof e.messageId === "string"
? e.messageId
: "";
if (s && end && activeRangeKeys.has(rangeKey(s, end))) return true;
}
return false;
}

// Numeric position of a padded ref (mNNNNN -> NNNNN); null when unmapped/blocked
// so the caller treats it as "position unknown" and never sweeps on a guess.
// Exported: applyCompression derives the fold sweep marker from covered message
// refs through the same helper so both sides agree on "unknown" (#3769).
export function refNum(ref: string | undefined): number | null {
if (!ref || ref === BLOCKED_REF) return null;
const n = Number(ref.replace(/\D/g, ""));
return Number.isFinite(n) ? n : null;
}

// Named return type (not inline): with the object literal in the signature,
// prettier's layout heuristic was not idempotent here (#416).
interface ParsedCallText {
Expand Down Expand Up @@ -240,33 +311,49 @@ export function hideConsumedCompressCalls(
},
);

const lastOrphaned = new Set<string>();
for (
let i = messages.length - 1;
i >= 0 && lastOrphaned.size < KEEP_LAST_ORPHANED;
i--
) {
const info = infos[i]!;
const callId = messages[i]!.toolCallId;
if (info.isCompressCall && callId && info.fate === "orphan") {
lastOrphaned.add(callId);
}
}
// #3769: orphans are removed only where a real fold already moved the cached
// breakpoint. sweptSet holds orphans removed on earlier folds (persistent +
// monotonic across turns/restarts/upgrades); pendingOrphanSweepMinRef carries
// this fold's earliest start ref so we additionally sweep orphans at/after it.
// Orphans before it sit in the still-cached prefix and stay visible; with no
// fold pending nothing new is swept, so a quiet turn never breaks the prefix.
// Orphans quoting an ACTIVE block's folded range are that block's summary
// carrier (#1567) and are exempt from the sweep (quotesActiveBlock).
const sweptSet = new Set(state.sweptOrphanCallIds ?? []);
const foldMinRef = state.pendingOrphanSweepMinRef;
const newlySweptCallIds: string[] = [];

const keepCallIds = new Set<string>();
const hiddenCallIds = new Set<string>();
for (let i = 0; i < messages.length; i++) {
const info = infos[i]!;
const callId = messages[i]!.toolCallId;
if (!info.isCompressCall || !callId) continue;
if (
info.fate === "inactive" ||
(info.fate === "orphan" && !lastOrphaned.has(callId))
) {
hiddenCallIds.add(callId);
} else {
keepCallIds.add(callId);
let hide = false;
if (info.fate === "inactive") {
hide = true;
} else if (info.fate === "orphan") {
const ref = state.messageRefs.byRaw[messages[i]!.id];
const rememberedHidden: string[] = state.hiddenOrphanRefs ?? [];
if (sweptSet.has(callId)) {
hide = true;
} else if (ref && rememberedHidden.includes(ref)) {
// Pre-#3769 states persisted the rotation's hidden set as message refs
// (hiddenOrphanRefs). Honor it on upgrade: resurrecting a pair the old
// kernel already removed would re-insert it mid-history and break the
// cached prefix again. The set self-perpetuates — rememberHiddenRef
// re-records these refs on every pass they stay hidden.
hide = true;
} else if (foldMinRef !== undefined && !quotesActiveBlock(messages[i]!.text, activeRangeKeys)) {
const num = refNum(ref);
if (num !== null && num >= foldMinRef) {
hide = true;
newlySweptCallIds.push(callId);
}
}
}
if (hide) hiddenCallIds.add(callId);
else keepCallIds.add(callId);
}

const fateByCallId = new Map<string, Fate>();
Expand Down Expand Up @@ -349,5 +436,5 @@ export function hideConsumedCompressCalls(
result.push(message);
}

return { messages: result, hidden, hiddenOrphanRefs };
return { messages: result, hidden, hiddenOrphanRefs, newlySweptCallIds };
}
4 changes: 4 additions & 0 deletions kernel/src/pipeline.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ export interface NodeEffects {
* foldability judgment — on lanes whose processTurn input is itself a
* pruned/render view, even the pre-prune input is not the apply class. */
applyView?: CoreMessage[];
/** #3939 ⑤: the post-prune, pre-transform resent view, stashed by the
* pre-transform-snapshot node so the view-latch node can replay latched
* bytes for messages the transform nodes dropped or rewrote. */
preTransformMessages?: CoreMessage[];
readonly [key: string]: unknown;
}

Expand Down
27 changes: 24 additions & 3 deletions kernel/src/refs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ export interface AssignRefsResult {
map: MessageRefMap;
nextIndex: number;
newlyAssigned: number;
/** Ids whose stale BLOCKED entry migrated to a fresh numeric ref this pass
* (#2607). The caller defers their carrier re-render (#3939 ②/#3805). */
migratedIds: string[];
}

export interface AssignRefsOptions {
Expand All @@ -85,6 +88,18 @@ export function assignRefs(
? options.nextIndex
: MIN_INDEX;
let newlyAssigned = 0;
const migratedIds: string[] = [];

// #3945: a BLOCKED binding is keyed by id but SHARED by every message
// carrying that id, while protection is decided per message. Migrating the
// binding the moment ONE sibling loses protection would re-tag a
// still-protected sibling's already-sent bytes. Migration requires that NO
// message with that id is protected this pass.
const siblingProtected = new Set<string>();
for (const message of messages) {
if (!message.id || options.shouldSkip?.(message)) continue;
if (options.isProtected?.(message)) siblingProtected.add(message.id);
}

for (const message of messages) {
if (!message.id || options.shouldSkip?.(message)) continue;
Expand All @@ -96,8 +111,14 @@ export function assignRefs(
// is lifted (#2607: #1188 pinned all media; archivable images now fold
// and must be citable via decompress({ imageRef })). Fresh number only —
// the id-never-reused contract holds.
if (prev === BLOCKED_REF && !protectedNow) delete map.byRaw[message.id];
else continue;
if (
prev === BLOCKED_REF &&
!protectedNow &&
!siblingProtected.has(message.id)
) {
delete map.byRaw[message.id];
migratedIds.push(message.id);
} else continue;
}

if (protectedNow) {
Expand All @@ -112,7 +133,7 @@ export function assignRefs(
newlyAssigned++;
}

return { map, nextIndex: cursor, newlyAssigned };
return { map, nextIndex: cursor, newlyAssigned, migratedIds };
}

function allocateFreeRef(
Expand Down
16 changes: 14 additions & 2 deletions kernel/src/render-refs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,16 @@ function renderMessage(
countTokens: (text: string) => number,
strategy: RenderStrategy,
snapshot: Record<string, number> | null = null,
deferred: ReadonlySet<string> | null = null,
): CoreMessage {
const ref = refForRaw(map, message.id);
if (!ref || ref === BLOCKED_REF) return message;

// #3939 ②/#3805: refs minted by the BLOCKED→numeric migration keep their
// pre-migration UNTAGGED bytes until the next fold boundary — stamping the
// tag now would rewrite an old message mid-history on a quiet turn.
if (deferred?.has(ref)) return message;

// "none": host reads the ref map directly — never pollute text.
if (strategy === "none") return message;

Expand Down Expand Up @@ -110,8 +116,11 @@ export function renderVisibleRefs(
): CoreMessage[] {
// Legacy behavior: recompute tokens every render (snapshot = null).
const map = state.messageRefs;
const deferred = state.renderDeferredRefs
? new Set(state.renderDeferredRefs)
: null;
return messages.map((message) =>
renderMessage(message, map, countTokens, strategy),
renderMessage(message, map, countTokens, strategy, null, deferred),
);
}

Expand All @@ -132,8 +141,11 @@ export function renderWithSnapshot(
): RenderWithSnapshotResult {
const map = state.messageRefs;
const snapshot = { ...(state.tokenSnapshot ?? {}) };
const deferred = state.renderDeferredRefs
? new Set(state.renderDeferredRefs)
: null;
const rendered = messages.map((message) =>
renderMessage(message, map, countTokens, strategy, snapshot),
renderMessage(message, map, countTokens, strategy, snapshot, deferred),
);
return { messages: rendered, tokenSnapshot: snapshot };
}
Expand Down
26 changes: 26 additions & 0 deletions kernel/src/sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,32 @@ export function syncBlocks(
nextBlockId: state.nextBlockId,
nextRunId: state.nextRunId,
deadRefs: state.deadRefs ? [...state.deadRefs] : undefined,
// #3769 orphan-cleanup ledgers — same carry set as cloneState: dropping any
// of these here would wipe them before the hide-compress-calls node reads
// them (this node runs first), silently disabling both the fold-gated sweep
// and the legacy hiddenOrphanRefs upgrade-compat clause every turn.
hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined,
sweptOrphanCallIds: state.sweptOrphanCallIds ? [...state.sweptOrphanCallIds] : undefined,
pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef,
// #3939 ③: the truncation latch must survive the sync node — dropping it
// here would re-expose clipped bytes mid-history after a sync carried the
// rest of the state (#3691 oscillation via the sync path).
emergencyTruncateLatch: state.emergencyTruncateLatch
? state.emergencyTruncateLatch.map((e) => ({ ref: e.ref, text: e.text }))
: undefined,
// #3939 ②: the render deferral must survive sync — losing it would stamp
// migration tags mid-history after a sync carried the ref map (#3805).
renderDeferredRefs: state.renderDeferredRefs
? [...state.renderDeferredRefs]
: undefined,
// #3939 ⑤: the view latch must survive sync — losing it re-derives
// transform output mid-history after a sync carried the state (#3805).
viewLatches: state.viewLatches
? Object.fromEntries(
Object.entries(state.viewLatches).map(([k, v]) => [k, { ...v }]),
)
: undefined,
viewLatchConfigDigest: state.viewLatchConfigDigest,
};

// Refs are additive (assignRefs never removes them from messageRefs), so
Expand Down
Loading
Loading