Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.201",
"version": "0.0.202",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
8 changes: 7 additions & 1 deletion kernel/src/compress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1970,7 +1970,13 @@ function applySingleRange(input: SingleRangeInput): SingleRangeOutcome {

for (const consumedId of consumedBlockIds) {
const consumed = blockById(input.state, consumedId);
if (consumed) consumed.active = false;
if (consumed) {
consumed.active = false;
// Lineage: name the consumer so "inactive" stays diagnosable — an
// orphaned inactive block with no supersedes id is a GC/orphan/expand
// event, one WITH it was absorbed into that parent fold.
consumed.supersedesBlockId = blockId;
}
}

return { tokens: compressedTokens, warnings };
Expand Down
17 changes: 14 additions & 3 deletions kernel/src/nudge-text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ export function formatRanges(
textPct: number;
dangerous: boolean;
noMergeBefore: boolean;
spansFolded: string[];
}
const refNum = (ref: string): number => {
const m = ref.match(/\d+/);
Expand All @@ -147,6 +148,7 @@ export function formatRanges(
protectedTools: [],
dangerous: r.dangerous ?? false,
noMergeBefore: r.noMergeBefore ?? false,
spansFolded: r.crossesCoveredBy ?? [],
});
}
for (const r of protectedRanges) {
Expand All @@ -169,6 +171,7 @@ export function formatRanges(
protectedTools: [...r.tools],
dangerous: false,
noMergeBefore: r.noMergeBefore ?? false,
spansFolded: [],
});
}
// Order/merge by POSITION, never ref number: refs can be non-monotonic vs array
Expand All @@ -193,6 +196,9 @@ export function formatRanges(
last.protectedTokens += e.protectedTokens;
last.protectedCount += e.protectedCount;
if (e.dangerous) last.dangerous = true;
for (const s of e.spansFolded) {
if (!last.spansFolded.includes(s)) last.spansFolded.push(s);
}
for (const t of e.protectedTools) {
if (!last.protectedTools.includes(t)) last.protectedTools.push(t);
}
Expand All @@ -211,13 +217,18 @@ export function formatRanges(
e.dangerous && e.compressibleTokens > 0
? " ⚠️ NOT recommended unless you are certain."
: "";
// The span label can bridge folded territory invisible on surface-
// replacing views — name the hidden blocks so a resubmit reads as a
// re-fold, not a fresh fold.
const spanNote =
e.spansFolded.length > 0 ? ` · spans folded ${e.spansFolded.join(", ")}` : "";
if (e.protectedTokens > 0 && e.compressibleTokens === 0) {
return ` ${lo}–${hi} ${e.count} msgs ${formatK(e.tokens)} [PROTECTED: ${e.protectedTools.join(", ")} — not compressible]${suffix}`;
return ` ${lo}–${hi} ${e.count} msgs ${formatK(e.tokens)} [PROTECTED: ${e.protectedTools.join(", ")} — not compressible]${spanNote}${suffix}`;
}
if (e.protectedTokens > 0 && e.compressibleTokens > 0) {
return ` ${lo}–${hi} ${e.count} msgs ${formatK(e.tokens)} [${formatK(e.compressibleTokens)} compressible | ${formatK(e.protectedTokens)} protected: ${e.protectedTools.join(", ")}]${userNote(e.userMsgs)}${suffix}`;
return ` ${lo}–${hi} ${e.count} msgs ${formatK(e.tokens)} [${formatK(e.compressibleTokens)} compressible | ${formatK(e.protectedTokens)} protected: ${e.protectedTools.join(", ")}]${spanNote}${userNote(e.userMsgs)}${suffix}`;
}
return ` ${lo}–${hi} ${e.count} msgs ${formatK(e.tokens)} [tool ${e.toolPct}% | text ${e.textPct}%]${userNote(e.userMsgs)}${suffix}`;
return ` ${lo}–${hi} ${e.count} msgs ${formatK(e.tokens)} [tool ${e.toolPct}% | text ${e.textPct}%]${spanNote}${userNote(e.userMsgs)}${suffix}`;
});
return `Compressible ranges (${merged.length}, oldest first):\n${lines.join("\n")}`;
}
Expand Down
69 changes: 68 additions & 1 deletion kernel/src/recommend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -198,11 +198,51 @@ export function computeProtectedRefs(
// ─── 2. Build Compressible + Protected Ranges ────────────────────────────────

/** Numeric part of a message ref ("m00005" → 5); null for non-m refs. */
function refNumber(ref: string): number | null {
export function refNumber(ref: string): number | null {
const m = /^m(\d+)$/i.exec(ref);
return m ? Number(m[1]) : null;
}

/** (#5508 family, display-only) ref-number holes owned by ACTIVE blocks:
* covered ids that surface-replacing views drop from the array without
* leaving an array slot or gap flag. Sorted by ref number. Shared by the
* recommend and report span-label surfaces so the two can't diverge. */
export function ownedHoles(
state: CompressionState,
): { n: number; owner: string }[] {
const ownerByMsg = new Map<string, string>();
for (const blk of state.blocks) {
if (!blk.active) continue;
for (const id of blk.effectiveMessageIds) ownerByMsg.set(id, blk.blockId);
}
const covered = coveredMessageIds(state);
const holes: { n: number; owner: string }[] = [];
for (const [ref, raw] of Object.entries(state.messageRefs.byRef)) {
const n = refNumber(ref);
if (n === null || !covered.has(raw)) continue;
const owner = ownerByMsg.get(raw);
if (owner !== undefined) holes.push({ n, owner });
}
holes.sort((a, b) => a.n - b.n);
return holes;
}

/** Owners of holes STRICTLY between two ref numbers; undefined when the pair
* is too narrow (≤1 hides nothing) or too wide (>512 scan cap) to label. */
export function crossingOwnersBetween(
holes: { n: number; owner: string }[],
lo: number,
hi: number,
): string[] | undefined {
if (hi - lo <= 1 || hi - lo > 512) return undefined;
const found: string[] = [];
for (const h of holes) {
if (h.n >= hi) break;
if (h.n > lo && !found.includes(h.owner)) found.push(h.owner);
}
return found.length > 0 ? found : undefined;
}

/** True when a recorded-dead ref sits STRICTLY between two live ref numbers —
* the endpoints themselves are live (a dead endpoint can't be a compressible
* message), so only interior holes matter (#3192/#3205). */
Expand Down Expand Up @@ -283,6 +323,7 @@ export function buildCompressibleRanges(
isTool: boolean;
isUser: boolean;
index: number;
crosses?: string[];
}[] = [];
const protectedMsgs: {
ref: string;
Expand All @@ -294,6 +335,15 @@ export function buildCompressibleRanges(

const covered = coveredMessageIds(state);

// Display-only honesty (#5508 family): on surface-replacing hosts the ids
// an active block already owns leave the array leaving NO slot and NO skip
// flag, so compressible runs render as one contiguous label spanning
// folded territory. Eligibility is never touched (KDD #12 keeps the
// projection as the single gate); this only NAMES which active blocks sit
// inside such a label, so blindly resubmitting it is recognizable as a
// re-fold instead of a fresh fold.
const holes = ownedHoles(state);

// Pairing: a tool-result may carry only toolCallId (no toolName). Collect the
// callIds of protected tool-calls first, then protect matching results too.
const protectedCallIds = collectProtectedToolCallIds(messages, config);
Expand Down Expand Up @@ -406,6 +456,18 @@ export function buildCompressibleRanges(
skipSinceCompressible = true;
}
}
// Only a message CONTINUING the open group can hide covered territory in
// its ref-number hole (a broken group bridges nothing by definition).
const crosses =
!skipSinceCompressible &&
curRefNum !== null &&
lastCompressibleRefNum !== null
? crossingOwnersBetween(
holes,
Math.min(curRefNum, lastCompressibleRefNum),
Math.max(curRefNum, lastCompressibleRefNum),
)
: undefined;
compressibleMsgs.push({
id: msg.id,
ref,
Expand All @@ -415,6 +477,7 @@ export function buildCompressibleRanges(
isTool: isToolMessage(msg),
isUser: msg.role === "user",
index: msgIndex,
crosses,
});
if (curRefNum !== null) lastCompressibleRefNum = curRefNum;
skipSinceCompressible = false;
Expand Down Expand Up @@ -494,6 +557,8 @@ export function buildCompressibleRanges(
}
range.textPct = 100 - range.toolPct;
}
const crossed = [...new Set(group.flatMap((info) => info.crosses ?? []))];
if (crossed.length > 0) range.crossesCoveredBy = crossed;
compressible.push(range);
}

Expand Down Expand Up @@ -566,6 +631,8 @@ function mergeBatch(batch: CompressibleRange[]): CompressibleRange {
// position-based merging (formatRanges) still sees it after this batch is
// re-emitted as a single range.
if (first.noMergeBefore === true) merged.noMergeBefore = true;
const crossed = batch.flatMap((r) => r.crossesCoveredBy ?? []);
if (crossed.length > 0) merged.crossesCoveredBy = [...new Set(crossed)];
return merged;
}

Expand Down
42 changes: 36 additions & 6 deletions kernel/src/report.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import { clampPrefix } from "./truncate.js";
import { clampPrefix, scrubLoneSurrogates } from "./truncate.js";
import { isToolMessage } from "./message-kind.js";
import { BLOCKED_REF, orderedRefPair, refForRaw } from "./refs.js";
import { countMessageTokens } from "./tokenize.js";
import { segmentGroups } from "./segment.js";
import { isLiveCheckpointCarrier } from "./state.js";
import { computeProtectedRefs } from "./recommend.js";
import {
computeProtectedRefs,
crossingOwnersBetween,
ownedHoles,
refNumber,
} from "./recommend.js";
import {
buildProjectionShared,
collectCoverage,
Expand Down Expand Up @@ -464,6 +469,17 @@ function renderUncompressedRanges(
// ordering would mis-sort equal-size ranges.
if (sort !== "time")
merged.sort((a, b) => b.tokens - a.tokens || a.startIndex - b.startIndex);
// Honest span labels: rows derived from surface-replacing views can bridge
// folded territory (the covered ids left no array slot or gap flag). Name
// the hiding active blocks inline — display only, eligibility untouched.
const holes = ownedHoles(state);
const crossedFolded = (r: Merged): string | undefined => {
const [lo, hi] = orderedRefPair(r.startRef, r.endRef);
const loN = refNumber(lo);
const hiN = refNumber(hi);
if (loN === null || hiN === null) return undefined;
return crossingOwnersBetween(holes, loN, hiN)?.join(", ");
};
// #3236: screen displayed rows through the SAME projection applySingleRange
// executes so a row that would be rejected wholesale carries a ⚠ flag. The
// diagnostic view flags rather than drops — it must keep showing what is
Expand Down Expand Up @@ -504,8 +520,19 @@ function renderUncompressedRanges(
protectedMessageIds: protectedRefs,
shared,
});
} catch {
return "\u26a0 not foldable: refs unresolvable";
} catch (err) {
// A thrown resolution failure means the row's endpoints cannot be
// mapped onto the apply array class at all (stale/BLOCKED/surface-
// drift refs). Surface the underlying reason verbatim-ish: the flat
// "refs unresolvable" verdict sent agents down dead-end theories
// (they read it as ledger corruption when it was one bad endpoint).
const detail = scrubLoneSurrogates(
err instanceof Error ? err.message : String(err),
)
.replace(/[\r\n]+/g, " ")
.trim()
.slice(0, 120);
return `\u26a0 not foldable: projection failed${detail ? ` (${detail})` : ""}`;
}
if (proj.fatal?.kind === "zone")
return "\u26a0 not foldable: protected zone only";
Expand All @@ -526,8 +553,9 @@ function renderUncompressedRanges(
? r.startRef
: orderedRefPair(r.startRef, r.endRef).join("–");
const flag = flagFor ? flagFor(r) : null;
const crossed = crossedFolded(r);
lines.push(
` ${range} (${r.count} msgs, ${formatTokens(r.tokens)}${r.count > 1 ? ` (${Math.round(r.tokens / r.count)}/msg)` : ""}) ${dominantTool(r.toolTokens)}${flag ? ` ${flag}` : ""}`,
` ${range} (${r.count} msgs, ${formatTokens(r.tokens)}${r.count > 1 ? ` (${Math.round(r.tokens / r.count)}/msg)` : ""}) ${dominantTool(r.toolTokens)}${crossed ? ` · spans folded ${crossed}` : ""}${flag ? ` ${flag}` : ""}`,
);
}
if (merged.length > limit) {
Expand Down Expand Up @@ -646,7 +674,9 @@ export function buildRecap(state: CompressionState, blockId?: string): string {
return `Block ${blockId} not found. Active blocks: ${activeList}`;
}
if (!block.active) {
return `Block ${blockId} is inactive (deactivated by nested compression).`;
return block.supersedesBlockId
? `Block ${blockId} is inactive (deactivated by nested compression, superseded by ${block.supersedesBlockId}).`
: `Block ${blockId} is inactive (deactivated by nested compression).`;
}
const range = `${block.effectiveMessageIds.length} messages`;
return `[Compressed conversation section]\n${block.summary}\n\n[${blockId} | ${range} | topic: "${block.topic ?? "(none)"}"]`;
Expand Down
19 changes: 19 additions & 0 deletions kernel/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,15 @@ export interface CompressionBlock {
* Undefined = unstamped traffic or provably-loose coverage; never guessed. */
startSeq?: number;
endSeq?: number;
/** Nested-compression lineage: id of the newer block that CONSUMED this one.
* Set once at applySingleRange's single consumption site (which deactivates
* every input block). Absent = never consumed — all other deactivation causes
* (orphan GC, host expand, stale-view reconciliation) leave no lineage, so
* storage/web/stats derivations can tell "absorbed into parent" apart from
* every other reason a block went inactive. Round-trips through the
* payload_json serialization; the read-side column mapping lives in
* src/storage/ingest.ts. */
supersedesBlockId?: string;
}

/** A compression block's current ref span, pre-resolved for display. */
Expand Down Expand Up @@ -708,6 +717,16 @@ export interface CompressibleRange {
* has no index gap and the index check alone would let mergeRangesToThreshold
* bridge back into one doomed span bracketing the dead ref. */
noMergeBefore?: boolean;
/** Display-only honesty flag (#5508 family): active block ids whose owned
* messages sit in ref-number holes INSIDE this range's label — territory
* invisible on surface-replacing (pruned) views, where the folded messages
* left the array without leaving a gap flag, so a contiguous run renders as
* one label that actually spans an already-folded block. NEVER participates
* in eligibility (KDD #12: the projection/fold gate is the single source of
* truth); it only tells the model which advertised spans already contain
* folded territory (blindly resubmitting such a span nests those blocks
* into a redundant parent fold). Absent on hand-built ranges. */
crossesCoveredBy?: string[];
}

export interface ProtectedRange {
Expand Down
Loading
Loading