Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.129",
"version": "0.0.130",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
7 changes: 6 additions & 1 deletion kernel/src/nudge-text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ export function formatRanges(
toolPct: number;
textPct: number;
dangerous: boolean;
noMergeBefore: boolean;
}
const refNum = (ref: string): number => {
const m = ref.match(/\d+/);
Expand All @@ -139,6 +140,7 @@ export function formatRanges(
protectedCount: 0,
protectedTools: [],
dangerous: r.dangerous ?? false,
noMergeBefore: r.noMergeBefore ?? false,
});
}
for (const r of protectedRanges) {
Expand All @@ -160,6 +162,7 @@ export function formatRanges(
protectedCount: r.count,
protectedTools: [...r.tools],
dangerous: false,
noMergeBefore: r.noMergeBefore ?? false,
});
}
// Order/merge by POSITION, never ref number: refs can be non-monotonic vs array
Expand All @@ -170,7 +173,9 @@ export function formatRanges(
const merged: Merged[] = [];
for (const e of entries) {
const last = merged[merged.length - 1];
if (last && e.startPos <= last.endPos + 1) {
// #3192/#3205: a dead-hole boundary has no index gap, so position adjacency
// alone would bridge across it — honor the flag set by buildCompressibleRanges.
if (last && !e.noMergeBefore && e.startPos <= last.endPos + 1) {
last.endRef = e.endRef;
last.endNum = Math.max(last.endNum, e.endNum);
last.endPos = Math.max(last.endPos, e.endPos);
Expand Down
93 changes: 90 additions & 3 deletions kernel/src/recommend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,58 @@ export function computeProtectedRefs(

// ─── 2. Build Compressible + Protected Ranges ────────────────────────────────

/** Numeric part of a message ref ("m00005" → 5); null for non-m refs. */
function refNumber(ref: string): number | null {
const m = /^m(\d+)$/i.exec(ref);
return m ? Number(m[1]) : null;
}

/** True when a recorded-dead ref sits STRICTLY between two live ref numbers —
* the endpoints themselves are live (a dead endpoint can't be a compressible
* message), so only interior holes matter (#3192/#3205). */
function hasDeadRefBetween(
deadRefNums: ReadonlySet<number>,
lo: number,
hi: number,
): boolean {
for (const n of deadRefNums) if (n > lo && n < hi) return true;
return false;
}

interface NoMergeTarget {
startRef: string;
endRef: string;
startIndex?: number;
endIndex?: number;
noMergeBefore?: boolean;
}

/** #3192/#3205: tag every range that opens right after a dead-ref hole so the
* position-based mergers (mergeRangesToThreshold, formatRanges) cannot bridge
* across it — a dropped message leaves no array slot, so the hole has no index
* gap and position adjacency alone would re-emit one doomed span bracketing the
* dead ref. Runs over compressible + protected together so cross-type
* boundaries are covered too. */
function markNoMergeAcrossDeadHoles(
ranges: NoMergeTarget[],
deadRefNums: ReadonlySet<number>,
): void {
if (deadRefNums.size === 0 || ranges.length < 2) return;
const startPos = (r: NoMergeTarget): number =>
r.startIndex ?? refNumber(r.startRef) ?? 0;
const order = [...ranges].sort((a, b) => startPos(a) - startPos(b));
for (let i = 1; i < order.length; i++) {
const prevEnd = refNumber(order[i - 1]!.endRef);
const curStart = refNumber(order[i]!.startRef);
if (prevEnd === null || curStart === null) continue;
const lo = Math.min(prevEnd, curStart);
const hi = Math.max(prevEnd, curStart);
if (hi - lo > 1 && hasDeadRefBetween(deadRefNums, lo, hi)) {
order[i]!.noMergeBefore = true;
}
}
}

/**
* Build compressible and protected range groups from the message list.
*
Expand Down Expand Up @@ -257,6 +309,15 @@ export function buildCompressibleRanges(
let skipSinceCompressible = false;
let skipSinceProtected = false;
let msgIndex = -1;
// #3192/#3205: numeric set of recorded-dead refs. A dropped message leaves a
// hole in the ref map that array-adjacency segmentation bridges, so we break
// groups at those holes (checked right before each compressible push below).
const deadRefNums = new Set<number>(
(state.deadRefs ?? [])
.map((ref) => refNumber(ref))
.filter((n): n is number => n !== null),
);
let lastCompressibleRefNum: number | null = null;

for (const msg of messages) {
msgIndex++;
Expand Down Expand Up @@ -315,6 +376,21 @@ export function buildCompressibleRanges(
continue;
}

// #3192/#3205: break the group at a dead-ref hole between this message and
// the previous compressible one (a dropped message leaves no array slot, so
// no earlier gap flag fires). Same level as covered/pruned/carrier gaps.
const curRefNum = refNumber(ref);
if (
deadRefNums.size > 0 &&
curRefNum !== null &&
lastCompressibleRefNum !== null
) {
const lo = Math.min(curRefNum, lastCompressibleRefNum);
const hi = Math.max(curRefNum, lastCompressibleRefNum);
if (hi - lo > 1 && hasDeadRefBetween(deadRefNums, lo, hi)) {
skipSinceCompressible = true;
}
}
compressibleMsgs.push({
id: msg.id,
ref,
Expand All @@ -325,6 +401,7 @@ export function buildCompressibleRanges(
isUser: msg.role === "user",
index: msgIndex,
});
if (curRefNum !== null) lastCompressibleRefNum = curRefNum;
skipSinceCompressible = false;
skipSinceProtected = true;
}
Expand Down Expand Up @@ -428,6 +505,8 @@ export function buildCompressibleRanges(
}
if (pcur) protectedRanges.push(pcur);

markNoMergeAcrossDeadHoles([...compressible, ...protectedRanges], deadRefNums);

return {
compressible: compressible.filter((g) => g.tokens > 0),
protected: protectedRanges,
Expand Down Expand Up @@ -460,6 +539,10 @@ function mergeBatch(batch: CompressibleRange[]): CompressibleRange {
if (batch.some((r) => r.dangerous === true)) {
merged.dangerous = true;
}
// #3192/#3205: carry the leading dead-hole boundary forward so downstream
// position-based merging (formatRanges) still sees it after this batch is
// re-emitted as a single range.
if (first.noMergeBefore === true) merged.noMergeBefore = true;
return merged;
}

Expand Down Expand Up @@ -502,6 +585,9 @@ export function mergeRangesToThreshold(
}
};
for (const r of ranges) {
// #3192/#3205: hard boundary at a dead-ref hole — never bridge across it,
// even though the hole carries no array-index gap (see noMergeBefore).
if (r.noMergeBefore) closeBatch();
const prev = batch[batch.length - 1];
// A physical array-index gap between the previous range and this one marks
// a block / protected / pruned boundary (buildCompressibleRanges leaves a
Expand All @@ -522,9 +608,10 @@ export function mergeRangesToThreshold(
if (batch.length > 0 && result.length > 0) {
const prev = result[result.length - 1]!;
const contiguous =
prev.endIndex === undefined ||
batch[0]!.startIndex === undefined ||
batch[0]!.startIndex <= prev.endIndex + 1;
!batch[0]!.noMergeBefore &&
(prev.endIndex === undefined ||
batch[0]!.startIndex === undefined ||
batch[0]!.startIndex <= prev.endIndex + 1);
if (contiguous) {
result[result.length - 1] = mergeBatch([prev, ...batch]);
} else if (batchChars >= minChars) {
Expand Down
9 changes: 9 additions & 0 deletions kernel/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,11 @@ export interface CompressibleRange {
startIndex?: number;
/** Messages-array index of this range's last message. */
endIndex?: number;
/** #3192/#3205: buildCompressibleRanges sets this when the range opens right
* after a dead-ref hole. A dropped message leaves NO array slot, so the hole
* has no index gap and the index check alone would let mergeRangesToThreshold
* bridge back into one doomed span bracketing the dead ref. */
noMergeBefore?: boolean;
}

export interface ProtectedRange {
Expand All @@ -481,6 +486,10 @@ export interface ProtectedRange {
startIndex?: number;
/** Messages-array index of last message. */
endIndex?: number;
/** #3192/#3205: see CompressibleRange.noMergeBefore — set when this protected
* range opens right after a dead-ref hole, so formatRanges does not bridge a
* preceding compressible range across the hole into one doomed span. */
noMergeBefore?: boolean;
}

export interface ContextRanges {
Expand Down
127 changes: 127 additions & 0 deletions kernel/tests/dead-ref-split.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import {
buildCompressibleRanges,
mergeRangesToThreshold,
} from "../src/recommend.js";
import { createInitialState } from "../src/state.js";
import { assignRefs } from "../src/refs.js";
import type { Config, CoreMessage } from "../src/types.js";

// #3192/#3205: a client-dropped message leaves a hole in the ref map that
// array-adjacency segmentation bridges, so buildCompressibleRanges must break
// groups at recorded-dead refs and emit adjacent live sub-spans — and
// mergeRangesToThreshold must not bridge them back across the hole.

function config(overrides: Partial<Config> = {}): Config {
return {
tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 },
nudge: {
maxContextLimitPct: 0.55,
minContextLimitPct: 0.45,
frequency: 5,
iterationThreshold: 15,
force: "soft",
growthRatio: 0.05,
},
promotionThreshold: 5,
truncate: { threshold: 1 },
merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 },
compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 },
protectedTools: [],
preserveRecentMessages: 0,
preserveRecentTokens: 0,
modelContextLimit: 100000,
...overrides,
};
}

function msg(id: string, text: string): CoreMessage {
return { id, role: "assistant", contentType: "text", text };
}

/** Mint refs m00001.. over the FULL id list, then hand back the state plus the
* source messages so a test can drop entries from the visible view. */
function setup(ids: string[]): { messages: CoreMessage[]; state: ReturnType<typeof createInitialState> } {
const messages = ids.map((id) => msg(id, "x".repeat(2000)));
const state = createInitialState();
state.messageRefs = assignRefs(messages, {
existing: state.messageRefs,
nextIndex: 1,
}).map;
return { messages, state };
}

function spans(ranges: { startRef: string; endRef: string }[]): [string, string][] {
return ranges.map((r) => [r.startRef, r.endRef]);
}

test("buildCompressibleRanges splits around an interior dead ref into adjacent live sub-spans", () => {
const { messages, state } = setup(["a", "b", "c", "d", "e"]); // m00001..m00005
state.deadRefs = ["m00003"]; // 'c' dropped by the client
const visible = [messages[0], messages[1], messages[3], messages[4]]; // a,b,d,e
const { compressible } = buildCompressibleRanges(visible, state, config());
assert.deepEqual(spans(compressible), [
["m00001", "m00002"],
["m00004", "m00005"],
]);
assert.equal(compressible[0]!.noMergeBefore, undefined);
assert.equal(compressible[1]!.noMergeBefore, true);
});

test("control: a NON-dead ref-map hole stays bridged (deadRefs empty → unchanged behavior)", () => {
const { messages, state } = setup(["a", "b", "c", "d", "e"]);
// Same drop, but 'c' is NOT recorded dead → array adjacency bridges it as before.
const visible = [messages[0], messages[1], messages[3], messages[4]];
const { compressible } = buildCompressibleRanges(visible, state, config());
assert.deepEqual(spans(compressible), [["m00001", "m00005"]]);
});

test("multiple dead holes → one sub-span per surviving run, none brackets a dead ref", () => {
const { messages, state } = setup(["a", "b", "c", "d", "e", "f", "g"]); // m1..m7
state.deadRefs = ["m00003", "m00006"]; // drop c and f
const visible = [
messages[0],
messages[1],
messages[3],
messages[4],
messages[6],
]; // a,b,d,e,g
const { compressible } = buildCompressibleRanges(visible, state, config());
assert.deepEqual(spans(compressible), [
["m00001", "m00002"],
["m00004", "m00005"],
["m00007", "m00007"],
]);
const dead = new Set([3, 6]);
for (const r of compressible) {
const s = Number(r.startRef.slice(1));
const e = Number(r.endRef.slice(1));
const [lo, hi] = s < e ? [s, e] : [e, s];
for (let n = lo + 1; n < hi; n++) {
assert.ok(!dead.has(n), `span ${r.startRef}–${r.endRef} brackets dead ${n}`);
}
}
});

test("trailing dead ref never becomes a span endpoint", () => {
const { messages, state } = setup(["a", "b", "c", "d", "e"]);
state.deadRefs = ["m00005"]; // drop 'e'
const visible = [messages[0], messages[1], messages[2], messages[3]]; // a,b,c,d
const { compressible } = buildCompressibleRanges(visible, state, config());
assert.deepEqual(spans(compressible), [["m00001", "m00004"]]);
});

test("mergeRangesToThreshold does NOT re-bridge across a dead-ref hole", () => {
const { messages, state } = setup(["a", "b", "c", "d", "e"]);
state.deadRefs = ["m00003"];
const visible = [messages[0], messages[1], messages[3], messages[4]];
const { compressible } = buildCompressibleRanges(visible, state, config());
// Each sub-span is ~4000 chars, well above the 1000 gate — without the
// noMergeBefore flag these would merge back into one doomed m00001–m00005.
const merged = mergeRangesToThreshold(compressible, 1000);
assert.deepEqual(spans(merged), [
["m00001", "m00002"],
["m00004", "m00005"],
]);
});
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

23 changes: 18 additions & 5 deletions tests/compress-recovery-ranges.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -183,11 +183,24 @@ test("#3192: a dropped message's DEAD ref is never re-advertised by the receipt
const rejected = request(ctx, "m00001", "m00005");
assert.equal(rejected.outcome, "refused");
assert.match(rejected.text, /cannot be anchored/);
// No advertised compressible span may cross the dead ref m00005.
assert.doesNotMatch(rejected.text, /m0000[1-4]\s*[–-]\s*m0000[5-9]/);
// The only live span bracketed the dead ref -> stop instruction, not a doomed retry span.
assert.match(rejected.text, /continue the task without compressing/);
assert.doesNotMatch(rejected.text, /use these refs exactly as listed/);
// No advertised compressible span may cross the dead ref m00005 — the red gate
// on the unfixed kernel (it advertised one span m00001–m00007 crossing m00005).
const advertised = advertisedRanges(rejected.text);
assert.ok(advertised.length > 0, `receipt must advertise the surviving sub-spans: ${rejected.text}`);
for (const [start, end] of advertised) {
assert.ok(!spanContains(start, end, 5), `${start}..${end} must not span the dead ref m00005`);
}
// The kernel SPLIT around the hole into adjacent live sub-spans (not a whole-span
// drop): the run before the hole ends at/before m00004, the run after starts at/
// after m00006. Both present proves the split rather than a silent drop.
assert.ok(
advertised.some(([start, end]) => Number(start.slice(1)) <= 4 && Number(end.slice(1)) <= 4),
`pre-hole sub-span must be advertised: ${JSON.stringify(advertised)}`,
);
assert.ok(
advertised.some(([start, end]) => Number(start.slice(1)) >= 6 && Number(end.slice(1)) >= 6),
`post-hole sub-span must be advertised: ${JSON.stringify(advertised)}`,
);

// Receipt and acp_status share ONE eligibility source (liveCompressionRanges);
// pin that source so the two surfaces cannot drift apart again (#2778 promise).
Expand Down
Loading