diff --git a/kernel/package.json b/kernel/package.json index 71da725e34..a318eb703b 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.129", + "version": "0.0.130", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/nudge-text.ts b/kernel/src/nudge-text.ts index d3caa47b83..7d656b728d 100644 --- a/kernel/src/nudge-text.ts +++ b/kernel/src/nudge-text.ts @@ -114,6 +114,7 @@ export function formatRanges( toolPct: number; textPct: number; dangerous: boolean; + noMergeBefore: boolean; } const refNum = (ref: string): number => { const m = ref.match(/\d+/); @@ -139,6 +140,7 @@ export function formatRanges( protectedCount: 0, protectedTools: [], dangerous: r.dangerous ?? false, + noMergeBefore: r.noMergeBefore ?? false, }); } for (const r of protectedRanges) { @@ -160,6 +162,7 @@ export function formatRanges( protectedCount: r.count, protectedTools: [...r.tools], dangerous: false, + noMergeBefore: r.noMergeBefore ?? false, }); } // Order/merge by POSITION, never ref number: refs can be non-monotonic vs array @@ -170,7 +173,9 @@ export function formatRanges( const merged: Merged[] = []; for (const e of entries) { const last = merged[merged.length - 1]; - if (last && e.startPos <= last.endPos + 1) { + // #3192/#3205: a dead-hole boundary has no index gap, so position adjacency + // alone would bridge across it — honor the flag set by buildCompressibleRanges. + if (last && !e.noMergeBefore && e.startPos <= last.endPos + 1) { last.endRef = e.endRef; last.endNum = Math.max(last.endNum, e.endNum); last.endPos = Math.max(last.endPos, e.endPos); diff --git a/kernel/src/recommend.ts b/kernel/src/recommend.ts index 8517176aeb..3f4030eae6 100644 --- a/kernel/src/recommend.ts +++ b/kernel/src/recommend.ts @@ -193,6 +193,58 @@ export function computeProtectedRefs( // ─── 2. Build Compressible + Protected Ranges ──────────────────────────────── +/** Numeric part of a message ref ("m00005" → 5); null for non-m refs. */ +function refNumber(ref: string): number | null { + const m = /^m(\d+)$/i.exec(ref); + return m ? Number(m[1]) : null; +} + +/** True when a recorded-dead ref sits STRICTLY between two live ref numbers — + * the endpoints themselves are live (a dead endpoint can't be a compressible + * message), so only interior holes matter (#3192/#3205). */ +function hasDeadRefBetween( + deadRefNums: ReadonlySet, + lo: number, + hi: number, +): boolean { + for (const n of deadRefNums) if (n > lo && n < hi) return true; + return false; +} + +interface NoMergeTarget { + startRef: string; + endRef: string; + startIndex?: number; + endIndex?: number; + noMergeBefore?: boolean; +} + +/** #3192/#3205: tag every range that opens right after a dead-ref hole so the + * position-based mergers (mergeRangesToThreshold, formatRanges) cannot bridge + * across it — a dropped message leaves no array slot, so the hole has no index + * gap and position adjacency alone would re-emit one doomed span bracketing the + * dead ref. Runs over compressible + protected together so cross-type + * boundaries are covered too. */ +function markNoMergeAcrossDeadHoles( + ranges: NoMergeTarget[], + deadRefNums: ReadonlySet, +): void { + if (deadRefNums.size === 0 || ranges.length < 2) return; + const startPos = (r: NoMergeTarget): number => + r.startIndex ?? refNumber(r.startRef) ?? 0; + const order = [...ranges].sort((a, b) => startPos(a) - startPos(b)); + for (let i = 1; i < order.length; i++) { + const prevEnd = refNumber(order[i - 1]!.endRef); + const curStart = refNumber(order[i]!.startRef); + if (prevEnd === null || curStart === null) continue; + const lo = Math.min(prevEnd, curStart); + const hi = Math.max(prevEnd, curStart); + if (hi - lo > 1 && hasDeadRefBetween(deadRefNums, lo, hi)) { + order[i]!.noMergeBefore = true; + } + } +} + /** * Build compressible and protected range groups from the message list. * @@ -257,6 +309,15 @@ export function buildCompressibleRanges( let skipSinceCompressible = false; let skipSinceProtected = false; let msgIndex = -1; + // #3192/#3205: numeric set of recorded-dead refs. A dropped message leaves a + // hole in the ref map that array-adjacency segmentation bridges, so we break + // groups at those holes (checked right before each compressible push below). + const deadRefNums = new Set( + (state.deadRefs ?? []) + .map((ref) => refNumber(ref)) + .filter((n): n is number => n !== null), + ); + let lastCompressibleRefNum: number | null = null; for (const msg of messages) { msgIndex++; @@ -315,6 +376,21 @@ export function buildCompressibleRanges( continue; } + // #3192/#3205: break the group at a dead-ref hole between this message and + // the previous compressible one (a dropped message leaves no array slot, so + // no earlier gap flag fires). Same level as covered/pruned/carrier gaps. + const curRefNum = refNumber(ref); + if ( + deadRefNums.size > 0 && + curRefNum !== null && + lastCompressibleRefNum !== null + ) { + const lo = Math.min(curRefNum, lastCompressibleRefNum); + const hi = Math.max(curRefNum, lastCompressibleRefNum); + if (hi - lo > 1 && hasDeadRefBetween(deadRefNums, lo, hi)) { + skipSinceCompressible = true; + } + } compressibleMsgs.push({ id: msg.id, ref, @@ -325,6 +401,7 @@ export function buildCompressibleRanges( isUser: msg.role === "user", index: msgIndex, }); + if (curRefNum !== null) lastCompressibleRefNum = curRefNum; skipSinceCompressible = false; skipSinceProtected = true; } @@ -428,6 +505,8 @@ export function buildCompressibleRanges( } if (pcur) protectedRanges.push(pcur); + markNoMergeAcrossDeadHoles([...compressible, ...protectedRanges], deadRefNums); + return { compressible: compressible.filter((g) => g.tokens > 0), protected: protectedRanges, @@ -460,6 +539,10 @@ function mergeBatch(batch: CompressibleRange[]): CompressibleRange { if (batch.some((r) => r.dangerous === true)) { merged.dangerous = true; } + // #3192/#3205: carry the leading dead-hole boundary forward so downstream + // position-based merging (formatRanges) still sees it after this batch is + // re-emitted as a single range. + if (first.noMergeBefore === true) merged.noMergeBefore = true; return merged; } @@ -502,6 +585,9 @@ export function mergeRangesToThreshold( } }; for (const r of ranges) { + // #3192/#3205: hard boundary at a dead-ref hole — never bridge across it, + // even though the hole carries no array-index gap (see noMergeBefore). + if (r.noMergeBefore) closeBatch(); const prev = batch[batch.length - 1]; // A physical array-index gap between the previous range and this one marks // a block / protected / pruned boundary (buildCompressibleRanges leaves a @@ -522,9 +608,10 @@ export function mergeRangesToThreshold( if (batch.length > 0 && result.length > 0) { const prev = result[result.length - 1]!; const contiguous = - prev.endIndex === undefined || - batch[0]!.startIndex === undefined || - batch[0]!.startIndex <= prev.endIndex + 1; + !batch[0]!.noMergeBefore && + (prev.endIndex === undefined || + batch[0]!.startIndex === undefined || + batch[0]!.startIndex <= prev.endIndex + 1); if (contiguous) { result[result.length - 1] = mergeBatch([prev, ...batch]); } else if (batchChars >= minChars) { diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 5968b0ae6a..22523eeea7 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -469,6 +469,11 @@ export interface CompressibleRange { startIndex?: number; /** Messages-array index of this range's last message. */ endIndex?: number; + /** #3192/#3205: buildCompressibleRanges sets this when the range opens right + * after a dead-ref hole. A dropped message leaves NO array slot, so the hole + * has no index gap and the index check alone would let mergeRangesToThreshold + * bridge back into one doomed span bracketing the dead ref. */ + noMergeBefore?: boolean; } export interface ProtectedRange { @@ -481,6 +486,10 @@ export interface ProtectedRange { startIndex?: number; /** Messages-array index of last message. */ endIndex?: number; + /** #3192/#3205: see CompressibleRange.noMergeBefore — set when this protected + * range opens right after a dead-ref hole, so formatRanges does not bridge a + * preceding compressible range across the hole into one doomed span. */ + noMergeBefore?: boolean; } export interface ContextRanges { diff --git a/kernel/tests/dead-ref-split.test.ts b/kernel/tests/dead-ref-split.test.ts new file mode 100644 index 0000000000..667f69490f --- /dev/null +++ b/kernel/tests/dead-ref-split.test.ts @@ -0,0 +1,127 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + buildCompressibleRanges, + mergeRangesToThreshold, +} from "../src/recommend.js"; +import { createInitialState } from "../src/state.js"; +import { assignRefs } from "../src/refs.js"; +import type { Config, CoreMessage } from "../src/types.js"; + +// #3192/#3205: a client-dropped message leaves a hole in the ref map that +// array-adjacency segmentation bridges, so buildCompressibleRanges must break +// groups at recorded-dead refs and emit adjacent live sub-spans — and +// mergeRangesToThreshold must not bridge them back across the hole. + +function config(overrides: Partial = {}): Config { + return { + tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 }, + nudge: { + maxContextLimitPct: 0.55, + minContextLimitPct: 0.45, + frequency: 5, + iterationThreshold: 15, + force: "soft", + growthRatio: 0.05, + }, + promotionThreshold: 5, + truncate: { threshold: 1 }, + merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 }, + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + protectedTools: [], + preserveRecentMessages: 0, + preserveRecentTokens: 0, + modelContextLimit: 100000, + ...overrides, + }; +} + +function msg(id: string, text: string): CoreMessage { + return { id, role: "assistant", contentType: "text", text }; +} + +/** Mint refs m00001.. over the FULL id list, then hand back the state plus the + * source messages so a test can drop entries from the visible view. */ +function setup(ids: string[]): { messages: CoreMessage[]; state: ReturnType } { + const messages = ids.map((id) => msg(id, "x".repeat(2000))); + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + return { messages, state }; +} + +function spans(ranges: { startRef: string; endRef: string }[]): [string, string][] { + return ranges.map((r) => [r.startRef, r.endRef]); +} + +test("buildCompressibleRanges splits around an interior dead ref into adjacent live sub-spans", () => { + const { messages, state } = setup(["a", "b", "c", "d", "e"]); // m00001..m00005 + state.deadRefs = ["m00003"]; // 'c' dropped by the client + const visible = [messages[0], messages[1], messages[3], messages[4]]; // a,b,d,e + const { compressible } = buildCompressibleRanges(visible, state, config()); + assert.deepEqual(spans(compressible), [ + ["m00001", "m00002"], + ["m00004", "m00005"], + ]); + assert.equal(compressible[0]!.noMergeBefore, undefined); + assert.equal(compressible[1]!.noMergeBefore, true); +}); + +test("control: a NON-dead ref-map hole stays bridged (deadRefs empty → unchanged behavior)", () => { + const { messages, state } = setup(["a", "b", "c", "d", "e"]); + // Same drop, but 'c' is NOT recorded dead → array adjacency bridges it as before. + const visible = [messages[0], messages[1], messages[3], messages[4]]; + const { compressible } = buildCompressibleRanges(visible, state, config()); + assert.deepEqual(spans(compressible), [["m00001", "m00005"]]); +}); + +test("multiple dead holes → one sub-span per surviving run, none brackets a dead ref", () => { + const { messages, state } = setup(["a", "b", "c", "d", "e", "f", "g"]); // m1..m7 + state.deadRefs = ["m00003", "m00006"]; // drop c and f + const visible = [ + messages[0], + messages[1], + messages[3], + messages[4], + messages[6], + ]; // a,b,d,e,g + const { compressible } = buildCompressibleRanges(visible, state, config()); + assert.deepEqual(spans(compressible), [ + ["m00001", "m00002"], + ["m00004", "m00005"], + ["m00007", "m00007"], + ]); + const dead = new Set([3, 6]); + for (const r of compressible) { + const s = Number(r.startRef.slice(1)); + const e = Number(r.endRef.slice(1)); + const [lo, hi] = s < e ? [s, e] : [e, s]; + for (let n = lo + 1; n < hi; n++) { + assert.ok(!dead.has(n), `span ${r.startRef}–${r.endRef} brackets dead ${n}`); + } + } +}); + +test("trailing dead ref never becomes a span endpoint", () => { + const { messages, state } = setup(["a", "b", "c", "d", "e"]); + state.deadRefs = ["m00005"]; // drop 'e' + const visible = [messages[0], messages[1], messages[2], messages[3]]; // a,b,c,d + const { compressible } = buildCompressibleRanges(visible, state, config()); + assert.deepEqual(spans(compressible), [["m00001", "m00004"]]); +}); + +test("mergeRangesToThreshold does NOT re-bridge across a dead-ref hole", () => { + const { messages, state } = setup(["a", "b", "c", "d", "e"]); + state.deadRefs = ["m00003"]; + const visible = [messages[0], messages[1], messages[3], messages[4]]; + const { compressible } = buildCompressibleRanges(visible, state, config()); + // Each sub-span is ~4000 chars, well above the 1000 gate — without the + // noMergeBefore flag these would merge back into one doomed m00001–m00005. + const merged = mergeRangesToThreshold(compressible, 1000); + assert.deepEqual(spans(merged), [ + ["m00001", "m00002"], + ["m00004", "m00005"], + ]); +}); diff --git a/package-lock.json b/package-lock.json index f74ad3c502..abc0afabfe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,7 +46,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.129", + "version": "0.0.130", "dev": true, "license": "MIT", "devDependencies": { diff --git a/tests/compress-recovery-ranges.test.ts b/tests/compress-recovery-ranges.test.ts index 6ae147b928..f934138b83 100644 --- a/tests/compress-recovery-ranges.test.ts +++ b/tests/compress-recovery-ranges.test.ts @@ -183,11 +183,24 @@ test("#3192: a dropped message's DEAD ref is never re-advertised by the receipt const rejected = request(ctx, "m00001", "m00005"); assert.equal(rejected.outcome, "refused"); assert.match(rejected.text, /cannot be anchored/); - // No advertised compressible span may cross the dead ref m00005. - assert.doesNotMatch(rejected.text, /m0000[1-4]\s*[–-]\s*m0000[5-9]/); - // The only live span bracketed the dead ref -> stop instruction, not a doomed retry span. - assert.match(rejected.text, /continue the task without compressing/); - assert.doesNotMatch(rejected.text, /use these refs exactly as listed/); + // No advertised compressible span may cross the dead ref m00005 — the red gate + // on the unfixed kernel (it advertised one span m00001–m00007 crossing m00005). + const advertised = advertisedRanges(rejected.text); + assert.ok(advertised.length > 0, `receipt must advertise the surviving sub-spans: ${rejected.text}`); + for (const [start, end] of advertised) { + assert.ok(!spanContains(start, end, 5), `${start}..${end} must not span the dead ref m00005`); + } + // The kernel SPLIT around the hole into adjacent live sub-spans (not a whole-span + // drop): the run before the hole ends at/before m00004, the run after starts at/ + // after m00006. Both present proves the split rather than a silent drop. + assert.ok( + advertised.some(([start, end]) => Number(start.slice(1)) <= 4 && Number(end.slice(1)) <= 4), + `pre-hole sub-span must be advertised: ${JSON.stringify(advertised)}`, + ); + assert.ok( + advertised.some(([start, end]) => Number(start.slice(1)) >= 6 && Number(end.slice(1)) >= 6), + `post-hole sub-span must be advertised: ${JSON.stringify(advertised)}`, + ); // Receipt and acp_status share ONE eligibility source (liveCompressionRanges); // pin that source so the two surfaces cannot drift apart again (#2778 promise).