Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions kernel/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.177",
"version": "0.0.178",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
1 change: 1 addition & 0 deletions kernel/src/compress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1186,6 +1186,7 @@ const assignRefsNode: PipelineNode = {
isProtected: protectedFn,
// Ephemeral retrieval injections never consume a ref slot.
shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX),
deadRefs: io.state.deadRefs,
});
// #3939 ②/#3805: a BLOCKED→numeric migration mints the ref NOW but the
// carrier's <acp> tag render is DEFERRED to the next fold boundary —
Expand Down
1 change: 1 addition & 0 deletions kernel/src/rebuild.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ export function rebuildCompressionState(
existing: state.messageRefs,
nextIndex: highestUsedIndex(state.messageRefs) + 1,
shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX),
deadRefs: state.deadRefs,
});
let working: CompressionState = { ...state, messageRefs: refResult.map };

Expand Down
73 changes: 62 additions & 11 deletions kernel/src/refs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,20 +73,57 @@ export interface AssignRefsOptions {
nextIndex: number;
isProtected?: (message: CoreMessage) => boolean;
shouldSkip?: (message: CoreMessage) => boolean;
/** Refs that belonged to pruned messages. They are never handed out again. */
deadRefs?: readonly string[];
}

function reservedRefIndices(deadRefs: readonly string[] | undefined): Set<number> {
const reserved = new Set<number>();
for (const ref of deadRefs ?? []) {
const index = refToIndex(ref);
if (index !== null) reserved.add(index);
}
return reserved;
}

/** Highest numeric number present on either side of the map. byRef can hold
* orphans whose byRaw owner is gone; those numbers are still issued. */
function highestIssuedIndex(map: MessageRefMap): number {
let highest = highestUsedIndex(map);
for (const ref of Object.keys(map.byRef)) {
const index = refToIndex(ref);
if (index !== null && index > highest) highest = index;
}
return highest;
}

export function assignRefs(
messages: CoreMessage[],
options: AssignRefsOptions,
): AssignRefsResult {
const existingByRaw = options.existing?.byRaw ?? {};
const existingByRef = options.existing?.byRef ?? {};
const map: MessageRefMap = {
byRaw: { ...options.existing.byRaw },
byRef: { ...options.existing.byRef },
byRaw: { ...existingByRaw },
byRef: { ...existingByRef },
};
let cursor =
const reserved = reservedRefIndices(options.deadRefs);
// The high-water mark is the highest ref this map or its tombstones have
// ever carried. A caller-supplied cursor below it must not reopen that
// range: a low ledgerSeq or a counter restart would otherwise reissue a
// number whose previous owner is gone from byRef.
let historicalHighWater = highestIssuedIndex({ byRaw: existingByRaw, byRef: existingByRef });
for (const index of reserved) {
if (index > historicalHighWater) historicalHighWater = index;
}
const requestedCursor =
Number.isInteger(options.nextIndex) && options.nextIndex >= MIN_INDEX
? options.nextIndex
: MIN_INDEX;
if (requestedCursor - 1 > historicalHighWater) {
historicalHighWater = requestedCursor - 1;
}
let cursor = Math.max(requestedCursor, historicalHighWater + 1);
let newlyAssigned = 0;
const migratedIds: string[] = [];

Expand Down Expand Up @@ -140,7 +177,15 @@ export function assignRefs(
ledgerSeq <= MAX_INDEX
) {
const text = indexToRef(ledgerSeq);
if (!map.byRef[text]) {
// A ledger stamp only proves identity when it lands above every ref
// already issued for this state and outside the dead-ref tombstones.
// Below the high-water mark the sequence was reused or reset; trusting
// seq+1 there would mint a collision with a historical owner.
if (
ledgerSeq > historicalHighWater &&
!map.byRef[text] &&
!reserved.has(ledgerSeq)
) {
map.byRaw[message.id] = text;
map.byRef[text] = message.id;
newlyAssigned++;
Expand All @@ -149,7 +194,7 @@ export function assignRefs(
}
}

const ref = allocateFreeRef(map, cursor);
const ref = allocateFreeRef(map, cursor, reserved);
cursor = ref.index + 1;
map.byRaw[message.id] = ref.text;
map.byRef[ref.text] = message.id;
Expand All @@ -162,11 +207,12 @@ export function assignRefs(
function allocateFreeRef(
map: MessageRefMap,
start: number,
reserved: ReadonlySet<number> = new Set(),
): { text: string; index: number } {
let candidate = Math.max(start, MIN_INDEX);
while (candidate <= MAX_INDEX) {
const text = indexToRef(candidate);
if (!map.byRef[text]) {
if (!map.byRef[text] && !reserved.has(candidate)) {
return { text, index: candidate };
}
candidate++;
Expand Down Expand Up @@ -250,12 +296,17 @@ export function restampRefsToLedger(
const byRef: Record<string, string> = { ...refs.byRef };
const scratch: MessageRefMap = { byRaw, byRef };
const displaced: string[] = [];
const reserved = reservedRefIndices(state.deadRefs);
const retiredThisPass = new Set<string>();
const leftovers: string[] = [];
// MessageRefMap carries no cursor field — the caller recomputes it from the
// highest used index (compress.ts:1047 pattern). We still track a local
// cursor for free-number allocation during this pass.
let nextIndex = highestUsedIndex(refs) + 1;
// cursor for free-number allocation during this pass. Tombstoned refs are
// part of the historical floor even though no byRef key survives for them.
let nextIndex = highestIssuedIndex(refs) + 1;
for (const index of reserved) {
if (index + 1 > nextIndex) nextIndex = index + 1;
}

for (const [rawId, seq] of seqByRawId) {
const current = byRaw[rawId];
Expand Down Expand Up @@ -309,8 +360,8 @@ export function restampRefsToLedger(
for (const rawId of displaced) {
const held = byRaw[rawId];
if (held === undefined) continue;
if (byRef[held] === rawId) continue; // back where we found it (no-op guard)
const free = allocateFreeRef(scratch, nextIndex);
if (byRef[held] === rawId) continue;
const free = allocateFreeRef(scratch, nextIndex, reserved);
out.superseded.set(held, rawId);
byRaw[rawId] = free.text;
byRef[free.text] = rawId;
Expand All @@ -323,7 +374,7 @@ export function restampRefsToLedger(
const held = byRaw[rawId];
if (held === undefined) continue;
if (byRef[held] === rawId) delete byRef[held];
const free = allocateFreeRef(scratch, nextIndex);
const free = allocateFreeRef(scratch, nextIndex, reserved);
out.superseded.set(held, rawId);
byRaw[rawId] = free.text;
byRef[free.text] = rawId;
Expand Down
89 changes: 85 additions & 4 deletions kernel/tests/refs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ import {
refToIndex,
rebuildRefIndex,
} from "../src/refs.js";
import type { CoreMessage } from "../src/types.js";
import { createInitialState } from "../src/state.js";
import type { CompressionState, CoreMessage } from "../src/types.js";

function msg(id: string, role: CoreMessage["role"] = "user"): CoreMessage {
return { id, role, contentType: "text", text: id };
Expand Down Expand Up @@ -121,14 +122,94 @@ test("assignRefs skips messages per shouldSkip", () => {
assert.equal(refForRaw(result.map, "c"), "m00002");
});

test("assignRefs skips free indices already taken (no collision)", () => {
test("assignRefs starts above an orphan byRef high-water", () => {
const existing = emptyRefMap();
existing.byRaw["old"] = "m00002";
existing.byRef["m00002"] = "old";
// An orphan reservation can keep a byRef key without a byRaw owner. Its
// number is still issued, so the new allocation starts above it.
existing.byRef["m00003"] = "ghost";

const result = assignRefs([msg("x")], { existing, nextIndex: 1 });
assert.equal(refForRaw(result.map, "x"), "m00001");
assert.equal(result.nextIndex, 2);
assert.equal(refForRaw(result.map, "x"), "m00004");
assert.equal(result.nextIndex, 5);
});

test("assignRefs counter never fills below an orphan byRef high-water", () => {
const existing = emptyRefMap();
existing.byRef["m00100"] = "ghost";

const result = assignRefs([msg("new")], { existing, nextIndex: 1 });

assert.equal(refForRaw(result.map, "new"), "m00101");
assert.equal(result.nextIndex, 102);
});

test("assignRefs refuses a reused ledger stamp below the historical floor", () => {
const existing = emptyRefMap();
existing.byRaw["floor"] = "m00010";
existing.byRef["m00010"] = "floor";
const newMessage: CoreMessage = { ...msg("new"), ledgerSeq: 7 };

const result = assignRefs([newMessage], {
existing,
nextIndex: 1,
deadRefs: ["m00007"],
});

assert.notEqual(refForRaw(result.map, "new"), "m00007", "dead ref never reissued");
assert.equal(refForRaw(result.map, "new"), "m00011", "counter allocation stays above the floor");
assert.equal(result.nextIndex, 12);
});

test("assignRefs counter path never drops below the historical floor", () => {
const existing = emptyRefMap();
existing.byRaw["floor"] = "m00010";
existing.byRef["m00010"] = "floor";

const result = assignRefs([msg("new")], {
existing,
nextIndex: 1,
deadRefs: ["m00007"],
});

assert.equal(refForRaw(result.map, "new"), "m00011");
assert.equal(result.nextIndex, 12);
});

test("assignRefs still accepts a ledger stamp above the historical floor", () => {
const existing = emptyRefMap();
existing.byRaw["floor"] = "m00010";
existing.byRef["m00010"] = "floor";
const newMessage: CoreMessage = { ...msg("new"), ledgerSeq: 11 };

const result = assignRefs([newMessage], {
existing,
nextIndex: 1,
deadRefs: ["m00007"],
});

assert.equal(refForRaw(result.map, "new"), "m00011");
assert.equal(result.nextIndex, 12);
});

test("assignRefs fresh allocation still starts at seq+1", () => {
const stamped: CoreMessage = { ...msg("first"), ledgerSeq: 1 };
const result = assignRefs([stamped], { existing: emptyRefMap(), nextIndex: 1 });
assert.equal(refForRaw(result.map, "first"), "m00001");
});

test("restampRefsToLedger re-homes displaced refs above dead tombstones", async () => {
const { restampRefsToLedger } = await import("../src/refs.js");
const state: CompressionState = createInitialState();
state.messageRefs.byRaw.a = "m00001"; state.messageRefs.byRef["m00001"] = "a";
state.messageRefs.byRaw.b = "m00003"; state.messageRefs.byRef["m00003"] = "b";
state.messageRefs.byRaw.d = "m00002"; state.messageRefs.byRef["m00002"] = "d";
state.deadRefs = ["m00004"];

const outcome = restampRefsToLedger(state, new Map([["a", 0], ["b", 1]]));
assert.equal(state.messageRefs.byRaw.d, "m00005", "dead m00004 is not a rehome target");
assert.equal(outcome.rehomed, 1);
});

test("rebuildRefIndex drops stale byRef entries and ignores BLOCKED", () => {
Expand Down
43 changes: 43 additions & 0 deletions kernel/tests/seq-restamp.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -218,3 +218,46 @@ test("#5196: the parked layout survives the next cold load untouched", () => {
assert.equal(state.messageRefs.byRaw.z, "m00003");
});

for (const fixture of [
{ name: "retired-seat leftover", byRaw: { a: "m00007", b: "m00009" }, seqs: [["a", 4], ["b", 6]], parked: "b", mover: "a", target: "m00005" },
{ name: "displaced unbacked owner", byRaw: { a: "m00004", b: "m00005" }, seqs: [["a", 4]], parked: "b", mover: "a", target: "m00005" },
{ name: "displaced and leftover overlap", byRaw: { a: "m00002", b: "m00001" }, seqs: [["a", 0], ["b", 1]], parked: "b", mover: "a", target: "m00001" },
] satisfies Array<{ name: string; byRaw: Record<string, string>; seqs: Array<[string, number]>; parked: string; mover: string; target: string }>) {
test(`#5196 composition: ${fixture.name} parks above orphan and tombstone history`, () => {
const state = stateWith({ ...fixture.byRaw, protected: "BLOCKED" }, { deadRefs: ["m00090"] });
state.messageRefs.byRef.m00070 = "historical-orphan";
const seqs = new Map([...fixture.seqs, ["protected", 99] as [string, number]]);
const out = restampRefsToLedger(state, seqs);

assert.equal(state.messageRefs.byRaw[fixture.parked], "m00091");
assert.equal(state.messageRefs.byRef.m00091, fixture.parked);
assert.equal(state.messageRefs.byRaw[fixture.mover], fixture.target);
assert.equal(state.messageRefs.byRef[fixture.target], fixture.mover);
assert.equal(state.messageRefs.byRef.m00070, "historical-orphan");
assert.equal(state.messageRefs.byRef.m00090, undefined);
assert.equal(state.messageRefs.byRaw.protected, "BLOCKED");
assert.equal(out.rehomed, 1, "an owner in both settlement lists parks only once");
assert.equal(out.blocked, 1);
assert.deepEqual(state.deadRefs, ["m00090"]);
if (fixture.name === "retired-seat leftover") {
assert.equal(state.messageRefs.byRef.m00007, undefined);
assert.equal(out.superseded.get("m00007"), "a");
}
const after = structuredClone(state);
const again = restampRefsToLedger(state, seqs);
assert.equal(again.kept + again.restamped + again.rehomed, 0);
assert.deepEqual(state, after, "the parked layout and history survive a second pass");
});
}

for (const seqs of [
new Map([["a", 4], ["b", 6]]),
new Map([["a", 8]]),
]) {
test(`#5196 composition: exhausted ${seqs.size === 2 ? "leftover" : "displaced"} parking is atomic`, () => {
const state = stateWith({ a: "m00007", b: "m00009" }, { deadRefs: ["m9999999"] });
const before = structuredClone(state);
assert.throws(() => restampRefsToLedger(state, seqs), /ref capacity exhausted/);
assert.deepEqual(state, before, "no partial ref swap or aligned marker escapes a failed pass");
});
}
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion src/absorb.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { PROXY_TOOL_NAMES, READ_ORIGINAL_IMAGE_TOOL_NAME, RULE_TOOL_NAME } from
import { effectiveRulesEnabled } from "./rules-feature.js";
import { ccrEnabled, retrieveToolName } from "./store.js";
import { log as loggerLog } from "./logger.js";
import type { Session } from "./session.js";
import { hasRefLedgerConflict, type Session } from "./session.js";
import { toolFail, toolOk, type ProxyToolResult } from "./proxy-tool-result.js";

const EFFECTIVE_ABSORB_KEY = "effectiveAbsorb";
Expand Down Expand Up @@ -84,6 +84,9 @@ type AbsorbExecCtx = {
// repeat-failure guard reasons about them uniformly with compress failures.
export function executeAbsorb(args: unknown, callId: string | undefined, absorb: AbsorbConfig, ctx: AbsorbExecCtx): ProxyToolResult {
const log = ctx.log ?? ((msg: string) => loggerLog("info", msg));
if (hasRefLedgerConflict(ctx.session)) {
return { ...toolFail("[absorb FAILED: this session's retained ref history is ambiguous. Absorbing could hide the wrong message, so nothing was changed. Raw messages, blocks, and the ref ledger are preserved.]"), reason: "ref-ledger-conflict" };
}
const parsed = parseAbsorbInput(args, callId, (w) => log(`[acp-absorb] ${w}`));
if (!parsed) {
return toolFail('[absorb FAILED: invalid input — expected { ref: "mNNNNN", summary: "..." }]');
Expand Down
4 changes: 3 additions & 1 deletion src/external-summary-compress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { buildCompressSystemPrompt, parseCompressInput, type ParsedRange } from
import { messageTimestampSections } from "./external-summary-surface.js";
import { configuredSummaryPlan } from "./external-summary-runtime.js";
import type { ResolvedKernelConfig } from "./compress-settings.js";
import { applyRanges, noVisibleContextRefusal, normalizeRangeOrder, recoverEvictedView, type RewriteCtx } from "./stream.js";
import { applyRanges, refLedgerConflictRefusal, noVisibleContextRefusal, normalizeRangeOrder, recoverEvictedView, type RewriteCtx } from "./stream.js";
import { compressResult, type ProxyToolResult } from "./proxy-tool-result.js";
import { imagePlaceholders } from "./image-note.js";
import { describeExternalSummaryFailure, summaryLengthBudgetClause, type SummaryWork } from "./external-summary.js";
Expand Down Expand Up @@ -35,6 +35,8 @@ function withOptionalSummaries(input: unknown): unknown {

/** Called under the host session lock; generation is separate from kernel commit. */
export async function applyConfiguredCompression(input: unknown, ctx: RewriteCtx, callId?: string, signal?: AbortSignal): Promise<ProxyToolResult> {
const conflict = refLedgerConflictRefusal(ctx);
if (conflict) return conflict;
if (signal?.aborted) return compressResult("[Compression FAILED: cancelled. Nothing compressed.]", "refused", 0);
// The plan rides the request Config rail (#833: ctx.config is the
// session's effective Config — same cascade that produced the wire).
Expand Down
4 changes: 3 additions & 1 deletion src/fork-adoption.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { stripRawStage } from "./marker/engine.js";
import { normalizeResponsesMessageItems, sanitizeResponsesInputIds, dropWhitespaceResponsesMessages } from "./loop/adapter-responses.js";
import { responsesToCoreWithToolImages } from "./responses-tool-output.js";
import { replaceBiliCompactionItems } from "./codex-compact.js";
import { peekSession, markDirty, reserveRefsThrough, type Session } from "./session.js";
import { peekSession, markDirty, reserveRefsThrough, hasRefLedgerConflict, type Session } from "./session.js";
import { getStore } from "./persist.js";
import { adoptContentStore, cloneStoreForRefs, contentStoreOf } from "./store.js";
import type { WireProtocol } from "./util.js";
Expand Down Expand Up @@ -241,6 +241,8 @@ export function planForkAdoption(
/** Seed a fresh fork session from the plan. Copy-on-fork: every value is a
* clone; the parent session is never touched. */
export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, parent: Session): void {
// Filtered adoption does not copy the parent's historical ref rows.
if (hasRefLedgerConflict(parent)) session.metadata.refLedgerConflict = true;
for (const b of plan.blocks) {
session.state.blocks.push(b);
const content = parent.blockContents.get(b.blockId);
Expand Down
Loading
Loading