Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion kernel/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "acp-kernel",
"version": "0.0.151",
"version": "0.0.152",
"description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.",
"license": "MIT",
"author": "ranxianglei",
Expand Down
25 changes: 22 additions & 3 deletions kernel/src/compress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import { activeBlockSpans } from "./block-map.js";
import { createRenderRefsNode } from "./render-refs.js";
import type { RenderStrategy } from "./render-refs.js";
import {
collectFailedCompressCallIds,
collectLatestProtected,
collectProtectedToolCallIds,
hasUnrecoverableMediaPayload,
Expand Down Expand Up @@ -1000,6 +1001,15 @@ const assignRefsNode: PipelineNode = {
const latest = hasProtection
? collectLatestProtected(io.messages, ctx.config)
: undefined;
// #4289: total-failure compress exchanges (receipt "[Compression FAILED",
// zero ranges applied) carry no summary carrier — lift their hard-coded
// exemption so they fold and get reclaimed like ordinary messages.
// Success/partial receipts and in-flight calls stay exempt. Computed only
// when some protection machinery is active, mirroring `latest` (otherwise
// nothing is BLOCKED anyway and behavior stays byte-identical).
const failedCompressCallIds = hasProtection
? collectFailedCompressCallIds(io.messages, ctx.config)
: undefined;
// Refs keep the exact pre-#1947 layout for name-only patterns (a
// protected call's unnamed result keeps a foldable ref; actual folding
// paths pair via filterProtectedToolMessages). #1947 path patterns
Expand All @@ -1011,7 +1021,11 @@ const assignRefsNode: PipelineNode = {
);
const pathProtectedCallIds =
hasProtection && hasPathPatterns
? collectProtectedToolCallIds(io.messages, ctx.config)
? collectProtectedToolCallIds(
io.messages,
ctx.config,
failedCompressCallIds,
)
: undefined;
// Unrecoverable media (remote URLs / file refs with no stored bytes) ride
// outside msg.text; folding one destroys it permanently with no restore
Expand All @@ -1022,8 +1036,13 @@ const assignRefsNode: PipelineNode = {
hasUnrecoverableMediaPayload(m) ||
(hasProtection
? (pathProtectedCallIds
? isMessageProtectedWithPairing(m, ctx.config, pathProtectedCallIds)
: isMessageProtected(m, ctx.config)) ||
? isMessageProtectedWithPairing(
m,
ctx.config,
pathProtectedCallIds,
failedCompressCallIds,
)
: isMessageProtected(m, ctx.config, failedCompressCallIds)) ||
(latest ? isMessageLatestProtected(m, latest) : false)
: false);
const refResult = assignRefs(io.messages, {
Expand Down
3 changes: 3 additions & 0 deletions kernel/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,10 +326,13 @@ export {
listSearchAlgorithms,
} from "./search.js";
export {
COMPRESS_FAILED_RECEIPT_PREFIX,
collectFailedCompressCallIds,
collectLatestProtected,
collectProtectedToolCallIds,
hasMediaPayload,
hasUnrecoverableMediaPayload,
isFailedCompressReceiptText,
isMessageLatestProtected,
isMessageProtected,
isMessageProtectedWithPairing,
Expand Down
102 changes: 96 additions & 6 deletions kernel/src/protected.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,78 @@ import type { Config, CoreMessage } from "./types.js";
* treats past compress calls as load-bearing metadata. acp_rule calls record
* persistent user rules that are re-injected every turn — compressing them
* away would silently lose what the rules state. Letting any of these be
* compressed away breaks decompress and the "summary is historical" contract. */
* compressed away breaks decompress and the "summary is historical" contract.
*
* Exception (billion-context#4289): a compress exchange whose paired result
* is a TOTAL-FAILURE receipt (COMPRESS_FAILED_RECEIPT_PREFIX — zero ranges
* applied) carries no summary carrier: no block cites its arguments, so
* decompress/search lose nothing folding it. Keeping such exchanges exempt
* forever pins every failed retry into context and raises the fold floor
* (measured: ~75% of a 227K context was compression residue). isMessageProtected
* lifts the exemption for exactly those exchanges when the caller passes the
* matching collectFailedCompressCallIds set; success/partial receipts
* ("[Compressed …") and result-less in-flight calls keep it. */
export const ALWAYS_PROTECTED_TOOLS = ["compress", "acp_rule"] as const;

/** Receipt prefix emitted by a compress invocation that applied ZERO ranges —
* the total-failure form. Successful AND partial-success receipts start with
* "[Compressed …" instead, so this prefix identifies total failure without
* parsing the body (billion-context#4289). */
export const COMPRESS_FAILED_RECEIPT_PREFIX = "[Compression FAILED";

/** True when a tool-result text is a total-failure compress receipt. */
export function isFailedCompressReceiptText(
text: string | undefined,
): boolean {
return (
typeof text === "string" &&
text.trimStart().startsWith(COMPRESS_FAILED_RECEIPT_PREFIX)
);
}

/** True when `name` denotes a `compress` call: the host dialect recognizer
* (#2970 isCompressTool covers e.g. MCP's "mcp__bili__compress") wins when
* set; otherwise the canonical bare name only. */
function isCompressToolName(
name: string | undefined,
config: Pick<Config, "isCompressTool">,
): boolean {
if (!name) return false;
if (config.isCompressTool) return config.isCompressTool(name);
return name === "compress";
}

/** Tool-call ids of compress exchanges whose paired result is a TOTAL-FAILURE
* receipt (#4289). Such exchanges carry no summary carrier — zero ranges were
* applied, so no block cites their arguments and decompress/search never read
* them — yet they enjoyed compress's permanent exemption, pinning every
* failed retry into the context forever. Callers feed the returned set into
* isMessageProtected / collectProtectedToolCallIds /
* isMessageProtectedWithPairing to lift the exemption for exactly those
* exchanges. Success/partial receipts and result-less (in-flight) calls do
* NOT enter the set: their summaries are load-bearing (conservative). A
* result lacking any recognizable tool name (own or paired) is not marked —
* it stays protected, as before. O(n) + one id→name map. */
export function collectFailedCompressCallIds(
messages: CoreMessage[],
config: Pick<Config, "isCompressTool">,
): Set<string> {
const ids = new Set<string>();
const callNameById = new Map<string, string>();
for (const m of messages) {
if (m.contentType === "tool-call" && m.toolCallId && m.toolName) {
callNameById.set(m.toolCallId, m.toolName);
}
}
for (const m of messages) {
if (m.contentType !== "tool-result" || !m.toolCallId) continue;
if (!isFailedCompressReceiptText(m.text)) continue;
const name = m.toolName ?? callNameById.get(m.toolCallId);
if (isCompressToolName(name, config)) ids.add(m.toolCallId);
}
return ids;
}

/** Tool results that must NEVER participate in the soft-protected recent zone
* (preserveRecentMessages / preserveRecentTokens / last user message).
*
Expand Down Expand Up @@ -202,6 +271,10 @@ export function virtualToolPathOf(
export function isMessageProtected(
msg: CoreMessage,
config: Pick<Config, "protectedTools" | "isToolProtected" | "isMessageProtected" | "messagePathOf">,
/** #4289: total-failure compress exchanges (from collectFailedCompressCallIds
* over the SAME message array class) lose their hard-coded exemption — see
* the ALWAYS_PROTECTED_TOOLS note above. Unset = legacy behavior. */
failedCompressCallIds?: ReadonlySet<string>,
): boolean {
// Host-declared message-level protection (#2419): applies to ANY content
// type and outranks everything below — the host owns the decision.
Expand All @@ -226,16 +299,31 @@ export function isMessageProtected(
return false;
}

// Hard-coded protection: ACP metadata tools are never compressible.
if ((ALWAYS_PROTECTED_TOOLS as readonly string[]).includes(msg.toolName)) {
// #4289: total-failure compress exchange (both halves identified by their
// shared toolCallId in the failed set). Lifts ONLY the hard-coded exemption
// below — the pattern hooks still run, so a host that explicitly pins these
// via protectedTools/isToolProtected wins (conservative).
const failedCompress =
failedCompressCallIds !== undefined &&
!!msg.toolCallId &&
failedCompressCallIds.has(msg.toolCallId);

// Hard-coded protection: ACP metadata tools are never compressible (except
// failed compress exchanges above — they carry no summary carrier).
if (
!failedCompress &&
(ALWAYS_PROTECTED_TOOLS as readonly string[]).includes(msg.toolName)
) {
return true;
}

for (const pattern of config.protectedTools) {
if (matchToolMessagePattern(msg, pattern)) return true;
}

if (config.isToolProtected?.(msg.toolName, msg.text)) return true;
if (!failedCompress && config.isToolProtected?.(msg.toolName, msg.text)) {
return true;
}

return false;
}
Expand All @@ -247,13 +335,14 @@ export function isMessageProtected(
export function collectProtectedToolCallIds(
messages: CoreMessage[],
config: Pick<Config, "protectedTools" | "isToolProtected">,
failedCompressCallIds?: ReadonlySet<string>,
): Set<string> {
const ids = new Set<string>();
for (const m of messages) {
if (
m.contentType === "tool-call" &&
m.toolCallId &&
isMessageProtected(m, config)
isMessageProtected(m, config, failedCompressCallIds)
) {
ids.add(m.toolCallId);
}
Expand All @@ -267,8 +356,9 @@ export function isMessageProtectedWithPairing(
msg: CoreMessage,
config: Pick<Config, "protectedTools" | "isToolProtected">,
protectedCallIds: Set<string>,
failedCompressCallIds?: ReadonlySet<string>,
): boolean {
if (isMessageProtected(msg, config)) return true;
if (isMessageProtected(msg, config, failedCompressCallIds)) return true;
if (
msg.contentType === "tool-result" &&
msg.toolCallId &&
Expand Down
17 changes: 15 additions & 2 deletions kernel/src/recommend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import { isToolMessage } from "./message-kind.js";
import { coveredMessageIds, isLiveCheckpointCarrier } from "./state.js";
import { SUMMARY_HEADER } from "./prune.js";
import {
collectFailedCompressCallIds,
collectLatestProtected,
collectProtectedToolCallIds,
hasUnrecoverableMediaPayload,
Expand Down Expand Up @@ -290,9 +291,16 @@ export function buildCompressibleRanges(

const covered = coveredMessageIds(state);

// #4289: total-failure compress exchanges are foldable — same set the refs
// site computed, so advertised ranges agree with assigned refs.
const failedCompressCallIds = collectFailedCompressCallIds(messages, config);
// Pairing: a tool-result may carry only toolCallId (no toolName). Collect the
// callIds of protected tool-calls first, then protect matching results too.
const protectedCallIds = collectProtectedToolCallIds(messages, config);
const protectedCallIds = collectProtectedToolCallIds(
messages,
config,
failedCompressCallIds,
);
// Latest-only protected calls: their results are covered by the pairing
// union; the calls themselves need the explicit check below (pairing only
// matches tool-results).
Expand Down Expand Up @@ -355,7 +363,12 @@ export function buildCompressibleRanges(
}

if (
isMessageProtectedWithPairing(msg, config, protectedCallIds) ||
isMessageProtectedWithPairing(
msg,
config,
protectedCallIds,
failedCompressCallIds,
) ||
isMessageLatestProtected(msg, latest)
) {
protectedMsgs.push({
Expand Down
8 changes: 6 additions & 2 deletions kernel/src/unit-projection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { activeBlocks, blockById, isLiveCheckpointCarrier } from "./state.js";
import { resolveBoundaries, blockVisibleInRange } from "./boundaries.js";
import { isSummaryMessageId } from "./prune.js";
import {
collectFailedCompressCallIds,
collectLatestProtected,
hasUnrecoverableMediaPayload,
isMessageLatestProtected,
Expand Down Expand Up @@ -461,9 +462,12 @@ function filterProtectedToolMessages(
const protectedCallIds = new Set<string>();
const removedIds = new Set<string>();
const latest = collectLatestProtected(messages, config);
// #4289: same failed-compress exemption as the refs/recommend sites — the
// apply-side judgment must agree with what assignRefs made foldable.
const failedCompressCallIds = collectFailedCompressCallIds(messages, config);
for (const id of latest.callIds) protectedCallIds.add(id);
for (const msg of messages) {
if (isMessageProtected(msg, config) && msg.toolCallId) {
if (isMessageProtected(msg, config, failedCompressCallIds) && msg.toolCallId) {
protectedCallIds.add(msg.toolCallId);
}
}
Expand All @@ -473,7 +477,7 @@ function filterProtectedToolMessages(
if (!msg) continue;
if (
hasUnrecoverableMediaPayload(msg) ||
isMessageProtected(msg, config) ||
isMessageProtected(msg, config, failedCompressCallIds) ||
isMessageLatestProtected(msg, latest)
) {
removedIds.add(id);
Expand Down
Loading