diff --git a/AGENTS.md b/AGENTS.md index fa56058..a882a51 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,8 @@ acp-kernel/ │ ├── rebuild.ts # Fork recovery + state rebuilding │ ├── hide.ts # Hide compressed messages from visible context │ ├── keep-markers.ts # KEEP/REF marker preservation in summaries +│ ├── ccr.ts # CCR: placeholder contract, storeLargeResults, retrieveByRef, applyRetrieve, ccrStoreNode +│ ├── content-store.ts # MessageContentStore — per-session content-addressed dedup store (per-ref append-only) │ ├── prompts.ts # Prompts (4 load-bearing rules) + defaultPrompts + resolvePrompts │ ├── compress-tools.ts # ACP tool schemas (3 wire shapes) + standing-prompt builders │ ├── surface-config.ts # Surface-text overrides: applySectionOverrides, cloneWithDescriptions, applyAcpToolOverrides diff --git a/DESIGN.md b/DESIGN.md index ea4d187..a6fe6b6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -113,20 +113,31 @@ type Config = { interface CompressionCore { // Per-turn node pipeline (replaces the message-transform hook's algorithm part). // Runs every turn (canonical order): assign-refs → sync-blocks → merge-blocks → - // prune → filter → hide-compress-calls → nudge-inject → emergency-truncate → - // render-refs. Survives/promotes blocks via advanceSurvival (no age-based - // deactivation). Returns transformed messages + updated state + nudge decision. + // prune → ccr-store → filter → hide-compress-calls → nudge-inject → + // emergency-truncate → render-refs. Survives/promotes blocks via advanceSurvival + // (no age-based deactivation). Returns transformed messages + updated state + + // nudge decision + the (possibly grown) content store. processTurn(input: { messages: CoreMessage[]; state: CompressionState; config: Config; tokenCount: number; + /** Per-session CCR content store from the previous turn (optional; an + * empty store is created when omitted). */ + contentStore?: MessageContentStore; }): { messages: CoreMessage[]; state: CompressionState; nudge?: NudgeDecision; + /** Always present; pass back as input.contentStore next turn. */ + contentStore: MessageContentStore; }; + // When the model calls acp_retrieve. Pure lookup over the host-provided store + // plus an ephemeral injection message (id prefix acp_retrieved_*) that consumes + // no ref and never enters fold space. Hallucinated refs → not-found ack. + retrieve(store: MessageContentStore, ref: string): ApplyRetrieveResult; + // When the model calls compress. `ranges[].summary` is model-produced text. // Allocates block(s), deactivates consumed blocks, updates indices, resets the // nudge growth baseline on success (§5.7 feedback-loop fix). diff --git a/README.md b/README.md index 2fea04a..63d6f94 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,12 @@ const state = createInitialState(); const config = defaultConfig(200000); // modelContextLimit (positional); optional overrides as 2nd arg // processTurn runs the canonical node pipeline every turn: -// assign-refs → sync-blocks → prune → absorb-hide → crush → absorb-prompt -// → filter → hide-compress-calls → recommend → nudge-inject +// assign-refs → sync-blocks → prune → ccr-store → absorb-hide → crush → +// absorb-prompt → filter → hide-compress-calls → recommend → nudge-inject // → emergency-truncate → render-refs -const { messages, state: nextState, nudge } = core.processTurn({ +const { messages, state: nextState, nudge, contentStore } = core.processTurn({ messages, state, config, tokenCount, + contentStore, // previous turn's content store (optional; see CCR below) }); // When the model emits a compress decision (summary written by the model): @@ -86,6 +87,74 @@ processTurn({ messages, state, config, tokenCount, renderTags: "none" }); `renderTags` is optional and defaults to `"all"`, so existing call sites keep working unchanged. +#### CCR — content-cached retrieval (lossless tool-result offload) + +CCR is the lossless alternative to absorb's lossy distillation. When enabled, +a tool result at or above `ccr.minToolTokens` is stored **once, at arrival** in +the per-session content store, and its visible copy is replaced with a +deterministic placeholder carrying enough signal (kind, size, command/head +preview, ref) to judge relevance without retrieving. The model pulls the +original back via the `acp_retrieve` tool; the retrieved text rides back as an +ephemeral trailing message that never consumes a ref and never enters the fold +space. Disabled by default — no behavior change unless opted in. + +```ts +const config = defaultConfig(200000, { ccr: { enabled: true } }); // opt-in + +// Turn N: pass the previous turn's store back in. +const { messages, state, contentStore } = core.processTurn({ + messages, state, config, tokenCount, + contentStore, // persisted from turn N-1 (host owns persistence) +}); + +// When the model calls acp_retrieve({ ref }): +const hit = core.retrieve(contentStore, "m00042"); +if (hit.ok) { + // hit.text — the original bytes + // hit.injection — trailing request-only message to append for this request + // hit.ackText — short tool-result ack string +} else { + // hit.ackText — not-found notice (hallucinated ref costs one tool call) +} +``` + +Placeholder wire shape (deterministic — same inputs always produce identical +bytes, so the visible text is byte-stable after arrival and prefix-cache +friendly): + +``` +📦 [acp-stored #m00423 · shell output · 4,213 tok] `npm run build` + → acp_retrieve("m000423") returns the full text +``` + +Contract guarantees: + +- **Replace-once-at-arrival.** The visible bytes change exactly once + (original → placeholder); every later turn sees identical bytes. +- **Tool-pair integrity.** Only the tool-result's own content shrinks; the + paired assistant `tool_calls` survive untouched (OpenAI-family wire pairing). +- **id-never-reused safe.** The store is per-ref append-only, first-write-wins, + and never reissues or recycles refs. Refs stay retrievable even if a host + prunes its own `messageRefs` map after compaction (e.g. billion-context + archive), because lookup happens against the store, not the ref map. +- **Ephemeral retrieval.** `acp_retrieved_*` messages are skipped by assign-refs + and excluded from block coverage — they consume no ref and enter no fold + space. Hosts may strip them after the request (nudge channel); if they + round-trip anyway, the kernel handles them safely. +- **Coexists with absorb.** Placeholder-marked results are never absorb + candidates (ID-reference wins); absorb keeps handling semantic distillation + of everything else. + +Persistence: the store is plain JSON (`{ version: 1, byHash, byRef }`) — hosts +persist it alongside `CompressionState` (same StateStore envelope path as +blockContents). Content-addressed dedup means identical bytes are written once. +`core.status(...)` reports `breakdown.storedMessages` and +`breakdown.retrievals` so a host can measure retrieve rate. + +For folded-range retention (v2 groundwork), hosts can call +`storeCoveredOriginals(store, messages, compressedState, [blockId], countTokens)` +right after `applyCompression` so pruned originals stay retrievable by ref. + **Token counts in rendered tags are snapshots.** Each message's `` attribute is frozen at the tag's first render (the per-message `tokenSnapshot` in state) and is not recomputed when the message @@ -103,6 +172,7 @@ live-recomputed tags, use `renderVisibleRefs` directly. | `buildStatusReport` / `buildRecap` | Context-usage report + block recap | | `mergeMarkedBlocks` / `collectOldGenBlocks` | Batch merge old-gen blocks into one summary | | `rebuildCompressionState` | Fork-recovery: replay historical compress calls | +| `MessageContentStore` / `storeLargeResults` / `retrieveByRef` / `storeCoveredOriginals` | CCR content store: per-session, content-addressed dedup of tool-result originals + on-demand retrieval (see "CCR" above) | | `applyMessageFilters` | Pluggable message-filter framework | | `resolveTransformChannel` | Channel-selection policy: an explicit preference wins; the default is the wire channel only when the caller reports it viable | | `applySectionOverrides` / `cloneWithDescriptions` / `applyAcpToolOverrides` | Prompt/tool *surface* customization (see below) | diff --git a/src/absorb.ts b/src/absorb.ts index adc30dd..92281d7 100644 --- a/src/absorb.ts +++ b/src/absorb.ts @@ -1,5 +1,6 @@ import { rawForRef, refForRaw, BLOCKED_REF } from "./refs.js"; import { ACP_TOOL_NAMES, ABSORB_TOOL_NAME } from "./compress-tools.js"; +import { isStoredPlaceholderText } from "./ccr.js"; import { collectLatestProtected, isMessageLatestProtected, @@ -31,7 +32,9 @@ export const ABSORB_PROMPT_MARKER = "[ACP absorb]"; export const DEFAULT_ABSORB_CONFIG: AbsorbConfig = { enabled: false, toolName: ABSORB_TOOL_NAME, - minToolTokens: 1000, + // Raised 1000 → 4000 (issue #352): lossless CCR takes over large-result + // handling; absorb's forced distillation only fires above the new bar. + minToolTokens: 4000, contextThresholdPct: 0, excludeTools: [], }; @@ -86,6 +89,7 @@ export function isAbsorbCandidate( latest?: LatestProtected, ): boolean { if (msg.contentType !== "tool-result" || !msg.toolCallId) return false; + if (isStoredPlaceholderText(msg.text ?? "")) return false; const cfg = resolveAbsorbConfig(config); if (isAcpOrConfiguredTool(msg.toolName, cfg)) return false; if (isMessageProtected(msg, config)) return false; diff --git a/src/boundaries.ts b/src/boundaries.ts index 1854c68..a58fd76 100644 --- a/src/boundaries.ts +++ b/src/boundaries.ts @@ -1,5 +1,6 @@ import { activeBlocks, blockById } from "./state.js"; import { isRenderedSummaryMessage, summaryMessageId } from "./prune.js"; +import { isRetrievedMessage } from "./ccr.js"; import type { CompressionBlock, CompressionState, @@ -118,10 +119,14 @@ export function resolveBoundaries( const messageIds: string[] = []; for (let index = startIndex; index <= endIndex; index++) { const message = input.messages[index]; - // Synthetic summary messages are transient view representations, not - // compressible content: exclude them so they never leak into a new - // block's effectiveMessageIds/directMessageIds. - if (message && !isRenderedSummaryMessage(message)) + // Synthetic summary and retrieval-injection messages are transient view + // representations, not compressible content: exclude them so they never + // leak into a new block's effectiveMessageIds/directMessageIds. + if ( + message && + !isRenderedSummaryMessage(message) && + !isRetrievedMessage(message) + ) messageIds.push(message.id); } diff --git a/src/ccr.ts b/src/ccr.ts new file mode 100644 index 0000000..22c3502 --- /dev/null +++ b/src/ccr.ts @@ -0,0 +1,421 @@ +import type { + CcrConfig, + Config, + CompressionState, + CoreMessage, +} from "./types.js"; +import { refForRaw, BLOCKED_REF } from "./refs.js"; +import { ACP_TOOL_NAMES } from "./compress-tools.js"; +import { isMessageProtected, matchToolPattern } from "./protected.js"; +import { hasStoredRef, retrieveByRef, storeOriginal } from "./content-store.js"; +import type { + MessageContentStore, + RetrieveResult, + StoredEntry, +} from "./content-store.js"; +import type { NodeIO, PipelineContext, PipelineNode } from "./pipeline.js"; + +/** + * CCR — content-cached retrieval (issue #352 / billion-context#1097). + * + * Lossless alternative to absorb's lossy distillation: oversized tool results + * are stored once at arrival in the per-session MessageContentStore, and the + * visible copy is replaced with a deterministic placeholder carrying enough + * signal (kind, size, command/head preview, ref) to judge relevance without + * retrieving. The model pulls the original back via the retrieve tool; the + * retrieved text rides back as an ephemeral trailing message that never + * consumes a ref and never enters the fold space. + * + * Replace-once-at-arrival: the pipeline node only acts on results it has not + * stored yet (marker check + store lookup). After replacement the visible + * bytes never change again → prefix-cache stable. + */ + +export const RETRIEVE_TOOL_NAME = "acp_retrieve"; + +export const DEFAULT_CCR_CONFIG: CcrConfig = { + enabled: false, + toolName: RETRIEVE_TOOL_NAME, + minToolTokens: 4000, + excludeTools: [], + maxHeadChars: 96, +}; + +export function resolveCcrConfig(config: Config): CcrConfig { + return { ...DEFAULT_CCR_CONFIG, ...config.ccr }; +} + +/** Marker embedded in every placeholder. Idempotency check for the pipeline + * node AND the exclusion signal for absorb (ID-reference wins over distill). */ +export const STORED_PLACEHOLDER_MARKER = "[acp-stored"; + +/** Reserved id prefix for ephemeral retrieval injections (mirrors + * acp_summary_): no ref is ever assigned, fold-space collection skips them. */ +export const RETRIEVED_ID_PREFIX = "acp_retrieved_"; + +const KIND_LABELS: Record = { + bash: "shell output", + shell: "shell output", + exec: "shell output", + execute_command: "shell output", + run: "shell output", + terminal: "shell output", + read: "file read", + read_file: "file read", + cat: "file read", + open: "file read", + grep: "search output", + rg: "search output", + search: "search output", + glob: "search output", + find: "search output", + webfetch: "web fetch", + web_fetch: "web fetch", + fetch: "web fetch", + curl: "web fetch", +}; + +export function classifyKind(toolName?: string): string { + if (!toolName) return "tool result"; + return KIND_LABELS[toolName.toLowerCase()] ?? "tool result"; +} + +function groupThousands(value: number): string { + return String(Math.max(0, Math.round(value))).replace( + /\B(?=(\d{3})+(?!\d))/g, + ",", + ); +} + +/** Collapse whitespace to single spaces and cap length. Deterministic — the + * preview is baked into the placeholder at arrival and must be byte-stable. */ +export function normalizeHead(text: string, maxChars: number): string { + const singleLine = (text || "").replace(/\s+/g, " ").trim(); + if (singleLine.length <= maxChars) return singleLine; + return singleLine.slice(0, maxChars) + "…"; +} + +const COMMAND_FIELDS = ["command", "cmd", "script", "query", "path", "url"]; + +/** Best-effort extraction of the call's subject (command/path/query) from the + * paired tool-call args JSON. Returns undefined when args are not JSON or no + * known field holds a short string — the head preview covers that case. */ +export function extractCommand( + toolCallText: string | undefined, + maxChars: number, +): string | undefined { + if (!toolCallText) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(toolCallText); + } catch { + return undefined; + } + if (typeof parsed !== "object" || parsed === null) return undefined; + const record = parsed as Record; + for (const field of COMMAND_FIELDS) { + const value = record[field]; + if (typeof value === "string" && value.trim().length > 0) { + const normalized = value.replace(/\s+/g, " ").trim(); + return normalized.length <= maxChars + ? normalized + : normalized.slice(0, maxChars) + "…"; + } + } + return undefined; +} + +export interface StoredPlaceholderInput { + ref: string; + kind: string; + tokens: number; + head: string; + command?: string; + retrieveToolName: string; +} + +/** Deterministic two-line placeholder. No timestamps, no randomness — the + * same inputs always produce identical bytes (prefix-cache friendly). */ +export function buildStoredPlaceholder(input: StoredPlaceholderInput): string { + const title = input.command ?? input.head; + const titlePart = title ? ` \`${title}\`` : ""; + return ( + `📦 ${STORED_PLACEHOLDER_MARKER} #${input.ref} · ${input.kind} · ` + + `${groupThousands(input.tokens)} tok]${titlePart}` + + `\n → ${input.retrieveToolName}("${input.ref}") returns the full text` + ); +} + +// Built from hex escapes: literal history-tag sequences in source get mangled +// by content sanitizers upstream of git push (same reason render-refs.ts uses LT/GT). +const LEADING_TAG_RE = new RegExp("^\\x3cacp [^>]*>[^\\x3c]*\\x3c\\/acp>\\n?"); + +export function stripLeadingTag(text: string): string { + return text.replace(LEADING_TAG_RE, ""); +} + +/** Marker check over the untagged body: hosts round-trip rendered output, so + * placeholders arrive prefixed with their own history tag. */ +export function isStoredPlaceholderText(text: string): boolean { + return stripLeadingTag(text).includes(STORED_PLACEHOLDER_MARKER); +} + +export function retrievedMessageId(ref: string): string { + return RETRIEVED_ID_PREFIX + ref; +} + +export function isRetrievedMessage(message: CoreMessage): boolean { + return ( + message.id.startsWith(RETRIEVED_ID_PREFIX) && + message.role === "system" && + message.contentType === "text" + ); +} + +export interface RetrievalInjection { + /** Short deterministic ack — rides as the tool result so OpenAI-family + * wire pairing (every tool_call needs a response) stays intact. */ + ackText: string; + /** Full-text trailing request-only message. Hosts strip it before + * persisting (same channel as nudge); if it round-trips anyway it is + * structurally excluded from refs and the fold space. */ + injection: CoreMessage; +} + +export function buildRetrievalInjection( + ref: string, + entry: StoredEntry, + text: string, +): RetrievalInjection { + const header = `[acp-retrieved #${ref} · ${entry.kind} · ${groupThousands(entry.tokens)} tok]`; + return { + ackText: `retrieved ${ref}: ${groupThousands(entry.tokens)} tok (${entry.chars} chars)`, + injection: { + id: retrievedMessageId(ref), + role: "system", + contentType: "text", + text: `${header}\n${text}`, + }, + }; +} + +export interface ApplyRetrieveInput { + store: MessageContentStore; + ref: string; +} + +export type ApplyRetrieveResult = + | { + ok: true; + text: string; + ackText: string; + injection: CoreMessage; + entry: StoredEntry; + } + | { ok: false; reason: "not-found"; ackText: string }; + +/** Kernel primitive behind acp_retrieve: resolve a ref to its stored + * original plus the wire-safe injection pair. Hallucinated refs → not-found + * (cost: one tool call, by design). */ +export function applyRetrieve(input: ApplyRetrieveInput): ApplyRetrieveResult { + const ref = input.ref.trim(); + const found: RetrieveResult = retrieveByRef(input.store, ref); + if (!found.ok) { + return { + ok: false, + reason: "not-found", + ackText: `retrieve ${ref}: not found — no stored original for this ref`, + }; + } + const built = buildRetrievalInjection(ref, found.entry, found.text); + return { + ok: true, + text: found.text, + ackText: built.ackText, + injection: built.injection, + entry: found.entry, + }; +} + +export interface StoreLargeResultsInput { + messages: CoreMessage[]; + state: CompressionState; + store: MessageContentStore; + config: Config; + countTokens: (text: string) => number; +} + +export interface StoreLargeResultsResult { + messages: CoreMessage[]; + store: MessageContentStore; + storedCount: number; +} + +/** Core transform: store oversized tool-result originals and replace their + * visible text with placeholders. Pure over its inputs; returns new arrays. + * Only shrinks tool-result content in place — message ids, roles, toolCallId + * and the assistant tool_calls half are untouched (tool-pair integrity). */ +export function storeLargeResults( + input: StoreLargeResultsInput, +): StoreLargeResultsResult { + const cfg = resolveCcrConfig(input.config); + if (!cfg.enabled) + return { messages: input.messages, store: input.store, storedCount: 0 }; + + let current = input.store; + let storedCount = 0; + const callById = new Map(); + for (const message of input.messages) { + if (message.contentType === "tool-call" && message.toolCallId) { + callById.set(message.toolCallId, message); + } + } + + const updated = input.messages.map((message) => { + if (message.contentType !== "tool-result") return message; + const text = message.text ?? ""; + if (text.length === 0) return message; + if (isStoredPlaceholderText(text)) return message; + if (!message.toolCallId) return message; + const toolName = message.toolName; + if ( + toolName && + (ACP_TOOL_NAMES.has(toolName) || toolName === cfg.toolName) + ) { + return message; + } + if ( + toolName && + cfg.excludeTools.some((pattern) => matchToolPattern(toolName, pattern)) + ) { + return message; + } + if (isMessageProtected(message, input.config)) return message; + const tokens = input.countTokens(text); + if (tokens < cfg.minToolTokens) return message; + const ref = refForRaw(input.state.messageRefs, message.id); + if (!ref || ref === BLOCKED_REF) return message; + if (hasStoredRef(current, ref)) return message; + const kind = classifyKind(toolName); + const head = normalizeHead(text, cfg.maxHeadChars); + const command = extractCommand( + callById.get(message.toolCallId)?.text, + cfg.maxHeadChars, + ); + current = storeOriginal(current, { + ref, + rawId: message.id, + text, + kind, + toolName, + tokens, + head, + }); + storedCount += 1; + return { + ...message, + text: buildStoredPlaceholder({ + ref, + kind, + tokens, + head, + command, + retrieveToolName: cfg.toolName, + }), + }; + }); + + return { messages: updated, store: current, storedCount }; +} + +/** Effect shape carried in NodeIO.effects.ccr by ccrStoreNode. */ +export interface CcrEffect { + store: MessageContentStore; + storedCount: number; +} + +/** Pipeline node: runs after prune (new results live in the preserved tail, + * already referenced by assign-refs) and before absorb (placeholder-marked + * text is not an absorb candidate → ID-reference priority). */ +export const ccrStoreNode: PipelineNode = { + name: "ccr-store", + enabled: (_io, ctx) => resolveCcrConfig(ctx.config).enabled, + run(io: NodeIO, ctx: PipelineContext): NodeIO { + const applied = storeLargeResults({ + messages: io.messages, + state: io.state, + store: ctx.contentStore, + config: ctx.config, + countTokens: ctx.countTokens, + }); + const effect: CcrEffect = { + store: applied.store, + storedCount: applied.storedCount, + }; + const stats = + applied.storedCount > 0 + ? { + ...io.state.stats, + storedCount: + (io.state.stats.storedCount ?? 0) + applied.storedCount, + } + : io.state.stats; + return { + ...io, + messages: applied.messages, + state: stats === io.state.stats ? io.state : { ...io.state, stats }, + effects: { ...io.effects, ccr: effect }, + }; + }, +}; + +/** Adapter glue for v2 groundwork: persist originals of newly covered + * messages after applyCompression succeeds, so retrieve-by-ref works for + * folded content. First-write-wins keeps earlier CCR arrivals authoritative. + * Reasoning messages are skipped (thinking content is not retrievable). */ +export function storeCoveredOriginals( + store: MessageContentStore, + messages: CoreMessage[], + state: CompressionState, + blockIds: readonly string[], + countTokens: (text: string) => number, + maxHeadChars: number = DEFAULT_CCR_CONFIG.maxHeadChars, +): MessageContentStore { + const covered = new Set(); + for (const block of state.blocks) { + if (!block.active || !blockIds.includes(block.blockId)) continue; + for (const id of block.effectiveMessageIds) covered.add(id); + } + let current = store; + for (const message of messages) { + if (!covered.has(message.id)) continue; + if (message.contentType === "reasoning") continue; + const text = stripLeadingTag(message.text ?? ""); + if (text.length === 0) continue; + const ref = refForRaw(state.messageRefs, message.id); + if (!ref || ref === BLOCKED_REF) continue; + if (isStoredPlaceholderText(text) && hasStoredRef(current, ref)) continue; + current = storeOriginal(current, { + ref, + rawId: message.id, + text, + kind: classifyKind(message.toolName), + toolName: message.toolName, + tokens: countTokens(text), + head: normalizeHead(text, maxHeadChars), + }); + } + return current; +} + +/** Bump the cumulative retrieval counter (host calls after a successful + * applyRetrieve; feeds the status report's retrieve-rate metric). */ +export function noteRetrieval(state: CompressionState): CompressionState { + return { + ...state, + stats: { + ...state.stats, + retrievalCount: (state.stats.retrievalCount ?? 0) + 1, + }, + }; +} diff --git a/src/compress-tools.ts b/src/compress-tools.ts index 26adb5e..f1b55fa 100644 --- a/src/compress-tools.ts +++ b/src/compress-tools.ts @@ -557,8 +557,8 @@ export const ACP_READONLY_TOOLS_RESPONSES = [ ] as const; /** All ACP tool names (dynamic membership — Set, not a static record). Does - * NOT include absorb: it is opt-in (config.absorb.enabled) and hosts only - * register/inject it when the feature is on. */ + * NOT include absorb or acp_retrieve: both are opt-in (config.absorb.enabled + * / config.ccr.enabled) and hosts only register/inject them when on. */ export const ACP_TOOL_NAMES: ReadonlySet = new Set([ COMPRESS_TOOL_NAME, DECOMPRESS_TOOL_NAME, @@ -716,6 +716,25 @@ const IMAGE_FULL_PARAMETERS = { required: ["ref"], }; +/** Opt-in retrieve tool (lossless content-cached retrieval). Inject/register + * only when config.ccr.enabled — NOT part of ACP_TOOLS_* arrays (same + * opt-in pattern as absorb). */ +export const RETRIEVE_TOOL_NAME = "acp_retrieve"; + +export const RETRIEVE_TOOL_DESCRIPTION = + "Retrieve the full original text of a stored tool result. REQUIRED when you need details from a placeholder that starts with 📦 [acp-stored #mNNNNN: pass its ref. Returns the exact original text; the placeholder stays in context. Unknown refs return not-found."; + +const RETRIEVE_PARAMETERS = { + type: "object" as const, + properties: { + ref: { + type: "string", + description: "mNNNNN ref shown in the 📦 [acp-stored placeholder", + }, + }, + required: ["ref"], +}; + export const IMAGE_FULL_TOOL = { name: IMAGE_FULL_TOOL_NAME, description: IMAGE_FULL_TOOL_DESCRIPTION, @@ -737,3 +756,26 @@ export const IMAGE_FULL_TOOL_RESPONSES = { description: IMAGE_FULL_TOOL_OPENAI.function.description, parameters: IMAGE_FULL_TOOL_OPENAI.function.parameters, }; + +export const RETRIEVE_TOOL = { + name: RETRIEVE_TOOL_NAME, + description: RETRIEVE_TOOL_DESCRIPTION, + input_schema: RETRIEVE_PARAMETERS, +}; + +export const RETRIEVE_TOOL_OPENAI = { + type: "function" as const, + function: { + name: RETRIEVE_TOOL_NAME, + description: RETRIEVE_TOOL_DESCRIPTION, + parameters: RETRIEVE_PARAMETERS, + }, +}; + +export const RETRIEVE_TOOL_RESPONSES = { + type: "function" as const, + name: RETRIEVE_TOOL_NAME, + description: RETRIEVE_TOOL_DESCRIPTION, + parameters: RETRIEVE_PARAMETERS, +}; + diff --git a/src/compress.ts b/src/compress.ts index a00b487..45e303a 100644 --- a/src/compress.ts +++ b/src/compress.ts @@ -16,6 +16,10 @@ import { truncateLargeToolOutputs } from "./truncate-tools.js"; import { hideConsumedCompressCalls } from "./hide-consumed.js"; import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyCrushToMessages } from "./crush.js"; +import { applyRetrieve, ccrStoreNode, RETRIEVED_ID_PREFIX } from "./ccr.js"; +import type { ApplyRetrieveResult, CcrEffect } from "./ccr.js"; +import { createContentStore } from "./content-store.js"; +import type { MessageContentStore } from "./content-store.js"; import { applyMessageFilters, listMessageFilters } from "./filter/index.js"; import { activeBlockSpans } from "./block-map.js"; import { createRenderRefsNode } from "./render-refs.js"; @@ -61,6 +65,9 @@ export interface Ports { export interface CompressionCore { processTurn(input: ProcessTurnInput): ProcessTurnResult; + /** Kernel primitive behind acp_retrieve: resolve a ref to its stored + * original plus the wire-safe injection pair (see ccr.ts). */ + retrieve(store: MessageContentStore, ref: string): ApplyRetrieveResult; applyCompression(input: ApplyCompressionInput): ApplyCompressionResult; defaultNodes(): PipelineNode[]; decompress( @@ -93,6 +100,9 @@ export interface ProcessTurnInput { * directly from result.state.messageRefs. */ renderTags?: RenderStrategy; + /** Per-session CCR content store (pass result.contentStore back here). + * Omit for pre-CCR sessions — an empty store is used. */ + contentStore?: MessageContentStore; } export interface ApplyCompressionInput { @@ -458,10 +468,12 @@ export function createCore(ports: Ports = {}): CompressionCore { `[acp-kernel] Config validation warnings: ${configErrors.join("; ")}. Thresholds may not fire correctly.`, ); } + const contentStore = input.contentStore ?? createContentStore(); const ctx: PipelineContext = { config: input.config, tokenCount: input.tokenCount, countTokens, + contentStore, }; const initial: NodeIO = { messages: input.messages, @@ -474,15 +486,24 @@ export function createCore(ports: Ports = {}): CompressionCore { const strategy: RenderStrategy = input.renderTags ?? "all"; const nodes = buildNodes(strategy); const result = runPipeline(nodes, initial, ctx); + const ccrEffect = result.effects.ccr as CcrEffect | undefined; return { messages: result.messages, state: result.state, nudge: result.effects.nudge, terminalEscape: result.effects.terminalEscape, truncationSkipped: result.effects.truncationSkipped, + contentStore: ccrEffect?.store ?? contentStore, }; } + function retrieve( + store: MessageContentStore, + ref: string, + ): ApplyRetrieveResult { + return applyRetrieve({ store, ref }); + } + function decompress(blockId: string, state: CompressionState) { return blockById(state, blockId); } @@ -515,7 +536,12 @@ export function createCore(ports: Ports = {}): CompressionCore { activeBlocks: active.length, totalBlocks: state.blocks.length, tokensCompressed: state.stats.tokensCompressed, - breakdown: { active: active.length, total: state.blocks.length }, + breakdown: { + active: active.length, + total: state.blocks.length, + storedMessages: state.stats.storedCount ?? 0, + retrievals: state.stats.retrievalCount ?? 0, + }, }; } @@ -531,6 +557,7 @@ export function createCore(ports: Ports = {}): CompressionCore { assignRefsNode, syncBlocksNode, pruneNode, + ccrStoreNode, absorbHideNode, crushNode, absorbPromptNode, @@ -546,6 +573,7 @@ export function createCore(ports: Ports = {}): CompressionCore { return { processTurn, + retrieve, applyCompression, defaultNodes, decompress, @@ -579,6 +607,8 @@ const assignRefsNode: PipelineNode = { existing: io.state.messageRefs, nextIndex: highestUsedIndex(io.state.messageRefs) + 1, isProtected: protectedFn, + // Ephemeral retrieval injections never consume a ref slot. + shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), }); return { ...io, state: { ...io.state, messageRefs: refResult.map } }; }, diff --git a/src/config.ts b/src/config.ts index 90ad95d..9d0ce72 100644 --- a/src/config.ts +++ b/src/config.ts @@ -35,7 +35,9 @@ export function defaultConfig( absorb: { enabled: false, toolName: "absorb", - minToolTokens: 1000, + // Raised 1000 → 4000 (issue #352): lossless CCR takes over large-result + // handling; absorb's forced distillation only fires above the new bar. + minToolTokens: 4000, contextThresholdPct: 0, excludeTools: [], }, @@ -50,6 +52,13 @@ export function defaultConfig( quality: 80, format: "webp", }, + ccr: { + enabled: false, + toolName: "acp_retrieve", + minToolTokens: 4000, + excludeTools: [], + maxHeadChars: 96, + }, }; return { ...base, @@ -65,6 +74,7 @@ export function defaultConfig( imageCompression: overrides.imageCompression ? { ...base.imageCompression, ...overrides.imageCompression } : base.imageCompression, + ccr: overrides.ccr ? { ...base.ccr, ...overrides.ccr } : base.ccr, }; } @@ -193,5 +203,22 @@ export function validateConfig(config: Config): string[] { errors.push('imageCompression.format must be "webp", "jpeg", or "png"'); } } + if (config.ccr) { + if (config.ccr.enabled && !config.ccr.toolName) { + errors.push("ccr.toolName must be a non-empty string when enabled"); + } + if ( + !Number.isFinite(config.ccr.minToolTokens) || + config.ccr.minToolTokens < 0 + ) { + errors.push("ccr.minToolTokens must be >= 0"); + } + if ( + !Number.isFinite(config.ccr.maxHeadChars) || + config.ccr.maxHeadChars < 0 + ) { + errors.push("ccr.maxHeadChars must be >= 0"); + } + } return errors; } diff --git a/src/content-store.ts b/src/content-store.ts new file mode 100644 index 0000000..b675887 --- /dev/null +++ b/src/content-store.ts @@ -0,0 +1,138 @@ +import { createHash } from "node:crypto"; + +/** + * Content-addressed message store (CCR foundation). + * + * Pure data + pure functions — zero I/O (DESIGN.md: "the core holds no state + * and performs no I/O"). The host persists this structure with its own + * mechanism (StateStore envelope, same path as block-level original caches); + * it deliberately does NOT live inside CompressionState so multi-megabyte + * originals don't get rewritten on every per-turn state save. + * + * Invariants: + * - Content-addressed dedup: identical text is stored once (byHash). + * - Append-only per ref: first write wins, entries are never removed or + * overwritten by fold/edit/truncate. Refs are never reissued by the kernel + * (id-never-reused contract), so ref → original is a stable function for + * the life of the session generation. + * - The ref → rawId alias is frozen at store time, so retrieval keeps working + * even after a host compaction archive prunes messageRefs (the store's own + * byRef index is the source of truth for retrieval). + * - Generation boundary: when a host rebase resets refs to m00001 (full-state + * rebase), the host resets the store with the state — otherwise old and new + * generations would share one ref namespace. Session-scoped compaction that + * keeps state (refs continue) leaves the store untouched on purpose. + * - Retention (cleanup/eviction) is a downstream policy decision, like all + * persisted state. + */ + +export interface StoredEntry { + /** sha256 hex of the stored original text (key into byHash). */ + hash: string; + /** Raw message id at store time. Frozen alias — survives ref-map pruning. */ + rawId: string; + /** Placeholder kind label ("shell output", "file read", ...). */ + kind: string; + /** Tool name at arrival, if any. */ + toolName?: string; + /** Token count of the original, fixed at arrival (deterministic metadata). */ + tokens: number; + /** Character length of the original. */ + chars: number; + /** Deterministic head preview embedded in the placeholder. */ + head: string; +} + +export interface MessageContentStore { + version: 1; + /** Content-addressed originals: sha256(text) → text. Dedup lives here. */ + byHash: Record; + /** Per-ref index: ref → entry. Append-only; first write wins. */ + byRef: Record; +} + +export interface StoreSpec { + ref: string; + rawId: string; + text: string; + kind: string; + toolName?: string; + tokens: number; + head: string; +} + +export function createContentStore(): MessageContentStore { + return { version: 1, byHash: {}, byRef: {} }; +} + +export function hashContent(text: string): string { + return createHash("sha256").update(text, "utf8").digest("hex"); +} + +/** Append-only insert. Returns the input unchanged when the ref is already + * stored (first write wins); otherwise adds the entry, deduping content by + * hash. Immutably styled like the rest of the kernel. */ +export function storeOriginal( + store: MessageContentStore, + spec: StoreSpec, +): MessageContentStore { + if (!spec.ref || store.byRef[spec.ref]) return store; + const hash = hashContent(spec.text); + const entry: StoredEntry = { + hash, + rawId: spec.rawId, + kind: spec.kind, + tokens: spec.tokens, + chars: spec.text.length, + head: spec.head, + }; + if (spec.toolName !== undefined) entry.toolName = spec.toolName; + const byHash = + store.byHash[hash] === undefined + ? { ...store.byHash, [hash]: spec.text } + : store.byHash; + return { ...store, byHash, byRef: { ...store.byRef, [spec.ref]: entry } }; +} + +export type RetrieveResult = + | { ok: true; text: string; entry: StoredEntry } + | { ok: false; reason: "not-found" }; + +/** Fetch an original by ref. Unknown/hallucinated refs → not-found (the cost + * is one tool call, by design). Resolves through the store's own byRef index + * — independent of messageRefs, which hosts may prune. */ +export function retrieveByRef( + store: MessageContentStore, + ref: string, +): RetrieveResult { + const entry = store.byRef[ref]; + if (!entry) return { ok: false, reason: "not-found" }; + const text = store.byHash[entry.hash]; + if (text === undefined) return { ok: false, reason: "not-found" }; + return { ok: true, text, entry }; +} + +export function hasStoredRef(store: MessageContentStore, ref: string): boolean { + return store.byRef[ref] !== undefined; +} + +export interface ContentStoreStats { + /** Number of stored refs. */ + entries: number; + /** Number of unique contents after dedup. */ + uniqueContents: number; + /** Total characters held across unique contents. */ + totalChars: number; +} + +export function contentStoreStats( + store: MessageContentStore, +): ContentStoreStats { + let totalChars = 0; + for (const text of Object.values(store.byHash)) totalChars += text.length; + return { + entries: Object.keys(store.byRef).length, + uniqueContents: Object.keys(store.byHash).length, + totalChars, + }; +} diff --git a/src/index.ts b/src/index.ts index 318f922..32087dd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -150,6 +150,56 @@ export { IMAGE_FULL_TOOL_OPENAI, IMAGE_FULL_TOOL_RESPONSES, } from "./compress-tools.js"; +export { + RETRIEVE_TOOL_NAME, + RETRIEVE_TOOL, + RETRIEVE_TOOL_OPENAI, + RETRIEVE_TOOL_RESPONSES, + RETRIEVE_TOOL_DESCRIPTION, +} from "./compress-tools.js"; +export { + createContentStore, + hashContent, + storeOriginal, + retrieveByRef, + hasStoredRef, + contentStoreStats, +} from "./content-store.js"; +export type { + MessageContentStore, + StoredEntry, + StoreSpec, + RetrieveResult, + ContentStoreStats, +} from "./content-store.js"; +export { + DEFAULT_CCR_CONFIG, + resolveCcrConfig, + STORED_PLACEHOLDER_MARKER, + RETRIEVED_ID_PREFIX, + classifyKind, + normalizeHead, + extractCommand, + buildStoredPlaceholder, + isStoredPlaceholderText, + retrievedMessageId, + isRetrievedMessage, + buildRetrievalInjection, + applyRetrieve, + storeLargeResults, + storeCoveredOriginals, + noteRetrieval, + ccrStoreNode, +} from "./ccr.js"; +export type { + StoredPlaceholderInput, + RetrievalInjection, + ApplyRetrieveInput, + ApplyRetrieveResult, + StoreLargeResultsInput, + StoreLargeResultsResult, + CcrEffect, +} from "./ccr.js"; export { ABSORB_PROMPT_MARKER, DEFAULT_ABSORB_CONFIG, diff --git a/src/pipeline.ts b/src/pipeline.ts index 9453b56..1ddcaf7 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -4,6 +4,9 @@ export interface PipelineContext { readonly config: import("./types.js").Config; readonly tokenCount: number; readonly countTokens: (text: string) => number; + /** Per-session CCR content store for this turn (ccr-store node updates it + * via effects.ccr; processTurn echoes it back in the result). */ + readonly contentStore: import("./content-store.js").MessageContentStore; } export interface NodeEffects { diff --git a/src/rebuild.ts b/src/rebuild.ts index 6fe10c3..e8fd205 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -1,6 +1,7 @@ import { createCore } from "./compress.js"; import { parseCompressArgs } from "./parse-compress-input.js"; import { assignRefs, highestUsedIndex } from "./refs.js"; +import { RETRIEVED_ID_PREFIX } from "./ccr.js"; import { defaultCountTokens } from "./tokenize.js"; import type { CompressionState, CoreMessage } from "./types.js"; @@ -33,6 +34,7 @@ export function rebuildCompressionState( const refResult = assignRefs(messages, { existing: state.messageRefs, nextIndex: highestUsedIndex(state.messageRefs) + 1, + shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), }); let working: CompressionState = { ...state, messageRefs: refResult.map }; diff --git a/src/state.ts b/src/state.ts index f9c068c..cb4d2a2 100644 --- a/src/state.ts +++ b/src/state.ts @@ -19,6 +19,8 @@ export function createInitialState(): CompressionState { imagesShrunk: 0, imageBytesSaved: 0, imageTokensSaved: 0, + storedCount: 0, + retrievalCount: 0, }, absorbed: [], rules: [], diff --git a/src/types.ts b/src/types.ts index 4ace8c4..4b577e6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,5 @@ +import type { MessageContentStore } from "./content-store.js"; + export type MessageRole = "user" | "assistant" | "system" | "tool"; export type MessageContentType = @@ -98,6 +100,10 @@ export interface CompressionStats { imageBytesSaved?: number; /** Cumulative estimated-token savings from image downsampling. */ imageTokensSaved?: number; + /** Cumulative messages stored in the CCR content store. Optional for pre-CCR persisted states. */ + storedCount?: number; + /** Cumulative successful acp_retrieve calls. Optional for pre-CCR persisted states. */ + retrievalCount?: number; } /** One instant tool-result absorption: the original tool-call + tool-result @@ -215,6 +221,18 @@ export interface ImageShrinkRecord { createdAt: number; } +export interface CcrConfig { + enabled: boolean; + /** Model-facing retrieve tool name (adapters may rename, e.g. acp_retrieve). */ + toolName: string; + /** Only tool results >= this many tokens are stored + replaced. 0 = all. */ + minToolTokens: number; + /** Tool-name patterns (glob suffix allowed) never CCR-stored, independent of protectedTools. */ + excludeTools: string[]; + /** Max characters for the placeholder's head/command preview. */ + maxHeadChars: number; +} + export interface CompressionState { blocks: CompressionBlock[]; messageRefs: MessageRefMap; @@ -338,6 +356,8 @@ export interface Config { crush?: CrushConfig; /** Image pre-compression routing/downsample + image_full restore (see image-compress.ts). Absent/disabled = feature off. */ imageCompression?: ImageCompressionConfig; + /** Lossless content-cached retrieval for oversized tool results (see ccr.ts). Absent/disabled = feature off. */ + ccr?: CcrConfig; messageFilters?: import("./filter/types.js").MessageFiltersConfig; } @@ -497,6 +517,10 @@ export interface ProcessTurnResult { /** Set when emergency truncation ran at/above its threshold but reclaimed * nothing — hosts should warn-log this (it was silent before, #300). */ truncationSkipped?: string; + /** Per-session content store after this turn (pass back as input.contentStore + * next turn and persist via the host's own mechanism). Always present; + * empty until CCR stores something. */ + contentStore: MessageContentStore; } export interface StatusReport { diff --git a/tests/ccr.test.ts b/tests/ccr.test.ts new file mode 100644 index 0000000..39ae962 --- /dev/null +++ b/tests/ccr.test.ts @@ -0,0 +1,862 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + createContentStore, + hashContent, + storeOriginal, + retrieveByRef, + hasStoredRef, + contentStoreStats, +} from "../src/content-store.js"; +import { + DEFAULT_CCR_CONFIG, + resolveCcrConfig, + STORED_PLACEHOLDER_MARKER, + RETRIEVED_ID_PREFIX, + classifyKind, + normalizeHead, + extractCommand, + buildStoredPlaceholder, + isStoredPlaceholderText, + retrievedMessageId, + isRetrievedMessage, + buildRetrievalInjection, + applyRetrieve, + storeLargeResults, + storeCoveredOriginals, + noteRetrieval, +} from "../src/ccr.js"; +import { createCore } from "../src/compress.js"; +import { createInitialState } from "../src/state.js"; +import { defaultConfig, validateConfig } from "../src/config.js"; +import { assignRefs, refForRaw } from "../src/refs.js"; +import { isAbsorbCandidate } from "../src/absorb.js"; +import { DEFAULT_ABSORB_CONFIG } from "../src/absorb.js"; +import { + RETRIEVE_TOOL_NAME, + RETRIEVE_TOOL, + RETRIEVE_TOOL_OPENAI, + RETRIEVE_TOOL_RESPONSES, + ACP_TOOLS_OPENAI, + ACP_TOOL_NAMES, +} from "../src/compress-tools.js"; +import type { Config, CoreMessage } from "../src/types.js"; + +const countTokens = (text: string) => Math.ceil(text.length / 4); + +function bigText(): string { + return "line of output ".repeat(2000); +} + +function toolCall(id: string, callId: string, toolName = "bash"): CoreMessage { + return { + id, + role: "assistant", + contentType: "tool-call", + toolName, + toolCallId: callId, + text: JSON.stringify({ command: `run ${callId}` }), + }; +} + +function toolResult( + id: string, + callId: string, + toolName = "bash", + text = bigText(), +): CoreMessage { + return { + id, + role: "tool", + contentType: "tool-result", + toolName, + toolCallId: callId, + text, + }; +} + +function ccrConfig(overrides: Partial = {}): Config { + return defaultConfig(100000, { + ccr: { ...DEFAULT_CCR_CONFIG, enabled: true, ...overrides }, + }); +} + +test("createContentStore yields an empty v1 store", () => { + const store = createContentStore(); + assert.deepEqual(store, { version: 1, byHash: {}, byRef: {} }); +}); + +test("storeOriginal + retrieveByRef round-trip", () => { + let store = createContentStore(); + const text = "hello world"; + store = storeOriginal(store, { + ref: "m00042", + rawId: "h_raw1", + text, + kind: "shell output", + toolName: "bash", + tokens: 3, + head: "hello world", + }); + assert.ok(hasStoredRef(store, "m00042")); + const found = retrieveByRef(store, "m00042"); + assert.ok(found.ok); + if (found.ok) { + assert.equal(found.text, text); + assert.equal(found.entry.rawId, "h_raw1"); + assert.equal(found.entry.kind, "shell output"); + assert.equal(found.entry.toolName, "bash"); + assert.equal(found.entry.tokens, 3); + assert.equal(found.entry.chars, text.length); + } +}); + +test("store dedups identical content across refs (content-addressed)", () => { + let store = createContentStore(); + const text = "same bytes twice"; + store = storeOriginal(store, { + ref: "m00001", + rawId: "a", + text, + kind: "k", + tokens: 1, + head: "", + }); + store = storeOriginal(store, { + ref: "m00002", + rawId: "b", + text, + kind: "k", + tokens: 1, + head: "", + }); + assert.equal(Object.keys(store.byHash).length, 1); + assert.equal(Object.keys(store.byRef).length, 2); + assert.deepEqual(contentStoreStats(store), { + entries: 2, + uniqueContents: 1, + totalChars: text.length, + }); +}); + +test("store is append-only per ref: first write wins", () => { + let store = createContentStore(); + store = storeOriginal(store, { + ref: "m00001", + rawId: "a", + text: "original", + kind: "k", + tokens: 1, + head: "", + }); + store = storeOriginal(store, { + ref: "m00001", + rawId: "b", + text: "MUTATED", + kind: "k2", + tokens: 9, + head: "", + }); + const found = retrieveByRef(store, "m00001"); + assert.ok(found.ok); + if (found.ok) { + assert.equal(found.text, "original"); + assert.equal(found.entry.rawId, "a"); + } +}); + +test("retrieveByRef returns not-found for unknown and empty refs", () => { + const store = createContentStore(); + assert.deepEqual(retrieveByRef(store, "m99999"), { + ok: false, + reason: "not-found", + }); + assert.deepEqual(retrieveByRef(store, ""), { + ok: false, + reason: "not-found", + }); +}); + +test("hashContent is deterministic sha256 hex", () => { + assert.equal(hashContent("abc"), hashContent("abc")); + assert.notEqual(hashContent("abc"), hashContent("abd")); + assert.match(hashContent("abc"), /^[0-9a-f]{64}$/); +}); + +test("buildStoredPlaceholder matches the deterministic wire shape", () => { + const placeholder = buildStoredPlaceholder({ + ref: "m00423", + kind: "shell output", + tokens: 4213, + head: "some preview", + command: "npm run build", + retrieveToolName: "acp_retrieve", + }); + assert.equal( + placeholder, + `📦 ${STORED_PLACEHOLDER_MARKER} #m00423 · shell output · 4,213 tok] \`npm run build\`\n → acp_retrieve("m00423") returns the full text`, + ); +}); + +test("buildStoredPlaceholder falls back to head when no command; byte-stable", () => { + const input = { + ref: "m00007", + kind: "file read", + tokens: 5000, + head: "head preview here", + retrieveToolName: "acp_retrieve", + }; + const first = buildStoredPlaceholder(input); + const second = buildStoredPlaceholder({ ...input }); + assert.equal(first, second); + assert.ok(first.includes("`head preview here`")); + assert.ok(isStoredPlaceholderText(first)); +}); + +test("classifyKind maps common tools and defaults to 'tool result'", () => { + assert.equal(classifyKind("bash"), "shell output"); + assert.equal(classifyKind("Bash"), "shell output"); + assert.equal(classifyKind("read"), "file read"); + assert.equal(classifyKind("grep"), "search output"); + assert.equal(classifyKind("webfetch"), "web fetch"); + assert.equal(classifyKind("some_unknown_tool"), "tool result"); + assert.equal(classifyKind(undefined), "tool result"); +}); + +test("normalizeHead collapses whitespace and truncates deterministically", () => { + assert.equal(normalizeHead(" a\n\n b \t c ", 96), "a b c"); + const long = "w".repeat(200); + const head = normalizeHead(long, 10); + assert.equal(head, "wwwwwwwwww…"); + assert.equal(normalizeHead("", 10), ""); +}); + +test("extractCommand pulls known fields from args JSON; undefined otherwise", () => { + assert.equal( + extractCommand(JSON.stringify({ command: "ls -la /tmp" }), 96), + "ls -la /tmp", + ); + assert.equal( + extractCommand(JSON.stringify({ path: "/etc/hosts" }), 96), + "/etc/hosts", + ); + assert.equal( + extractCommand(JSON.stringify({ url: "https://example.com/a b" }), 96), + "https://example.com/a b", + ); + assert.equal(extractCommand("not json {", 96), undefined); + assert.equal(extractCommand(JSON.stringify({ other: 1 }), 96), undefined); + assert.equal(extractCommand(undefined, 96), undefined); + const longCmd = "x".repeat(200); + assert.equal( + extractCommand(JSON.stringify({ command: longCmd }), 10), + "xxxxxxxxxx…", + ); +}); + +function refsFor(messages: CoreMessage[]) { + const refResult = assignRefs(messages, { existing: {}, nextIndex: 1 }); + return refResult.map; +} + +test("storeLargeResults: disabled config leaves everything untouched", () => { + const messages = [toolCall("c1", "call1"), toolResult("r1", "call1")]; + const map = refsFor(messages); + const state = { ...createInitialState(), messageRefs: map }; + const config = defaultConfig(100000); + const result = storeLargeResults({ + messages, + state, + store: createContentStore(), + config, + countTokens, + }); + assert.equal(result.storedCount, 0); + assert.equal(result.messages[1]!.text, bigText()); +}); + +test("storeLargeResults: below-threshold results are untouched", () => { + const small = "short output"; + const messages = [ + toolCall("c1", "call1"), + toolResult("r1", "call1", "bash", small), + ]; + const state = { ...createInitialState(), messageRefs: refsFor(messages) }; + const result = storeLargeResults({ + messages, + state, + store: createContentStore(), + config: ccrConfig(), + countTokens, + }); + assert.equal(result.storedCount, 0); + assert.equal(result.messages[1]!.text, small); +}); + +test("storeLargeResults: over-threshold result is stored + replaced once", () => { + const original = bigText(); + const messages = [ + toolCall("c1", "call1"), + toolResult("r1", "call1", "bash", original), + ]; + const state = { ...createInitialState(), messageRefs: refsFor(messages) }; + const result = storeLargeResults({ + messages, + state, + store: createContentStore(), + config: ccrConfig(), + countTokens, + }); + assert.equal(result.storedCount, 1); + const replaced = result.messages[1]!.text ?? ""; + assert.ok(isStoredPlaceholderText(replaced)); + assert.ok(replaced.includes("#m00002")); + assert.ok(replaced.includes("· shell output ·")); + assert.ok(replaced.includes('acp_retrieve("m00002")')); + assert.ok(replaced.includes("`run call1`")); + assert.equal(result.messages[1]!.id, "r1"); + assert.equal(result.messages[1]!.toolCallId, "call1"); + assert.equal( + result.messages[0]!.text, + JSON.stringify({ command: "run call1" }), + ); + const found = retrieveByRef(result.store, "m00002"); + assert.ok(found.ok); + if (found.ok) assert.equal(found.text, original); +}); + +test("storeLargeResults: idempotent on already-replaced text", () => { + const messages = [ + toolCall("c1", "call1"), + toolResult("r1", "call1", "bash", bigText()), + ]; + const state = { ...createInitialState(), messageRefs: refsFor(messages) }; + const first = storeLargeResults({ + messages, + state, + store: createContentStore(), + config: ccrConfig(), + countTokens, + }); + const second = storeLargeResults({ + messages: first.messages, + state, + store: first.store, + config: ccrConfig(), + countTokens, + }); + assert.equal(second.storedCount, 0); + assert.deepEqual(second.messages, first.messages); + assert.deepEqual(second.store, first.store); +}); + +test("storeLargeResults: ACP tools, own retrieve tool, excludeTools, protected tools are skipped", () => { + const mk = (id: string, name: string) => [ + toolCall(`c-${id}`, `call-${id}`, name), + toolResult(`r-${id}`, `call-${id}`, name), + ]; + const all = [ + ...mk("compress", "compress"), + ...mk("retrieve", "acp_retrieve"), + ...mk("excluded", "secret_tool"), + ...mk("protected", "vault_tool"), + ]; + const state = { ...createInitialState(), messageRefs: refsFor(all) }; + const config = ccrConfig({ + excludeTools: ["secret_*", "vault_*"], + }); + const withProtected = defaultConfig(100000, { + ccr: { + ...DEFAULT_CCR_CONFIG, + enabled: true, + excludeTools: ["secret_*", "vault_*"], + }, + protectedTools: ["vault_tool"], + }); + for (const cfg of [config, withProtected]) { + const result = storeLargeResults({ + messages: all, + state, + store: createContentStore(), + config: cfg as Config, + countTokens, + }); + assert.equal( + result.storedCount, + 0, + `expected zero stored for ${JSON.stringify(cfg.protectedTools)}`, + ); + } +}); + +test("storeLargeResults: results without toolCallId or with BLOCKED refs are skipped", () => { + const orphan = { ...toolResult("r1", "", "bash"), toolCallId: undefined }; + const blockedMessages = [toolResult("r2", "call2", "bash")]; + const blockedState = { + ...createInitialState(), + messageRefs: { byRaw: { r2: "BLOCKED" }, byRef: { BLOCKED: "r2" } }, + }; + const base = { + store: createContentStore(), + config: ccrConfig(), + countTokens, + }; + const r1 = storeLargeResults({ + messages: [orphan], + state: { ...createInitialState(), messageRefs: refsFor([orphan]) }, + ...base, + }); + assert.equal(r1.storedCount, 0); + const r2 = storeLargeResults({ + messages: blockedMessages, + state: blockedState, + ...base, + }); + assert.equal(r2.storedCount, 0); +}); + +function sessionWithBigBash(): CoreMessage[] { + return [ + { id: "u1", role: "user", contentType: "text", text: "build the project" }, + { + id: "a1", + role: "assistant", + contentType: "text", + text: "running the build now", + }, + toolCall("c1", "call1", "bash"), + toolResult("r1", "call1", "bash", bigText()), + ]; +} + +test("processTurn without CCR: empty store echoed back, messages unchanged", () => { + const core = createCore({ countTokens }); + const messages = sessionWithBigBash(); + const result = core.processTurn({ + messages, + state: createInitialState(), + config: defaultConfig(100000), + tokenCount: 1000, + renderTags: "none", + }); + assert.deepEqual(result.contentStore, { version: 1, byHash: {}, byRef: {} }); + assert.equal(result.messages[3]!.text, bigText()); + assert.equal(result.state.stats.storedCount, 0); +}); + +test("processTurn with CCR: stores at arrival, replaces with placeholder, bumps stats", () => { + const core = createCore({ countTokens }); + const original = bigText(); + const result = core.processTurn({ + messages: sessionWithBigBash(), + state: createInitialState(), + config: ccrConfig(), + tokenCount: 1000, + }); + assert.equal(result.state.stats.storedCount, 1); + const replaced = result.messages[3]!.text ?? ""; + assert.ok(isStoredPlaceholderText(replaced)); + assert.ok(replaced.includes("#m00004")); + const found = retrieveByRef(result.contentStore, "m00004"); + assert.ok(found.ok); + if (found.ok) assert.equal(found.text, original); +}); + +test("processTurn is byte-stable across turns after replacement", () => { + const core = createCore({ countTokens }); + const turn1 = core.processTurn({ + messages: sessionWithBigBash(), + state: createInitialState(), + config: ccrConfig(), + tokenCount: 1000, + }); + const placeholderTurn1 = turn1.messages[3]!.text!; + const followUp: CoreMessage = { + id: "u2", + role: "user", + contentType: "text", + text: "did it work?", + }; + const turn2 = core.processTurn({ + messages: [...turn1.messages, followUp], + state: turn1.state, + config: ccrConfig(), + tokenCount: 1200, + contentStore: turn1.contentStore, + }); + assert.equal(turn2.messages[3]!.text, placeholderTurn1); + assert.equal(turn2.state.stats.storedCount, 1); + assert.deepEqual(turn2.contentStore, turn1.contentStore); +}); + +test("core.retrieve resolves originals; hallucinated refs are not-found", () => { + const core = createCore({ countTokens }); + const turn = core.processTurn({ + messages: sessionWithBigBash(), + state: createInitialState(), + config: ccrConfig(), + tokenCount: 1000, + }); + const hit = core.retrieve(turn.contentStore, "m00004"); + assert.ok(hit.ok); + if (hit.ok) { + assert.equal(hit.text, bigText()); + assert.equal(hit.injection.id, retrievedMessageId("m00004")); + assert.equal(hit.injection.role, "system"); + assert.ok(hit.injection.text!.startsWith("[acp-retrieved #m00004")); + assert.ok(hit.injection.text!.endsWith(bigText())); + assert.ok(hit.ackText.includes("m00004")); + } + const miss = core.retrieve(turn.contentStore, "m99999"); + assert.equal(miss.ok, false); + if (!miss.ok) { + assert.equal(miss.reason, "not-found"); + assert.ok(miss.ackText.includes("not found")); + } +}); + +test("retrieved refs survive host-side ref-map pruning (archive resilience)", () => { + const core = createCore({ countTokens }); + const turn = core.processTurn({ + messages: sessionWithBigBash(), + state: createInitialState(), + config: ccrConfig(), + tokenCount: 1000, + }); + assert.equal(refForRaw(turn.state.messageRefs, "r1"), "m00004"); + const pruned = { ...turn.state, messageRefs: { byRaw: {}, byRef: {} } }; + assert.equal(refForRaw(pruned.messageRefs, "r1"), null); + const found = core.retrieve(turn.contentStore, "m00004"); + assert.ok(found.ok); + if (found.ok) assert.equal(found.text, bigText()); +}); + +test("ephemeral retrieval injections consume no ref and survive the pipeline", () => { + const core = createCore({ countTokens }); + const injection = buildRetrievalInjection( + "m00004", + { + hash: "h", + rawId: "r1", + kind: "shell output", + tokens: 5000, + chars: 20000, + head: "", + }, + "FULL TEXT HERE", + ); + assert.ok(isRetrievedMessage(injection.injection)); + assert.equal(injection.injection.id, `${RETRIEVED_ID_PREFIX}m00004`); + assert.ok( + !isRetrievedMessage({ + id: "x", + role: "system", + contentType: "text", + text: "y", + }), + ); + + const roundTripped: CoreMessage[] = [ + ...sessionWithBigBash().slice(0, 3), + injection.injection, + { id: "u2", role: "user", contentType: "text", text: "next question" }, + ]; + const result = core.processTurn({ + messages: roundTripped, + state: createInitialState(), + config: ccrConfig(), + tokenCount: 1000, + }); + assert.equal( + refForRaw(result.state.messageRefs, injection.injection.id), + null, + ); + const survived = result.messages.find((m) => m.id === injection.injection.id); + assert.ok(survived, "retrieval injection must survive processTurn"); +}); + +test("retrieval injections never enter fold space (block effectiveMessageIds exclusion)", () => { + const core = createCore({ countTokens }); + const injection = buildRetrievalInjection( + "m00004", + { + hash: "h", + rawId: "r1", + kind: "shell output", + tokens: 5000, + chars: 20000, + head: "", + }, + "FULL TEXT HERE", + ).injection; + const messages: CoreMessage[] = [ + { id: "u1", role: "user", contentType: "text", text: "start" }, + { id: "a1", role: "assistant", contentType: "text", text: "working on it" }, + injection, + { id: "u2", role: "user", contentType: "text", text: "continue please" }, + { + id: "a2", + role: "assistant", + contentType: "text", + text: "done with the work here", + }, + { + id: "u3", + role: "user", + contentType: "text", + text: "final question to keep tail recent", + }, + { + id: "a3", + role: "assistant", + contentType: "text", + text: "final answer to the final question", + }, + ]; + const seeded = core.processTurn({ + messages, + state: createInitialState(), + config: defaultConfig(100000), + tokenCount: 1000, + }); + const compressed = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00005", summary: "S".repeat(60) }], + messages: seeded.messages, + state: seeded.state, + config: defaultConfig(100000, { + compress: { minCompressRange: 0 }, + preserveRecentMessages: 1, + preserveRecentTokens: 0, + }), + }); + assert.equal(compressed.result.blocksCreated, 1); + const block = compressed.state.blocks.find((b) => b.active)!; + assert.ok(block.effectiveMessageIds.includes("u1")); + assert.ok(!block.effectiveMessageIds.includes(injection.id)); +}); + +test("placeholder-marked results are not absorb candidates (ID-reference priority)", () => { + const placeholder = buildStoredPlaceholder({ + ref: "m00004", + kind: "shell output", + tokens: 5000, + head: "preview", + retrieveToolName: "acp_retrieve", + }); + const msg: CoreMessage = { + id: "r1", + role: "tool", + contentType: "tool-result", + toolName: "bash", + toolCallId: "call1", + text: placeholder, + }; + const config = defaultConfig(100000, { + absorb: { ...DEFAULT_ABSORB_CONFIG, enabled: true }, + }); + assert.equal(isAbsorbCandidate(msg, config), false); + const plain: CoreMessage = { ...msg, text: bigText() }; + assert.equal(isAbsorbCandidate(plain, config), true); +}); + +test("noteRetrieval increments stats.retrievalCount; status reports both metrics", () => { + const core = createCore({ countTokens }); + const turn = core.processTurn({ + messages: sessionWithBigBash(), + state: createInitialState(), + config: ccrConfig(), + tokenCount: 1000, + }); + const bumped = noteRetrieval(noteRetrieval(turn.state)); + const report = core.status(bumped, 1000, ccrConfig()); + assert.equal(report.breakdown.storedMessages, 1); + assert.equal(report.breakdown.retrievals, 2); +}); + +test("storeCoveredOriginals stores folded message originals for retrieve-by-ref", () => { + const core = createCore({ countTokens }); + const messages: CoreMessage[] = [ + { + id: "u1", + role: "user", + contentType: "text", + text: "first user message here", + }, + { + id: "a1", + role: "assistant", + contentType: "text", + text: "first assistant reply here", + }, + { + id: "u2", + role: "user", + contentType: "text", + text: "second user message here", + }, + { + id: "a2", + role: "assistant", + contentType: "text", + text: "second assistant reply here", + }, + { + id: "u3", + role: "user", + contentType: "text", + text: "recent tail user message", + }, + { + id: "a3", + role: "assistant", + contentType: "text", + text: "recent tail assistant reply", + }, + ]; + const seeded = core.processTurn({ + messages, + state: createInitialState(), + config: defaultConfig(100000), + tokenCount: 1000, + }); + const compressed = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00004", summary: "S".repeat(60) }], + messages: seeded.messages, + state: seeded.state, + config: defaultConfig(100000, { + compress: { minCompressRange: 0 }, + preserveRecentMessages: 1, + preserveRecentTokens: 0, + }), + }); + const block = compressed.state.blocks.find((b) => b.active)!; + let store = createContentStore(); + store = storeCoveredOriginals( + store, + seeded.messages, + compressed.state, + [block.blockId], + countTokens, + ); + const foundU1 = retrieveByRef(store, "m00001"); + assert.ok(foundU1.ok); + if (foundU1.ok) assert.equal(foundU1.text, "first user message here"); + assert.ok(!hasStoredRef(store, "m00005")); +}); + +test("storeCoveredOriginals skips reasoning and keeps earlier CCR originals (first-write-wins)", () => { + const core = createCore({ countTokens }); + const original = bigText(); + const messages: CoreMessage[] = [ + toolCall("c1", "call1", "bash"), + toolResult("r1", "call1", "bash", original), + { + id: "th1", + role: "assistant", + contentType: "reasoning", + text: "private thinking", + }, + { id: "u2", role: "user", contentType: "text", text: "follow up question" }, + { + id: "a2", + role: "assistant", + contentType: "text", + text: "follow up answer here", + }, + { + id: "u3", + role: "user", + contentType: "text", + text: "tail user message stays recent", + }, + { + id: "a3", + role: "assistant", + contentType: "text", + text: "tail assistant reply stays recent", + }, + ]; + const seeded = core.processTurn({ + messages, + state: createInitialState(), + config: ccrConfig(), + tokenCount: 1000, + }); + const compressed = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00004", summary: "S".repeat(60) }], + messages: seeded.messages, + state: seeded.state, + config: defaultConfig(100000, { + compress: { minCompressRange: 0 }, + preserveRecentMessages: 1, + preserveRecentTokens: 0, + }), + }); + const block = compressed.state.blocks.find((b) => b.active)!; + let store = seeded.contentStore; + store = storeCoveredOriginals( + store, + seeded.messages, + compressed.state, + [block.blockId], + countTokens, + ); + const foundR1 = retrieveByRef(store, "m00002"); + assert.ok(foundR1.ok); + if (foundR1.ok) assert.equal(foundR1.text, original); + assert.ok(!hasStoredRef(store, "m00003")); +}); + +test("absorb minToolTokens default raised 1000 -> 4000 (issue #352 disclosure)", () => { + assert.equal(DEFAULT_ABSORB_CONFIG.minToolTokens, 4000); + assert.equal(defaultConfig(100000).absorb?.minToolTokens, 4000); + assert.equal(DEFAULT_ABSORB_CONFIG.enabled, false); +}); + +test("ccr defaults: disabled, acp_retrieve, 4000 tok, mergeable overrides", () => { + assert.deepEqual(DEFAULT_CCR_CONFIG, { + enabled: false, + toolName: "acp_retrieve", + minToolTokens: 4000, + excludeTools: [], + maxHeadChars: 96, + }); + const base = defaultConfig(100000); + assert.equal(base.ccr?.enabled, false); + assert.equal(base.ccr?.minToolTokens, 4000); + const merged = defaultConfig(100000, { + ccr: { enabled: true, minToolTokens: 100 }, + }); + assert.equal(resolveCcrConfig(merged).enabled, true); + assert.equal(resolveCcrConfig(merged).minToolTokens, 100); + assert.equal(resolveCcrConfig(merged).toolName, "acp_retrieve"); +}); + +test("validateConfig rejects invalid ccr settings", () => { + assert.equal(validateConfig(ccrConfig()).length, 0); + const bad1 = defaultConfig(100000, { + ccr: { ...DEFAULT_CCR_CONFIG, enabled: true, toolName: "" }, + }); + assert.ok(validateConfig(bad1).some((e) => e.includes("ccr.toolName"))); + const bad2 = defaultConfig(100000, { + ccr: { ...DEFAULT_CCR_CONFIG, minToolTokens: -1 }, + }); + assert.ok(validateConfig(bad2).some((e) => e.includes("ccr.minToolTokens"))); + const bad3 = defaultConfig(100000, { + ccr: { ...DEFAULT_CCR_CONFIG, maxHeadChars: -5 }, + }); + assert.ok(validateConfig(bad3).some((e) => e.includes("ccr.maxHeadChars"))); +}); + +test("RETRIEVE_TOOL schemas exist in all three wire shapes and stay opt-in", () => { + assert.equal(RETRIEVE_TOOL_NAME, "acp_retrieve"); + assert.equal(RETRIEVE_TOOL.name, "acp_retrieve"); + assert.deepEqual(RETRIEVE_TOOL.input_schema.required, ["ref"]); + assert.equal(RETRIEVE_TOOL_OPENAI.type, "function"); + assert.equal(RETRIEVE_TOOL_OPENAI.function.name, "acp_retrieve"); + assert.equal(RETRIEVE_TOOL_RESPONSES.type, "function"); + assert.equal(RETRIEVE_TOOL_RESPONSES.name, "acp_retrieve"); + const openaiNames = ACP_TOOLS_OPENAI.map((t) => t.function.name); + assert.ok(!openaiNames.includes("acp_retrieve")); + assert.ok(!ACP_TOOL_NAMES.has("acp_retrieve")); +}); diff --git a/tests/image-compress.test.ts b/tests/image-compress.test.ts index 556aafa..1785881 100644 --- a/tests/image-compress.test.ts +++ b/tests/image-compress.test.ts @@ -592,6 +592,8 @@ test("createInitialState seeds image counters and empty sticky arrays", () => { imagesShrunk: 0, imageBytesSaved: 0, imageTokensSaved: 0, + storedCount: 0, + retrievalCount: 0, }); assert.deepEqual(s.imageFullRestored, []); assert.deepEqual(s.imageShrinks, []); diff --git a/tests/persist.test.ts b/tests/persist.test.ts index d84a4c8..d7e681a 100644 --- a/tests/persist.test.ts +++ b/tests/persist.test.ts @@ -370,14 +370,7 @@ test("mergeCompressionState keeps parsed values over defaults", () => { }; const merged = mergeCompressionState(parsed); assert.equal(merged.nextBlockId, 41); - assert.deepEqual(merged.stats, { - tokensCompressed: 1234, - compressionCount: 7, - absorbedTokens: 0, - imagesShrunk: 0, - imageBytesSaved: 0, - imageTokensSaved: 0, - }); + assert.deepEqual(merged.stats, { tokensCompressed: 1234, compressionCount: 7, absorbedTokens: 0, imagesShrunk: 0, imageBytesSaved: 0, imageTokensSaved: 0, storedCount: 0, retrievalCount: 0 }); }); test("legacy hook adopts pre-envelope records on loadAll and loadSync", async () => { diff --git a/tests/pipeline.test.ts b/tests/pipeline.test.ts index f7f1fbf..1bb0786 100644 --- a/tests/pipeline.test.ts +++ b/tests/pipeline.test.ts @@ -28,6 +28,7 @@ test("defaultNodes exposes the canonical ordered pipeline", () => { "assign-refs", "sync-blocks", "prune", + "ccr-store", "absorb-hide", "crush", "absorb-prompt",