From b8c17c2c5da575824641bfff170a4a0fdda2e7bd Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 23 Sep 2026 02:07:37 +0800 Subject: [PATCH 1/2] feat: tool-result valuator + deterministic crush plugin interface (two-tier absorb gate) --- AGENTS.md | 1 + README.md | 34 +- src/compress.ts | 29 + src/config.ts | 34 + src/crush.ts | 1753 ++++++++++++++++++++++++++++++++++++++++ src/index.ts | 26 + src/types.ts | 19 + tests/crush.test.ts | 1114 +++++++++++++++++++++++++ tests/pipeline.test.ts | 1 + 9 files changed, 3009 insertions(+), 2 deletions(-) create mode 100644 src/crush.ts create mode 100644 tests/crush.test.ts diff --git a/AGENTS.md b/AGENTS.md index 0c848d2..fa56058 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,7 @@ acp-kernel/ ├── src/ │ ├── index.ts # Barrel export │ ├── compress.ts # Core: processTurn, applyCompression, decideNudge, pipeline nodes +│ ├── crush.ts # Tool-result valuator + deterministic crush strategies (two-tier absorb gate) │ ├── boundaries.ts # Range boundary resolution (startId/endId → message indices) │ ├── filter.ts # Protected tool message filtering │ ├── nudge.ts # Nudge text rendering diff --git a/README.md b/README.md index a0cc423..9ad0cfc 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,9 @@ const state = createInitialState(); const config = defaultConfig(200000); // modelContextLimit (positional); optional overrides as 2nd arg // processTurn runs the canonical node pipeline every turn: -// assign-refs → sync-blocks → merge-blocks → prune → filter → -// hide-compress-calls → nudge-inject → emergency-truncate → render-refs +// assign-refs → sync-blocks → prune → absorb-hide → crush → absorb-prompt +// → filter → hide-compress-calls → recommend → nudge-inject +// → emergency-truncate → render-refs const { messages, state: nextState, nudge } = core.processTurn({ messages, state, config, tokenCount, }); @@ -162,6 +163,35 @@ The nudge system tells the model *when* to compress. It implements: - **Compressible-range computation**: reports the actual compressible ranges (excluding covered + preserved-recent messages) so the model knows what to target. - **Baseline reset on compress**: `applyCompression` clears the growth baseline on success, preventing the feedback-loop bug where the nudge re-fires post-compress. +### Tool-result crush gate + +Deterministic first tier of the two-tier absorb gate. When a tool result qualifies as an absorb candidate and context pressure is on, the `crush` node (between `absorb-hide` and `absorb-prompt`) re-evaluates it **in view** with pure, deterministic strategies before the absorb-prompt decision runs: + +- **skip** — below `absorb.minToolTokens` or under `absorb.contextThresholdPct`; bytes untouched. +- **crushed** — deterministic compression brought it under `minToolTokens`; no model round-trip. The wire carries the crushed payload (e.g. `__acp_crush` rows/identical-run envelopes for JSON, decodable byte-for-byte). +- **distill** — still over (or uncrushable); forwarded to the `[ACP absorb]` prompt path, carrying the crushed payload when crushing succeeded. + +Built-in strategies form an ordered plugin registry (hosts can add/replace/disable via `registerCrushPlugin` / config): + +| Strategy | Kind | Lossy | What it does | +|----------|------|-------|--------------| +| `json-fold` | json | no | constant-field hoisting + identical row/run collapse into annotated envelopes | +| `code-trim` | code | yes | comment/docstring elision behind a string-literal-aware scanner; template literals and assigned triple-quoted strings survive verbatim; unterminated constructs fail open | +| `log-select` | log | yes | level-classified line selection; **every distinct ERROR/FAIL line survives verbatim** (kernel-enforced invariant — output dropping one is rejected); warnings deduped, stack frames collapsed, honest `[N lines omitted]` footer | + +Kernel-enforced guarantees: dispatch guards keep each payload kind on its own strategies; plugins must be pure and deterministic (same payload + config ⇒ same bytes, prefix-cache friendly); any throw, sub-`minReduction` result, or invariant violation fails open to the next candidate or pass-through. **Default off** (`crush.enabled: false`) and requires `absorb.enabled: true`; when disabled the pipeline is byte-identical to pre-crush behavior. + +```json +"crush": { + "enabled": false, + "minReduction": 0.1, + "strategies": { + "json-fold": { "enabled": false }, + "code-trim": { "excludeTools": ["bash"] } + } +} +``` + ## Wire codec (`acp-kernel/wire`) The `acp-kernel/wire` subpath ships the provider wire-body codecs (lossless diff --git a/src/compress.ts b/src/compress.ts index 0f2fad5..a00b487 100644 --- a/src/compress.ts +++ b/src/compress.ts @@ -15,6 +15,7 @@ import type { ResolvedRange } from "./boundaries.js"; import { truncateLargeToolOutputs } from "./truncate-tools.js"; import { hideConsumedCompressCalls } from "./hide-consumed.js"; import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; +import { applyCrushToMessages } from "./crush.js"; import { applyMessageFilters, listMessageFilters } from "./filter/index.js"; import { activeBlockSpans } from "./block-map.js"; import { createRenderRefsNode } from "./render-refs.js"; @@ -531,6 +532,7 @@ export function createCore(ports: Ports = {}): CompressionCore { syncBlocksNode, pruneNode, absorbHideNode, + crushNode, absorbPromptNode, filterNode, hideCompressCallsNode, @@ -625,6 +627,33 @@ const absorbPromptNode: PipelineNode = { }, }; +// Tier 1 of the two-tier absorb gate: deterministic compression of eligible +// oversized tool results. Runs after absorb-hide (already-absorbed pairs are +// gone) and before absorb-prompt, which then re-decides on post-crush sizes — +// results crushed below minToolTokens stop triggering model round-trips. +const crushNode: PipelineNode = { + name: "crush", + enabled: (_io, ctx) => + ctx.config.crush?.enabled === true && ctx.config.absorb?.enabled === true, + run(io, ctx) { + const applied = applyCrushToMessages( + io.messages, + ctx.config, + ctx.tokenCount, + ctx.countTokens, + ); + return { + ...io, + messages: applied.messages, + effects: { + ...io.effects, + crushCount: applied.crushedCount, + crushDistilledCount: applied.distilledCount, + }, + }; + }, +}; + const filterNode: PipelineNode = { name: "filter", enabled: (_io, ctx) => diff --git a/src/config.ts b/src/config.ts index 75a1465..365a0b2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -39,6 +39,10 @@ export function defaultConfig( contextThresholdPct: 0, excludeTools: [], }, + crush: { + enabled: false, + minReduction: 0.1, + }, }; return { ...base, @@ -50,6 +54,7 @@ export function defaultConfig( absorb: overrides.absorb ? { ...base.absorb, ...overrides.absorb } : base.absorb, + crush: overrides.crush ? { ...base.crush, ...overrides.crush } : base.crush, }; } @@ -129,5 +134,34 @@ export function validateConfig(config: Config): string[] { errors.push("rules.maxRuleChars must be >= 1"); } } + if (config.crush) { + if ( + !Number.isFinite(config.crush.minReduction) || + config.crush.minReduction <= 0 || + config.crush.minReduction > 1 + ) { + errors.push("crush.minReduction must be in (0, 1]"); + } + if (config.crush.strategies) { + for (const [id, ov] of Object.entries(config.crush.strategies)) { + if (!ov || typeof ov !== "object" || Array.isArray(ov)) { + errors.push(`crush.strategies.${id} must be an object`); + continue; + } + if (ov.enabled !== undefined && typeof ov.enabled !== "boolean") { + errors.push(`crush.strategies.${id}.enabled must be a boolean`); + } + if ( + ov.excludeTools !== undefined && + (!Array.isArray(ov.excludeTools) || + ov.excludeTools.some((t) => typeof t !== "string")) + ) { + errors.push( + `crush.strategies.${id}.excludeTools must be a string array`, + ); + } + } + } + } return errors; } diff --git a/src/crush.ts b/src/crush.ts new file mode 100644 index 0000000..db268af --- /dev/null +++ b/src/crush.ts @@ -0,0 +1,1753 @@ +/** + * Tool-result valuator + deterministic compression ("crush") channel (#354): + * the two-tier absorb gate owned by the kernel. Tier 1: mechanical size + * reduction of eligible oversized tool results via registered crush plugins; + * tier 2: model distillation (the existing `[ACP absorb]` prompt path) only + * when a result is still over threshold after crushing. Pure function of + * (payload, config) — identical bytes on every turn a result is re-sent + * (prefix-cache stable); view-only like the absorb prompt, no kernel state + * is mutated. + * + * Plugin contract (CrushPluginDef.run): pure & deterministic, input never + * mutated, fail-open on any anomaly, sub-minimum-size inputs untouched. The + * kernel enforces what plugins must not each own: dispatch guards (JSON / + * source / log payloads never reach a strategy for another kind), the + * minReduction floor, and — for lossy log output — that every ERROR/FAIL line + * of the input survives in the output verbatim, else the result is rejected + * and the payload passes through unchanged. + * + * Default strategies (dispatched by content sniffing): + * - json-fold: field-variance folding (concept adapted from headroom + * SmartCrusher, headroomlabs-ai/headroom, Apache-2.0; implementation + * original TS). LOSSLESS BY CONSTRUCTION. Annotation vocabulary (reserved + * key `__acp_crush`): + * {"__acp_crush":"rows","rows":N,"const":{k:v,...},"items":[...]} + * array of N rows; every row = `const` merged with its `items` entry + * (same order); `items` omitted iff no row has a non-constant field. + * {"__acp_crush":"identical-run","count":N,"item":X} + * N consecutive identical elements X. + * Payloads legitimately containing a `__acp_crush` key cannot be told apart + * from crushed output — disable crush for them. + * - code-trim: structural trimmer (Python, JavaScript/TypeScript; sound for + * other // -comment languages). LOSSY: full-line comments, docstrings and + * blank-line runs are elided into one marker line (`# [acp-crush: elided N + * lines]` / `// [acp-crush: elided N lines]`) and NOT retained. Code lines, + * string literals and trailing comments stay byte-exact. A stronger AST + * implementation can register under kinds ["code"] to replace it. + * - log-select: build/test log condenser (ported from headroom LogCompressor, + * Apache-2.0): classifies every line (level / summary / stack-trace + * membership), keeps errors+fails first (within a keep budget; the kernel + * invariant above rejects any lossy result that drops an error line), + * deduped warnings, bounded stack traces (runtime frames collapsed to a + * marker), summary lines and ±3 context, caps the rest by score. LOSSY: + * noise lines are dropped, not retained. Upstream deviations: no CCR store + * — the omission footer carries honest counts instead of a retrieve hash; + * the keep budget is the fixed 100-line cap instead of headroom's adaptive + * Kneedle sizing (the cap only bounds output; the absorb gate and model + * compression already bound cost downstream). Only attempted when the + * payload is neither JSON-shaped nor confidently detected source code. + * + * Fail-open: parse errors, scanner bails (unterminated string/comment/ + * template), low-confidence language detection, plugin exceptions, or + * reduction below the configured minimum leave the payload byte-identical. + * Query-aware relevance selection (headroom layer b) is deliberately not + * ported: hosts re-send tool results every turn, so the crush must stay + * query-independent or provider prefix caches churn. + */ + +import { + ABSORB_PROMPT_MARKER, + isAbsorbCandidate, + resolveAbsorbConfig, +} from "./absorb.js"; +import { matchToolPattern } from "./protected.js"; +import { defaultCountTokens, type TokenCountFn } from "./tokenize.js"; +import type { + AbsorbConfig, + Config, + CoreMessage, + CrushConfig, +} from "./types.js"; + +export const DEFAULT_CRUSH_CONFIG: CrushConfig = { + enabled: false, + minReduction: 0.1, +}; + +export type CrushKind = "json" | "code" | "log"; + +/** Caller context handed to every plugin run (tool identity etc.). */ +export interface CrushMeta { + toolName?: string; +} + +export interface CrushOutput { + text: string; + strategy: string; + lossy: boolean; + stats?: Record; +} + +/** Narrow synchronous compression contract. `null` = pass-through. */ +export interface CrushPluginDef { + readonly id: string; + /** Content kinds this plugin may be dispatched on; the kernel refuses others. */ + readonly kinds: readonly CrushKind[]; + run(text: string, meta: CrushMeta): Omit | null; +} + +export interface CrushOptions { + minReduction?: number; + countTokens?: TokenCountFn; + meta?: CrushMeta; + /** Plugins to try, in order. Default: all registered, registry order. */ + plugins?: readonly CrushPluginDef[]; +} + +export type CrushDecisionKind = "skip" | "crushed" | "distill"; + +/** Valuator decision for one tool-result payload. */ +export interface CrushEvaluation { + kind: CrushDecisionKind; + /** Final text: original for skip, crushed otherwise. */ + text: string; + rawTokens: number; + newTokens?: number; + reduction?: number; + strategy?: string; + lossy?: boolean; +} + +const MAX_INPUT_CHARS = 2_000_000; +const MAX_DEPTH = 12; +const MAX_FOLDS = 1000; +const RUN_MIN = 4; +const HOIST_MIN = 4; +const CODE_KEEP_RATIO = 0.97; + +const CRUSH_KEY = "__acp_crush"; + +function isPlainObject(v: unknown): v is Record { + if (typeof v !== "object" || v === null || Array.isArray(v)) return false; + const p = Object.getPrototypeOf(v); + return p === null || p === Object.prototype; +} + +export function classifyCrushText(text: string): CrushKind { + const trimmed = text.trim(); + if ( + (trimmed.startsWith("{") && trimmed.endsWith("}")) || + (trimmed.startsWith("[") && trimmed.endsWith("]")) + ) + return "json"; + if (detectLanguage(text) !== null) return "code"; + return "log"; +} + +/** Single-call entry shared by the arrival path (processTurn crush node) and + * the future retrieve path (#1097): classify, dispatch in order to the first + * kind-matched plugin that produces an accepted result. `null` = pass-through. */ +export function crushText( + text: string, + options: CrushOptions = {}, +): CrushOutput | null { + const countTokens = options.countTokens ?? defaultCountTokens; + const minReduction = + options.minReduction ?? DEFAULT_CRUSH_CONFIG.minReduction; + const meta = options.meta ?? {}; + if (text.length === 0 || text.length > MAX_INPUT_CHARS) return null; + const rawTok = countTokens(text); + if (rawTok <= 0) return null; + const kind = classifyCrushText(text); + const wanted = options.plugins ?? registeredPlugins(); + for (const plugin of wanted) { + if (!plugin.kinds.includes(kind)) continue; + let res: Omit | null; + try { + res = plugin.run(text, meta); + } catch { + continue; + } + if (!res || !res.text || res.text === text) continue; + const newTok = countTokens(res.text); + const reduction = (rawTok - newTok) / rawTok; + if (reduction < minReduction) continue; + if (kind === "log" && res.lossy && !errorLinesSurvive(text, res.text)) + continue; + const out: CrushOutput = { + text: res.text, + strategy: plugin.id, + lossy: res.lossy, + }; + if (res.stats) out.stats = res.stats; + return out; + } + return null; +} + +interface FoldStats { + folds: number; +} + +function canonOf(v: unknown, cache: Map): string { + if (!isPlainObject(v) && !Array.isArray(v)) + return JSON.stringify(v) ?? "null"; + const key = v as object; + const hit = cache.get(key); + if (hit !== undefined) return hit; + let s: string; + if (Array.isArray(v)) { + s = `[${v.map((e) => canonOf(e, cache)).join(",")}]`; + } else { + const entries = Object.entries(v as Record) + .map(([k, val]) => `${JSON.stringify(k)}:${canonOf(val, cache)}`) + .sort(); + s = `{${entries.join(",")}}`; + } + cache.set(key, s); + return s; +} + +function strLen(v: unknown): number { + return JSON.stringify(v).length; +} + +function compactValue( + v: unknown, + depth: number, + stats: FoldStats, + cache: Map, +): unknown { + if (depth > MAX_DEPTH || stats.folds >= MAX_FOLDS) return v; + if (Array.isArray(v)) return compactArray(v, depth, stats, cache); + if (isPlainObject(v)) { + let changed = false; + const out: Record = {}; + for (const [k, val] of Object.entries(v)) { + const nv = compactValue(val, depth + 1, stats, cache); + out[k] = nv; + if (nv !== val) changed = true; + } + return changed ? out : v; + } + return v; +} + +function compactArray( + arr: unknown[], + depth: number, + stats: FoldStats, + cache: Map, +): unknown { + const n = arr.length; + if (n < RUN_MIN && n < HOIST_MIN) + return arr.map((e) => compactValue(e, depth + 1, stats, cache)); + if (stats.folds >= MAX_FOLDS) return arr; + + const plainStats: FoldStats = { folds: 0 }; + const plain = arr.map((e) => compactValue(e, depth + 1, plainStats, cache)); + let winner: unknown = plain; + let winnerLen = strLen(plain); + let winnerFolds = plainStats.folds; + + if (n >= RUN_MIN) { + const rf = buildRunFold(arr, depth, cache); + if (rf && rf.len < winnerLen) { + winner = rf.value; + winnerLen = rf.len; + winnerFolds = rf.folds; + } + } + if (n >= HOIST_MIN && arr.every((e) => isPlainObject(e))) { + const hc = buildConstHoist(arr, depth, cache); + if (hc && hc.len < winnerLen) { + winner = hc.value; + winnerLen = hc.len; + winnerFolds = hc.folds; + } + } + stats.folds += winnerFolds; + return winner; +} + +interface FoldCandidate { + value: unknown; + len: number; + folds: number; +} + +function isRunMarker( + v: unknown, +): v is { [CRUSH_KEY]: "identical-run"; count: number; item: unknown } { + return ( + isPlainObject(v) && + (v as Record)[CRUSH_KEY] === "identical-run" + ); +} + +function buildRunFold( + arr: unknown[], + depth: number, + cache: Map, +): FoldCandidate | null { + const n = arr.length; + const canon = arr.map((e) => canonOf(e, cache)); + const segs: unknown[] = []; + let runs = 0; + let i = 0; + while (i < n) { + let j = i; + while (j + 1 < n && canon[j + 1] === canon[i]) j++; + const runLen = j - i + 1; + if (runLen >= RUN_MIN) { + segs.push({ [CRUSH_KEY]: "identical-run", count: runLen, item: arr[i] }); + runs++; + } else { + for (let k = i; k <= j; k++) segs.push(arr[k]); + } + i = j + 1; + } + if (runs === 0) return null; + const inner: FoldStats = { folds: 0 }; + const final = segs.map((s) => + isRunMarker(s) + ? { ...s, item: compactValue(s.item, depth + 1, inner, cache) } + : compactValue(s, depth + 1, inner, cache), + ); + return { value: final, len: strLen(final), folds: runs + inner.folds }; +} + +function buildConstHoist( + arr: Record[], + depth: number, + cache: Map, +): FoldCandidate | null { + const n = arr.length; + const keys: string[] = []; + const seen = new Set(); + for (const e of arr) { + for (const k of Object.keys(e)) { + if (!seen.has(k)) { + seen.add(k); + keys.push(k); + } + } + } + const first = arr[0]; + if (!first) return null; + const constKeys: string[] = []; + for (const k of keys) { + const c0 = canonOf(first[k], cache); + let constant = true; + for (let i = 1; i < n; i++) { + const row = arr[i]!; + if (!(k in row) || canonOf(row[k], cache) !== c0) { + constant = false; + break; + } + } + if (constant) constKeys.push(k); + } + if (constKeys.length === 0) return null; + const constSet = new Set(constKeys); + const constObj: Record = {}; + for (const k of constKeys) constObj[k] = first[k]; + const items = arr.map((e) => { + const o: Record = {}; + for (const [k, v] of Object.entries(e)) { + if (!constSet.has(k)) o[k] = v; + } + return o; + }); + const hasVarying = items.some((o) => Object.keys(o).length > 0); + const inner: FoldStats = { folds: 0 }; + const env: Record = { + [CRUSH_KEY]: "rows", + rows: n, + const: compactValue(constObj, depth + 1, inner, cache), + }; + if (hasVarying) + env.items = items.map((it) => compactValue(it, depth + 1, inner, cache)); + const len = strLen(env); + if (len >= strLen(arr)) return null; + return { value: env, len, folds: 1 + inner.folds }; +} + +export function crushJson(text: string): string | null { + const parsed: unknown = JSON.parse(text); + const cache = new Map(); + const stats: FoldStats = { folds: 0 }; + const out = compactValue(parsed, 0, stats, cache); + if (stats.folds === 0) return null; + const s = JSON.stringify(out); + return s.length < text.length ? s : null; +} + +type Lang = "python" | "js"; + +function detectLanguage(text: string): Lang | null { + const sample = text.split(/\r?\n/).slice(0, 300); + let py = 0; + let js = 0; + for (const line of sample) { + if (/^\s*#!.*python/.test(line)) py += 4; + if (/^\s*(async\s+)?def\s+\w/.test(line) || /^\s*class\s+\w/.test(line)) + py += 2; + if (/\bfrom\s+['"]/.test(line) || /^\s*import\s+['"]/.test(line)) js += 2; + else if (/^\s*(import|from)\s+[A-Za-z_]\w*/.test(line)) py += 1; + if (/^\s*(function\b|const\s|let\s|var\s)/.test(line)) js += 2; + if (/=>/.test(line)) js += 1; + if (/^\s*(public|private|protected|static|final|void|return)\b/.test(line)) + js += 1; + if (/;\s*$/.test(line) && line.trim().length > 0) js += 1; + if (/\bself\./.test(line) || /^\s*(elif|except|yield)\b/.test(line)) + py += 1; + if (/\bconsole\.log\b|\brequire\s*\(|module\.exports|\bawait\s/.test(line)) + js += 1; + } + const MARGIN = 1.5; + if (py >= 3 && py > js * MARGIN) return "python"; + if (js >= 4 && js > py * MARGIN) return "js"; + return null; +} + +function finalizeTrimmed(original: string, emitted: string[]): string | null { + if (emitted.length >= original.split(/\r?\n/).length) return null; + const out = emitted.join("\n") + (original.endsWith("\n") ? "\n" : ""); + return out.length < original.length * CODE_KEEP_RATIO ? out : null; +} + +function trimPython(text: string): string | null { + const lines = text.split(/\r?\n/); + const emitted: string[] = []; + let pending = 0; + const flush = () => { + if (pending > 0) { + emitted.push( + `# [acp-crush: elided ${pending} line${pending === 1 ? "" : "s"}]`, + ); + pending = 0; + } + }; + let state: "code" | "triple" = "code"; + let quote = ""; + let isDocstring = false; + let blankRun = 0; + let atModuleStart = true; + let pendingDefClass = false; + let docstringSlot = false; + + const classifySig = (codePart: string) => { + const t = codePart.trim(); + if (/^(async\s+)?(def|class)\b/.test(t)) pendingDefClass = true; + if (t.endsWith(":")) { + docstringSlot = pendingDefClass; + pendingDefClass = false; + } else if (t.length > 0 && !t.startsWith("@")) { + docstringSlot = false; + } + atModuleStart = false; + }; + + for (let li = 0; li < lines.length; li++) { + const line = lines[li]!; + if (state === "triple") { + const close = findTripleClose(line, quote); + if (close === -1) { + if (isDocstring) pending++; + else emitted.push(line); + continue; + } + state = "code"; + const rest = line.slice(close + 3).trim(); + if (rest.length > 0) { + flush(); + emitted.push(line); + classifySig(rest); + } else { + if (isDocstring) pending++; + else emitted.push(line); + } + isDocstring = false; + docstringSlot = false; + pendingDefClass = false; + atModuleStart = false; + blankRun = 0; + continue; + } + if (line.trim().length === 0) { + blankRun++; + if (blankRun >= 3) { + flush(); + emitted.push(""); + blankRun = 1; + } + continue; + } + if (li === 0 && line.startsWith("#!")) { + flush(); + emitted.push(line); + blankRun = 0; + continue; + } + const m = line.match(/^\s*("""|''')/); + if (m) { + flush(); + // m.index points at the leading whitespace (\s* in the regex) — + // the triple itself starts further right; slicing from m.index + // would make closeOnSameLine find the opening triple itself. + const q = m[1]!; + const opensHere = line.indexOf(q); + const closeOnSameLine = findTripleClose(line.slice(opensHere + 3), q); + if (closeOnSameLine !== -1) { + const restAfter = line + .slice(opensHere + 3 + closeOnSameLine + 3) + .trim(); + if (restAfter.length > 0) { + emitted.push(line); + classifySig(restAfter); + } else if (atModuleStart || docstringSlot) { + pending++; + atModuleStart = false; + docstringSlot = false; + pendingDefClass = false; + } else { + emitted.push(line); + } + blankRun = 0; + continue; + } + state = "triple"; + quote = q; + isDocstring = atModuleStart || docstringSlot; + atModuleStart = false; + docstringSlot = false; + if (!isDocstring) emitted.push(line); + blankRun = 0; + continue; + } + const probe = findCommentOrTriple(line); + if (probe.kind === "comment-full") { + pending++; + continue; + } + if (probe.kind === "triple-mid") { + flush(); + state = "triple"; + quote = probe.quote; + isDocstring = false; + emitted.push(line); + classifySig(line.slice(0, probe.idx)); + blankRun = 0; + continue; + } + flush(); + emitted.push(line); + classifySig(line); + blankRun = 0; + } + if (state === "triple") return null; + flush(); + return finalizeTrimmed(text, emitted); +} + +function findTripleClose(line: string, quote: string): number { + let i = 0; + while (i < line.length) { + if (line[i] === "\\") { + i += 2; + continue; + } + if (line.startsWith(quote, i)) return i; + i++; + } + return -1; +} + +function findCommentOrTriple(line: string): { + kind: "none" | "comment-full" | "triple-mid"; + idx: number; + quote: string; +} { + let i = 0; + let inStr: string | null = null; + while (i < line.length) { + const c = line[i]; + if (inStr) { + if (c === "\\") i += 2; + else if (c === inStr) inStr = null; + else i++; + continue; + } + if (c === "'" || c === '"') { + inStr = c; + i++; + continue; + } + if (c === "#") { + return { + kind: line.slice(0, i).trim().length === 0 ? "comment-full" : "none", + idx: i, + quote: "", + }; + } + if (line.startsWith('"""', i) || line.startsWith("'''", i)) { + return { kind: "triple-mid", idx: i, quote: line.slice(i, i + 3) }; + } + i++; + } + return { kind: "none", idx: -1, quote: "" }; +} + +interface TmplState { + mode: "tpl" | "expr" | "str"; + quote: string; + depth: number; +} + +function scanTemplateRest( + line: string, + from: number, + st: TmplState, +): TmplState & { closed: boolean; bail: boolean; consumed: number } { + let m = st.mode; + let q = st.quote; + let d = st.depth; + let i = from; + while (i < line.length) { + const c = line[i]; + if (m === "str") { + if (c === "\\") i += 2; + else if (c === q) m = "expr"; + else i++; + continue; + } + if (m === "expr") { + if (c === "'" || c === '"') { + q = c; + m = "str"; + i++; + continue; + } + if (c === "`") + return { + mode: m, + quote: q, + depth: d, + closed: false, + bail: true, + consumed: 0, + }; + if (c === "{") d++; + else if (c === "}") { + d--; + if (d === 0) m = "tpl"; + } + i++; + continue; + } + if (c === "\\") { + i += 2; + continue; + } + if (c === "`") + return { + mode: "tpl", + quote: "", + depth: 0, + closed: true, + bail: false, + consumed: i - from + 1, + }; + if (c === "$" && line[i + 1] === "{") { + m = "expr"; + d = 1; + i += 2; + continue; + } + i++; + } + return { + mode: m, + quote: q, + depth: d, + closed: false, + bail: false, + consumed: 0, + }; +} + +type JsLineKind = + | "code" + | "comment-full" + | "block-open-only" + | "block-open-mid" + | "template-open"; + +function scanJsCodeLine(line: string): { + kind: JsLineKind; + tmpl?: TmplState; + bail?: boolean; +} { + let i = 0; + let inStr: string | null = null; + while (i < line.length) { + const c = line[i]; + if (inStr) { + if (c === "\\") i += 2; + else if (c === inStr) inStr = null; + else i++; + continue; + } + if (c === "'" || c === '"') { + inStr = c; + i++; + continue; + } + if (c === "`") { + const r = scanTemplateRest(line, i, { mode: "tpl", quote: "", depth: 0 }); + if (r.bail) return { kind: "code", bail: true }; + if (!r.closed) + return { + kind: "template-open", + tmpl: { mode: r.mode, quote: r.quote, depth: r.depth }, + }; + i += r.consumed; + continue; + } + if (c === "/" && line[i + 1] === "/") { + return { + kind: line.slice(0, i).trim().length === 0 ? "comment-full" : "code", + }; + } + if (c === "/" && line[i + 1] === "*") { + const close = line.indexOf("*/", i + 2); + const before = line.slice(0, i).trim().length > 0; + if (close === -1) + return { kind: before ? "block-open-mid" : "block-open-only" }; + const rest = line.slice(close + 2).trim(); + if (rest.length === 0) return { kind: before ? "code" : "comment-full" }; + i = close + 2; + continue; + } + i++; + } + return { kind: "code" }; +} + +function trimJsTs(text: string): string | null { + const lines = text.split(/\r?\n/); + const emitted: string[] = []; + let pending = 0; + const flush = () => { + if (pending > 0) { + emitted.push( + `// [acp-crush: elided ${pending} line${pending === 1 ? "" : "s"}]`, + ); + pending = 0; + } + }; + let state: "code" | "block" | "template" = "code"; + let tmpl: TmplState = { mode: "tpl", quote: "", depth: 0 }; + let blankRun = 0; + + for (const line of lines) { + if (state === "block") { + const close = line.indexOf("*/"); + if (close === -1) { + pending++; + continue; + } + state = "code"; + if (line.slice(close + 2).trim().length === 0) pending++; + else { + flush(); + emitted.push(line); + } + blankRun = 0; + continue; + } + if (state === "template") { + const r = scanTemplateRest(line, 0, tmpl); + if (r.bail) return null; + tmpl = { mode: r.mode, quote: r.quote, depth: r.depth }; + if (r.closed) state = "code"; + emitted.push(line); + blankRun = 0; + continue; + } + if (line.trim().length === 0) { + blankRun++; + if (blankRun >= 3) { + flush(); + emitted.push(""); + blankRun = 1; + } + continue; + } + const scan = scanJsCodeLine(line); + if (scan.bail) return null; + if (scan.kind === "comment-full") { + pending++; + continue; + } + if (scan.kind === "block-open-only") { + pending++; + state = "block"; + continue; + } + if (scan.kind === "block-open-mid") { + flush(); + state = "block"; + emitted.push(line); + blankRun = 0; + continue; + } + if (scan.kind === "template-open") { + flush(); + state = "template"; + tmpl = scan.tmpl ?? { mode: "tpl", quote: "", depth: 0 }; + emitted.push(line); + blankRun = 0; + continue; + } + flush(); + emitted.push(line); + blankRun = 0; + } + if (state !== "code") return null; + flush(); + return finalizeTrimmed(text, emitted); +} + +export function crushCode(text: string): string | null { + const lang = detectLanguage(text); + if (lang === null) return null; + if (lang === "python") return trimPython(text); + return trimJsTs(text); +} + +// ── Log strategy (ported from headroom LogCompressor, Apache-2.0) ────────── + +const LOG_MIN_LINES = 50; +const LOG_MAX_TOTAL_LINES = 100; +// Keep budget for error/fail lines. The kernel invariant rejects any lossy +// result that drops an error line, so logs with more distinct errors than +// this budget fail open to pass-through rather than losing error content. +const LOG_MAX_ERRORS = 20; +const LOG_ERROR_CONTEXT = 3; +const LOG_MAX_WARNINGS = 5; +const LOG_MAX_STACK_TRACES = 3; +const LOG_STACK_MAX_LINES = 20; +const LOG_TRACE_HEAD_FRAMES = 3; +const LOG_TRACE_APP_FRAMES = 5; +const LOG_CLASSIFIED_GATE = 5; +const LOG_SUMMARY_GATE = 3; + +type LogLevelName = + "error" | "fail" | "warn" | "info" | "debug" | "trace" | "unknown"; + +interface LogLine { + i: number; + content: string; + level: LogLevelName; + isStack: boolean; + isSummary: boolean; + score: number; +} + +// Pattern priority (higher severity wins), not leftmost match: a line +// containing both a warn- and an error-word must classify as error so it is +// kept by the error budget rather than dropped as noise. +const LEVEL_PATTERNS: Array<[LogLevelName, RegExp]> = [ + ["error", /\b(ERROR|FATAL|CRITICAL)\b/i], + ["fail", /\b(FAIL|FAILED)\b/i], + ["warn", /\b(WARN|WARNING)\b/i], + ["info", /\bINFO\b/i], + ["debug", /\bDEBUG\b/i], + ["trace", /\bTRACE\b/i], +]; + +function classifyLevel(line: string): LogLevelName { + for (const [level, re] of LEVEL_PATTERNS) { + if (re.test(line)) return level; + } + return "unknown"; +} + +function isLogSummaryLine(line: string): boolean { + if (line.startsWith("===") || line.startsWith("---")) return true; + let d = 0; + while ( + d < line.length && + line.charCodeAt(d) >= 48 && + line.charCodeAt(d) <= 57 + ) + d++; + if (d > 0 && line[d] === " ") { + const rest = line.slice(d + 1); + if ( + rest.startsWith("passed") || + rest.startsWith("failed") || + rest.startsWith("skipped") || + rest.startsWith("error") || + rest.startsWith("warning") + ) + return true; + } + for (const prefix of [ + "Test ", + "Tests ", + "Tests:", + "Test:", + "Suite ", + "Suites ", + "Suites:", + "Suite:", + ]) { + if (line.startsWith(prefix)) { + const m = line.slice(prefix.length).match(/\S/); + if (m !== null && m[0].charCodeAt(0) >= 48 && m[0].charCodeAt(0) <= 57) + return true; + } + } + if ( + line.startsWith("TOTAL") || + line.startsWith("Total") || + line.startsWith("Summary") + ) + return true; + for (const prefix of ["Build", "Compile", "Test"]) { + if ( + line.startsWith(prefix) && + (line.includes("succeeded") || + line.includes("failed") || + line.includes("complete")) + ) + return true; + } + return false; +} + +function isDigitChar(c: string): boolean { + return c >= "0" && c <= "9"; +} + +function isAsciiAlnum(c: string): boolean { + const n = c.charCodeAt(0); + return (n >= 97 && n <= 122) || (n >= 65 && n <= 90) || (n >= 48 && n <= 57); +} + +function hasLineColSuffix(s: string): boolean { + for (let i = 0; i + 1 < s.length; i++) { + if (s[i] === ":" && isDigitChar(s[i + 1]!)) { + let j = i + 1; + while (j < s.length && isDigitChar(s[j]!)) j++; + if (j + 1 < s.length && s[j] === ":" && isDigitChar(s[j + 1]!)) + return true; + } + } + return false; +} + +type TraceFlavor = + | "py" + | "js" + | "java" + | "dotnet" + | "rust-error" + | "rust-backtrace" + | "go-panic"; + +function isPythonFileFrame(s: string): boolean { + return ( + s.startsWith('File "') && + s.includes('", line ') && + s.length > 0 && + isDigitChar(s[s.length - 1]!) + ); +} + +function isJsAtFrame(s: string): boolean { + return ( + s.startsWith("at ") && + s.includes("(") && + s.includes(")") && + hasLineColSuffix(s) + ); +} + +function isJavaAtFrame(s: string): boolean { + if (!s.startsWith("at ") || !s.includes("(")) return false; + const open = s.indexOf("("); + const body = s.slice(3, open); + if (body.length === 0) return false; + for (const c of body) { + if (!(isAsciiAlnum(c) || c === "." || c === "_" || c === "$" || c === "/")) + return false; + } + return true; +} + +function isRustPanicOpener(s: string): boolean { + return s.startsWith("thread '") && s.includes("panicked at"); +} + +function isGoroutineHeader(line: string): boolean { + if (!line.startsWith("goroutine ")) return false; + const rest = line.slice(10); + let d = 0; + while (d < rest.length && isDigitChar(rest[d]!)) d++; + return d > 0 && rest.slice(d).startsWith(" ["); +} + +function isGoPanicOpener(line: string): boolean { + return ( + line.startsWith("panic: ") || + line.startsWith("fatal error: ") || + isGoroutineHeader(line) + ); +} + +function isGoFileFrame(line: string): boolean { + return ( + line.startsWith("\t") && line.includes(".go:") && line.includes(" +0x") + ); +} + +function isGoCallFrame(line: string): boolean { + if (line.startsWith("created by ")) return true; + if (line.startsWith(" ") || line.startsWith("\t") || !line.endsWith(")")) + return false; + const open = line.indexOf("("); + if (open === -1) return false; + const symbol = line.slice(0, open); + if (symbol.length === 0 || !symbol.includes(".")) return false; + for (const c of symbol) { + if (!(isAsciiAlnum(c) || c === "." || c === "_" || c === "/" || c === "*")) + return false; + } + return true; +} + +function isDotnetFrame(s: string): boolean { + return s.startsWith("at ") && s.includes(") in ") && s.includes(":line "); +} + +function isDotnetExceptionHead(trimmed: string): boolean { + const colon = trimmed.indexOf(":"); + if (colon === -1) return false; + const head = trimmed.slice(0, colon); + if (!head.endsWith("Exception") || !head.includes(".")) return false; + for (const c of head) { + if (!(isAsciiAlnum(c) || c === "." || c === "_" || c === "`" || c === "+")) + return false; + } + return true; +} + +function isRustBacktraceFrame(s: string): boolean { + const t = s.trimStart(); + let i = 0; + while (i < t.length && isDigitChar(t[i]!)) i++; + if (i === 0 || t[i] !== ":") return false; + i++; + while (i < t.length && t[i] === " ") i++; + const rest = t.slice(i); + if (!rest.startsWith("0x")) return false; + let h = 0; + for (const c of rest.slice(2)) { + if ( + (c >= "0" && c <= "9") || + (c >= "a" && c <= "f") || + (c >= "A" && c <= "F") + ) + h++; + else break; + } + return h > 0; +} + +function isJavaMoreSummary(trimmed: string): boolean { + if (!trimmed.startsWith("... ")) return false; + const rest = trimmed.slice(4); + let d = 0; + while (d < rest.length && isDigitChar(rest[d]!)) d++; + return d > 0 && rest.slice(d).trim() === "more"; +} + +function traceFlavorFor(line: string): TraceFlavor | null { + const t = line.trimStart(); + if (t.startsWith("Traceback (most recent call last)") || isPythonFileFrame(t)) + return "py"; + // Before js/java: .NET `at Ns.Class.Method(...) in File.cs:line N` also + // satisfies the Java `at (` shape. + if (t.startsWith("Unhandled exception.") || isDotnetFrame(t)) return "dotnet"; + if (isJsAtFrame(t)) return "js"; + if (isJavaAtFrame(t)) return "java"; + if (t.startsWith("--> ") && hasLineColSuffix(t)) return "rust-error"; + if ( + isRustPanicOpener(t) || + t.startsWith("stack backtrace:") || + isRustBacktraceFrame(line) + ) + return "rust-backtrace"; + if (isGoPanicOpener(line)) return "go-panic"; + return null; +} + +function traceTerminates( + flavor: TraceFlavor, + line: string, + linesSoFar: number, +): boolean { + const t = line.trimStart(); + switch (flavor) { + case "py": { + const indentedOrBlank = + line.startsWith(" ") || line.startsWith("\t") || line.length === 0; + const continuation = + t.startsWith("Traceback") || + t.startsWith("File ") || + t.startsWith("During handling") || + t.startsWith("The above exception"); + if (indentedOrBlank || continuation) return false; + return !(t.length > 0 && t[0]! >= "A" && t[0]! <= "Z"); + } + case "js": + return !t.startsWith("at ") && line.length !== 0; + case "java": { + const chain = + t.startsWith("Caused by:") || + t.startsWith("Suppressed:") || + isJavaMoreSummary(t); + return !t.startsWith("at ") && !chain && line.length !== 0; + } + case "dotnet": { + if (line.length === 0) return false; + const continues = + t.startsWith("at ") || + t.startsWith("--->") || + t.startsWith("--- End of") || + isDotnetExceptionHead(t); + return !continues; + } + case "rust-error": + return !t.startsWith("--> ") && line.length !== 0; + case "rust-backtrace": { + if (line.length === 0 || linesSoFar === 1) return false; + const isFrame = t.length > 0 && isDigitChar(t[0]!); + const continuation = + line.startsWith(" ") || + line.startsWith("\t") || + t.startsWith("stack backtrace:") || + t.startsWith("note: run with"); + return !isFrame && !continuation; + } + case "go-panic": { + if (line.length === 0) return false; + const continues = + line.startsWith("\t") || + isGoroutineHeader(line) || + isGoCallFrame(line) || + line.startsWith("panic: ") || + line.startsWith("fatal error: ") || + line.startsWith("[signal "); + return !continues; + } + } +} + +function scoreLogLine(l: LogLine): number { + const base = + l.level === "error" || l.level === "fail" + ? 1.0 + : l.level === "warn" + ? 0.5 + : l.level === "info" || l.level === "unknown" + ? 0.1 + : l.level === "debug" + ? 0.05 + : 0.02; + return Math.min(base + (l.isStack ? 0.3 : 0) + (l.isSummary ? 0.4 : 0), 1.0); +} + +function parseLogLines(lines: string[]): LogLine[] { + const out: LogLine[] = new Array(lines.length); + let active: TraceFlavor | null = null; + let traceLines = 0; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]!; + const entry: LogLine = { + i, + content: line, + level: classifyLevel(line), + isStack: false, + isSummary: isLogSummaryLine(line), + score: 0, + }; + if (active !== null) { + const flavor: TraceFlavor = active; + if ( + traceLines >= LOG_STACK_MAX_LINES || + traceTerminates(flavor, line, traceLines) + ) { + const capHit = traceLines >= LOG_STACK_MAX_LINES; + active = null; + traceLines = 0; + const nf = traceFlavorFor(line); + if (nf !== null) { + active = nf; + traceLines = 1; + entry.isStack = true; + } else if (capHit && !traceTerminates(flavor, line, 2)) { + // A capped trace does not end at its own cap line: chained + // traces reopen on the same line. + active = flavor; + traceLines = 1; + entry.isStack = true; + } + } else { + entry.isStack = true; + traceLines++; + } + } else { + const f = traceFlavorFor(line); + if (f !== null) { + active = f; + traceLines = 1; + entry.isStack = true; + } + } + entry.score = scoreLogLine(entry); + out[i] = entry; + } + return out; +} + +function selectWithFirstLast(arr: LogLine[], maxCount: number): LogLine[] { + if (arr.length <= maxCount) return arr.slice(); + const out: LogLine[] = []; + const seen = new Set(); + const push = (l: LogLine) => { + if (seen.add(l.i)) out.push(l); + }; + push(arr[0]!); + push(arr[arr.length - 1]!); + if (out.length < maxCount) { + const byScore = arr.slice().sort((a, b) => b.score - a.score || a.i - b.i); + for (const l of byScore) { + if (!seen.has(l.i)) { + push(l); + if (out.length >= maxCount) break; + } + } + } + return out; +} + +// Conservative dedupe key: the message identifier before the first `:`/`=` +// stays intact; only variable parts (digits, addresses, paths) in the suffix +// are normalised, so distinct messages sharing a trailing shape stay distinct. +function normalizeForDedupe(content: string): string { + let splitAt = content.length; + for (let i = 0; i < content.length; i++) { + if (content[i] === ":" || content[i] === "=") { + splitAt = i; + break; + } + } + const suffix = content + .slice(splitAt) + .replace(/\d+/g, "N") + .replace(/0x[0-9a-fA-F]+/g, "ADDR") + .replace(/\/[\w/]+\//g, "/PATH/"); + return content.slice(0, splitAt) + suffix; +} + +const RUNTIME_FRAME_PREFIXES = [ + "at java.", + "at jdk.", + "at sun.", + "at javax.", + "at scala.", + "at System.", + "at Microsoft.", + "runtime.", + "created by runtime.", +]; +const RUNTIME_FRAME_MARKERS = [ + "site-packages/", + "/usr/lib/python", + "lib/python3.", + "node:internal/", + "node_modules/", + "(internal/", + "core::", + "std::", + "alloc::", + "rust_begin_unwind", + "__rust_", + "/rustc/", + "/usr/local/go/src/", + "/libexec/src/runtime/", +]; + +function isFrameLine(content: string): boolean { + const t = content.trimStart(); + return ( + t.startsWith("at ") || + (t.startsWith('File "') && t.includes('", line ')) || + isRustBacktraceFrame(content) || + isGoFileFrame(content) || + isGoCallFrame(content) + ); +} + +function isChainHeadLine(content: string): boolean { + const t = content.trimStart(); + return ( + t.startsWith("Caused by:") || + t.startsWith("Suppressed:") || + t.startsWith("... ") || + t.startsWith("--->") || + t.startsWith("--- End of") || + t.startsWith("During handling") || + t.startsWith("The above exception") + ); +} + +function isRuntimeFrame(content: string): boolean { + const t = content.trimStart(); + return ( + RUNTIME_FRAME_PREFIXES.some((p) => t.startsWith(p)) || + RUNTIME_FRAME_MARKERS.some((m) => content.includes(m)) + ); +} + +interface CollapsedTrace { + kept: LogLine[]; + dropped: Set; +} + +function collapseTraceFrames( + stack: LogLine[], + headFrames: number, + appFrames: number, +): CollapsedTrace { + const kept: LogLine[] = []; + const dropped = new Set(); + let framesSeen = 0; + let appKept = 0; + let runStart = -1; + let runLen = 0; + let prevDropped = false; + const flushRun = () => { + if (runStart !== -1) { + kept.push({ + i: runStart, + content: ` [... ${runLen} frames collapsed]`, + level: "unknown", + isStack: true, + isSummary: false, + score: 0.8, + }); + runStart = -1; + runLen = 0; + } + }; + for (const line of stack) { + if (isFrameLine(line.content) && !isChainHeadLine(line.content)) { + framesSeen++; + const runtime = isRuntimeFrame(line.content); + const keep = + framesSeen <= headFrames || (!runtime && appKept < appFrames); + if (keep) { + if (!runtime) appKept++; + flushRun(); + kept.push(line); + prevDropped = false; + } else { + if (runStart === -1) runStart = line.i; + runLen++; + dropped.add(line.i); + prevDropped = true; + } + } else if ( + prevDropped && + (line.content.startsWith(" ") || line.content.startsWith("\t")) && + !isChainHeadLine(line.content) + ) { + runLen++; + dropped.add(line.i); + } else { + flushRun(); + kept.push(line); + prevDropped = false; + } + } + flushRun(); + return { kept, dropped }; +} + +function selectLogLines(all: LogLine[]): LogLine[] { + const errors: LogLine[] = []; + const fails: LogLine[] = []; + const warnings: LogLine[] = []; + const summaries: LogLine[] = []; + const stacks: LogLine[][] = []; + let current: LogLine[] = []; + for (const l of all) { + if (l.level === "error") errors.push(l); + else if (l.level === "fail") fails.push(l); + else if (l.level === "warn") warnings.push(l); + if (l.isStack) current.push(l); + else if (current.length > 0) { + stacks.push(current); + current = []; + } + if (l.isSummary) summaries.push(l); + } + if (current.length > 0) stacks.push(current); + + const selected = new Map(); + for (const l of selectWithFirstLast(errors, LOG_MAX_ERRORS)) + selected.set(l.i, l); + for (const l of selectWithFirstLast(fails, LOG_MAX_ERRORS)) + selected.set(l.i, l); + const seenWarn = new Set(); + const dedupedWarnings: LogLine[] = []; + for (const w of warnings) { + const key = normalizeForDedupe(w.content); + if (!seenWarn.has(key)) { + seenWarn.add(key); + dedupedWarnings.push(w); + } + } + for (const w of dedupedWarnings.slice(0, LOG_MAX_WARNINGS)) + selected.set(w.i, w); + for (const stack of stacks.slice(0, LOG_MAX_STACK_TRACES)) { + if (stack.length > LOG_STACK_MAX_LINES) { + const collapsed = collapseTraceFrames( + stack, + LOG_TRACE_HEAD_FRAMES, + LOG_TRACE_APP_FRAMES, + ); + for (const i of collapsed.dropped) selected.delete(i); + for (const l of collapsed.kept.slice(0, LOG_STACK_MAX_LINES)) + selected.set(l.i, l); + } else { + for (const l of stack) selected.set(l.i, l); + } + } + for (const s of summaries) selected.set(s.i, s); + + for (const idx of Array.from(selected.keys())) { + const lo = Math.max(0, idx - LOG_ERROR_CONTEXT); + const hi = Math.min(all.length, idx + LOG_ERROR_CONTEXT + 1); + for (let i = lo; i < hi; i++) { + if (i !== idx && !selected.has(i)) selected.set(i, all[i]!); + } + } + + let ordered = Array.from(selected.values()); + if (ordered.length > LOG_MAX_TOTAL_LINES) { + ordered.sort((a, b) => b.score - a.score || a.i - b.i); + ordered = ordered.slice(0, LOG_MAX_TOTAL_LINES); + ordered.sort((a, b) => a.i - b.i); + } + return ordered; +} + +function formatLogOutput(selected: LogLine[], all: LogLine[]): string { + const count = (lv: LogLevelName) => + all.reduce((n, l) => (l.level === lv ? n + 1 : n), 0); + const output = selected.map((l) => l.content); + const omitted = all.length - selected.length; + if (omitted > 0) { + const parts: string[] = []; + const e = count("error"); + const f = count("fail"); + const w = count("warn"); + const inf = count("info"); + if (e > 0) parts.push(`${e} ERROR`); + if (f > 0) parts.push(`${f} FAIL`); + if (w > 0) parts.push(`${w} WARN`); + if (inf > 0) parts.push(`${inf} INFO`); + if (parts.length > 0) + output.push(`[${omitted} lines omitted: ${parts.join(", ")}]`); + } + return output.join("\n"); +} + +const LOG_FORMAT_TABLE: Array<[string, string[]]> = [ + [ + "pytest", + [ + "=== FAILURES", + "=== ERRORS", + "=== test session", + "=== short test summary", + "PASSED [", + "FAILED [", + "ERROR [", + "SKIPPED [", + "collected ", + ], + ], + ["npm", ["npm ERR!", "npm WARN", "npm info", "npm http"]], + ["cargo", ["Compiling ", "Finished ", "Running ", "warning: ", "error[E"]], + ["jest", ["PASS ", "FAIL ", "Test Suites:"]], + ["make", ["make[", "make:", "gcc ", "g++ ", "clang "]], +]; + +function detectLogFormat(lines: string[]): string { + const sample = lines.slice(0, 100); + let best = "generic"; + let bestScore = 0; + for (const [name, pats] of LOG_FORMAT_TABLE) { + let score = 0; + for (const line of sample) { + if (pats.some((p) => line.includes(p))) score++; + } + if (score > bestScore) { + bestScore = score; + best = name; + } + } + return best; +} + +export function crushLog(text: string): string | null { + const lines = text.split(/\r?\n/); + if (lines.length < LOG_MIN_LINES) return null; + let classified = 0; + let summaries = 0; + let traces = 0; + for (const line of lines) { + const lv = classifyLevel(line); + if (lv !== "unknown") classified++; + if (isLogSummaryLine(line)) summaries++; + if (traceFlavorFor(line) !== null) traces++; + } + // Gate: require positive log structure before any line may be dropped — + // prose and structured data must never reach the selector. + if (!( + classified >= LOG_CLASSIFIED_GATE || + traces >= 1 || + summaries >= LOG_SUMMARY_GATE || + detectLogFormat(lines) !== "generic" + )) + return null; + const parsed = parseLogLines(lines); + const selected = selectLogLines(parsed); + if (selected.length >= lines.length) return null; + const out = formatLogOutput(selected, parsed); + if (out.length >= text.length) return null; + return out; +} + +function errorLinesSurvive(input: string, output: string): boolean { + const kept = new Set( + output + .split(/\r?\n/) + .map((l) => l.trim()) + .filter((l) => l.length > 0), + ); + for (const line of input.split(/\r?\n/)) { + const level = classifyLevel(line); + if (level !== "error" && level !== "fail") continue; + const t = line.trim(); + if (t.length > 0 && !kept.has(t)) return false; + } + return true; +} + +export const jsonFoldPlugin: CrushPluginDef = { + id: "json-fold", + kinds: ["json"], + run(text) { + const out = crushJson(text); + return out ? { text: out, lossy: false } : null; + }, +}; + +export const codeTrimPlugin: CrushPluginDef = { + id: "code-trim", + kinds: ["code"], + run(text) { + const out = crushCode(text); + return out ? { text: out, lossy: true } : null; + }, +}; + +export const logSelectPlugin: CrushPluginDef = { + id: "log-select", + kinds: ["log"], + run(text) { + const out = crushLog(text); + return out ? { text: out, lossy: true } : null; + }, +}; + +const builtInPlugins: readonly CrushPluginDef[] = [ + jsonFoldPlugin, + codeTrimPlugin, + logSelectPlugin, +]; +const crushRegistry = new Map(); +for (const p of builtInPlugins) crushRegistry.set(p.id, p); + +export function registerCrushPlugin(def: CrushPluginDef): void { + crushRegistry.set(def.id, def); +} + +export function unregisterCrushPlugin(id: string): void { + crushRegistry.delete(id); +} + +export function listCrushPlugins(): Array< + Pick +> { + return Array.from(crushRegistry.values()).map((p) => ({ + id: p.id, + kinds: p.kinds, + })); +} + +/** Test support: restore the built-in set exactly, in built-in order. */ +export function resetCrushPlugins(): void { + crushRegistry.clear(); + for (const p of builtInPlugins) crushRegistry.set(p.id, p); +} + +function registeredPlugins(): readonly CrushPluginDef[] { + return Array.from(crushRegistry.values()); +} + +export function resolveCrushConfig(config: Config): CrushConfig { + return { ...DEFAULT_CRUSH_CONFIG, ...(config.crush ?? {}) }; +} + +function effectivePlugins( + crush: CrushConfig, + meta: CrushMeta, +): readonly CrushPluginDef[] { + const toolName = meta.toolName; + return registeredPlugins().filter((p) => { + const ov = crush.strategies?.[p.id]; + if (ov?.enabled === false) return false; + if ( + toolName && + ov?.excludeTools?.some((pat) => matchToolPattern(toolName, pat)) + ) + return false; + return true; + }); +} + +export interface EvaluateCrushInput { + text: string; + absorb: AbsorbConfig; + crush: CrushConfig; + /** Current context token count (context-pressure signal). */ + tokenCount: number; + modelContextLimit: number; + meta?: CrushMeta; + countTokens?: TokenCountFn; +} + +/** Two-tier gate decision for one tool-result payload. Pure: same input ⇒ + * same bytes. `skip` = under the absorb gates; `crushed` = deterministic + * compression brought it under minToolTokens (no model round-trip); + * `distill` = still over (or uncrushable) — hand `text` to the absorb prompt + * path; when crushing succeeded `text` is the crushed payload. */ +export function evaluateToolResult(input: EvaluateCrushInput): CrushEvaluation { + const { absorb, crush, tokenCount, modelContextLimit, meta = {} } = input; + const countTokens = input.countTokens ?? defaultCountTokens; + const text = input.text; + const rawTokens = countTokens(text); + if ( + absorb.contextThresholdPct > 0 && + modelContextLimit > 0 && + tokenCount < absorb.contextThresholdPct * modelContextLimit + ) { + return { kind: "skip", text, rawTokens }; + } + if (rawTokens < absorb.minToolTokens) { + return { kind: "skip", text, rawTokens }; + } + const out = crushText(text, { + minReduction: crush.minReduction, + countTokens, + meta, + plugins: effectivePlugins(crush, meta), + }); + if (!out) return { kind: "distill", text, rawTokens }; + const newTokens = countTokens(out.text); + return { + kind: newTokens < absorb.minToolTokens ? "crushed" : "distill", + text: out.text, + rawTokens, + newTokens, + reduction: (rawTokens - newTokens) / rawTokens, + strategy: out.strategy, + lossy: out.lossy, + }; +} + +export interface ApplyCrushResult { + messages: CoreMessage[]; + /** Results crushed below minToolTokens (no model round-trip). */ + crushedCount: number; + /** Results forwarded to the absorb prompt path with crushed text. */ + distilledCount: number; +} + +/** Pipeline-facing application: view-only, deterministic per turn. Runs + * between absorb-hide and absorb-prompt so the prompt node re-decides on + * post-crush sizes in the same turn (atomic two-tier gate). Returns the + * input array unchanged when nothing qualifies. */ +export function applyCrushToMessages( + messages: CoreMessage[], + config: Config, + tokenCount: number, + countTokens: TokenCountFn, +): ApplyCrushResult { + const absorb = resolveAbsorbConfig(config); + const crush = resolveCrushConfig(config); + let crushedCount = 0; + let distilledCount = 0; + let changed = false; + const out: CoreMessage[] = []; + for (const msg of messages) { + if (!isAbsorbCandidate(msg, config)) { + out.push(msg); + continue; + } + const text = msg.text ?? ""; + // If a marker is present anyway (host-resent rendered text), crush the + // payload only and drop the stale marker — appendAbsorbPrompts + // re-decides on the post-crush size later in this turn. + const markerAt = text.indexOf(ABSORB_PROMPT_MARKER); + const payload = markerAt >= 0 ? text.slice(0, markerAt) : text; + const ev = evaluateToolResult({ + text: payload, + absorb, + crush, + tokenCount, + modelContextLimit: config.modelContextLimit, + meta: { toolName: msg.toolName }, + countTokens, + }); + if (ev.kind === "skip") { + out.push(msg); + continue; + } + changed = true; + if (ev.kind === "crushed") crushedCount++; + else distilledCount++; + out.push({ ...msg, text: ev.text }); + } + return { messages: changed ? out : messages, crushedCount, distilledCount }; +} diff --git a/src/index.ts b/src/index.ts index 6af20e0..1dcb67a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -183,6 +183,32 @@ export type { RemoveRuleResult, ClearRulesResult, } from "./rules.js"; +export { + DEFAULT_CRUSH_CONFIG, + classifyCrushText, + crushText, + evaluateToolResult, + applyCrushToMessages, + resolveCrushConfig, + registerCrushPlugin, + unregisterCrushPlugin, + listCrushPlugins, + resetCrushPlugins, + jsonFoldPlugin, + codeTrimPlugin, + logSelectPlugin, +} from "./crush.js"; +export type { + CrushKind, + CrushMeta, + CrushOutput, + CrushOptions, + CrushPluginDef, + CrushDecisionKind, + CrushEvaluation, + EvaluateCrushInput, + ApplyCrushResult, +} from "./crush.js"; export { rebuildCompressionState } from "./rebuild.js"; export type { RebuildResult, RebuildPorts } from "./rebuild.js"; export { parseCompressArgs } from "./parse-compress-input.js"; diff --git a/src/types.ts b/src/types.ts index 78e8bf4..95961b9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -157,6 +157,23 @@ export interface RulesConfig { maxRuleChars?: number; } +export interface CrushStrategyOverride { + /** Set false to disable this strategy even when crush is enabled. Default true. */ + enabled?: boolean; + /** Tool-name patterns (glob suffix allowed) this strategy skips. */ + excludeTools?: string[]; +} + +/** Deterministic tool-result compression ("crush") — tier 1 of the absorb + * gate (see crush.ts). Absent/disabled = feature off. */ +export interface CrushConfig { + enabled: boolean; + /** Minimum relative size reduction to accept a crushed result. In (0, 1]. */ + minReduction: number; + /** Per-strategy toggles/exclusions keyed by plugin id (json-fold, code-trim, log-select, ...). */ + strategies?: Record; +} + export interface CompressionState { blocks: CompressionBlock[]; messageRefs: MessageRefMap; @@ -272,6 +289,8 @@ export interface Config { absorb?: AbsorbConfig; /** Persistent acp_rule reminders (see rules.ts). Absent/disabled = feature off. */ rules?: RulesConfig; + /** Deterministic tool-result compression, tier 1 of the absorb gate (see crush.ts). Requires absorb enabled. */ + crush?: CrushConfig; messageFilters?: import("./filter/types.js").MessageFiltersConfig; } diff --git a/tests/crush.test.ts b/tests/crush.test.ts new file mode 100644 index 0000000..ca5acf3 --- /dev/null +++ b/tests/crush.test.ts @@ -0,0 +1,1114 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { createInitialState } from "../src/state.js"; +import { defaultConfig, validateConfig } from "../src/config.js"; +import { ABSORB_PROMPT_MARKER, DEFAULT_ABSORB_CONFIG } from "../src/absorb.js"; +import { makeIO, runPipeline } from "../src/pipeline.js"; +import type { CoreMessage, CrushConfig } from "../src/types.js"; +import { + DEFAULT_CRUSH_CONFIG, + classifyCrushText, + crushText, + evaluateToolResult, + resolveCrushConfig, + registerCrushPlugin, + unregisterCrushPlugin, + listCrushPlugins, + resetCrushPlugins, + jsonFoldPlugin, + codeTrimPlugin, + logSelectPlugin, +} from "../src/crush.js"; + +const tok = (t: string): number => Math.ceil(t.length / 4); + +interface RowEnv { + __acp_crush: "rows"; + rows: number; + const: Record; + items?: Record[]; +} + +type AnyEnv = + | RowEnv + | { __acp_crush: "identical-run"; count: number; item: unknown } + | unknown[] + | Record; + +function decodeValue(v: unknown): unknown { + if (Array.isArray(v)) { + const outArr: unknown[] = []; + for (const e of v) { + if ( + e !== null && + typeof e === "object" && + (e as Record).__acp_crush === "identical-run" + ) { + const m = e as { count: number; item: unknown }; + for (let i = 0; i < m.count; i++) outArr.push(decodeValue(m.item)); + } else outArr.push(decodeValue(e)); + } + return outArr; + } + if (v !== null && typeof v === "object") { + const o = v as Record; + if (o.__acp_crush === "identical-run") { + const m = o as { count: number; item: unknown }; + return Array.from({ length: m.count }, () => decodeValue(m.item)); + } + if (o.__acp_crush === "rows") { + const r = o as RowEnv; + const items = r.items ?? []; + return Array.from({ length: r.rows }, (_, i) => ({ + ...(r.const as Record), + ...(items[i] as Record), + })); + } + const out: Record = {}; + for (const [k, val] of Object.entries(o)) out[k] = decodeValue(val); + return out; + } + return v; +} + +test("json-fold: constant-field hoist is lossless (rows envelope)", () => { + const rows = Array.from({ length: 8 }, (_, i) => ({ + id: i * 7, + ts: "2026-09-21T12:00:00Z", + level: "INFO", + name: `svc-${i}`, + })); + const src = JSON.stringify(rows); + const out = jsonFoldPlugin.run(src, {})?.text; + assert.ok(out, "must crush"); + assert.ok(out!.length < src.length); + const env = JSON.parse(out!) as RowEnv; + assert.equal(env.__acp_crush, "rows"); + assert.equal(env.rows, 8); + assert.equal(env.const.ts, "2026-09-21T12:00:00Z"); + assert.equal(env.const.level, "INFO"); + assert.equal( + (env.const as Record).id, + undefined, + "varying field must not be hoisted", + ); + assert.deepEqual( + decodeValue(JSON.parse(out!)), + rows, + "decode must reconstruct the original array", + ); +}); + +test("json-fold: identical-run fold is lossless", () => { + const arr = [ + ...Array(5).fill({ e: "boom", code: 500 }), + { e: "ok", code: 200 }, + { e: "ok", code: 200 }, + { e: "warn", code: 300 }, + { e: "warn", code: 300 }, + ]; + const src = JSON.stringify(arr); + const out = jsonFoldPlugin.run(src, {})?.text; + assert.ok(out, "must crush"); + assert.ok(out!.length < src.length); + assert.deepEqual( + decodeValue(JSON.parse(out!)), + arr, + "run expansion must reconstruct the original array", + ); +}); + +test("json-fold: fully identical array collapses and decodes back", () => { + const arr = Array.from({ length: 6 }, () => ({ + status: "healthy", + region: "us-east-1", + })); + const src = JSON.stringify(arr); + const out = jsonFoldPlugin.run(src, {})?.text; + assert.ok(out, "must crush"); + assert.ok(out!.length < src.length / 2); + assert.deepEqual(decodeValue(JSON.parse(out!)), arr); +}); + +test("json-fold: deterministic (byte-identical across calls)", () => { + const rows = Array.from({ length: 10 }, (_, i) => ({ + id: i, + host: `h${i}`, + zone: "z1", + })); + const src = JSON.stringify(rows); + assert.equal( + jsonFoldPlugin.run(src, {})?.text, + jsonFoldPlugin.run(src, {})?.text, + ); +}); + +test("json-fold: nested envelopes recurse and stay lossless", () => { + const doc = { + meta: "x", + logs: Array.from({ length: 6 }, (_, i) => ({ + seq: i, + src: "api", + msg: `m${i}`, + })), + }; + const src = JSON.stringify(doc); + const out = jsonFoldPlugin.run(src, {})?.text; + assert.ok(out, "must crush"); + const parsed = JSON.parse(out!) as { logs: AnyEnv }; + assert.equal((parsed.logs as RowEnv).__acp_crush, "rows"); + assert.deepEqual(decodeValue(parsed), doc); +}); + +test("json-fold: no redundancy -> null (never reformats)", () => { + const rows = Array.from({ length: 8 }, (_, i) => ({ + id: i, + a: `a${i}`, + b: `b${i}`, + c: i % 3, + })); + assert.equal(jsonFoldPlugin.run(JSON.stringify(rows), {}), null); + assert.equal(jsonFoldPlugin.run('{"a":1,"b":[1,2,3]}', {}), null); +}); + +test("crushText: malformed JSON falls through, prose passes through untouched", () => { + assert.equal( + crushText("[1,2,", { countTokens: tok }), + null, + "malformed JSON must fail open", + ); + assert.equal( + crushText("The quick brown fox jumps over the lazy dog. ".repeat(20), { + countTokens: tok, + }), + null, + "prose is not a crush target", + ); + assert.equal(crushText("", { countTokens: tok }), null); +}); + +test("crushText: minReduction gate rejects marginal savings", () => { + const arr = Array.from({ length: 4 }, () => "x".repeat(100)); + const src = JSON.stringify(arr); + assert.ok( + crushText(src, { minReduction: 0.5, countTokens: tok }), + "~66% reduction passes a 0.5 gate", + ); + assert.equal( + crushText(src, { minReduction: 0.9, countTokens: tok }), + null, + "same payload rejected at a 0.9 gate", + ); +}); + +test("crushText: dispatches json-first, then code", () => { + const rows = Array.from({ length: 8 }, (_, i) => ({ id: i * 3, zone: "z1" })); + assert.equal( + crushText(JSON.stringify(rows), { countTokens: tok })?.strategy, + "json-fold", + ); + const py = [ + "import os", + "", + "def f():", + " # c1", + " # c2", + " # c3", + " # c4", + " return os.getcwd()", + ].join("\n"); + assert.equal(crushText(py, { countTokens: tok })?.strategy, "code-trim"); +}); + +test("classifyCrushText: json / code / log buckets", () => { + assert.equal(classifyCrushText(JSON.stringify([{ a: 1 }])), "json"); + assert.equal( + classifyCrushText(["import os", "", "def f():", " pass"].join("\n")), + "code", + ); + assert.equal( + classifyCrushText( + Array.from({ length: 60 }, (_, i) => `INFO tick ${i}`).join("\n"), + ), + "log", + ); + assert.equal( + classifyCrushText("plain english paragraph without structure"), + "log", + ); +}); + +test("config: defaultConfig ships crush disabled with default minReduction", () => { + const cfg = defaultConfig(200000); + assert.deepEqual(cfg.crush, { enabled: false, minReduction: 0.1 }); + const merged = defaultConfig(200000, { crush: { enabled: true } }); + assert.deepEqual(merged.crush, { enabled: true, minReduction: 0.1 }); + const full = defaultConfig(200000, { + crush: { + enabled: true, + minReduction: 0.4, + strategies: { "json-fold": { enabled: false } }, + }, + }); + assert.deepEqual(full.crush, { + enabled: true, + minReduction: 0.4, + strategies: { "json-fold": { enabled: false } }, + }); +}); + +test("validateConfig: crush block validation", () => { + assert.deepEqual( + validateConfig( + defaultConfig(200000, { crush: { enabled: true, minReduction: 0.5 } }), + ), + [], + ); + assert.deepEqual( + validateConfig( + defaultConfig(200000, { crush: { enabled: true, minReduction: 0 } }), + ), + ["crush.minReduction must be in (0, 1]"], + ); + assert.deepEqual( + validateConfig( + defaultConfig(200000, { crush: { enabled: true, minReduction: 1.5 } }), + ), + ["crush.minReduction must be in (0, 1]"], + ); + assert.deepEqual( + validateConfig( + defaultConfig(200000, { + crush: { enabled: true, minReduction: Number.NaN }, + }), + ), + ["crush.minReduction must be in (0, 1]"], + ); + assert.deepEqual( + validateConfig( + defaultConfig(200000, { + crush: { enabled: true, strategies: { x: "no" } }, + }), + ), + ["crush.strategies.x must be an object"], + ); + assert.deepEqual( + validateConfig( + defaultConfig(200000, { + crush: { enabled: true, strategies: { x: { enabled: "yes" } } }, + }), + ), + ["crush.strategies.x.enabled must be a boolean"], + ); + assert.deepEqual( + validateConfig( + defaultConfig(200000, { + crush: { enabled: true, strategies: { x: { excludeTools: 5 } } }, + }), + ), + ["crush.strategies.x.excludeTools must be a string array"], + ); +}); + +test("resolveCrushConfig: fills defaults over partial blocks", () => { + assert.deepEqual( + resolveCrushConfig(defaultConfig(200000)), + DEFAULT_CRUSH_CONFIG, + ); + assert.deepEqual( + resolveCrushConfig(defaultConfig(200000, { crush: { enabled: true } })), + { enabled: true, minReduction: 0.1 }, + ); + assert.deepEqual( + resolveCrushConfig( + defaultConfig(200000, { crush: { enabled: true, minReduction: 0.7 } }), + ), + { + enabled: true, + minReduction: 0.7, + }, + ); +}); + +const PY_DOCSTRING = [ + "#!/usr/bin/env python3", + "# module-level comment", + "", + "", + "def fetch(url):", + ' """Fetch the url.', + "", + " Multi-line body.", + ' """', + " import urllib.request", + " return urllib.request.urlopen(url).read()", +].join("\n"); + +test("code-trim: python docstrings/comments elided, code lines kept verbatim", () => { + const out = codeTrimPlugin.run(PY_DOCSTRING, {})?.text; + assert.ok(out, "must crush"); + const lines = out!.split("\n"); + assert.equal(lines[0], "#!/usr/bin/env python3", "shebang kept"); + assert.ok(lines.includes("def fetch(url):"), "signature kept"); + assert.ok(lines.includes(" import urllib.request"), "body kept"); + assert.ok( + lines.includes(" return urllib.request.urlopen(url).read()"), + "body kept", + ); + assert.ok(!out!.includes("Multi-line body"), "docstring body elided"); + assert.ok(!out!.includes("module-level comment"), "comment elided"); + assert.match(out!, /\[acp-crush: elided \d+ lines?\]/); +}); + +test("code-trim: assigned triple-quoted string is NOT a docstring and is kept", () => { + const src = [ + "import os", + "# comment one", + "# comment two", + "# comment three", + "# comment four", + "# comment five", + "", + "def gen():", + ' """Docstring body.', + " second line.", + ' """', + ' x = """kept string"""', + " return x", + ].join("\n"); + const out = codeTrimPlugin.run(src, {})?.text; + assert.ok(out, "must still crush (comments + docstring elided)"); + assert.ok( + out!.includes('x = """kept string"""'), + "non-docstring triple literal must survive verbatim", + ); + assert.ok(!out!.includes("Docstring body"), "docstring elided"); +}); + +test("code-trim: unterminated triple quote fails open", () => { + const src = [ + "import os", + "", + "def f():", + ' """unclosed docstring', + " return x", + ].join("\n"); + assert.equal(codeTrimPlugin.run(src, {}), null); +}); + +const JS_TPL = [ + "// top comment one", + "// top comment two", + "// top comment three", + "// top comment four", + "function greet(name) {", + " // build a safe url", + ' const url = `https://x/${name.split("//")[0]}/y`;', + " /* block", + " comment line two", + " comment */", + " console.log(`hi ${url}`);", + " return url;", + "}", +].join("\n"); + +test("code-trim: js/ts comments elided, template literals with ${} intact", () => { + const out = codeTrimPlugin.run(JS_TPL, {})?.text; + assert.ok(out, "must crush"); + assert.ok(!out!.includes("top comment")); + assert.ok(!out!.includes("comment */")); + assert.ok( + out!.includes('const url = `https://x/${name.split("//")[0]}/y`;'), + "template literal kept verbatim", + ); + assert.ok( + out!.includes("console.log(`hi ${url}`);"), + "second template kept verbatim", + ); + assert.match(out!, /\/\/ \[acp-crush: elided \d+ lines?\]/); +}); + +test("code-trim: prose is not detected as code", () => { + assert.equal( + codeTrimPlugin.run("This is a plain English paragraph. ".repeat(30), {}), + null, + ); +}); + +function pytestLog(): string { + const lines: string[] = []; + lines.push("=== test session starts ==="); + lines.push("platform linux -- Python 3.12.4, pytest-8.3.2"); + for (let i = 0; i < 40; i++) + lines.push(`tests/test_api.py::test_case_${i} PASSED`); + for (let i = 0; i < 12; i++) { + lines.push(`___________ test_case_fail_${i} ___________`); + lines.push(`E AssertionError: expected status 200, got ${500 + i}`); + lines.push(`ERROR tests/test_api.py::test_case_fail_${i}`); + lines.push(""); + } + for (let i = 0; i < 500; i++) + lines.push( + "WARNING urllib3.connectionpool:connectionpool.py:869 - Connection pool is full, discarding connection", + ); + for (let i = 0; i < 100; i++) + lines.push(`INFO app.api:api.py:${i} - request handled in ${i}ms`); + lines.push(""); + lines.push("=== short test summary info ==="); + lines.push( + "FAILED tests/test_api.py::test_case_fail_0 - AssertionError: expected status 200, got 500", + ); + lines.push("======== 12 failed, 40 passed in 3.42s ========"); + return lines.join("\n"); +} + +test("log-select: every distinct error line survives, warnings deduped, noise dropped", () => { + const src = pytestLog(); + const out = logSelectPlugin.run(src, {})?.text; + assert.ok(out, "must crush"); + assert.ok(out!.length < src.length / 2, "big reduction expected"); + const errorLines = out! + .split("\n") + .filter((l) => l.startsWith("ERROR tests/test_api.py::")); + assert.equal( + errorLines.length, + 12, + "kernel invariant: all distinct ERROR lines survive (budget 20 >= 12)", + ); + assert.ok( + out!.includes("ERROR tests/test_api.py::test_case_fail_0"), + "first error kept", + ); + assert.ok( + out!.includes("ERROR tests/test_api.py::test_case_fail_11"), + "last error kept", + ); + const warnCount = out! + .split("\n") + .filter((l) => l.includes("Connection pool is full")).length; + assert.ok( + warnCount >= 1 && warnCount <= 15, + `identical warnings deduped (${warnCount} of 500)`, + ); + const infoCount = out! + .split("\n") + .filter((l) => l.includes("request handled in")).length; + assert.ok(infoCount <= 15, `INFO noise dropped (${infoCount} of 100)`); + assert.match(out!, /\[\d+ lines omitted:/, "honest omission footer present"); + assert.ok( + out!.includes("=== short test summary info ==="), + "summary line kept", + ); +}); + +test("log-select: deterministic (byte-identical across calls)", () => { + const src = pytestLog(); + assert.equal( + logSelectPlugin.run(src, {})?.text, + logSelectPlugin.run(src, {})?.text, + ); +}); + +test("log-select: short logs pass through untouched", () => { + const short = [ + "INFO start", + "ERROR boom", + "ERROR bang", + ...Array(30).fill("INFO tick"), + ].join("\n"); + assert.equal(short.split("\n").length, 33); + assert.equal(logSelectPlugin.run(short, {}), null, "<50 lines never crushed"); +}); + +test("log-select: unstructured prose never reaches the selector", () => { + const prose = Array.from( + { length: 60 }, + (_, i) => + `Sentence number ${i} talks about ordinary topics without any structure.`, + ).join("\n"); + assert.equal(logSelectPlugin.run(prose, {}), null); + assert.equal( + crushText(prose, { countTokens: tok }), + null, + "and crushText fails open on prose", + ); +}); + +function jsTraceLog(): string { + const lines: string[] = ["ERROR unhandled exception in worker pool"]; + for (let i = 0; i < 25; i++) + lines.push( + ` at Worker.task (/app/node_modules/pkg/dist/worker.js:${10 + i}:${i})`, + ); + lines.push( + " at process.processTicksAndRejections (node:internal/process/task_queues:95:5)", + ); + for (let i = 0; i < 40; i++) lines.push("info heartbeat ok"); + return lines.join("\n"); +} + +test("log-select: long runtime stack traces collapse to head frames + marker", () => { + const src = jsTraceLog(); + const out = logSelectPlugin.run(src, {})?.text; + assert.ok(out, "must crush"); + assert.ok( + out!.includes("at Worker.task (/app/node_modules/pkg/dist/worker.js:10:0)"), + "head frame kept", + ); + assert.match( + out!, + /\[\.\.\. \d+ frames collapsed\]/, + "collapse marker present", + ); + assert.ok(!out!.includes("worker.js:20:10"), "middle runtime frames dropped"); + assert.ok(!out!.includes("worker.js:34:24"), "tail runtime frames dropped"); +}); + +test("crushText: log-shaped payloads dispatch to the log strategy", () => { + assert.equal( + crushText(pytestLog(), { countTokens: tok })?.strategy, + "log-select", + ); +}); + +test("crushText: error-line invariant rejects lossy log output that drops errors", () => { + // Errors are spaced > 2*LOG_ERROR_CONTEXT apart so the ±context fill around + // selected lines cannot bridge over the dropped ones. + const lines: string[] = []; + for (let i = 0; i < 25; i++) { + lines.push( + `ERROR db_${String.fromCharCode(97 + (i % 26))}: connection lost shard ${i}`, + ); + for (let j = 0; j < 8; j++) + lines.push(`INFO heartbeat ok shard-${i} tick-${j}`); + } + const src = lines.join("\n"); + assert.ok( + logSelectPlugin.run(src, {}) !== null, + "the plugin itself produces output", + ); + assert.equal( + crushText(src, { countTokens: tok }), + null, + ">20 distinct errors cannot all survive -> rejected -> fail open", + ); +}); + +test("registry: list/register/unregister/reset built-ins", () => { + resetCrushPlugins(); + assert.deepEqual(listCrushPlugins(), [ + { id: "json-fold", kinds: ["json"] }, + { id: "code-trim", kinds: ["code"] }, + { id: "log-select", kinds: ["log"] }, + ]); + const spy = { id: "spy", kinds: ["json"], run: (_t: string) => null }; + registerCrushPlugin(spy); + assert.equal( + listCrushPlugins().find((p) => p.id === "spy")?.kinds[0], + "json", + ); + unregisterCrushPlugin("spy"); + assert.equal( + listCrushPlugins().some((p) => p.id === "spy"), + false, + ); + registerCrushPlugin(spy); + resetCrushPlugins(); + assert.equal( + listCrushPlugins().some((p) => p.id === "spy"), + false, + "reset restores exactly the built-ins", + ); +}); + +test("crushText: dispatch guard — plugins only see their own kind", () => { + const seen: string[] = []; + const spy = { + id: "spy-code-only", + kinds: ["code"], + run: (t: string) => { + seen.push(t); + return null; + }, + }; + const rows = Array.from({ length: 8 }, (_, i) => ({ id: i * 3, zone: "z1" })); + const json = JSON.stringify(rows); + assert.equal( + crushText(json, { countTokens: tok, plugins: [spy] }), + null, + "json payload never reaches a code-only plugin", + ); + assert.equal(seen.length, 0, "plugin body never invoked for a foreign kind"); +}); + +test("crushText: throwing plugin fails open to the next candidate", () => { + const boom = { + id: "boom", + kinds: ["json"], + run: () => { + throw new Error("boom"); + }, + }; + const rows = Array.from({ length: 8 }, (_, i) => ({ id: i * 3, zone: "z1" })); + const out = crushText(JSON.stringify(rows), { + countTokens: tok, + plugins: [boom, jsonFoldPlugin], + }); + assert.equal( + out?.strategy, + "json-fold", + "built-in still crushes after the thrower failed open", + ); +}); + +test("crushText: plugin returning input-identical text is a no-op", () => { + const identity = { + id: "identity", + kinds: ["json"], + run: (t: string) => ({ text: t, lossy: false }), + }; + const rows = Array.from({ length: 8 }, (_, i) => ({ id: i * 3, zone: "z1" })); + assert.equal( + crushText(JSON.stringify(rows), { countTokens: tok, plugins: [identity] }), + null, + ); +}); + +const absorbOn = ( + overrides: Partial = {}, +): typeof DEFAULT_ABSORB_CONFIG => ({ + ...DEFAULT_ABSORB_CONFIG, + enabled: true, + ...overrides, +}); + +const crushOn = (overrides: Partial = {}): CrushConfig => ({ + ...DEFAULT_CRUSH_CONFIG, + enabled: true, + ...overrides, +}); + +test("evaluateToolResult: skip when below minToolTokens or under context pressure", () => { + const small = JSON.stringify([{ id: 1 }]); + const ev1 = evaluateToolResult({ + text: small, + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + tokenCount: 0, + modelContextLimit: 200000, + countTokens: tok, + }); + assert.deepEqual(ev1, { + kind: "skip", + text: small, + rawTokens: ev1.rawTokens, + }); + const rows = Array.from({ length: 10 }, (_, i) => ({ + id: i * 7, + ts: "2026-09-21T12:00:00Z", + level: "INFO", + name: `svc-${i}`, + })); + const ev2 = evaluateToolResult({ + text: JSON.stringify(rows), + absorb: absorbOn({ contextThresholdPct: 0.8 }), + crush: crushOn(), + tokenCount: 1000, + modelContextLimit: 100000, + countTokens: tok, + }); + assert.equal( + ev2.kind, + "skip", + "context below threshold % -> whole feature idle", + ); +}); + +test("evaluateToolResult: crushed when deterministic compression lands under minToolTokens", () => { + const rows = Array.from({ length: 10 }, (_, i) => ({ + id: i * 7, + ts: "2026-09-21T12:00:00Z", + level: "INFO", + name: `svc-${i}`, + })); + const src = JSON.stringify(rows); + const ev = evaluateToolResult({ + text: src, + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + tokenCount: 0, + modelContextLimit: 200000, + countTokens: tok, + }); + assert.equal(ev.kind, "crushed"); + assert.ok(ev.newTokens !== undefined && ev.newTokens < 100); + assert.ok(ev.reduction !== undefined && ev.reduction > 0); + assert.equal(ev.strategy, "json-fold"); + assert.equal(ev.lossy, false); + assert.ok(ev.text.includes("__acp_crush")); +}); + +test("evaluateToolResult: distill carries the crushed payload when still over threshold", () => { + const rows = Array.from({ length: 40 }, (_, i) => ({ + id: i, + level: "INFO", + msg: `request handled with status code and correlation value ${i}`, + })); + const src = JSON.stringify(rows); + const ev = evaluateToolResult({ + text: src, + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + tokenCount: 0, + modelContextLimit: 200000, + countTokens: tok, + }); + assert.equal(ev.kind, "distill"); + assert.ok( + ev.text.includes("__acp_crush"), + "crushed payload forwarded to the absorb prompt path", + ); + assert.notEqual(ev.text, src); +}); + +test("evaluateToolResult: uncrushable payload distills with the ORIGINAL text", () => { + const prose = "Sentence about ordinary topics. ".repeat(200); + const ev = evaluateToolResult({ + text: prose, + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + tokenCount: 0, + modelContextLimit: 200000, + countTokens: tok, + }); + assert.equal(ev.kind, "distill"); + assert.equal( + ev.text, + prose, + "no strategy applied -> original bytes preserved", + ); +}); + +test("evaluateToolResult: per-strategy disable and excludeTools honored", () => { + const rows = Array.from({ length: 10 }, (_, i) => ({ + id: i * 7, + ts: "2026-09-21T12:00:00Z", + level: "INFO", + name: `svc-${i}`, + })); + const src = JSON.stringify(rows); + const off = evaluateToolResult({ + text: src, + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn({ strategies: { "json-fold": { enabled: false } } }), + tokenCount: 0, + modelContextLimit: 200000, + countTokens: tok, + }); + assert.equal(off.kind, "distill"); + assert.equal(off.text, src, "disabled strategy -> original text"); + const code = + [ + "import os", + "", + "def f():", + " # c1", + " # c2", + " # c3", + " # c4", + " # c5", + " # c6", + " return os.getcwd()", + ].join("\n") + "\n"; + const excl = evaluateToolResult({ + text: code.repeat(30), + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn({ strategies: { "code-trim": { excludeTools: ["bash"] } } }), + tokenCount: 0, + modelContextLimit: 200000, + meta: { toolName: "bash" }, + countTokens: tok, + }); + assert.equal(excl.kind, "distill"); + assert.equal(excl.text, code.repeat(30), "excluded tool -> original text"); + const notExcl = evaluateToolResult({ + text: code.repeat(30), + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn({ strategies: { "code-trim": { excludeTools: ["bash"] } } }), + tokenCount: 0, + modelContextLimit: 200000, + meta: { toolName: "python" }, + countTokens: tok, + }); + assert.notEqual( + notExcl.text, + code.repeat(30), + "non-excluded tool still crushed", + ); +}); + +function turnMessages(resultText: string): CoreMessage[] { + return [ + { id: "u1", role: "user", contentType: "text", text: "run the job" }, + { + id: "a-tc", + role: "assistant", + contentType: "tool-call", + toolName: "bash", + toolCallId: "call_1", + text: JSON.stringify({ command: "job" }), + }, + { + id: "t-res", + role: "tool", + contentType: "tool-result", + toolCallId: "call_1", + text: resultText, + }, + ]; +} + +test("processTurn: feature off (either switch) leaves messages byte-identical", () => { + const rows = Array.from({ length: 10 }, (_, i) => ({ + id: i, + ts: "2026-09-21T12:00:00Z", + level: "INFO", + })); + const src = JSON.stringify(rows); + const core = createCore({ countTokens: tok }); + const absorbHigh = absorbOn({ minToolTokens: 100000 }); + const cases = [ + defaultConfig(200000, { absorb: absorbHigh }), + defaultConfig(200000, { absorb: absorbHigh, crush: { enabled: false } }), + defaultConfig(200000, { + absorb: { ...DEFAULT_ABSORB_CONFIG, enabled: false }, + crush: crushOn(), + }), + ]; + for (const config of cases) { + const turn = core.processTurn({ + messages: turnMessages(src), + state: createInitialState(), + config, + tokenCount: 0, + renderTags: "none", + }); + assert.deepEqual( + turn.messages.map((m) => m.text), + ["run the job", JSON.stringify({ command: "job" }), src], + "byte-identical result text", + ); + } +}); + +test("processTurn: sub-threshold results are never touched", () => { + const small = JSON.stringify([{ id: 1, zone: "z1" }]); + const core = createCore({ countTokens: tok }); + const config = defaultConfig(200000, { + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + }); + const turn = core.processTurn({ + messages: turnMessages(small), + state: createInitialState(), + config, + tokenCount: 0, + renderTags: "none", + }); + assert.deepEqual( + turn.messages.map((m) => m.text), + ["run the job", JSON.stringify({ command: "job" }), small], + ); +}); + +test("processTurn: crushed-below-threshold result skips the model absorb round-trip", () => { + const rows = Array.from({ length: 10 }, (_, i) => ({ + id: i * 7, + ts: "2026-09-21T12:00:00Z", + level: "INFO", + name: `svc-${i}`, + })); + const src = JSON.stringify(rows); + assert.ok(tok(src) >= 100, "fixture must start above the absorb threshold"); + const core = createCore({ countTokens: tok }); + const config = defaultConfig(200000, { + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + }); + const turn = core.processTurn({ + messages: turnMessages(src), + state: createInitialState(), + config, + tokenCount: 0, + renderTags: "none", + }); + const res = turn.messages.find((m) => m.id === "t-res")!; + const env = JSON.parse(res.text!) as RowEnv; + assert.equal(env.__acp_crush, "rows", "wire carries the crushed envelope"); + assert.ok( + !turn.messages.some((m) => (m.text ?? "").includes(ABSORB_PROMPT_MARKER)), + "below threshold: no model absorb round-trip", + ); + assert.deepEqual( + turn.messages.filter((m) => m.id !== "t-res").map((m) => m.text), + ["run the job", JSON.stringify({ command: "job" })], + "non-result messages untouched", + ); +}); + +test("processTurn: still-over-threshold result is forwarded crushed WITH the absorb prompt", () => { + const rows = Array.from({ length: 40 }, (_, i) => ({ + id: i, + level: "INFO", + msg: `request handled with status code and correlation value ${i}`, + })); + const src = JSON.stringify(rows); + const core = createCore({ countTokens: tok }); + const config = defaultConfig(200000, { + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + }); + const turn = core.processTurn({ + messages: turnMessages(src), + state: createInitialState(), + config, + tokenCount: 0, + renderTags: "none", + }); + const res = turn.messages.find((m) => m.id === "t-res")!; + const markerAt = res.text!.indexOf(ABSORB_PROMPT_MARKER); + assert.ok(markerAt > 0, "absorb prompt re-attached on the crushed result"); + const env = JSON.parse(res.text!.slice(0, markerAt)) as RowEnv; + assert.equal(env.__acp_crush, "rows", "crushed before the prompt decision"); + assert.ok( + tok(res.text!.slice(0, markerAt)) >= 100, + "still above threshold after crushing", + ); +}); + +test("processTurn: log result crushed below threshold skips the absorb round-trip", () => { + const log = [ + "INFO boot ok", + ...Array(55).fill("warn retry x"), + "ERROR db connection lost", + "ERROR transaction aborted", + "Summary: 2 errors, 55 warnings", + ].join("\n"); + assert.ok(tok(log) >= 150, "fixture must start above the absorb threshold"); + const core = createCore({ countTokens: tok }); + const config = defaultConfig(200000, { + absorb: absorbOn({ minToolTokens: 150 }), + crush: crushOn(), + }); + const turn = core.processTurn({ + messages: turnMessages(log), + state: createInitialState(), + config, + tokenCount: 0, + renderTags: "none", + }); + const res = turn.messages.find((m) => m.id === "t-res")!; + assert.ok( + res.text && tok(res.text) < 150, + `crushed below threshold (${tok(res.text!)} tok)`, + ); + assert.ok( + res.text!.includes("ERROR db connection lost"), + "errors survive the crush", + ); + assert.ok( + res.text!.includes("ERROR transaction aborted"), + "errors survive the crush", + ); + assert.ok( + !turn.messages.some((m) => (m.text ?? "").includes(ABSORB_PROMPT_MARKER)), + "below threshold: no model absorb round-trip", + ); +}); + +test("processTurn: stale absorb marker is stripped and re-decided on the crushed payload", () => { + const rows = Array.from({ length: 10 }, (_, i) => ({ + id: i * 7, + ts: "2026-09-21T12:00:00Z", + level: "INFO", + name: `svc-${i}`, + })); + const src = JSON.stringify(rows); + const core = createCore({ countTokens: tok }); + let state = createInitialState(); + let messages = turnMessages(src); + const off = core.processTurn({ + messages, + state, + config: defaultConfig(200000, { absorb: absorbOn({ minToolTokens: 100 }) }), + tokenCount: 0, + renderTags: "none", + }); + state = off.state; + messages = off.messages; + const baked = messages.find((m) => m.id === "t-res")!.text!; + assert.ok( + baked.includes(ABSORB_PROMPT_MARKER), + "precondition: marker baked on turn 1", + ); + const on = core.processTurn({ + messages, + state, + config: defaultConfig(200000, { + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + }), + tokenCount: 0, + renderTags: "none", + }); + const res = on.messages.find((m) => m.id === "t-res")!.text!; + assert.ok(res.includes("__acp_crush"), "payload crushed on turn 2"); + assert.ok( + !res.includes(ABSORB_PROMPT_MARKER), + "stale marker stripped, re-decided below threshold -> no fresh prompt", + ); +}); + +test("crush node: reports effects and respects gating", () => { + const core = createCore({ countTokens: tok }); + const node = core.defaultNodes().find((n) => n.name === "crush")!; + assert.ok(node, "crush node present in the pipeline"); + const rows = Array.from({ length: 10 }, (_, i) => ({ + id: i * 7, + ts: "2026-09-21T12:00:00Z", + level: "INFO", + name: `svc-${i}`, + })); + const src = JSON.stringify(rows); + const msgs = turnMessages(src); + const ctx = { + config: defaultConfig(200000, { + absorb: absorbOn({ minToolTokens: 100 }), + crush: crushOn(), + }), + tokenCount: 0, + countTokens: tok, + }; + const io = runPipeline([node], makeIO(msgs, createInitialState()), ctx); + assert.equal(io.effects.crushCount, 1); + assert.equal(io.effects.crushDistilledCount, 0); + assert.ok( + io.messages.find((m) => m.id === "t-res")!.text!.includes("__acp_crush"), + ); + const offCtx = { + config: defaultConfig(200000, { + absorb: absorbOn({ minToolTokens: 100 }), + crush: { enabled: false }, + }), + tokenCount: 0, + countTokens: tok, + }; + const io2 = runPipeline([node], makeIO(msgs, createInitialState()), offCtx); + assert.equal( + io2.messages.find((m) => m.id === "t-res")!.text, + src, + "gated off -> untouched", + ); + assert.equal(io2.effects.crushCount, undefined); +}); diff --git a/tests/pipeline.test.ts b/tests/pipeline.test.ts index 9cdcee2..f7f1fbf 100644 --- a/tests/pipeline.test.ts +++ b/tests/pipeline.test.ts @@ -29,6 +29,7 @@ test("defaultNodes exposes the canonical ordered pipeline", () => { "sync-blocks", "prune", "absorb-hide", + "crush", "absorb-prompt", "filter", "hide-compress-calls", From c6dd2bf5e3526ac212b3af93c5a7fa406a317df9 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 23 Sep 2026 03:00:26 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20crush=20=E2=80=94=20json-fold=20spar?= =?UTF-8?q?se-null=20hoist=20loss,=20JS=20regex-literal=20comment=20mispar?= =?UTF-8?q?se,=20plugin=20fail-open=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- src/crush.ts | 94 ++++++++++++++++++++++++++++++++++++---- tests/crush.test.ts | 103 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 9ad0cfc..2fea04a 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ The nudge system tells the model *when* to compress. It implements: Deterministic first tier of the two-tier absorb gate. When a tool result qualifies as an absorb candidate and context pressure is on, the `crush` node (between `absorb-hide` and `absorb-prompt`) re-evaluates it **in view** with pure, deterministic strategies before the absorb-prompt decision runs: - **skip** — below `absorb.minToolTokens` or under `absorb.contextThresholdPct`; bytes untouched. -- **crushed** — deterministic compression brought it under `minToolTokens`; no model round-trip. The wire carries the crushed payload (e.g. `__acp_crush` rows/identical-run envelopes for JSON, decodable byte-for-byte). +- **crushed** — deterministic compression brought it under `minToolTokens`; no model round-trip. The wire carries the crushed payload (e.g. `__acp_crush` rows/identical-run envelopes for JSON, lossless — decodes back to an equal JSON document). - **distill** — still over (or uncrushable); forwarded to the `[ACP absorb]` prompt path, carrying the crushed payload when crushing succeeded. Built-in strategies form an ordered plugin registry (hosts can add/replace/disable via `registerCrushPlugin` / config): diff --git a/src/crush.ts b/src/crush.ts index db268af..02e198e 100644 --- a/src/crush.ts +++ b/src/crush.ts @@ -161,14 +161,24 @@ export function crushText( const kind = classifyCrushText(text); const wanted = options.plugins ?? registeredPlugins(); for (const plugin of wanted) { - if (!plugin.kinds.includes(kind)) continue; let res: Omit | null; try { + // Contract-violating host plugins (missing/malformed kinds, non-string + // result text) must fail open like any other anomaly, not throw out of + // the pipeline or poison msg.text with a non-string. + if (!Array.isArray(plugin.kinds) || !plugin.kinds.includes(kind)) + continue; res = plugin.run(text, meta); } catch { continue; } - if (!res || !res.text || res.text === text) continue; + if ( + !res || + typeof res.text !== "string" || + res.text === "" || + res.text === text + ) + continue; const newTok = countTokens(res.text); const reduction = (rawTok - newTok) / rawTok; if (reduction < minReduction) continue; @@ -337,6 +347,11 @@ function buildConstHoist( if (!first) return null; const constKeys: string[] = []; for (const k of keys) { + // A key absent from row 0 must not be hoisted: canonOf(undefined) === + // canonOf(null), so "absent in row 0, null elsewhere" would pass the + // const check, then JSON.stringify drops the undefined constObj value and + // the field vanishes from every decoded row (losslessness violation). + if (!(k in first)) continue; const c0 = canonOf(first[k], cache); let constant = true; for (let i = 1; i < n; i++) { @@ -684,6 +699,53 @@ type JsLineKind = | "block-open-mid" | "template-open"; +// A `/` can start a regex literal only where an expression is expected. If the +// previous significant char ends a token (identifier char, `.`, quote, `)`, +// `]`), the `/` is division — parsing a regex there would misread ordinary +// arithmetic, so the attempt is skipped. +function regexAllowedAfter(prev: string | null): boolean { + if (prev === null) return true; + if (/[A-Za-z0-9_]/.test(prev)) return false; + return !".'\"`) ]".includes(prev); +} + +function prevSigChar(line: string, i: number): string | null { + for (let j = i - 1; j >= 0; j--) { + const c = line[j]!; + if (c === " " || c === "\t") continue; + return c; + } + return null; +} + +// Scan a JS regex literal starting at line[i] === "/". Returns the index just +// past the closing slash and its flags, or null when it does not close on this +// line (regex literals never span lines) — i.e. the `/` was division, not a +// regex. Honors backslash escapes and [..] character classes (a `/` inside a +// class does not close the literal). +function scanRegexLiteral(line: string, i: number): number | null { + let j = i + 1; + let inClass = false; + while (j < line.length) { + const c = line[j]!; + if (c === "\\") { + j += 2; + continue; + } + if (inClass) { + if (c === "]") inClass = false; + } else if (c === "[") { + inClass = true; + } else if (c === "/") { + let f = j + 1; + while (f < line.length && /[a-z]/i.test(line[f]!)) f++; + return f; + } + j++; + } + return null; +} + function scanJsCodeLine(line: string): { kind: JsLineKind; tmpl?: TmplState; @@ -715,12 +777,28 @@ function scanJsCodeLine(line: string): { i += r.consumed; continue; } - if (c === "/" && line[i + 1] === "/") { - return { - kind: line.slice(0, i).trim().length === 0 ? "comment-full" : "code", - }; - } - if (c === "/" && line[i + 1] === "*") { + if (c === "/") { + const nxt = line[i + 1]; + if (nxt === "/") { + return { + kind: line.slice(0, i).trim().length === 0 ? "comment-full" : "code", + }; + } + if (nxt !== "*") { + // A regex pattern may not start with `*` ("Nothing to repeat"), so + // `/*` is unambiguously a comment open. Any other `/` where an + // expression is expected may start a regex literal — consume it whole + // so patterns like /[/*]/ cannot open a fake block comment later. + if (regexAllowedAfter(prevSigChar(line, i))) { + const end = scanRegexLiteral(line, i); + if (end !== null) { + i = end; + continue; + } + } + i++; + continue; + } const close = line.indexOf("*/", i + 2); const before = line.slice(0, i).trim().length > 0; if (close === -1) diff --git a/tests/crush.test.ts b/tests/crush.test.ts index ca5acf3..f19656d 100644 --- a/tests/crush.test.ts +++ b/tests/crush.test.ts @@ -9,6 +9,7 @@ import type { CoreMessage, CrushConfig } from "../src/types.js"; import { DEFAULT_CRUSH_CONFIG, classifyCrushText, + crushJson, crushText, evaluateToolResult, resolveCrushConfig, @@ -1112,3 +1113,105 @@ test("crush node: reports effects and respects gating", () => { ); assert.equal(io2.effects.crushCount, undefined); }); + +test("json-fold: sparse null key absent in row 0 survives rows-hoist decode", () => { + const rows = Array.from({ length: 60 }, (_, i) => ({ + zone: "z1", + id: `id-${String(i).padStart(3, "0")}`, + name: `item-${i}`.padEnd(90, "."), + ...(i === 0 ? {} : { a: null }), + })); + const out = crushJson(JSON.stringify(rows)); + assert.ok(out, "should crush via rows hoist"); + const env = JSON.parse(out) as RowEnv; + assert.equal(env.__acp_crush, "rows"); + assert.equal(env.const.zone, "z1", "genuine constant still hoists"); + const decoded = decodeValue(env) as Record[]; + for (let i = 0; i < rows.length; i++) + assert.deepEqual(decoded[i], rows[i], `row ${i} decodes value-lossless`); +}); + +test("code-trim: regex literal containing /* does not open a fake comment", () => { + const js = [ + "function f() {", + " var re = /[/*]/;", + " var a = 1;", + " var b = 2;", + " var c = 3;", + " var d = 4;", + ' var s = "has */ inside";', + " var e = 5;", + " var g = 6;", + " return a + b + c + d + e + g;", + "}", + ].join("\n"); + const out = crushText(js); + if (out !== null) { + for (const line of [ + "var re = /[/*]/;", + "var a = 1;", + "var b = 2;", + "var c = 3;", + "var d = 4;", + "var e = 5;", + "var g = 6;", + ]) + assert.ok(out.text.includes(line), `code line preserved: ${line}`); + } +}); + +test("code-trim: division is not a regex; full-line comments still elide", () => { + const js = [ + "function ratio(total, parts) {", + " var a = total / parts[0];", + " var b = (total + 1) / (parts[1] || 1);", + " var c = -total / parts[2];", + " /* first full-line comment */", + " /* second full-line comment */", + " var d = total / 2 * parts[3];", + " return a + b + c + d;", + "}", + ].join("\n"); + const out = crushText(js); + assert.ok(out !== null, "elided comments should pass the reduction floor"); + assert.ok(out.text.includes("var a = total / parts[0];")); + assert.ok(out.text.includes("var b = (total + 1) / (parts[1] || 1);")); + assert.ok(out.text.includes("var d = total / 2 * parts[3];")); + assert.ok(!out.text.includes("first full-line comment"), "comment elided"); +}); + +test("crushText: plugin with malformed kinds fails open instead of throwing", () => { + const bad = { + id: "bad", + kinds: undefined, + run: () => null, + } as unknown as Parameters[0]; + const rows = Array.from({ length: 8 }, (_, i) => ({ id: i * 3, zone: "z1" })); + const out = crushText(JSON.stringify(rows), { + countTokens: tok, + plugins: [bad, jsonFoldPlugin], + }); + assert.equal( + out?.strategy, + "json-fold", + "malformed plugin skipped, built-in still applies", + ); +}); + +test("crushText: plugin returning non-string text fails open", () => { + const liar = { + id: "liar", + kinds: ["json"], + run: () => ({ text: 123, lossy: false }), + } as unknown as Parameters[0]; + const rows = Array.from({ length: 8 }, (_, i) => ({ id: i * 3, zone: "z1" })); + const out = crushText(JSON.stringify(rows), { + countTokens: tok, + plugins: [liar, jsonFoldPlugin], + }); + assert.equal( + out?.strategy, + "json-fold", + "non-string result skipped, built-in still applies", + ); +});