Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ acp-kernel/
│ ├── keep-markers.ts # KEEP/REF marker preservation in summaries
│ ├── ccr.ts # CCR: placeholder contract, storeLargeResults, retrieveByRef, applyRetrieve, ccrStoreNode
│ ├── content-store.ts # MessageContentStore — per-session content-addressed dedup store (per-ref append-only)
│ ├── preserved-ids.ts # Mechanical fidelity (#481): verbatim id extraction from folded tool content + block appendix
│ ├── prompts.ts # Prompts (4 load-bearing rules) + defaultPrompts + resolvePrompts
│ ├── compress-tools.ts # ACP tool schemas (3 wire shapes) + standing-prompt builders
│ ├── surface-config.ts # Surface-text overrides: applySectionOverrides, cloneWithDescriptions, applyAcpToolOverrides
Expand Down
78 changes: 71 additions & 7 deletions src/compress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ import { isToolMessage } from "./message-kind.js";
import { adjustBoundariesForToolPairs } from "./tool-pairs.js";
import { adjustBoundariesForReasoningPairs } from "./reasoning-pairs.js";
import { computeIntegrityWithdrawals } from "./turn-integrity.js";
import {
buildFidelityAppendix,
extractFidelityIds,
fidelitySourceTexts,
} from "./preserved-ids.js";
import {
computeProtectedRefs,
buildCompressibleRanges,
Expand Down Expand Up @@ -312,27 +317,53 @@ function evaluateRefold(

/** In-place refold (#398): same block id/coverage/tier, new summary (+topic),
* fresh runId, restoredInline flag cleared. Summary passes the same length
* checks as a fresh compression. */
* checks as a fresh compression. Returns the #481 fidelity appendix appended
* ("" when none was needed). */
function applyRefolds(input: {
spec: ApplyCompressionInput["ranges"][number];
state: CompressionState;
runId: string;
config: Config;
blockIds: string[];
}): void {
messages: CoreMessage[];
}): string {
validateSummaryLength(input.spec, input.config.compress);
const targets = new Set(input.blockIds);
// #481: extract from the OLD summaries and still-visible tool messages
// before the map rewrites them — a re-fold must not drop ids.
const fidelityTexts: string[] = [];
for (const block of input.state.blocks) {
if (!targets.has(block.blockId)) continue;
fidelityTexts.push(block.summary);
const coverage = new Set(block.effectiveMessageIds);
for (const message of input.messages) {
if (!coverage.has(message.id)) continue;
fidelityTexts.push(...fidelitySourceTexts([message]));
}
}
const appendix = buildFidelityAppendix(
input.spec.summary,
extractFidelityIds(fidelityTexts),
);
const summary =
appendix === "" ? input.spec.summary : input.spec.summary + appendix;
input.state.blocks = input.state.blocks.map((block) =>
targets.has(block.blockId)
? {
...block,
summary: input.spec.summary,
summary,
topic: input.spec.topic ?? block.topic,
runId: input.runId,
restoredInline: false,
}
: block,
);
return appendix;
}

function fidelityNote(appendix: string): string | undefined {
if (appendix === "") return undefined;
return `Kernel appended mechanically-preserved identifier(s) to the block summary: ${appendix.trim()}`;
}

export function createCore(ports: Ports = {}): CompressionCore {
Expand Down Expand Up @@ -606,14 +637,17 @@ export function createCore(ports: Ports = {}): CompressionCore {
const decision = refoldDecisions.get(spec);
if (decision?.kind === "refold") {
try {
applyRefolds({
const appendix = applyRefolds({
spec,
state,
runId,
config: input.config,
blockIds: decision.blocks.map((block) => block.blockId),
messages: input.messages,
});
blocksCreated += decision.blocks.length;
const note = fidelityNote(appendix);
if (note) notes.push(note);
} catch (error) {
errors.push(
rangeError(
Expand Down Expand Up @@ -651,6 +685,7 @@ export function createCore(ports: Ports = {}): CompressionCore {
blocksCreated += outcome.refolded ? outcome.refolded.length : 1;
tokensCompressed += outcome.tokens;
warnings.push(...outcome.warnings);
if (outcome.notes) notes.push(...outcome.notes);
} catch (error) {
errors.push(
rangeError(
Expand Down Expand Up @@ -1135,6 +1170,8 @@ interface SingleRangeOutcome {
/** #400: set when the range ended as an in-place refold instead of creating
* a fresh block — the ids of the blocks updated (caller counts these, not 1). */
refolded?: string[];
/** Kernel-side transparency notes to surface to the caller (#481). */
notes?: string[];
}

function applySingleRange(input: SingleRangeInput): SingleRangeOutcome {
Expand Down Expand Up @@ -1373,17 +1410,20 @@ function applySingleRange(input: SingleRangeInput): SingleRangeOutcome {
// blocks in place; everything else keeps the legacy rejection verbatim.
const decision = evaluateRefold(input.state, input.spec);
if (decision.kind === "refold") {
applyRefolds({
const appendix = applyRefolds({
spec: input.spec,
state: input.state,
runId: input.runId,
config: input.config,
blockIds: decision.blocks.map((block) => block.blockId),
messages: input.messages,
});
const note = fidelityNote(appendix);
return {
tokens: 0,
warnings,
refolded: decision.blocks.map((block) => block.blockId),
...(note !== undefined ? { notes: [note] } : {}),
};
}
const first = consumedBlockIds[0]!;
Expand Down Expand Up @@ -1411,13 +1451,32 @@ function applySingleRange(input: SingleRangeInput): SingleRangeOutcome {
}
}

// Mechanical fidelity (#481): fold-commit post-processing. Extract verbatim
// identifiers from the folded source texts — live tool messages plus
// consumed blocks' summaries (tier distillation carries ids forward through
// summaries on pruned hosts) — and append any id missing from the model
// summary as a machine line the model never transcribes.
const foldedMessages = filteredIds
.map((id) => input.messages.find((entry) => entry.id === id))
.filter((m): m is CoreMessage => m !== undefined);
const fidelityTexts = [
...fidelitySourceTexts(foldedMessages),
...consumedBlockIds
.map((id) => blockById(input.state, id)?.summary)
.filter((s): s is string => s !== undefined),
];
const fidelityAppendix = buildFidelityAppendix(
input.spec.summary,
extractFidelityIds(fidelityTexts),
);

const blockId = allocateBlockId(input.state);
const block: CompressionBlock = {
blockId,
runId: input.runId,
tier: outputTier,
topic: input.spec.topic,
summary: input.spec.summary,
summary: input.spec.summary + fidelityAppendix,
directMessageIds: filteredIds,
effectiveMessageIds: [...effectiveMessageIds],
directBlockIds: [...consumedBlockIds],
Expand All @@ -1437,7 +1496,12 @@ function applySingleRange(input: SingleRangeInput): SingleRangeOutcome {
if (consumed) consumed.active = false;
}

return { tokens: compressedTokens, warnings };
const note = fidelityNote(fidelityAppendix);
return {
tokens: compressedTokens,
warnings,
...(note !== undefined ? { notes: [note] } : {}),
};
}

function applyPairBoundaryAdjustments(
Expand Down
7 changes: 7 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,13 @@ export type {
export { renderHandoff, renderMessage, matchSession } from "./handoff.js";
export type { HandoffInput, HandoffMeta, HandoffBlockFull } from "./handoff.js";
export { isToolMessage } from "./message-kind.js";
export {
DEFAULT_FIDELITY_EXTRACTORS,
fidelitySourceTexts,
extractFidelityIds,
buildFidelityAppendix,
} from "./preserved-ids.js";
export type { FidelityExtractor } from "./preserved-ids.js";
export { hideConsumedCompressCalls } from "./hide-consumed.js";
export type { HideConsumedResult } from "./hide-consumed.js";
export {
Expand Down
89 changes: 89 additions & 0 deletions src/preserved-ids.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import type { CoreMessage } from "./types.js";

/** One verbatim-identifier extractor (#481): a machine label plus a global
* pattern whose matches are preserved byte-for-byte into a compression block
* appendix so they survive folding regardless of what the model summary
* writes. `pattern` must be global; exec state is cloned per call. */
export interface FidelityExtractor {
label: string;
pattern: RegExp;
}

/** Active extractors, deliberately narrow per #481 direction: only opencode
* subagent session ids for now. Commit-hash / PR-number extractors can join
* this list later — the appendix format already carries the label. */
export const DEFAULT_FIDELITY_EXTRACTORS: readonly FidelityExtractor[] = [
{ label: "subagent-session", pattern: /\bses_[A-Za-z0-9]+\b/g },
];

/** Texts that can carry dispatch-pair identifiers (#481): tool-call and
* tool-result messages only. User/assistant prose is out of scope — the
* directed fix targets the subagent dispatch pair shape, not free text. */
export function fidelitySourceTexts(messages: CoreMessage[]): string[] {
const texts: string[] = [];
for (const message of messages) {
if (!message.text) continue;
if (
message.contentType !== "tool-call" &&
message.contentType !== "tool-result"
)
continue;
texts.push(message.text);
}
return texts;
}

/** Extract deduped identifiers from folded texts, grouped by extractor label.
* First-occurrence order is preserved within each label. */
export function extractFidelityIds(
texts: Iterable<string>,
extractors: readonly FidelityExtractor[] = DEFAULT_FIDELITY_EXTRACTORS,
): Map<string, string[]> {
const out = new Map<string, string[]>();
for (const extractor of extractors) {
const pattern = new RegExp(
extractor.pattern.source,
extractor.pattern.flags.replace("g", "") + "g",
);
const seen = new Set<string>();
const ids: string[] = [];
for (const text of texts) {
if (!text) continue;
let match: RegExpExecArray | null;
while ((match = pattern.exec(text)) !== null) {
const id = match[0];
if (!seen.has(id)) {
seen.add(id);
ids.push(id);
}
if (match.index === pattern.lastIndex) pattern.lastIndex++;
}
}
if (ids.length > 0) out.set(extractor.label, ids);
}
return out;
}

function idPresentIn(text: string, id: string): boolean {
const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
return new RegExp(`\\b${escaped}\\b`).test(text);
}

/** Machine appendix lines for extracted ids missing from the model summary;
* "" when nothing was extracted or every id is already present verbatim.
* One line per label — the format stays extensible as extractors are added. */
export function buildFidelityAppendix(
summary: string,
idsByLabel: Map<string, string[]>,
): string {
if (idsByLabel.size === 0) return "";
const lines: string[] = [];
for (const [label, ids] of idsByLabel) {
const missing = ids.filter((id) => !idPresentIn(summary, id));
if (missing.length > 0) {
lines.push(`[acp-preserved:${label}] ${missing.join(", ")}`);
}
}
if (lines.length === 0) return "";
return `\n${lines.join("\n")}`;
}
Loading
Loading