From 3759b53e72f35dcbc349c4466ff7980d7ef6569f Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 17:41:27 +0800 Subject: [PATCH 01/29] kernel: reject whole compress batch when any summary fails length validation (#3003) --- kernel/package-lock.json | 2 +- kernel/package.json | 2 +- kernel/src/compress.ts | 52 +++++ kernel/tests/compress.test.ts | 15 +- .../issue3003-all-or-nothing-length.test.ts | 196 ++++++++++++++++++ kernel/tests/validation.test.ts | 9 +- package-lock.json | 2 +- 7 files changed, 269 insertions(+), 9 deletions(-) create mode 100644 kernel/tests/issue3003-all-or-nothing-length.test.ts diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 1956cd54ec..4c282670af 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.138", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/kernel/package.json b/kernel/package.json index ed0ff46830..8ac891d931 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.138", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 7257814f02..6a9db1cd0b 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -600,6 +600,58 @@ export function createCore(ports: Ports = {}): CompressionCore { } } + // #3003: all-or-nothing summary-LENGTH pre-check. Length-class failures are + // pure functions of (spec.summary, spec.summaryMaxChars, cfg) — no state and + // no cross-range coverage — so they can be screened BEFORE any block commits. + // Screening them here turns the common PARTIAL outcome (one oversized summary + // rejects while sibling ranges land and open a seam) into a clean 0-commit + // reject: nothing lands, so the model's single retry pays ONE seam instead of + // two. Content-class failures (protected-zone / turn-integrity / no-new- + // messages) DO depend on state + cross-range coverage and stay per-range below. + // Scope is EXACTLY the fold-candidate set that mutates state in the loop — + // never every requested range, else a consumed range whose summary happens to + // be long would wrongly kill a batch whose real candidates all validate. + { + const lengthFailures: string[] = []; + for (const spec of input.ranges) { + if (skipSpecs.has(spec)) continue; + const resolution = classifications.get(spec); + if (resolution === undefined) continue; + const foldsFresh = resolution.status === "ok"; + const foldsRefold = + resolution.status === "consumed" && + refoldDecisions.get(spec)?.kind === "refold"; + if (!foldsFresh && !foldsRefold) continue; + try { + validateSummaryLength(spec, input.config.compress); + } catch (error) { + lengthFailures.push( + rangeError( + spec, + error instanceof Error ? error.message : String(error), + ), + ); + } + } + if (lengthFailures.length > 0) { + // Rejected batches carry their dead-ref tombstone side effect (#2362): + // the clone differs from input.state only in deadRefs before this point, + // so returning it is safe (same contract as the minCompressRange gate). + return { + state, + result: { + blocksCreated: 0, + tokensCompressed: 0, + errors: [ + `Batch rejected before folding any range — ${lengthFailures.length} of ${input.ranges.length} requested range(s) failed summary-length validation: ${lengthFailures.join(" | ")}. Nothing was folded, so no prefix was rewritten. Fix it one way or the other and resend: shorten or split each offending range's summary to fit its cap, then resend the whole batch; or drop the offending range(s) and resend only the rest that validate.`, + ...classificationErrors, + ], + warnings: [], + }, + }; + } + } + for (const spec of input.ranges) { if (skipSpecs.has(spec)) continue; const resolution = classifications.get(spec); diff --git a/kernel/tests/compress.test.ts b/kernel/tests/compress.test.ts index 7f65c0bae5..0d8383d033 100644 --- a/kernel/tests/compress.test.ts +++ b/kernel/tests/compress.test.ts @@ -486,7 +486,7 @@ test("applyCompression reports error for unknown boundary ref", () => { assert.equal(result.result.errors.length, 1); }); -test("batch compress attributes per-range errors and keeps partial success", () => { +test("batch compress rejects the whole batch when a summary fails length validation (#3003)", () => { const core = createCore(); const state = createInitialState(); const messages = [ @@ -526,11 +526,16 @@ test("batch compress attributes per-range errors and keeps partial success", () }), }); - assert.equal(result.result.blocksCreated, 1, "valid range still compresses"); + // #3003: a length-class failure in ANY candidate rejects the WHOLE batch — + // the valid sibling no longer lands (that partial commit opened a seam that + // the retry then paid twice). All-or-nothing means zero blocks here. + assert.equal(result.result.blocksCreated, 0, "no sibling may land while one summary fails length"); + assert.equal(result.state.blocks.length, 0); assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); assert.match( result.result.errors[0]!, - /^range m00003\.\.m00004: Summary too short \(22 chars, min 50\)/, + /range m00003\.\.m00004: Summary too short \(22 chars, min 50\)/, ); }); @@ -1024,9 +1029,11 @@ test("empty summary is attributed to its range", () => { assert.equal(result.result.blocksCreated, 0); assert.equal(result.result.errors.length, 1); + // #3003: single-range length failures now surface through the all-or-nothing + // batch gate; the per-range detail is embedded verbatim in that message. assert.match( result.result.errors[0]!, - /^range m00001\.\.m00002: Summary is empty/, + /range m00001\.\.m00002: Summary is empty/, ); }); diff --git a/kernel/tests/issue3003-all-or-nothing-length.test.ts b/kernel/tests/issue3003-all-or-nothing-length.test.ts new file mode 100644 index 0000000000..e7aabb6ca9 --- /dev/null +++ b/kernel/tests/issue3003-all-or-nothing-length.test.ts @@ -0,0 +1,196 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { createInitialState } from "../src/state.js"; +import { prune } from "../src/prune.js"; +import { assignRefs } from "../src/refs.js"; +import type { Config, CoreMessage } from "../src/types.js"; + +// #3003: PARTIAL fold double-seam cost. When a multi-range compress batch has +// one length-class failure (summary over/under cap) among otherwise-valid +// folds, the old per-range commit landed the valid siblings FIRST (opening a +// seam) and the model then retried the bad range into a SECOND seam. The kernel +// now screens summary LENGTH all-or-nothing BEFORE any block commits: any +// length-class failure rejects the WHOLE batch (0 commits, 0 seams), so one +// retry pays once. Content-class failures (protected-zone / turn-integrity / +// no-new-messages) stay per-range. These tests pin both directions of the gate +// (#3617): it fires on a candidate's length failure, and it does NOT fire for +// a non-candidate range whose summary merely happens to be out of range. + +function msg( + id: string, + text: string, + role: CoreMessage["role"] = "user", +): CoreMessage { + return { id, role, contentType: "text", text }; +} + +function config(overrides: Partial = {}): Config { + return { + tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 }, + nudge: { + maxContextLimitPct: 0.55, + minContextLimitPct: 0.45, + frequency: 5, + iterationThreshold: 15, + force: "soft", + growthRatio: 0.05, + growthFloor: 6000, + growthCap: 50000, + minGrowthFloor: 5000, + minGrowthRatio: 0.45, + emergencyThresholdPct: 0.98, + }, + promotionThreshold: 5, + truncate: { threshold: 1 }, + merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 }, + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + protectedTools: [], + preserveRecentMessages: 0, + preserveRecentTokens: 0, + modelContextLimit: 100000, + ...overrides, + }; +} + +test("a too-long summary in a multi-range batch rejects the WHOLE batch (#3003)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // Range A's summary exceeds maxSummaryLength (20); range B is within range. + // Without the pre-check, B would land (seam #1) and A would retry (seam #2). + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "x".repeat(40), topic: "too long" }, + { startRef: "m00003", endRef: "m00004", summary: "ok", topic: "fine" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 0, "nothing may commit"); + assert.equal(result.state.blocks.length, 0, "no block may exist after rejection"); + assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + result.result.errors[0]!, + /range m00001\.\.m00002: Summary too long \(40 chars, max 20\)/, + ); + assert.match(result.result.errors[0]!, /Nothing was folded, so no prefix was rewritten/); + // Both actionable exits must be offered. + assert.match(result.result.errors[0]!, /shorten or split/); + assert.match(result.result.errors[0]!, /drop the offending range\(s\) and resend only the rest/); +}); + +test("a too-short summary in a multi-range batch rejects the WHOLE batch (#3003)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "ok", topic: "fine" }, + { startRef: "m00003", endRef: "m00004", summary: "x".repeat(5), topic: "too short" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 50 }, + }), + }); + + assert.equal(result.result.blocksCreated, 0, "the valid sibling must NOT land either"); + assert.equal(result.state.blocks.length, 0); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + result.result.errors[0]!, + /range m00003\.\.m00004: Summary too short \(5 chars, min 50\)/, + ); +}); + +test("a multi-range batch whose summaries all validate still commits every range (#3003 control)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "ok-a", topic: "a" }, + { startRef: "m00003", endRef: "m00004", summary: "ok-b", topic: "b" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 2, "both valid ranges commit"); + assert.equal(result.state.blocks.length, 2); + assert.equal(result.result.errors.length, 0); +}); + +test("a non-candidate range with an oversized summary does NOT reject a valid fresh fold (#3003 scoping)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [msg("a", "alpha"), msg("b", "beta")]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // m00099..m00100 are valid-format but never allocated -> status "unknown", + // never a fold candidate. Its oversized summary must be ignored by the + // all-or-nothing length screen (checking it would wrongly kill the batch). + const result = core.applyCompression({ + ranges: [ + { startRef: "m00099", endRef: "m00100", summary: "z".repeat(500) }, + { startRef: "m00001", endRef: "m00002", summary: "ok", topic: "fine" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 1, "the valid candidate still commits"); + assert.equal(result.state.blocks.length, 1); + assert.doesNotMatch( + result.result.errors.join("\n"), + /Batch rejected before folding any range/, + ); + // The unknown ref is still reported per-range (content-class, unchanged). + assert.match(result.result.errors.join("\n"), /m00099/); +}); diff --git a/kernel/tests/validation.test.ts b/kernel/tests/validation.test.ts index 09286377cd..c32cb46c5a 100644 --- a/kernel/tests/validation.test.ts +++ b/kernel/tests/validation.test.ts @@ -259,7 +259,7 @@ test("validation: all checks disabled (0s) allows any input", () => { assert.equal(result.result.errors.length, 0); }); -test("validation: errors are collected per-range in batch compress", () => { +test("validation: an empty-summary range rejects the whole batch (#3003)", () => { const core = createCore(); const messages = [ msg("a", longText), @@ -282,7 +282,12 @@ test("validation: errors are collected per-range in batch compress", () => { state, config, }); - assert.equal(result.result.blocksCreated, 1); + // #3003: a length-class failure (empty summary) in ANY candidate rejects the + // WHOLE batch — the valid sibling no longer lands. Content-class failures + // (unknown/invalid refs) remain per-range; see the invalid-refs batch test. + assert.equal(result.result.blocksCreated, 0); + assert.equal(result.state.blocks.length, 0); assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); assert.match(result.result.errors[0]!, /empty/i); }); diff --git a/package-lock.json b/package-lock.json index 101b764601..7bcf65f2a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.138", "dev": true, "license": "MIT", "devDependencies": { From 69b2ad14473d34b57126f3c1757e2592a584e3c4 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 17:42:03 +0800 Subject: [PATCH 02/29] fix: classify batch summary-length rejections in gateReason + review pins (#3003, absorbs #3670) --- .../issue3003-all-or-nothing-length.test.ts | 59 ++++++++++++++++++- src/stream.ts | 1 + 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/kernel/tests/issue3003-all-or-nothing-length.test.ts b/kernel/tests/issue3003-all-or-nothing-length.test.ts index e7aabb6ca9..6e8b942945 100644 --- a/kernel/tests/issue3003-all-or-nothing-length.test.ts +++ b/kernel/tests/issue3003-all-or-nothing-length.test.ts @@ -111,7 +111,7 @@ test("a too-short summary in a multi-range batch rejects the WHOLE batch (#3003) const result = core.applyCompression({ ranges: [ - { startRef: "m00001", endRef: "m00002", summary: "ok", topic: "fine" }, + { startRef: "m00001", endRef: "m00002", summary: "y".repeat(60), topic: "fine" }, { startRef: "m00003", endRef: "m00004", summary: "x".repeat(5), topic: "too short" }, ], messages, @@ -194,3 +194,60 @@ test("a non-candidate range with an oversized summary does NOT reject a valid fr // The unknown ref is still reported per-range (content-class, unchanged). assert.match(result.result.errors.join("\n"), /m00099/); }); + +test("a too-long refold summary rejects the WHOLE batch incl. a fresh sibling (#3003 refold direction)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // First fold m00001..m00002 into a block. + const first = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "first fold", topic: "t" }, + ], + messages, + state, + config: config(), + }); + assert.equal(first.result.blocksCreated, 1); + const blockId = first.state.blocks[0]!.blockId; + + // Restore it inline so re-requesting its span classifies as a refold candidate. + first.state.blocks.find((b) => b.blockId === blockId)!.restoredInline = true; + + // Second call: refold with an OVERSIZED new summary + a fresh valid sibling. + // Without the pre-check the fresh sibling would land first (seam #1) and the + // refold retry would pay seam #2; the gate must reject before either commits. + const second = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "z".repeat(400) }, + { startRef: "m00003", endRef: "m00004", summary: "fresh ok", topic: "sib" }, + ], + messages, + state: first.state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(second.result.blocksCreated, 0, "nothing may commit"); + assert.equal(second.state.blocks.length, 1, "only the pre-existing block remains"); + assert.match(second.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + second.result.errors[0]!, + /range m00001\.\.m00002: Summary too long \(400 chars, max 20\)/, + ); + // Refold atomicity: the restored block keeps its old summary (no partial + // in-place update may precede the batch rejection). + const b1 = second.state.blocks.find((b) => b.blockId === blockId)!; + assert.equal(b1.summary, "first fold"); +}); diff --git a/src/stream.ts b/src/stream.ts index 74a8c6bf21..079170b960 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -913,6 +913,7 @@ export function applyRanges(parsed: ReturnType, ctx: : /already compressed/.test(errs) ? "gate:already-compressed" : /requested range\(s\) resolved/.test(errs) ? "gate:none-resolved" : /too small/.test(errs) ? "gate:too-small" + : /Batch rejected before folding any range/.test(errs) ? "gate:summary-length" : "gate:other"; const revNote = revs.length > 0 ? ` Note: startId > endId in range(s) ${revs.map((rg) => `${rg.startRef}→${rg.endRef}`).join(", ")} — your refs were reversed; they were normalized to ascending order before evaluation, so check your ref order.` From fd0b29ebff54ec0cc06f46040cf6ea2266303c44 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 17:48:35 +0800 Subject: [PATCH 03/29] =?UTF-8?q?kernel:=20all-or-nothing=20batch=20when?= =?UTF-8?q?=20a=20range=20dies=20during=20application=20(#3939=20=E2=91=A3?= =?UTF-8?q?,=20closes=20#3003=20double-seam)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/src/compress.ts | 37 ++++++++++++++++++- .../issue3939-fold-boundary-pins.test.ts | 35 +++++++++++++++++- kernel/tests/protect-recent.test.ts | 14 ++++--- 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 6a9db1cd0b..e33bb300fb 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -652,6 +652,21 @@ export function createCore(ports: Ports = {}): CompressionCore { } } + // #3939 (PR-B item ④, extends #3670/#3003 to content class): the length + // pre-check above cannot screen content-class fatals (protected-zone / + // turn-integrity / livelock / empty-range) because they depend on state + // plus cross-range coverage — range N+1's eligibility may hinge on range N + // having landed. So apply the loop and keep a pristine pre-loop snapshot; + // any apply-level throw rejects the WHOLE batch with zero commits instead + // of leaving earlier siblings landed (the #3003 double-seam shape: a fold + // seam PLUS the retry seam on the model's next attempt). Classification- + // level outcomes (unknown / invalid / consumed-skip) never mutate state, + // so they stay per-range and do NOT poison the batch. + const preLoopState = cloneState(state); + const applyErrors: string[] = []; + const warningsBase = warnings.length; + const notesBase = notes.length; + for (const spec of input.ranges) { if (skipSpecs.has(spec)) continue; const resolution = classifications.get(spec); @@ -669,7 +684,7 @@ export function createCore(ports: Ports = {}): CompressionCore { }); blocksCreated += decision.blocks.length; } catch (error) { - errors.push( + applyErrors.push( rangeError( spec, error instanceof Error ? error.message : String(error), @@ -709,7 +724,7 @@ export function createCore(ports: Ports = {}): CompressionCore { tokensCompressed += outcome.tokens; warnings.push(...outcome.warnings); } catch (error) { - errors.push( + applyErrors.push( rangeError( spec, error instanceof Error ? error.message : String(error), @@ -718,6 +733,24 @@ export function createCore(ports: Ports = {}): CompressionCore { } } + if (applyErrors.length > 0) { + // Same contract as the gate paths above: the returned state differs + // from input.state only in the #2362 dead-ref tombstones recorded during + // classification. Nothing else landed, so no prefix was rewritten. + return { + state: preLoopState, + result: { + blocksCreated: 0, + tokensCompressed: 0, + errors: [ + `Batch rejected before folding any range — ${applyErrors.length} of ${input.ranges.length} requested range(s) failed during application: ${applyErrors.join(" | ")}. Nothing was folded, so no prefix was rewritten. Fix or drop the offending range(s), then resend the whole batch.`, + ...classificationErrors, + ], + warnings: warnings.slice(0, warningsBase), + }, + }; + } + state.stats.compressionCount += blocksCreated; state.stats.tokensCompressed += tokensCompressed; diff --git a/kernel/tests/issue3939-fold-boundary-pins.test.ts b/kernel/tests/issue3939-fold-boundary-pins.test.ts index 2ed635b6d7..bfd1488b6d 100644 --- a/kernel/tests/issue3939-fold-boundary-pins.test.ts +++ b/kernel/tests/issue3939-fold-boundary-pins.test.ts @@ -295,7 +295,7 @@ test("pin: already-truncated text is skipped on the next pass (marker idempotenc assert.equal(second.truncatedCount, 0); }); -test("pin: applyCompression lands front ranges even when a later range errors (non-atomic partial landing, #3003 shape)", () => { +test("pin: unknown-ref sibling errors do not poison valid ranges (classification-level partial report)", () => { const core = createCore(); const state = createInitialState(); const messages: CoreMessage[] = [ @@ -326,3 +326,36 @@ test("pin: applyCompression lands front ranges even when a later range errors (n /m00009.*above this session's highest allocated ref/, ); }); + +test("pin: a range that dies DURING application rejects the whole batch all-or-nothing (#3939 ④, closes the #3003 double-seam)", () => { + const core = createCore(); + const state = createInitialState(); + const messages: CoreMessage[] = [ + { id: "a", role: "user", contentType: "text", text: "alpha" }, + { id: "b", role: "user", contentType: "text", text: "beta" }, + { id: "c", role: "user", contentType: "text", text: "gamma" }, + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "s1", topic: "t1" }, + { startRef: "m00003", endRef: "m00003", summary: "s2", topic: "t2" }, + ], + messages, + state, + config: config(), + protectedMessageIds: new Set(["m00003"]), + }); + + assert.equal(result.result.blocksCreated, 0); + assert.equal(result.state.blocks.length, 0); + assert.equal(result.state.stats.compressionCount, 0); + assert.match( + result.result.errors[0]!, + /Batch rejected before folding any range.*entirely within the protected zone/s, + ); +}); diff --git a/kernel/tests/protect-recent.test.ts b/kernel/tests/protect-recent.test.ts index 62319940b7..3e0ebd8d4a 100644 --- a/kernel/tests/protect-recent.test.ts +++ b/kernel/tests/protect-recent.test.ts @@ -346,8 +346,12 @@ test("warnings accumulate across multiple ranges in one batch", () => { const state = seededState(messages); const cfg = config({ preserveRecentMessages: 3 }); - // Two ranges: each partially overlaps the recent zone (m00008..m00010). - // Both should produce warnings, and both unprotected heads should compress. + // Two ranges: the second now ends inside the recent zone, which under the + // #3939 all-or-nothing contract would reject the whole batch (a fully- + // protected range is a fatal apply error, pinned in issue3939-fold-boundary- + // pins.test.ts), so it instead OVERLAPS the first range's span — overlap + // warn+skip keeps the batch valid while still exercising multi-range + // warning accumulation. const result = core.applyCompression({ ranges: [ { @@ -357,9 +361,9 @@ test("warnings accumulate across multiple ranges in one batch", () => { topic: "a", }, { - startRef: "m00010", + startRef: "m00008", endRef: "m00010", - summary: "second range fully protected tail", + summary: "second range overlaps the first", topic: "b", }, ], @@ -372,7 +376,7 @@ test("warnings accumulate across multiple ranges in one batch", () => { result.result.blocksCreated >= 1, "at least the unprotected head compresses", ); - assert.ok(result.result.warnings.length >= 1, "warnings surfaced"); + assert.ok(result.result.warnings.length >= 2, "warnings surfaced"); }); // --- neverPreserveRecentTools is configurable (billion-context#1277) --- From 5d6b22abb3b01d5f17b430da4d2f27172d9ee3e0 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:22:21 +0800 Subject: [PATCH 04/29] kernel: gate failed-compress orphan rotation on committed folds (#3769) The kernel kept only the newest TWO unmatched compress call+result pairs and hid older ones on EVERY turn, decoupled from real folds. On a third failed compress this deleted the oldest pair mid-history, rewriting an earlier cached prefix at full cost with no fold behind the break. Replace the keep-two rotation with fold-gated, position-aware, persistent sweeping: orphans stay visible between folds (append-only, cache-friendly); a committed fold stamps pendingOrphanSweepMinRef (earliest new-block startRef) and the next pass hides only orphans at/after that ref, so the sweep rides the fold's own rewrite (cache-free) while earlier orphans (still-cached prefix) survive; swept ids persist in sweptOrphanCallIds (monotonic) so a resumed session never resurrects them. Bump acp-kernel 0.0.136 -> 0.0.137. --- kernel/src/compress.ts | 44 ++++- kernel/src/hide-consumed.ts | 81 +++++---- kernel/src/types.ts | 25 ++- kernel/tests/hide-consumed-mcp.test.ts | 16 +- kernel/tests/hide-merge-rebuild.test.ts | 45 +++-- kernel/tests/issue3769-orphan-sweep.test.ts | 189 ++++++++++++++++++++ kernel/tests/orphan-endpoint-snap.test.ts | 3 +- 7 files changed, 347 insertions(+), 56 deletions(-) create mode 100644 kernel/tests/issue3769-orphan-sweep.test.ts diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index e33bb300fb..83a471c16b 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -666,6 +666,10 @@ export function createCore(ports: Ports = {}): CompressionCore { const applyErrors: string[] = []; const warningsBase = warnings.length; const notesBase = notes.length; + // #3769: capture which blocks already exist so, after the loop, we can tell + // which were created by THIS fold and where it starts (for the orphan sweep). + const preFoldBlockIds = new Set(state.blocks.map((b) => b.blockId)); + for (const spec of input.ranges) { if (skipSpecs.has(spec)) continue; @@ -755,6 +759,24 @@ export function createCore(ports: Ports = {}): CompressionCore { state.stats.tokensCompressed += tokensCompressed; if (blocksCreated > 0) { + // #3769: record where THIS fold starts (min over the newly-created + // blocks' startRef) so the next processTurn sweeps orphan failures at/after + // it cache-neutrally, leaving orphans before it in the still-cached prefix. + // Kept as the running min across folds that commit before a sweep consumes it. + let foldStartRef: number | undefined; + for (const b of state.blocks) { + if (preFoldBlockIds.has(b.blockId)) continue; + const n = Number((b.startRef ?? "").replace(/\D/g, "")); + if (Number.isFinite(n) && (foldStartRef === undefined || n < foldStartRef)) { + foldStartRef = n; + } + } + if (foldStartRef !== undefined) { + state.pendingOrphanSweepMinRef = + state.pendingOrphanSweepMinRef === undefined + ? foldStartRef + : Math.min(state.pendingOrphanSweepMinRef, foldStartRef); + } // Compress succeeded: clear the growth baseline so the next turn // re-establishes it at the new (lower) token count. Without this the // nudge re-fires in a feedback loop (the §5.7 baseline-reset bug). @@ -1081,10 +1103,26 @@ const hideCompressCallsNode: PipelineNode = { io.messages, ctx.config.isCompressTool, ); + // #3769: fold the newly-swept orphans into the persistent monotonic set so + // they stay hidden across turns/restarts/upgrades, then consume the fold + // marker (one-shot) — a quiet turn leaves both untouched except clearing a + // stale marker. + const patch: Partial = { + hiddenOrphanRefs: hidden.hiddenOrphanRefs, + pendingOrphanSweepMinRef: undefined, + }; + if (hidden.newlySweptCallIds.length > 0) { + patch.sweptOrphanCallIds = [ + ...new Set([ + ...(io.state.sweptOrphanCallIds ?? []), + ...hidden.newlySweptCallIds, + ]), + ]; + } return { ...io, messages: hidden.messages, - state: { ...io.state, hiddenOrphanRefs: hidden.hiddenOrphanRefs }, + state: { ...io.state, ...patch }, }; }, }; @@ -1879,6 +1917,10 @@ function cloneState(state: CompressionState): CompressionState { hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined, + sweptOrphanCallIds: state.sweptOrphanCallIds + ? [...state.sweptOrphanCallIds] + : undefined, + pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef, deadRefs: state.deadRefs ? [...state.deadRefs] : undefined, lastPassIds: state.lastPassIds ? [...state.lastPassIds] : undefined, }; diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index 25e674605d..d2db4722a0 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -4,28 +4,42 @@ import { COMPRESS_TOOL_NAME } from "./compress-tools.js"; import type { CompressionState, CoreMessage } from "./types.js"; // Orphaned compress calls (no matching block — failed attempts, or historical -// calls whose blocks predate compressCallId recording) keep their NEWEST two -// call+result pairs visible: failures must stay observable or a deterministic -// model re-issues the same no-op compress forever, pinned at a fixed point -// (billion-context-pi issue #9: 3,849 identical calls over 5h13m under -// KEEP_LAST_ORPHANED=0). Older orphans are hidden, so the residue is bounded -// at two pairs regardless of session length — PR #18's unbounded accumulation -// does not return (its own live check showed the cap: 10 in → 6 out). -const KEEP_LAST_ORPHANED = 2; +// calls whose blocks predate compressCallId recording) stay visible until a REAL +// fold gives a cache-safe place to remove them (#3769). Failures must remain +// observable or a deterministic model re-issues the same no-op compress forever +// (billion-context-pi #9: 3,849 identical calls over 5h13m under keep=0), AND a +// quiet turn must never move the cached prefix breakpoint forward just to delete +// a few hundred chars of old failure residue — the old keep-last-two rotation did +// exactly that (hid the oldest failure on EVERY turn, seam cost far exceeding the +// bytes removed). Now: between folds nothing is swept; on a fold the node sweeps +// orphans at/after the fold's earliest start ref onto the persistent +// sweptOrphanCallIds set (they stay hidden forever), leaving orphans before it +// visible. Residue is bounded by fold cadence, not a fixed pair count. export interface HideConsumedResult { messages: CoreMessage[]; hidden: number; - /** Padded refs of the orphan compress call/result messages hidden by this - * pass (orphans beyond the keep window). Consumed by boundary resolution to - * snap a range endpoint onto a hidden-but-intact orphan (#396). */ + /** Padded refs of ALL currently-hidden orphan compress call/result messages + * (the persistent swept set, not just this pass). Consumed by boundary + * resolution to snap a range endpoint onto a hidden-but-intact orphan (#396). */ hiddenOrphanRefs: string[]; + /** Orphan call ids newly swept by THIS pass (only when a fold was pending); + * the node unions these into state.sweptOrphanCallIds. Empty on quiet turns. */ + newlySweptCallIds: string[]; } function rangeKey(startRef: string, endRef: string): string { return `${startRef}::${endRef}`; } +// Numeric position of a padded ref (mNNNNN -> NNNNN); null when unmapped/blocked +// so the caller treats it as "position unknown" and never sweeps on a guess. +function refNum(ref: string | undefined): number | null { + if (!ref || ref === BLOCKED_REF) return null; + const n = Number(ref.replace(/\D/g, "")); + return Number.isFinite(n) ? n : null; +} + // Named return type (not inline): with the object literal in the signature, // prettier's layout heuristic was not idempotent here (#416). interface ParsedCallText { @@ -240,18 +254,15 @@ export function hideConsumedCompressCalls( }, ); - const lastOrphaned = new Set(); - for ( - let i = messages.length - 1; - i >= 0 && lastOrphaned.size < KEEP_LAST_ORPHANED; - i-- - ) { - const info = infos[i]!; - const callId = messages[i]!.toolCallId; - if (info.isCompressCall && callId && info.fate === "orphan") { - lastOrphaned.add(callId); - } - } + // #3769: orphans are removed only where a real fold already moved the cached + // breakpoint. sweptSet holds orphans removed on earlier folds (persistent + + // monotonic across turns/restarts/upgrades); pendingOrphanSweepMinRef carries + // this fold's earliest start ref so we additionally sweep orphans at/after it. + // Orphans before it sit in the still-cached prefix and stay visible; with no + // fold pending nothing new is swept, so a quiet turn never breaks the prefix. + const sweptSet = new Set(state.sweptOrphanCallIds ?? []); + const foldMinRef = state.pendingOrphanSweepMinRef; + const newlySweptCallIds: string[] = []; const keepCallIds = new Set(); const hiddenCallIds = new Set(); @@ -259,14 +270,22 @@ export function hideConsumedCompressCalls( const info = infos[i]!; const callId = messages[i]!.toolCallId; if (!info.isCompressCall || !callId) continue; - if ( - info.fate === "inactive" || - (info.fate === "orphan" && !lastOrphaned.has(callId)) - ) { - hiddenCallIds.add(callId); - } else { - keepCallIds.add(callId); + let hide = false; + if (info.fate === "inactive") { + hide = true; + } else if (info.fate === "orphan") { + if (sweptSet.has(callId)) { + hide = true; + } else if (foldMinRef !== undefined) { + const num = refNum(state.messageRefs.byRaw[messages[i]!.id]); + if (num !== null && num >= foldMinRef) { + hide = true; + newlySweptCallIds.push(callId); + } + } } + if (hide) hiddenCallIds.add(callId); + else keepCallIds.add(callId); } const fateByCallId = new Map(); @@ -349,5 +368,5 @@ export function hideConsumedCompressCalls( result.push(message); } - return { messages: result, hidden, hiddenOrphanRefs }; + return { messages: result, hidden, hiddenOrphanRefs, newlySweptCallIds }; } diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 22523eeea7..3c56b1a1c2 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -240,8 +240,29 @@ export interface CompressionState { * dead-ending the whole range (#396). A drifted/dangling ref (#387) is NOT in * this set and keeps the clear "cannot be anchored" error. Optional for * pre-feature persisted states. */ - hiddenOrphanRefs?: string[]; - /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but + hiddenOrphanRefs?: string[]; + /** Persistent, MONOTONIC set of orphan compress-call ids permanently swept + * (#3769). An orphan (a failed compress call/result with no matching block) + * is only removed from the visible view when a REAL fold gives a cache-safe + * place to do it: the hide node sweeps orphans at/after the fold's earliest + * start ref onto this set, and they stay hidden forever — across the next + * turn, repeated processTurn calls, restarts and upgrades. Monotonic (never + * shrinks) so a previously-hidden record can never reappear once the client + * resends full history. Replaces the old keep-last-two rotation, which hid + * the oldest failure on EVERY quiet turn and moved the cached prefix + * breakpoint forward for no new fold. Optional: pre-feature persisted states + * lack it (nothing was permanently hidden under the old per-pass rule). */ + sweptOrphanCallIds?: string[]; + /** Numeric part (mNNNNN -> NNNNN) of the EARLIEST start ref of the most + * recent fold that has not yet been consumed by an orphan sweep (#3769). + * Written by applyCompression when it creates >=1 block (min over the newly + * created blocks' startRef); consumed + cleared by the hide-compress-calls + * node on the next processTurn, which then sweeps orphans with ref >= this + * value (orphans before it sit in the still-cached prefix and stay visible). + * Carries the position so the sweep is cache-neutral: it only removes records + * the fold already invalidated. Optional: absent until the first post-fix fold. */ + pendingOrphanSweepMinRef?: number; + /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk * history rewrite drops them outright), so no range citing them can ever diff --git a/kernel/tests/hide-consumed-mcp.test.ts b/kernel/tests/hide-consumed-mcp.test.ts index fdeb460487..0f618de67c 100644 --- a/kernel/tests/hide-consumed-mcp.test.ts +++ b/kernel/tests/hide-consumed-mcp.test.ts @@ -89,8 +89,11 @@ test("#2970 L2: active synthetic-id carrier survives beyond the orphan window", const r = hideConsumedCompressCalls(state, msgs, isMcp); const kept = new Set(r.messages.map((m) => m.toolCallId)); assert.ok(kept.has("tool_a"), "active synthetic-id carrier must survive"); - assert.ok(!kept.has("tool_b"), "oldest true orphan still capped at two"); - assert.ok(kept.has("tool_c") && kept.has("tool_d"), "newest two orphans kept"); + assert.equal(r.hidden, 0, "no fold pending -> nothing swept (#3769)"); + assert.ok( + kept.has("tool_b") && kept.has("tool_c") && kept.has("tool_d"), + "orphans remain visible until a real fold", + ); }); test("#2970 L2: inactive synthetic-id carrier is hidden (stale receipt cleanup)", () => { @@ -112,7 +115,7 @@ test("#2970 L2: inactive synthetic-id carrier is hidden (stale receipt cleanup)" assert.equal(r.hidden, 2, "call + result removed"); }); -test("#2970/#2362: mcp__bili__compress orphans keep only the newest two", () => { +test("#2970/#2362: mcp__bili__compress orphans stay visible between folds (#3769)", () => { const state = createInitialState(); const msgs: CoreMessage[] = []; for (const id of ["o1", "o2", "o3", "o4"]) { @@ -121,8 +124,11 @@ test("#2970/#2362: mcp__bili__compress orphans keep only the newest two", () => const r = hideConsumedCompressCalls(state, msgs, isMcp); const kept = new Set(r.messages.map((m) => m.toolCallId)); - assert.ok(!kept.has("o1") && !kept.has("o2"), "older orphans hidden"); - assert.ok(kept.has("o3") && kept.has("o4"), "newest two kept"); + assert.equal(r.hidden, 0, "quiet turn sweeps nothing (#3769)"); + assert.ok( + ["o1", "o2", "o3", "o4"].every((id) => kept.has(id)), + "all orphans remain visible until a real fold (#3769)", + ); }); test("#2970: active synthetic-id carrier's long summary is stubbed", () => { diff --git a/kernel/tests/hide-merge-rebuild.test.ts b/kernel/tests/hide-merge-rebuild.test.ts index 96d5e9b4a9..1cfe694e5a 100644 --- a/kernel/tests/hide-merge-rebuild.test.ts +++ b/kernel/tests/hide-merge-rebuild.test.ts @@ -27,7 +27,7 @@ function block(overrides: Partial): CompressionBlock { }; } -test("hideConsumedCompressCalls keeps active-block calls, hides consumed, keeps newest two orphans", () => { +test("hideConsumedCompressCalls keeps active-block calls and orphans until a fold (#3769)", () => { const state: CompressionState = { ...createInitialState(), blocks: [ @@ -70,20 +70,18 @@ test("hideConsumedCompressCalls keeps active-block calls, hides consumed, keeps { id: "m5", role: "user", contentType: "text", text: "hello" }, ]; const result = hideConsumedCompressCalls(state, messages); - // active-block call kept; consumed hidden; newest KEEP_LAST_ORPHANED=2 - // orphans kept visible (failure observability — billion-context-pi #9). - assert.equal(result.hidden, 1); + assert.equal(result.hidden, 0, "no fold pending -> nothing swept (#3769)"); const remainingCallIds = result.messages .filter((m) => m.toolName === "compress") .map((m) => m.toolCallId); - assert.ok(remainingCallIds.includes("call-active")); - assert.ok(!remainingCallIds.includes("call-consumed")); + assert.ok(remainingCallIds.includes("call-active"), "active carrier kept"); + assert.ok(remainingCallIds.includes("call-consumed"), "orphan stays visible between folds"); assert.ok(remainingCallIds.includes("call-orphan1")); assert.ok(remainingCallIds.includes("call-orphan2")); assert.ok(result.messages.some((m) => m.text === "hello")); }); -test("orphaned compress residue is bounded: only the newest two pairs survive any session length", () => { +test("orphan residue is swept onto a committed fold, not rotated on quiet turns (#3769)", () => { const state: CompressionState = { ...createInitialState(), blocks: [ @@ -125,22 +123,37 @@ test("orphaned compress residue is bounded: only the newest two pairs survive an text: "tail", }); + messages.forEach((m, i) => { + state.messageRefs.byRaw[m.id] = `m${String(i + 1).padStart(5, "0")}`; + }); + state.pendingOrphanSweepMinRef = 10; + const result = hideConsumedCompressCalls(state, messages); - // 12 failed pairs in → only the newest 2 pairs + the active call remain: - // PR #18's unbounded accumulation cannot recur under KEEP_LAST_ORPHANED=2. - assert.equal(result.hidden, 20); + assert.equal( + result.hidden, + 16, + "orphans at/after fold start (ref 10) swept; before it kept (#3769)", + ); + assert.deepEqual(result.newlySweptCallIds, [ + "call-fail5", + "call-fail6", + "call-fail7", + "call-fail8", + "call-fail9", + "call-fail10", + "call-fail11", + "call-fail12", + ]); const remainingCallIds = result.messages .filter((m) => m.toolName === "compress" && m.contentType === "tool-call") .map((m) => m.toolCallId); assert.deepEqual(remainingCallIds, [ "call-active", - "call-fail11", - "call-fail12", + "call-fail1", + "call-fail2", + "call-fail3", + "call-fail4", ]); - const remainingResultIds = result.messages - .filter((m) => m.contentType === "tool-result") - .map((m) => m.toolCallId); - assert.deepEqual(remainingResultIds, ["call-fail11", "call-fail12"]); }); test("a failed compress call and its result stay visible (issue #9 fixed-point precondition removed)", () => { diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts new file mode 100644 index 0000000000..4d867190d5 --- /dev/null +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -0,0 +1,189 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { assignRefs } from "../src/refs.js"; +import { hideConsumedCompressCalls } from "../src/hide-consumed.js"; +import { createInitialState } from "../src/state.js"; +import type { CompressionState, Config, CoreMessage } from "../src/types.js"; + +// #3769 regression: orphaned failed compress calls are hidden only where a committed +// fold already broke the cached prefix (at/after its earliest start ref), tracked in a +// persistent monotonic set — never rotated away on a quiet turn. These cases pin that timing. + +const isMcp = (n: string) => n === "compress" || n === "mcp__bili__compress"; + +function msg(id: string, text: string): CoreMessage { + return { id, role: "user", contentType: "text", text }; +} + +function failPair(callId: string): [CoreMessage, CoreMessage] { + return [ + { + id: `c-${callId}`, + role: "assistant", + contentType: "tool-call", + toolName: "mcp__bili__compress", + toolCallId: callId, + text: JSON.stringify({ content: [{ startId: "mA", endId: "mB", summary: "s" }] }), + }, + { + id: `r-${callId}`, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text: "nothing to compress", + }, + ]; +} + +function seqRefs(state: CompressionState, messages: CoreMessage[]): void { + messages.forEach((m, i) => { + state.messageRefs.byRaw[m.id] = `m${String(i + 1).padStart(5, "0")}`; + }); +} + +function config(): Config { + return { + tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 }, + nudge: { + maxContextLimitPct: 0.55, + minContextLimitPct: 0.45, + frequency: 5, + iterationThreshold: 15, + force: "soft", + growthRatio: 0.05, + growthFloor: 6000, + growthCap: 50000, + minGrowthFloor: 5000, + minGrowthRatio: 0.45, + emergencyThresholdPct: 0.98, + }, + promotionThreshold: 5, + truncate: { threshold: 1 }, + merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 }, + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + protectedTools: [], + preserveRecentMessages: 0, + preserveRecentTokens: 0, + modelContextLimit: 100000, + }; +} + +test("a third failed compress appended to history does NOT rewrite mid-history on a quiet turn (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early context")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid context")); + messages.push(...failPair("f2")); + messages.push(msg("p3", "newer context")); + messages.push(...failPair("f3")); + seqRefs(state, messages); + + const inbound = messages.map((m) => m.id); + const r = hideConsumedCompressCalls(state, messages, isMcp); + + assert.equal(r.hidden, 0, "quiet turn sweeps nothing"); + assert.equal(r.newlySweptCallIds.length, 0, "no fold -> nothing newly swept"); + assert.deepEqual( + r.messages.map((m) => m.id), + inbound, + "outbound ids identical to inbound: oldest failure NOT rotated away mid-history", + ); +}); + +test("a fold sweeps orphans at/after its start but keeps earlier ones in the cached prefix (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid")); + messages.push(...failPair("f2")); + messages.push(msg("p3", "late")); + messages.push(...failPair("f3")); + seqRefs(state, messages); + // refs: p1=1 f1c=2 f1r=3 p2=4 f2c=5 f2r=6 p3=7 f3c=8 f3r=9 + state.pendingOrphanSweepMinRef = 5; + + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f2", "f3"], "only orphans at/after fold start swept"); + const ids = r.messages.map((m) => m.id); + assert.ok(ids.includes("c-f1"), "orphan before fold start stays visible"); + assert.ok(!ids.includes("c-f2") && !ids.includes("c-f3"), "orphans at/after fold start removed"); +}); + +test("once swept, an orphan stays hidden on later quiet turns (persistent, restart-safe) (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid")); + messages.push(...failPair("f2")); + seqRefs(state, messages); + + state.pendingOrphanSweepMinRef = 5; + const first = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(first.newlySweptCallIds, ["f2"], "the fold sweep removes only f2"); + + state.sweptOrphanCallIds = [...first.newlySweptCallIds]; + state.pendingOrphanSweepMinRef = undefined; + + const second = hideConsumedCompressCalls(state, messages, isMcp); + assert.equal(second.newlySweptCallIds.length, 0, "quiet turn adds nothing"); + const ids = second.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f2"), "previously-swept orphan stays hidden"); + assert.ok(ids.includes("c-f1"), "unswept orphan remains visible"); +}); + +test("failures do not roll-clean per attempt: N quiet turns sweep nothing, one fold sweeps once (#3769)", () => { + const base: CoreMessage[] = [msg("p1", "early")]; + for (let t = 1; t <= 3; t++) { + const state = createInitialState(); + const messages: CoreMessage[] = [...base]; + for (let k = 1; k <= t; k++) messages.push(...failPair(`f${k}`)); + seqRefs(state, messages); + + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.equal(r.hidden, 0, `turn ${t}: no fold -> nothing swept despite ${t} failures`); + assert.equal(r.newlySweptCallIds.length, 0, `turn ${t}: no new sweep`); + } + + const state = createInitialState(); + const messages: CoreMessage[] = [...base]; + for (let k = 1; k <= 3; k++) messages.push(...failPair(`f${k}`)); + seqRefs(state, messages); + state.pendingOrphanSweepMinRef = 1; + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f1", "f2", "f3"], "one fold sweeps all eligible orphans at once"); +}); + +test("a real fold writes the sweep marker and the next pass removes the orphan behind it (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("h1", "first detailed work item alpha content"), + msg("h2", "second detailed work item beta content"), + msg("h3", "third detailed work item gamma content"), + ...failPair("f1"), + msg("h4", "fourth detailed work item delta content"), + msg("last", "latest user intent theta"), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const out = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00003", summary: "fold the spent head", topic: "head" }], + messages, + state, + config: config(), + }); + assert.equal(out.result.errors.length, 0, `unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, "the head genuinely folds"); + assert.equal(out.state.pendingOrphanSweepMinRef, 1, "marker points at the fold's earliest start ref"); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f1"], "orphan behind the fold is swept"); + assert.ok(!r.messages.some((m) => m.toolCallId === "f1"), "swept orphan removed from outbound"); +}); diff --git a/kernel/tests/orphan-endpoint-snap.test.ts b/kernel/tests/orphan-endpoint-snap.test.ts index 33aa658d20..36a2533446 100644 --- a/kernel/tests/orphan-endpoint-snap.test.ts +++ b/kernel/tests/orphan-endpoint-snap.test.ts @@ -79,10 +79,11 @@ test("hideConsumedCompressCalls records refs of hidden orphan calls only", () => existing: state.messageRefs, nextIndex: 1, }).map; + state.sweptOrphanCallIds = ["c1"]; const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 2, "oldest orphan pair is hidden"); + assert.equal(result.hidden, 2, "previously-swept orphan pair stays hidden (#3769)"); assert.deepEqual( result.hiddenOrphanRefs, ["m00001", "m00002"], From 4bc5076786c34edd8e7f653f3f598a76c22c3cee Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:23:08 +0800 Subject: [PATCH 05/29] fix: attribute proactive orphan-cleanup breaks in the cache ledger (#3769) The #3769 seam landed in the 'no fold/switch/restart attribution' bucket because the break came from bili's own cleanup of failed-compress records, not a fold. Add an orphan-sweep attribution channel to the cache ledger: detect growth in the session's sweptOrphanCallIds set per sample and, when a mid-history break has no other claimant, label it prov='orphan-sweep' (count + missed tokens) instead of leaving it unexplained. A real fold claims its own break first, so there is no double-counting. New counters normalize through getCacheLedger for persistence compatibility. --- src/cache-ledger.ts | 60 ++++++++++++++-- tests/issue3769-orphan-sweep-ledger.test.ts | 77 +++++++++++++++++++++ 2 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 tests/issue3769-orphan-sweep-ledger.test.ts diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index 8d9486886b..609b87b905 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -119,7 +119,7 @@ interface LedgerLine { * of host churn). Sparse: omitted unless booked. The seam arm sets `seam` * above instead. Lets the web trajectory chart paint the same bucket * membership the economics card aggregates. */ - prov?: "ttl" | "side" | "rewind" | "client"; + prov?: "ttl" | "side" | "rewind" | "client" | "orphan-sweep"; /** #1535: 1 iff `model` differs from the previous sample's KNOWN model * (unknown sides never flag); marks the re-billed stable prefix on the * first request after a model switch. Sparse: omitted unless set. */ @@ -206,6 +206,13 @@ interface LedgerLine { interface CacheLedger { v: 1; lastBlockId: number; + /** #3769: high-water of session.state.sweptOrphanCallIds length — detects a + * proactive failed-compress cleanup between samples so detectSeam can + * attribute a mid-history break to the cleanup instead of "unexplained". */ + lastSweptCount?: number; + /** #3769: orphans swept on the CURRENT sample (0 = none); read by detectSeam's + * orphan-sweep arm, refreshed each recordCacheSample. Transient. */ + sweepThisSample?: number; consumedFoldSeq: number; sampleSeq: number; foldSeqCounter: number; @@ -278,6 +285,11 @@ interface CacheLedger { * bili rebuild seam. Charged only when the client arm wins. */ clientInserts: number; clientInsertMissed: number; + /** #3769: mid-history breaks attributed to a proactive failed-compress + * cleanup (orphan rotation) landing in-window without a fold claiming + * the break — kept out of the unexplained seam tally. */ + orphanSweepSeams: number; + orphanSweepMissed: number; /** #2929: the provider-side arm split by idle gap — samples settled more * than PROVIDER_TTL_GAP_MS after the previous sample are normal TTL * expiry, not an abnormal provider failure. */ @@ -943,6 +955,21 @@ function detectSeam(session: Session, led: CacheLedger): void { // Inbound prefix-stable or a clean tail boundary: the outbound // mid-list break is bili-side — fall through to the seam bucket. } + // #3769: a proactive failed-compress cleanup (orphan rotation) landed in this + // window. After the fold-gated fix such a cleanup only rides along a committed + // fold, which claims its own break above via foldSeq — so reaching here means a + // mid-history break coincided with a cleanup WITHOUT any fold/switch/restart/ + // client arm claiming it. Attribute it to the cleanup rather than leave it + // "unexplained". Additive: none of the earlier arms is touched. + if ((led.sweepThisSample ?? 0) > 0) { + line.prov = "orphan-sweep"; + agg.orphanSweepSeams += 1; + agg.orphanSweepMissed += line.tr; + if (agg.orphanSweepSeams === 1) { + loggerLog("warn", `[${session.id}] [cache-seam] mid-history break attributed to proactive failed-compress cleanup (${led.sweepThisSample} record(s) rotated out, #3769): hit ${line.hitPct}% (unexplained=${Math.round(line.tr)} tok) — no fold claimed this break`); + } + return; + } agg.seamSuspects += 1; agg.seamMissed += line.tr; line.seam = 1; @@ -974,7 +1001,7 @@ export function getCacheLedger(session: Session): CacheLedger { "restartDropMissed", "attributedMissed", "unknownSamples", "unknownInput", "nbSamples", "nbInput", "seamSuspects", "seamMissed", - "providerSideMisses", "providerSideMissed", "providerTtlMisses", "providerTtlMissed", "rewinds", "rewindMissed", "abortCorrelated", "clientInserts", "clientInsertMissed", + "providerSideMisses", "providerSideMissed", "providerTtlMisses", "providerTtlMissed", "rewinds", "rewindMissed", "abortCorrelated", "clientInserts", "clientInsertMissed", "orphanSweepSeams", "orphanSweepMissed", ] as const) { if (typeof g[key] !== "number") g[key] = 0; } @@ -991,7 +1018,7 @@ export function getCacheLedger(session: Session): CacheLedger { foldSeqCounter: 0, folds: [], lines: [], - agg: { requests: 0, input: 0, cached: 0, output: 0, nc: 0, cr: 0, tr: 0, switches: 0, switchMissed: 0, effSwitches: 0, effSwitchMissed: 0, keySwitches: 0, keySwitchMissed: 0, promptSwitches: 0, promptSwitchMissed: 0, projSwitches: 0, projSwitchMissed: 0, wireSwitches: 0, wireSwitchMissed: 0, upstreamSwitches: 0, upstreamSwitchMissed: 0, restartDrops: 0, restartDropMissed: 0, attributedMissed: 0, unknownSamples: 0, unknownInput: 0, nbSamples: 0, nbInput: 0, seamSuspects: 0, seamMissed: 0, providerSideMisses: 0, providerSideMissed: 0, providerTtlMisses: 0, providerTtlMissed: 0, rewinds: 0, rewindMissed: 0, abortCorrelated: 0, clientInserts: 0, clientInsertMissed: 0 }, + agg: { requests: 0, input: 0, cached: 0, output: 0, nc: 0, cr: 0, tr: 0, switches: 0, switchMissed: 0, effSwitches: 0, effSwitchMissed: 0, keySwitches: 0, keySwitchMissed: 0, promptSwitches: 0, promptSwitchMissed: 0, projSwitches: 0, projSwitchMissed: 0, wireSwitches: 0, wireSwitchMissed: 0, upstreamSwitches: 0, upstreamSwitchMissed: 0, restartDrops: 0, restartDropMissed: 0, attributedMissed: 0, unknownSamples: 0, unknownInput: 0, nbSamples: 0, nbInput: 0, seamSuspects: 0, seamMissed: 0, providerSideMisses: 0, providerSideMissed: 0, providerTtlMisses: 0, providerTtlMissed: 0, rewinds: 0, rewindMissed: 0, abortCorrelated: 0, clientInserts: 0, clientInsertMissed: 0, orphanSweepSeams: 0, orphanSweepMissed: 0 }, }; meta[LEDGER_KEY] = led; return led; @@ -1040,6 +1067,22 @@ function detectNewFolds(session: Session, led: CacheLedger): void { } } +// #3769: sweptOrphanCallIds is monotonic, so its length high-water makes growth +// exact. detectSeam's orphan-sweep arm reads sweepThisSample set here. +function detectOrphanSweeps(session: Session, led: CacheLedger): void { + const n = session.state?.sweptOrphanCallIds?.length ?? 0; + // First observation only baselines: a resumed session's pre-existing sweeps + // must not be charged to the current sample. + if (led.lastSweptCount === undefined) { + led.lastSweptCount = n; + led.sweepThisSample = 0; + return; + } + const delta = n - led.lastSweptCount; + led.lastSweptCount = n; + led.sweepThisSample = delta > 0 ? delta : 0; +} + /** Record one provider usage report into the session ledger. `input` must be * NORMALIZED (cached included — promptInputTotal semantics). `cached === null` * means the provider reported NO cache-hit tokens (unmeasurable): such samples @@ -1051,6 +1094,7 @@ export function recordCacheSample( ): void { const led = getCacheLedger(session); detectNewFolds(session, led); + detectOrphanSweeps(session, led); const prevLine = led.lines[led.lines.length - 1]; // #1536: unknown-cache samples still need a prev/cur pair so the fold chain // stays consistent, but decompose with cached=0 then QUARANTINE every derived @@ -1707,7 +1751,7 @@ interface BodyStability { interface BiliCacheReport extends CacheReport { stability: BodyStability; /** #2131: line set widened with the per-call body-stability fields. */ - lines: Array; + lines: Array; modelSwitches: ModelSwitchStats; /** #3803: host generation-parameter switches (reasoning effort / thinking config) — a provider cache-namespace flip that re-bills the whole stable prefix once, distinct from any byte change. */ effSwitches: ModelSwitchStats; @@ -1731,7 +1775,7 @@ interface BiliCacheReport extends CacheReport { * is booked as new content, never as a prefix re-pay. */ initialBills: { samples: number; inputTokens: number }; invalidation: InvalidationTokenBreakdown; - seam: { suspects: number; missed: number; events: SeamEvent[]; providerSide: { count: number; missed: number }; providerTtl: { count: number; missed: number }; rewinds: { count: number; missed: number }; abortCorrelated: number; client: { count: number; missed: number } }; + seam: { suspects: number; missed: number; events: SeamEvent[]; providerSide: { count: number; missed: number }; providerTtl: { count: number; missed: number }; rewinds: { count: number; missed: number }; abortCorrelated: number; client: { count: number; missed: number }; orphanSweep: { count: number; missed: number } }; } export function buildSessionCacheReport(session: Session): BiliCacheReport { @@ -1918,7 +1962,7 @@ export function buildSessionCacheReport(session: Session): BiliCacheReport { initialBills: { samples: a.nbSamples, inputTokens: a.nbInput }, invalidation, stability, - seam: { suspects: a.seamSuspects, missed: a.seamMissed, events: led.seamEvents ?? [], providerSide: { count: a.providerSideMisses, missed: a.providerSideMissed }, providerTtl: { count: a.providerTtlMisses, missed: a.providerTtlMissed }, rewinds: { count: a.rewinds, missed: a.rewindMissed }, abortCorrelated: a.abortCorrelated, client: { count: a.clientInserts, missed: a.clientInsertMissed } }, + seam: { suspects: a.seamSuspects, missed: a.seamMissed, events: led.seamEvents ?? [], providerSide: { count: a.providerSideMisses, missed: a.providerSideMissed }, providerTtl: { count: a.providerTtlMisses, missed: a.providerTtlMissed }, rewinds: { count: a.rewinds, missed: a.rewindMissed }, abortCorrelated: a.abortCorrelated, client: { count: a.clientInserts, missed: a.clientInsertMissed }, orphanSweep: { count: a.orphanSweepSeams, missed: a.orphanSweepMissed } }, }; } @@ -2092,6 +2136,10 @@ function formatSeam(r: BiliCacheReport): string { if (e.client) out.push(` #${e.seq} ${fmtTime(e.at)} hit ${e.hitPct.toFixed(1)}% · input ${fmtTok(e.input)} · inbound message[${e.client.idx}] ${e.client.role}/${e.client.cls} ${e.client.len}B JSON · outbound ${fmtTok(e.lcpBytes)}B @ message[${e.msgIndex ?? "?"}] of ${e.prevMsgs}→${e.curMsgs}`); } } + if (r.seam.orphanSweep.count > 0) { + out.push("🧹 ORPHAN CLEANUP SEAM (proactive failed-compress rotation — #3769)"); + out.push(` ${r.seam.orphanSweep.count} sample(s) · ${fmtTok(r.seam.orphanSweep.missed)} tok re-billed when failed-compress records were rotated out of history without a fold claiming the break — bili's own cleanup, not an upstream or host rewrite`); + } if (r.seam.abortCorrelated > 0) { out.push("⏻ ABORT-CORRELATED"); out.push(` ${r.seam.abortCorrelated} missed sample(s) within 30s of a client mid-stream abort — abort/retry churn rewrites the resent tail; correlation, not causation (see bili.log 'client aborted mid-stream')`); diff --git a/tests/issue3769-orphan-sweep-ledger.test.ts b/tests/issue3769-orphan-sweep-ledger.test.ts new file mode 100644 index 0000000000..d6182a4c21 --- /dev/null +++ b/tests/issue3769-orphan-sweep-ledger.test.ts @@ -0,0 +1,77 @@ +// #3769: attribute a proactive failed-compress cleanup (orphan rotation) in the +// cache ledger. detectOrphanSweeps watches the monotonic length of +// session.state.sweptOrphanCallIds between samples; when a mid-history break +// otherwise lands in the unexplained seam bucket AND a cleanup grew this turn, +// the residual books into its own bucket (prov:"orphan-sweep") instead of +// "no fold/switch/restart attribution". Gated on actual growth: no growth keeps +// the legacy seam bucket, and a resumed session's pre-existing sweeps baseline +// without being charged to the first sample. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { noteForwardedBody, settleUsageReport, getCacheLedger, buildSessionCacheReport, handleAcpCache } from "../src/cache-ledger.ts"; +import type { Session } from "../src/session.ts"; + +let seq = 0; +function makeSession(): Session { + seq += 1; + return { + id: `orphan-sweep-${seq}`, + metadata: {}, + stats: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, tokenSnapshot: {} }, + } as unknown as Session; +} + +const body = (msgs: string[]): string => JSON.stringify({ model: "m", messages: msgs.map((c) => ({ role: "user", content: c })) }); +const T0 = Date.parse("2026-10-19T10:49:00Z"); + +function settle(session: Session, at: number, input: number, cached: number): void { + settleUsageReport(session, { total: input, reportedCached: cached, output: 0, protocol: "openai", upstream: "http://u" }); +} + +test("cleanup growth + unattributed break books the orphan-sweep bucket, not the seam bucket", () => { + const s = makeSession(); + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); // baseline: swept empty -> high-water 0 + // A proactive cleanup rotated two failed-compress records out of history this turn. + s.state!.sweptOrphanCallIds = ["c1", "c2"]; + noteForwardedBody(s, body(["a", "B2", "c"])); // outbound mid-history break, no inbound capture + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + const line = led.lines[led.lines.length - 1]!; + assert.equal(line.prov, "orphan-sweep", "residual booked to the cleanup"); + assert.equal(line.seam, undefined, "NOT a seam suspect"); + assert.equal(led.agg.orphanSweepSeams, 1); + assert.ok(led.agg.orphanSweepMissed > 0); + assert.equal(led.agg.seamSuspects, 0, "kept out of the unexplained tally"); + const report = buildSessionCacheReport(s); + assert.equal(report.seam.orphanSweep.count, 1); + assert.ok(report.seam.orphanSweep.missed > 0); + assert.match(handleAcpCache(s).text, /ORPHAN CLEANUP SEAM/); +}); + +test("same break WITHOUT cleanup growth keeps the legacy seam bucket (arm is gated on growth)", () => { + const s = makeSession(); + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); + noteForwardedBody(s, body(["a", "B2", "c"])); // identical break, but nothing was swept + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + const line = led.lines[led.lines.length - 1]!; + assert.equal(line.seam, 1, "legacy seam bucket preserved"); + assert.equal(line.prov, undefined); + assert.equal(led.agg.orphanSweepSeams, 0); + assert.equal(led.agg.seamSuspects, 1); +}); + +test("a resumed session's pre-existing sweeps baseline without charging the first sample", () => { + const s = makeSession(); + s.state!.sweptOrphanCallIds = ["c1"]; // already swept before this process started + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); // first observation baselines high-water at 1 + noteForwardedBody(s, body(["a", "B2", "c"])); // no NEW growth since baseline + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + assert.equal(led.agg.orphanSweepSeams, 0, "pre-existing sweeps not charged to this sample"); + assert.equal(led.agg.seamSuspects, 1, "falls back to the legacy seam bucket"); +}); From df3b259cf2edf8b5dd04c80549e09576505c1b63 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:57:28 +0800 Subject: [PATCH 06/29] kernel: normalize indentation of #3769 edits (whitespace only) types.ts and two kernel test files sit on a prettier-clean master baseline; the #3769 additions landed over-indented by one space. Normalized to the surrounding 2-space style so the diff stays confined to the feature. No behavior change; typecheck + targeted kernel/host suites re-run green. --- kernel/src/types.ts | 46 +++++++++++------------ kernel/tests/hide-merge-rebuild.test.ts | 5 ++- kernel/tests/orphan-endpoint-snap.test.ts | 6 ++- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 3c56b1a1c2..91f98dd40b 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -240,29 +240,29 @@ export interface CompressionState { * dead-ending the whole range (#396). A drifted/dangling ref (#387) is NOT in * this set and keeps the clear "cannot be anchored" error. Optional for * pre-feature persisted states. */ - hiddenOrphanRefs?: string[]; - /** Persistent, MONOTONIC set of orphan compress-call ids permanently swept - * (#3769). An orphan (a failed compress call/result with no matching block) - * is only removed from the visible view when a REAL fold gives a cache-safe - * place to do it: the hide node sweeps orphans at/after the fold's earliest - * start ref onto this set, and they stay hidden forever — across the next - * turn, repeated processTurn calls, restarts and upgrades. Monotonic (never - * shrinks) so a previously-hidden record can never reappear once the client - * resends full history. Replaces the old keep-last-two rotation, which hid - * the oldest failure on EVERY quiet turn and moved the cached prefix - * breakpoint forward for no new fold. Optional: pre-feature persisted states - * lack it (nothing was permanently hidden under the old per-pass rule). */ - sweptOrphanCallIds?: string[]; - /** Numeric part (mNNNNN -> NNNNN) of the EARLIEST start ref of the most - * recent fold that has not yet been consumed by an orphan sweep (#3769). - * Written by applyCompression when it creates >=1 block (min over the newly - * created blocks' startRef); consumed + cleared by the hide-compress-calls - * node on the next processTurn, which then sweeps orphans with ref >= this - * value (orphans before it sit in the still-cached prefix and stay visible). - * Carries the position so the sweep is cache-neutral: it only removes records - * the fold already invalidated. Optional: absent until the first post-fix fold. */ - pendingOrphanSweepMinRef?: number; - /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but + hiddenOrphanRefs?: string[]; + /** Persistent, MONOTONIC set of orphan compress-call ids permanently swept + * (#3769). An orphan (a failed compress call/result with no matching block) + * is only removed from the visible view when a REAL fold gives a cache-safe + * place to do it: the hide node sweeps orphans at/after the fold's earliest + * start ref onto this set, and they stay hidden forever — across the next + * turn, repeated processTurn calls, restarts and upgrades. Monotonic (never + * shrinks) so a previously-hidden record can never reappear once the client + * resends full history. Replaces the old keep-last-two rotation, which hid + * the oldest failure on EVERY quiet turn and moved the cached prefix + * breakpoint forward for no new fold. Optional: pre-feature persisted states + * lack it (nothing was permanently hidden under the old per-pass rule). */ + sweptOrphanCallIds?: string[]; + /** Numeric part (mNNNNN -> NNNNN) of the EARLIEST start ref of the most + * recent fold that has not yet been consumed by an orphan sweep (#3769). + * Written by applyCompression when it creates >=1 block (min over the newly + * created blocks' startRef); consumed + cleared by the hide-compress-calls + * node on the next processTurn, which then sweeps orphans with ref >= this + * value (orphans before it sit in the still-cached prefix and stay visible). + * Carries the position so the sweep is cache-neutral: it only removes records + * the fold already invalidated. Optional: absent until the first post-fix fold. */ + pendingOrphanSweepMinRef?: number; + /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk * history rewrite drops them outright), so no range citing them can ever diff --git a/kernel/tests/hide-merge-rebuild.test.ts b/kernel/tests/hide-merge-rebuild.test.ts index 1cfe694e5a..f0a464dedc 100644 --- a/kernel/tests/hide-merge-rebuild.test.ts +++ b/kernel/tests/hide-merge-rebuild.test.ts @@ -75,7 +75,10 @@ test("hideConsumedCompressCalls keeps active-block calls and orphans until a fol .filter((m) => m.toolName === "compress") .map((m) => m.toolCallId); assert.ok(remainingCallIds.includes("call-active"), "active carrier kept"); - assert.ok(remainingCallIds.includes("call-consumed"), "orphan stays visible between folds"); + assert.ok( + remainingCallIds.includes("call-consumed"), + "orphan stays visible between folds", + ); assert.ok(remainingCallIds.includes("call-orphan1")); assert.ok(remainingCallIds.includes("call-orphan2")); assert.ok(result.messages.some((m) => m.text === "hello")); diff --git a/kernel/tests/orphan-endpoint-snap.test.ts b/kernel/tests/orphan-endpoint-snap.test.ts index 36a2533446..3bbb72c8d4 100644 --- a/kernel/tests/orphan-endpoint-snap.test.ts +++ b/kernel/tests/orphan-endpoint-snap.test.ts @@ -83,7 +83,11 @@ test("hideConsumedCompressCalls records refs of hidden orphan calls only", () => const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 2, "previously-swept orphan pair stays hidden (#3769)"); + assert.equal( + result.hidden, + 2, + "previously-swept orphan pair stays hidden (#3769)", + ); assert.deepEqual( result.hiddenOrphanRefs, ["m00001", "m00002"], From ce6c244ed2982e312376174270cfc854195303ee Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 16:48:20 +0800 Subject: [PATCH 07/29] kernel: derive orphan-sweep marker from covered message refs (#3769 review fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found a defect in the fold-marker computation: block-boundary specs (tier distillation compresses bN..bM) store startRef="bN" verbatim on the new block, so Number(startRef.replace(/\D/g,"")) parsed a BLOCK number, not the earliest covered MESSAGE ref. In long sessions that marks the sweep far too early: the next hide pass deletes failed pairs sitting in the still-cached prefix before the fold's own divergence point — re-introducing exactly the #3769 mid-history cache break on every tier-distillation fold. Compute foldStartRef as the min refNum over each new block's effectiveMessageIds via state.messageRefs.byRaw instead; ids missing from the map (pruned) contribute nothing, which only under-sweeps (safe direction). refNum is exported from hide-consumed.ts so both consumers share one "unknown" definition. Regression test 6 in issue3769-orphan-sweep.test.ts: m-boundary fold then block-boundary fold asserts pendingOrphanSweepMinRef === first covered ref (not the block number) and that the earlier pair stays visible. Verified to fail against the pre-fix code. --- kernel/src/compress.ts | 24 +++++++---- kernel/src/hide-consumed.ts | 4 +- kernel/tests/issue3769-orphan-sweep.test.ts | 45 +++++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 83a471c16b..2aea1377b2 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -14,7 +14,7 @@ import { import type { ResolvedRange } from "./boundaries.js"; import { truncateLargeToolOutputs } from "./truncate-tools.js"; import { scrubLoneSurrogates } from "./truncate.js"; -import { hideConsumedCompressCalls } from "./hide-consumed.js"; +import { hideConsumedCompressCalls, refNum } from "./hide-consumed.js"; import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyCrushToMessages } from "./crush.js"; import { applyRetrieve, ccrStoreNode, RETRIEVED_ID_PREFIX } from "./ccr.js"; @@ -759,16 +759,24 @@ export function createCore(ports: Ports = {}): CompressionCore { state.stats.tokensCompressed += tokensCompressed; if (blocksCreated > 0) { - // #3769: record where THIS fold starts (min over the newly-created - // blocks' startRef) so the next processTurn sweeps orphan failures at/after - // it cache-neutrally, leaving orphans before it in the still-cached prefix. - // Kept as the running min across folds that commit before a sweep consumes it. + // #3769: record where THIS fold starts so the next processTurn sweeps + // orphan failures at/after it cache-neutrally, leaving orphans before it + // in the still-cached prefix. Kept as the running min across folds that + // commit before a sweep consumes it. The position comes from the covered + // message refs of each new block, NOT its startRef string: a block-boundary + // spec stores startRef="bN", whose digits are a BLOCK number, not a message + // position — parsing it would mark the sweep far too early and rewrite the + // still-cached prefix (#3769 again). refNum returns null for unmapped/blocked + // ids, so a fully-pruned coverage simply sets no marker (under-sweep is safe; + // over-sweep is not). let foldStartRef: number | undefined; for (const b of state.blocks) { if (preFoldBlockIds.has(b.blockId)) continue; - const n = Number((b.startRef ?? "").replace(/\D/g, "")); - if (Number.isFinite(n) && (foldStartRef === undefined || n < foldStartRef)) { - foldStartRef = n; + for (const id of b.effectiveMessageIds) { + const num = refNum(state.messageRefs.byRaw[id]); + if (num !== null && (foldStartRef === undefined || num < foldStartRef)) { + foldStartRef = num; + } } } if (foldStartRef !== undefined) { diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index d2db4722a0..69819675db 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -34,7 +34,9 @@ function rangeKey(startRef: string, endRef: string): string { // Numeric position of a padded ref (mNNNNN -> NNNNN); null when unmapped/blocked // so the caller treats it as "position unknown" and never sweeps on a guess. -function refNum(ref: string | undefined): number | null { +// Exported: applyCompression derives the fold sweep marker from covered message +// refs through the same helper so both sides agree on "unknown" (#3769). +export function refNum(ref: string | undefined): number | null { if (!ref || ref === BLOCKED_REF) return null; const n = Number(ref.replace(/\D/g, "")); return Number.isFinite(n) ? n : null; diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts index 4d867190d5..abf97737f3 100644 --- a/kernel/tests/issue3769-orphan-sweep.test.ts +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -187,3 +187,48 @@ test("a real fold writes the sweep marker and the next pass removes the orphan b assert.deepEqual(r.newlySweptCallIds, ["f1"], "orphan behind the fold is swept"); assert.ok(!r.messages.some((m) => m.toolCallId === "f1"), "swept orphan removed from outbound"); }); + +test("a block-boundary fold marks the sweep by covered message refs, not the block number (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("p1", "early cached prefix alpha"), + ...failPair("f1"), + msg("p2", "mid detailed work item beta content"), + msg("p3", "mid detailed work item gamma content"), + msg("p4", "mid detailed work item delta content"), + msg("p5", "mid detailed work item epsilon content"), + msg("p6", "latest user intent theta"), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + // refs: p1=1 c-f1=2 r-f1=3 p2=4 p3=5 p4=6 p5=7 p6=8 + + const first = core.applyCompression({ + ranges: [{ startRef: "m00004", endRef: "m00005", summary: "fold the mid pair", topic: "mid" }], + messages, + state, + config: config(), + }); + assert.equal(first.result.errors.length, 0, `unexpected errors: ${first.result.errors.join("; ")}`); + assert.ok(first.result.blocksCreated >= 1, "the mid pair folds"); + const blockId = first.state.blocks[first.state.blocks.length - 1].blockId; + + const out = core.applyCompression({ + ranges: [{ startRef: blockId, endRef: "m00006", summary: "distill the block plus its tail", topic: "distill" }], + messages, + state: first.state, + config: config(), + }); + assert.equal(out.result.errors.length, 0, `unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, "the block-boundary range folds"); + // Parsing the new block's startRef string ("b1") yields the BLOCK number 1, which + // would sweep the cached-prefix orphan f1 (ref 2). Covered-ref derivation must say 4. + assert.equal(out.state.pendingOrphanSweepMinRef, 4, "marker sits at the earliest covered message ref"); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, [], "cached-prefix orphan before the fold is untouched"); + assert.ok(r.messages.some((m) => m.toolCallId === "f1"), "f1 remains visible"); +}); From 6544bf3fd96a0365cd8e5a64e66dd3b75dfd9ba8 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 16:48:30 +0800 Subject: [PATCH 08/29] fix: hydrate orphan-sweep state fields across restart (#3769 review fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the two new kernel state fields were dropped on every session load: buildRecord persists session.state wholesale, but the load path rebuilds it through mergeState's explicit field whitelist, which predated sweptOrphanCallIds / pendingOrphanSweepMinRef. Losing sweptOrphanCallIds resurrects already-cleaned failed pairs after a restart — a mid-history rewrite (cache break) that violates the PR's own persistence acceptance case. Losing the pending marker only defers a sweep by one fold (conservative). Extend the mergeState whitelist with both fields, degrading to undefined for old persisted states exactly like every other optional field. --- src/persist.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/persist.ts b/src/persist.ts index 56d4ef9130..9652ce41c8 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -238,6 +238,12 @@ function mergeState(parsed: CompressionState): CompressionState { // resurrects with absorbed=[] and hideAbsorbedMessages has nothing to hide. absorbed: parsed.absorbed ?? fresh.absorbed, rules: parsed.rules ?? fresh.rules, + // #3769: fold-gated orphan-sweep bookkeeping must survive hydration — + // dropping sweptOrphanCallIds resurrects already-cleaned pairs after a + // restart (mid-history rewrite = cache break); the pending marker is + // transient, losing it only defers the sweep to the next fold. + sweptOrphanCallIds: parsed.sweptOrphanCallIds ?? fresh.sweptOrphanCallIds, + pendingOrphanSweepMinRef: parsed.pendingOrphanSweepMinRef ?? fresh.pendingOrphanSweepMinRef, }; } From 4aaf6ee8a31f139467cac18295e43ff2dfd446f7 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 16:48:47 +0800 Subject: [PATCH 09/29] test: align #2749/#1567 fixtures and storage-render goldens with fold-gated sweeps (#3769) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three stale pins broke under the approved keep-two -> fold-gated behavior change: - stub-resubmission-guard (#2749): the fixture applied folds without a call id, unlike production (src/loop/core.ts passes callId), so the resent call classified as orphan and the new sweep hid it. Pass "call-b1" at each first fold; all summary-integrity assertions unchanged. 5/5 green. - cache-friendly-plugin: the two anthropic geometry subtests pinned the old KEEP_LAST_ORPHANED=2 rotation (oldest pair pruned on later turns). Rewritten to pin the new contract: failed pairs stay visible between folds, every anchor stays stripped while its own pair rides, quiet turns append-stable. A live-server probe plus the passing drop-pair/main-matrix subtests confirm the geometry expected values. Header docs updated to match the new mechanism. 11/11 green. - storage-render goldens: regenerated via scripts/update-storage-render-goldens.ts. Diff is additive only — the persisted envelope gained lastSweptCount (cache ledger) and the two new kernel state keys (empty digests); state/blockContents/stats section digests unchanged. --- tests/cache-friendly-plugin.test.ts | 69 +++++++++++----------- tests/golden/storage-render/anthropic.json | 8 ++- tests/golden/storage-render/chat.json | 8 ++- tests/golden/storage-render/google.json | 8 ++- tests/golden/storage-render/responses.json | 8 ++- tests/stub-resubmission-guard.test.ts | 13 ++-- 6 files changed, 64 insertions(+), 50 deletions(-) diff --git a/tests/cache-friendly-plugin.test.ts b/tests/cache-friendly-plugin.test.ts index 11d9927ebe..18bf5496ee 100644 --- a/tests/cache-friendly-plugin.test.ts +++ b/tests/cache-friendly-plugin.test.ts @@ -11,9 +11,10 @@ // therefore re-injects the in-place acp_summary carrier from block state on // every request. Since the #1567 fix the carrier is STRIPPED per-block, but // self-verified: only while that exact fold's compress pair actually rides -// the (post-prepare) inbound history. A pair pruned client-side or hidden by -// the kernel (KEEP_LAST_ORPHANED) keeps the anchor, so an active fold never -// has zero carriers. This suite pins that behavior byte-for-byte. +// the (post-prepare) inbound history. A pair pruned client-side or swept by +// the kernel's fold-gated orphan cleanup (#3769) keeps the anchor, so an +// active fold never has zero carriers. This suite pins that behavior +// byte-for-byte. // // Invariants pinned per wire (bodies = consecutive upstream-bound request // payloads of one session, compared as flat unit lists — text blocks and @@ -39,13 +40,13 @@ // fold spec echoed in the re-sent tool_call args + the tail // quote inside the persistently re-sent tool result) and ZERO // carrier markers (#1567 fix); -// P2c fail-safe (#1567 hardening): the anchor re-carries the summary -// whenever the pair does NOT ride inbound — the client stops -// re-sending it, or the kernel hid it (KEEP_LAST_ORPHANED) — -// pinned by the two hardening subtests at the bottom of this file; -// P3 multi-fold: two folds in one session; both pairs stay visible -// (≤ KEEP_LAST_ORPHANED) and the pair-quote counts stay constant across -// all later turns; + // P2c fail-safe (#1567 hardening): the anchor re-carries the summary + // whenever the pair does NOT ride inbound — the client stops + // re-sending it, or the kernel swept it on a later fold (#3769) — + // pinned by the hardening subtests at the bottom of this file; + // P3 multi-fold: two folds in one session; both pairs stay visible for the + // rest of the session (fold-gated sweeps, #3769 — keep-two rotation is + // gone) and the pair-quote counts stay constant across all later turns; // P4 no transient acp_loop_* artifact leaks into any forwarded body // (plugin mode has none by construction — pin it); // P5 marker/tag hygiene: every content text unit carries exactly one @@ -672,10 +673,10 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr const gamma = FOLD_E.summary; for (const i of [5, 6, 7]) { const raw = cans[i]!.raw; - // carrier is superseded on every turn; the two head/tail quotes ride - // the re-sent geo_a tool result until that pair is pruned at t7 - const expA = i === 5 ? 2 : 0; - assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA}`); + // carrier is superseded on every turn; the geo_a pair quotes ride + // every later turn — fold-gated sweeps (#3769) replaced keep-two + // rotation, so no later fold prunes this oldest pair here + assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != 2`); assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: carriers must be stripped while their pairs ride inbound (got ${countOcc(raw, SUM_MARKER)})`); assert.ok(!raw.includes("q2-marker"), `${L(i)}: swallowed content q2 must stay hidden`); assert.ok(!raw.includes("a2-marker"), `${L(i)}: swallowed content a2 must stay hidden`); @@ -694,8 +695,10 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assert.equal(countOcc(cans[7]!.raw, gamma), 2, `${L(7)}: resent gamma quotes must survive`); for (const i of [6, 7]) { const raw = cans[i]!.raw; - assert.ok(raw.includes("\"geo_wide\"") && raw.includes("\"geo_e\""), `${L(i)}: latest two pairs must stay visible`); - assert.ok(!raw.includes("\"geo_a\""), `${L(i)}: oldest pair must be pruned (KEEP_LAST_ORPHANED=2)`); + // all three failed pairs stay visible: fold-gated sweeps (#3769) only + // remove orphans at/after a committed fold's start, never rotate the + // newest failures away between folds + assert.ok(raw.includes("\"geo_a\"") && raw.includes("\"geo_wide\"") && raw.includes("\"geo_e\""), `${L(i)}: all three pairs must stay visible (no keep-two rotation)`); } // #1567 (fixed): no text unit carries the wide summary anymore (stripped // anchor); the quotes ride structured tool units, not text @@ -751,11 +754,14 @@ testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pai assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); -// #1567 hardening scenario B: three concurrent ACTIVE folds. The kernel hides -// consumed compress pairs beyond the newest KEEP_LAST_ORPHANED=2, so the -// oldest pair leaves the wire while its block is still active — the strip -// guard must see it gone and KEEP that anchor (kernel-side fail-safe). -testCompat(`cache-friendly plugin matrix: anthropic — 3rd fold prunes oldest pair → oldest anchor kept (#1567 hardening)`, async () => { +// #3769 contract pin (plugin lane): three concurrent ACTIVE folds. The old +// KEEP_LAST_ORPHANED=2 rotation hid the oldest failed pair on EVERY turn — +// a mid-history rewrite with no fold behind it, exactly #3769's cache seam. +// Fold-gated sweeps replaced it: failed pairs stay visible between folds, so +// every anchor stays stripped (its own pair rides) and later turns remain +// append-stable. The zero-carrier fail-safe itself (pair gone → anchor +// re-carries) is pinned by the drop-pair subtest above. +testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds keep all failed pairs riding (fold-gated sweeps, #3769)`, async () => { const SUM_P1 = "Prune-safe fold one summary covering m00001..m00002"; const SUM_P2 = "Prune-safe fold two summary covering m00003..m00004"; const SUM_P3 = "Prune-safe fold three summary covering m00005..m00006"; @@ -779,25 +785,22 @@ testCompat(`cache-friendly plugin matrix: anthropic — 3rd fold prunes oldest p assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-prune"); - // t3/t4: pairs ride (≤ 2) → anchors stripped. From t5 on, three pairs - // coexist and the kernel hides the oldest (KEEP_LAST_ORPHANED=2) — the - // guard sees it gone and keeps that anchor (kernel-side fail-safe) + // t3/t4: each fold's pair rides from its own turn on → anchors stripped for (const i of [2, 3]) { assert.equal(countOcc(cans[i]!.raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while pairs ride`); } assert.ok(cans[3]!.raw.includes('"fs1"') && cans[3]!.raw.includes('"fs2"'), `${L(3)}: both pairs visible`); - // t5..t7: fs1 hidden by KEEP_LAST_ORPHANED → anchor F1 re-carries; - // F2/F3 anchors stay stripped while their pairs ride + // t5..t7: three failed pairs coexist and ALL stay visible (no keep-two + // rotation, #3769) → every anchor stays stripped, no carrier needed for (const i of [4, 5, 6]) { const raw = cans[i]!.raw; - assert.ok(!raw.includes('"fs1"'), `${L(i)}: oldest pair pruned (KEEP_LAST_ORPHANED=2)`); - assert.ok(raw.includes('"fs2"') && raw.includes('"fs3"'), `${L(i)}: newest two pairs stay visible`); - assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(i)}: exactly one carrier (F1 anchor re-carried)`); - assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: SUM_P1 = carrier only (its pair quotes pruned with the pair)`); - assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: SUM_P2 quotes only`); - assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: SUM_P3 quotes only`); + assert.ok(raw.includes('"fs1"') && raw.includes('"fs2"') && raw.includes('"fs3"'), `${L(i)}: all three failed pairs must stay visible (fold-gated sweeps, #3769)`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: no carrier needed — every anchor's pair rides inbound`); + assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: SUM_P1 pair quotes only`); + assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: SUM_P2 pair quotes only`); + assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: SUM_P3 pair quotes only`); } - // once the oldest anchor re-carries, later turns are append-stable + // mid-history never mutates between quiet turns assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q8", "a8"]); }, { timeout: 120_000 }); diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index c565999118..934a26087d 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "1c7b7922230de1bc61ce9b909a9fbfd4a9a28e478d12178568d21e97946760b5", + "stateDigest": "f894d02089eb22567a0e5d332ab7c12153d2a3d92a37140f8fc3d4b3d3cfcc19", "sectionDigests": { "state": "a4b83f873d16bfc4", "blockContents": "dc7d13d250c94bb4", "stats": "cff3c2546898aeb3", - "metadata": "b6fe321d2ad536b0" + "metadata": "a10cb71754abe91e" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", @@ -101,7 +101,9 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f" }, "stateShape": { "refs": 49, diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 6ed76a04ec..816c8816a1 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "995db4a4701baf4b72504dedc8842a79fc17dc9bcc535e4367c8a87e732616dd", + "stateDigest": "4731e0ab9374388662bdc5ef54386d22cf446b71712d379cdbbdd530805d10cc", "sectionDigests": { "state": "463b3dc26a100c2a", "blockContents": "94a58f96d7c5cfc3", "stats": "59842ce5415d3a4d", - "metadata": "348ced08c027b22b" + "metadata": "0f57b6cba74a16d8" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", @@ -101,7 +101,9 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 19ddaf90da..2fd2f8942f 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "32b187839fe05ec69aeebade3bb8aad18587ec7018a1816df7c50471dc8a3187", + "stateDigest": "5b8b7f10031808c1cfcc6161a6c3881d485d4016f435e27ee89bd015a296fee4", "sectionDigests": { "state": "a9b9531d952b48d8", "blockContents": "473264d9df8ba338", "stats": "e358dc992395fc7e", - "metadata": "124f318e161ebf7f" + "metadata": "bf873598b93525e7" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", @@ -101,7 +101,9 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 2dd5340f2e..e8897cc195 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "96f5faf2e54f7d9257f4f423e4abfc089bd90140a1970f862061685707c6b6fa", + "stateDigest": "7013217e0a5669e3a4ddb8f3e02f70a4dad08cfe5d10170a962ff3f2fa8db3d7", "sectionDigests": { "state": "d7951fa36f9e6fe9", "blockContents": "b6701268a0bdcd8a", "stats": "67e33bcc612fdcc1", - "metadata": "ac1c297bf3b603eb" + "metadata": "f3bd0c932cc616b0" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", @@ -101,7 +101,9 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f" }, "stateShape": { "refs": 55, diff --git a/tests/stub-resubmission-guard.test.ts b/tests/stub-resubmission-guard.test.ts index a1864162d6..42c5e2a097 100644 --- a/tests/stub-resubmission-guard.test.ts +++ b/tests/stub-resubmission-guard.test.ts @@ -124,7 +124,10 @@ test("#2749 canonical integrity pin: full summary survives save → history redu ["user", "Follow-up questions about the configuration."], ["assistant", "y".repeat(3000)], ]); - const out = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00004", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + // The fold is linked to its tool-call id exactly as production does + // (src/loop/core.ts passes callId); the resent "call-b1" below then + // classifies active instead of orphan. + const out = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00004", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(out.startsWith("[Compressed m00001–m00004 → 1 block(s)"), out.split("\n")[0]); // (a) saved canonically in full — no trimming, no truncation. assert.equal(storedSummary(ctx.session.state, "b1"), LONG_SUMMARY); @@ -155,7 +158,7 @@ test("#2749 rejects a byte-identical stub re-submission; state unchanged, refusa ["user", "late part two"], ["assistant", "d".repeat(1000)], ["user", "late part three"], ["assistant", "e".repeat(1000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); // Next request: the model sees its own call reduced to the stub. ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); @@ -181,7 +184,7 @@ test("#2749 self-refold under the block's own stub is blocked (silent-shrink wor ["user", "setup steps discussed early"], ["assistant", "a".repeat(4000)], ["user", "more detail on the setup"], ["assistant", "b".repeat(4000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); // Re-fold the SAME span submitting the block's own stub as the new summary. @@ -204,7 +207,7 @@ test("#2749 legitimate summaries still pass: short, ellipsis-prose, and ±1-char ["user", "late four"], ["assistant", "f".repeat(4000)], ["user", "late five"], ["assistant", "g".repeat(4000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); @@ -234,7 +237,7 @@ test("#2749 host harvest tracks kernel-emitted stubs (format-drift tripwire)", ( seedTurn(ctx, [ ["user", "early part"], ["assistant", "a".repeat(1000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed"), ok.split("\n")[0]); const view = nextRequestView(ctx, "call-b1", "m00001", "m00002"); const visCall = view.find((m) => m.toolCallId === "call-b1")!; From 9488e484f7e6b26382caa010cbac76b6e8a5eb83 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:16 +0800 Subject: [PATCH 10/29] kernel: honor legacy hiddenOrphanRefs on upgrade (#3769 P0) Pre-#3769 states persisted the keep-two rotation's hidden set as message refs (hiddenOrphanRefs); the new hide pass ignored it, so upgrading would resurrect rotation-hidden failed pairs mid-history and break the cached prefix again. Honor the legacy set on upgrade; rememberHiddenRef re-records those refs every pass, so the clause self-perpetuates without any state migration. --- kernel/src/hide-consumed.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index 69819675db..888469bb32 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -276,10 +276,18 @@ export function hideConsumedCompressCalls( if (info.fate === "inactive") { hide = true; } else if (info.fate === "orphan") { + const ref = state.messageRefs.byRaw[messages[i]!.id]; if (sweptSet.has(callId)) { hide = true; + } else if (ref && state.hiddenOrphanRefs?.includes(ref)) { + // Pre-#3769 states persisted the rotation's hidden set as message refs + // (hiddenOrphanRefs). Honor it on upgrade: resurrecting a pair the old + // kernel already removed would re-insert it mid-history and break the + // cached prefix again. The set self-perpetuates — rememberHiddenRef + // re-records these refs on every pass they stay hidden. + hide = true; } else if (foldMinRef !== undefined) { - const num = refNum(state.messageRefs.byRaw[messages[i]!.id]); + const num = refNum(ref); if (num !== null && num >= foldMinRef) { hide = true; newlySweptCallIds.push(callId); From 3eaf0cde01895c88fffcf6d3b7492cdb2e55a5d6 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:16 +0800 Subject: [PATCH 11/29] kernel: carry orphan-cleanup fields through sync-blocks (#3769 P0) syncBlocks deep-clones CompressionState with an explicit field list that missed hiddenOrphanRefs / sweptOrphanCallIds / pendingOrphanSweepMinRef. It runs BEFORE the hide-compress-calls node in every processTurn, so all three ledgers were wiped each turn: the fold-gated sweep never fired end-to-end and the legacy upgrade-compat clause was unreachable. Carry them exactly like cloneState does (lastPassIds stays excluded: its only reader runs pre-sync and the field is re-stamped at processTurn exit). --- kernel/src/sync.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index 76dc42c259..9a12073f3e 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -46,6 +46,13 @@ export function syncBlocks( nextBlockId: state.nextBlockId, nextRunId: state.nextRunId, deadRefs: state.deadRefs ? [...state.deadRefs] : undefined, + // #3769 orphan-cleanup ledgers — same carry set as cloneState: dropping any + // of these here would wipe them before the hide-compress-calls node reads + // them (this node runs first), silently disabling both the fold-gated sweep + // and the legacy hiddenOrphanRefs upgrade-compat clause every turn. + hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined, + sweptOrphanCallIds: state.sweptOrphanCallIds ? [...state.sweptOrphanCallIds] : undefined, + pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef, }; // Refs are additive (assignRefs never removes them from messageRefs), so From a8d24aa8a37052f74bdbf1e96f296b4299bcb158 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:38 +0800 Subject: [PATCH 12/29] kernel: regression tests for upgrade compat and pipeline carry (#3769 P0) Two processTurn-level pins: an upgraded pre-fold-gate state keeps its rotation-hidden orphan hidden across quiet turns (legacy hiddenOrphanRefs clause), and a fold marker set by applyCompression survives the full pipeline to the hide node (sync-blocks carry) so the fold-gated sweep actually fires end-to-end. --- kernel/tests/issue3769-orphan-sweep.test.ts | 71 +++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts index abf97737f3..839ba6ac4b 100644 --- a/kernel/tests/issue3769-orphan-sweep.test.ts +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -232,3 +232,74 @@ test("a block-boundary fold marks the sweep by covered message refs, not the blo assert.deepEqual(r.newlySweptCallIds, [], "cached-prefix orphan before the fold is untouched"); assert.ok(r.messages.some((m) => m.toolCallId === "f1"), "f1 remains visible"); }); + +test("an upgraded pre-fold-gate state keeps its rotation-hidden orphan hidden (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = [ + msg("p1", "early"), + ...failPair("f1"), + msg("p2", "mid"), + ...failPair("f2"), + msg("p3", "late"), + ...failPair("f3"), + ]; + seqRefs(state, messages); + // refs: p1=1 c-f1=2 r-f1=3 p2=4 c-f2=5 r-f2=6 p3=7 c-f3=8 r-f3=9 + + // Exact shape of a state the old keep-two kernel persisted: with three failures + // the rotation had already removed f1 from mid-history and remembered its refs. + // sweptOrphanCallIds / pendingOrphanSweepMinRef do not exist yet. + state.hiddenOrphanRefs = ["m00002", "m00003"]; + + const first = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(first.newlySweptCallIds, [], "nothing newly swept without a fold"); + const ids = first.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f1") && !ids.includes("r-f1"), "previously-hidden pair does not resurface on upgrade"); + assert.ok(ids.includes("c-f2") && ids.includes("c-f3"), "recent failures stay visible"); + + // Pipeline write-back (hide-compress-calls node) then another quiet turn: the + // legacy set self-perpetuates through rememberHiddenRef, no new field required. + state.hiddenOrphanRefs = first.hiddenOrphanRefs; + const second = hideConsumedCompressCalls(state, messages, isMcp); + const ids2 = second.messages.map((m) => m.id); + assert.ok(!ids2.includes("c-f1") && !ids2.includes("r-f1"), "still hidden on the next quiet turn"); +}); + +test("the fold marker survives the full pipeline: sync-blocks carries it to the hide node (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("p1", "early cached prefix alpha"), + ...failPair("f2"), + msg("p2", "mid detailed work item beta content"), + ]; + const state = createInitialState(); + seqRefs(state, messages); + // refs: p1=1 c-f2=2 r-f2=3 p2=4 — a fold starting at ref 2 committed last turn. + // Before the sync-blocks carry fix this marker died on the FIRST pipeline node + // (syncBlocks rebuilt state without the #3769 fields), so the sweep never fired. + state.pendingOrphanSweepMinRef = 2; + + const cfg = { ...config(), isCompressTool: isMcp }; + const turn = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 1000, + renderTags: "none", + }); + + const ids = turn.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f2") && !ids.includes("r-f2"), "orphan at/after fold start swept inside the pipeline"); + assert.deepEqual(turn.state.sweptOrphanCallIds, ["f2"], "swept id folded into the persistent set"); + assert.equal(turn.state.pendingOrphanSweepMinRef, undefined, "one-shot marker consumed"); + + const quiet = core.processTurn({ + messages: [...messages, msg("p3", "latest user intent theta")], + state: turn.state, + config: cfg, + tokenCount: 1000, + renderTags: "none", + }); + const ids2 = quiet.messages.map((m) => m.id); + assert.ok(!ids2.includes("c-f2"), "previously-swept orphan stays hidden on the next pipeline pass"); +}); From 0be491fc5627a04a1b5cd277c5e9c169cd227a67 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:38 +0800 Subject: [PATCH 13/29] fix: inherit orphan-cleanup ledgers on resume/fork adoption (#3769 P0) applyForkAdoption rebuilds session state from the parent on resume/fork replay; without carrying hiddenOrphanRefs and sweptOrphanCallIds, failed compress pairs already hidden pre-upgrade or already swept would resurface on replay, rewriting mid-history at full cache cost. Both ledgers are monotonic sets re-recorded every pass, so the carry self-perpetuates. --- src/fork-adoption.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 5de73f8a2b..16222d9a61 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -262,6 +262,14 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare // dropped (its failed tail, a rewound branch) — and the model's older // text may still cite those. Reserve the parent's whole ref space. reserveRefsThrough(session, highestUsedIndex(parent.state.messageRefs)); + // #3769: carry the orphan-cleanup ledgers too — without this, the failed + // compress pairs the pre-upgrade keep-two rotation hid (hiddenOrphanRefs) + // or the fold-gated sweep already swept (sweptOrphanCallIds) would + // RESURFACE on resume/fork replay, rewriting mid-history at full cache + // cost. Both are monotonic id/ref sets; the hide pass re-records them + // every turn, so the carry self-perpetuates afterwards. + if (parent.state.hiddenOrphanRefs?.length) session.state.hiddenOrphanRefs = [...parent.state.hiddenOrphanRefs]; + if (parent.state.sweptOrphanCallIds?.length) session.state.sweptOrphanCallIds = [...parent.state.sweptOrphanCallIds]; markDirty(session); } From fe2c46e5c6f92aa53495fd576e7ed6162aff70b4 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:38 +0800 Subject: [PATCH 14/29] test: end-to-end upgrade compat for rotation-hidden orphans (#3769 P0) Real-server e2e: drive a session with a failed compress pair, persist it, patch the record back to an old-shape state (hiddenOrphanRefs set, no #3769 fields), reload under a fresh proxy, and assert the previously-hidden pair stays out of the outbound history while new turns append stably. --- tests/issue3769-upgrade-compat.test.ts | 215 +++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 tests/issue3769-upgrade-compat.test.ts diff --git a/tests/issue3769-upgrade-compat.test.ts b/tests/issue3769-upgrade-compat.test.ts new file mode 100644 index 0000000000..289528f6cd --- /dev/null +++ b/tests/issue3769-upgrade-compat.test.ts @@ -0,0 +1,215 @@ +// #3769 upgrade compatibility (owner merge condition): sessions persisted by a +// pre-fold-gate kernel carry their rotation-hidden orphan pairs ONLY in +// `hiddenOrphanRefs` (no sweptOrphanCallIds / pendingOrphanSweepMinRef). After +// upgrading, those pairs must stay hidden on the outbound history — resurrecting +// them re-inserts mid-history bytes and breaks the cached prefix again. +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; +import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +process.env.NODE_ENV = "test"; +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_CONFIG_FILE = path.join(mkdtempSync(path.join(tmpdir(), "bili-upg-hermetic-")), "no-config.json"); + +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions, _resetSessionsForTest } from "../src/session.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const LABEL = "upg3769"; +const ARG_JSON = JSON.stringify({ content: [{ startId: "m00001", endId: "m00002", topic: "upg", summary: "UPG_SUM_MARKER" }] }); +const RESULT_TEXT = "UPG_RESULT_MARKER"; + +type M = Record; + +const u1: M = { role: "user", content: "q one alpha" }; +const a1: M = { role: "assistant", content: "a one alpha" }; +const u2: M = { role: "user", content: "q two beta" }; +const a2: M = { role: "assistant", content: "a two beta" }; +const pairCall: M = { role: "assistant", content: null, tool_calls: [{ id: "call_upg1", type: "function", function: { name: "compress", arguments: ARG_JSON } }] }; +const pairResult: M = { role: "tool", tool_call_id: "call_upg1", content: RESULT_TEXT }; +const u3: M = { role: "user", content: "q three gamma" }; +const u4: M = { role: "user", content: "q four delta UPG_TAIL_MARKER" }; + +function sse(obj: unknown): string { + return `data: ${JSON.stringify(obj)}\n\n`; +} + +async function post(url: string, messages: M[], bodies: string[]): Promise { + const r = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": LABEL }, + body: JSON.stringify({ model: "gpt-test", messages, stream: true }), + }); + void bodies; + return r.status ?? 0; +} + +type PersistRecord = { + version: number; + savedAt: number; + id: string; + payload: { + meta: { label?: string }; + messages: { id: string; contentType?: string; toolName?: string; toolCallId?: string }[]; + state: { messageRefs: { byRaw: Record }; hiddenOrphanRefs?: string[]; sweptOrphanCallIds?: string[]; pendingOrphanSweepMinRef?: number }; + }; +}; + +function findSessionFile(root: string, label: string): string { + const stack = [root]; + while (stack.length > 0) { + const d = stack.pop()!; + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) { + stack.push(p); + } else if (e.name.endsWith(".json") && !e.name.endsWith(".content-store.json")) { + try { + const rec = JSON.parse(readFileSync(p, "utf8")) as PersistRecord; + if (rec.id === label || rec.payload?.meta?.label === label) return p; + } catch { /* not our file */ } + } + } + } + throw new Error(`no session file for ${label} under ${root}`); +} + +test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-end (#3769)", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "bili-upg-store-")); + const bodies: string[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + bodies.push(Buffer.concat(chunks).toString("utf8")); + const n = bodies.length; + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: `ok ${n}` } }] })); + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })); + res.write("data: [DONE]\n\n"); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upPort = (upstream.address() as { port: number }).port; + + const stores: SessionStore[] = []; + try { + // Phase 1: current code runs two turns; the failed pair rides visibly + // (fold-gated contract: no fold -> no sweep) and the record lands on disk. + const store1 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + stores.push(store1); + _setStoreForTest(store1); + _resetSessionsForTest(); + setRegistryForTest({}); + const proxy1 = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upPort}`]: { models: { "gpt-test": { context: 15_000 } } } }, + modelContextLimit: 15_000, + kernelConfig: defaultConfig(15_000), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + passthrough: false, + passthroughSource: null, + autoUpdate: false, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: true, + releaseNotesCheck: true, + compat: { roles: {} }, + streamErrorShape: "protocol", + mitm: { enabled: false, domains: [] }, + } as unknown as ProxyOptions); + await once(proxy1, "listening"); + const port1 = (proxy1.address() as { port: number }).port; + const url1 = `http://127.0.0.1:${port1}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; + + assert.equal(await post(url1, [u1, a1, u2, a2], bodies), 200); + assert.equal(await post(url1, [u1, a1, u2, a2, { role: "assistant", content: "ok 1" }, pairCall, pairResult, u3], bodies), 200); + // New-code baseline: without a fold the pair stays visible outbound. + assert.ok(bodies[bodies.length - 1]!.includes("UPG_SUM_MARKER"), "pre-upgrade baseline: failed pair rides visibly"); + + const s1 = listSessions().find((x) => x.meta.label === LABEL); + assert.ok(s1, "session exists"); + assert.ok(store1.flushSync(s1!)); + proxy1.close(); + proxy1.closeIdleConnections?.(); + await once(proxy1, "close"); + + // Patch the persisted record into the exact shape an old kernel left: + // rotation had hidden the pair (remembered by ref), new fields absent. + const file = findSessionFile(dir, LABEL); + const rec = JSON.parse(readFileSync(file, "utf8")) as PersistRecord; + const callMsg = rec.payload.messages.find((m) => m.contentType === "tool-call" && m.toolName === "compress"); + const resMsg = rec.payload.messages.find((m) => m.contentType === "tool-result" && m.toolCallId === "call_upg1"); + assert.ok(callMsg && resMsg, "record snapshot carries the failed pair"); + const callRef = rec.payload.state.messageRefs.byRaw[callMsg!.id]; + const resRef = rec.payload.state.messageRefs.byRaw[resMsg!.id]; + assert.ok(callRef && resRef, "refs resolvable from the persisted state"); + rec.payload.state.hiddenOrphanRefs = [callRef, resRef]; + delete rec.payload.state.sweptOrphanCallIds; + delete rec.payload.state.pendingOrphanSweepMinRef; + writeFileSync(file, JSON.stringify(rec)); + + // Phase 2: a fresh server instance hydrates the patched record. + _resetSessionsForTest(); + const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + stores.push(store2); + _setStoreForTest(store2); + const proxy2 = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upPort}`]: { models: { "gpt-test": { context: 15_000 } } } }, + modelContextLimit: 15_000, + kernelConfig: defaultConfig(15_000), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + passthrough: false, + passthroughSource: null, + autoUpdate: false, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: true, + releaseNotesCheck: true, + compat: { roles: {} }, + streamErrorShape: "protocol", + mitm: { enabled: false, domains: [] }, + } as unknown as ProxyOptions); + await once(proxy2, "listening"); + const port2 = (proxy2.address() as { port: number }).port; + const url2 = `http://127.0.0.1:${port2}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; + + const hist3: M[] = [u1, a1, u2, a2, { role: "assistant", content: "ok 1" }, pairCall, pairResult, u3, { role: "assistant", content: "ok 2" }, u4]; + assert.equal(await post(url2, hist3, bodies), 200); + const out = bodies[bodies.length - 1]!; + assert.ok(!out.includes("UPG_SUM_MARKER"), "upgrade: hidden compress args do not resurface"); + assert.ok(!out.includes("UPG_RESULT_MARKER"), "upgrade: hidden result does not resurface"); + assert.ok(out.includes("UPG_TAIL_MARKER"), "live tail rides normally"); + + const s2 = listSessions().find((x) => x.meta.label === LABEL); + assert.ok(s2, "hydrated session exists"); + assert.ok(s2.state.hiddenOrphanRefs?.includes(callRef!), "legacy hidden set survived hydration and self-perpetuates"); + proxy2.close(); + proxy2.closeIdleConnections?.(); + await once(proxy2, "close"); + } finally { + for (const s of stores) s.closeQuietly(); + upstream.close(); + upstream.closeIdleConnections?.(); + await once(upstream, "close").catch(() => undefined); + _setStoreForTest(new SessionStore({ enabled: false })); + void rmrf(dir); + } +}); From 8e8590ef3b770775be06be1bc304030ecc4741cc Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:51 +0800 Subject: [PATCH 15/29] test: pin fold-gated sweep timing through the live server (#3769) With the sync-blocks carry in place the sweep fires end-to-end: a failed compress pair appended at a fold step is swept by that step's own model request, so it never rides inbound and its anchor carries the summary for good (zero-carrier fail-safe, #1567). Re-pin the main matrix (all four wires), the swallow/back-to-back geometries (wide fold absorbs the alpha carrier; FOLD_E tail hits the protected zone), the drop-pair subtest (client-side drop is now a wire no-op) and the 3-concurrent-folds pin (carriers stack one per fold, stable thereafter), from observed live-server bodies. Google pair-absence proxy switches from the topic (which the carrier embeds) to the fold-spec refs that only the re-sent functionCall args carry. --- tests/cache-friendly-plugin.test.ts | 194 ++++++++++++++-------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/tests/cache-friendly-plugin.test.ts b/tests/cache-friendly-plugin.test.ts index 18bf5496ee..deeea63826 100644 --- a/tests/cache-friendly-plugin.test.ts +++ b/tests/cache-friendly-plugin.test.ts @@ -11,10 +11,12 @@ // therefore re-injects the in-place acp_summary carrier from block state on // every request. Since the #1567 fix the carrier is STRIPPED per-block, but // self-verified: only while that exact fold's compress pair actually rides -// the (post-prepare) inbound history. A pair pruned client-side or swept by -// the kernel's fold-gated orphan cleanup (#3769) keeps the anchor, so an -// active fold never has zero carriers. This suite pins that behavior -// byte-for-byte. +// the (post-prepare) inbound history. Under fold-gated sweeps (#3769) a pair +// whose call sits at/after a committed fold's start ref is hidden from that +// fold turn on — in this harness the fold POST and its model request share +// one step, so each failed pair is swept at its own fold step, never rides, +// and its anchor carries the summary for good (zero-carrier fail-safe). This +// suite pins that behavior byte-for-byte. // // Invariants pinned per wire (bodies = consecutive upstream-bound request // payloads of one session, compared as flat unit lists — text blocks and @@ -34,19 +36,17 @@ // /__bili/plugin/tool fold): // P2a the pre-fold span is replaced by exactly the expected carrier // units and nothing else moves or mutates (exact reconstruction); -// P2b the in-place carrier is stripped from every post-fold turn -// while the client's pair rides inbound — occurrence counts -// pinned: exactly 2 summary-text hits per post-fold turn (the -// fold spec echoed in the re-sent tool_call args + the tail -// quote inside the persistently re-sent tool result) and ZERO -// carrier markers (#1567 fix); - // P2c fail-safe (#1567 hardening): the anchor re-carries the summary - // whenever the pair does NOT ride inbound — the client stops - // re-sending it, or the kernel swept it on a later fold (#3769) — - // pinned by the hardening subtests at the bottom of this file; - // P3 multi-fold: two folds in one session; both pairs stay visible for the - // rest of the session (fold-gated sweeps, #3769 — keep-two rotation is - // gone) and the pair-quote counts stay constant across all later turns; +// P2b the pair is swept at its own fold step (#3769) and never rides +// inbound — occurrence counts pinned: exactly ONE summary-text hit +// per post-fold turn (the anchor carrier) and ZERO pair quotes, +// one carrier marker per committed fold (#1567 fail-safe); +// P2c fail-safe (#1567 hardening): the anchor carries the summary +// whenever the pair does NOT ride inbound — under #3769 that is +// every post-fold turn in this harness; the drop-pair subtest +// pins that a client-side pair drop afterwards is a wire no-op; +// P3 multi-fold: two folds in one session; each anchor carries its own +// summary from its fold turn on, the carrier count grows by one per +// fold and then stays constant (keep-two rotation is gone, #3769); // P4 no transient acp_loop_* artifact leaks into any forwarded body // (plugin mode has none by construction — pin it); // P5 marker/tag hygiene: every content text unit carries exactly one @@ -69,7 +69,8 @@ // // Scenario (main, all wires): t1=[q1,a1,q2,a2], t2=+q3,a3, t3=+q4,a4, // foldA(m00001..m00002)+pairA@t4, t5=+q5,a5, foldB(m00003..m00005)+pairB@t6, -// t7=+q7,a7, t8=+q8,a8 — every historical pair rides every later request. +// t7=+q7,a7, t8=+q8,a8 — each failed pair is swept at its own fold step +// (#3769); its anchor carries the summary from there on. // // Scope notes: tool protocol only (the default). The swallow/back-to-back // fold geometries get a dedicated anthropic subtest (kernel-side behavior, @@ -614,28 +615,29 @@ for (const wire of ["anthropic", "openai", "responses", "google"] as Wire[]) { assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // head boundary survives; anchor stripped (#1567) - assertPrefix(cans[3]!.core, cans[4]!.core, L(4), wire === "google" ? ["a5"] : ["q5", "a5"], wire === "google"); // google: #3159 — q5 leads the fold-pair result - assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], [], L(5)); // anchor stripped (#1567) - assertPrefix(cans[5]!.core, cans[6]!.core, L(6), wire === "google" ? ["a7"] : ["q7", "a7"], wire === "google"); // google: #3159 — q7 leads the fold-pair result + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1", "SA"], L(3)); // head boundary survives; anchor A carries — its pair was swept at fold time (#3769) + assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); + assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], ["SB"], L(5)); // span consumed; anchor B carries (#3769) + assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q7", "a7"]); assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); for (let i = 3; i < cans.length; i++) { const raw = cans[i]!.raw; - // #1567 (fixed): the in-place carrier is stripped per-block while - // the client's re-sent pair rides inbound — only the two pair - // quotes (tool_call args + tool result) keep the summary text. - const expA = 2; - const expB = i < 5 ? 0 : 2; - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: in-place carrier must be stripped while the client's pair rides inbound (got ${countOcc(raw, SUM_MARKER)})`); - assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA} (pair quotes only, #1567)`); - assert.equal(countOcc(raw, SUM_B), expB, `${L(i)}: SUM_B occurrences ${countOcc(raw, SUM_B)} != ${expB} (pair quotes only, #1567)`); - // google part-shapes carry no call id; pin pair presence through - // the fold topic embedded in the re-sent functionCall args - const pairA = wire === "google" ? "CFP-TOPIC-A" : "\"cfp_a\""; - const pairB = wire === "google" ? "CFP-TOPIC-B" : "\"cfp_b\""; - assert.ok(raw.includes(pairA), `${L(i)}: pairA lost from re-sent history`); - if (i >= 5) assert.ok(raw.includes(pairB), `${L(i)}: pairB lost from re-sent history`); + // #3769: each failed pair is swept at its own fold step, so no + // pair quotes ride inbound — each anchor carries its summary for + // good (zero-carrier fail-safe, #1567) + const expA = 1; + const expB = i < 5 ? 0 : 1; + assert.equal(countOcc(raw, SUM_MARKER), i < 5 ? 1 : 2, `${L(i)}: one carrier per committed fold (pairs swept, #3769)`); + assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA} (anchor carrier only, #3769)`); + assert.equal(countOcc(raw, SUM_B), expB, `${L(i)}: SUM_B occurrences ${countOcc(raw, SUM_B)} != ${expB} (anchor carrier only, #3769)`); + // google part-shapes carry no call id; pin pair ABSENCE through + // the fold spec refs that would sit in the re-sent functionCall + // args (the anchor carrier embeds topic+summary, never the spec) + const pairA = wire === "google" ? "\"startId\":\"m00001\"" : "\"cfp_a\""; + const pairB = wire === "google" ? "\"startId\":\"m00003\"" : "\"cfp_b\""; + assert.ok(!raw.includes(pairA), `${L(i)}: swept pairA must stay hidden (#3769)`); + assert.ok(!raw.includes(pairB), `${L(i)}: swept pairB must stay hidden (#3769)`); } }, { timeout: 120_000 }); @@ -666,21 +668,32 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // head boundary survives; anchor stripped (#1567) + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1", "SA"], L(3)); // head boundary survives; anchor A carries (pair swept at fold time, #3769) assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); const wide = FOLD_WIDE.summary; const gamma = FOLD_E.summary; + // Fold-gated sweeps (#3769): every failed pair is hidden from its own + // fold step on, so no pair quotes ride inbound and each anchor carries + // its summary. The wide fold (m00001..m00004) swallows the alpha fold's + // range entirely — the alpha summary merges away and the wide carrier + // takes over. FOLD_E's tail sits in the protected zone, so only its + // first message folds and the gamma carrier lands where q3 used to sit. for (const i of [5, 6, 7]) { const raw = cans[i]!.raw; - // carrier is superseded on every turn; the geo_a pair quotes ride - // every later turn — fold-gated sweeps (#3769) replaced keep-two - // rotation, so no later fold prunes this oldest pair here - assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != 2`); - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: carriers must be stripped while their pairs ride inbound (got ${countOcc(raw, SUM_MARKER)})`); assert.ok(!raw.includes("q2-marker"), `${L(i)}: swallowed content q2 must stay hidden`); assert.ok(!raw.includes("a2-marker"), `${L(i)}: swallowed content a2 must stay hidden`); + assert.ok(!raw.includes("\"geo_a\"") && !raw.includes("\"geo_wide\"") && !raw.includes("\"geo_e\""), `${L(i)}: all three failed pairs swept at their fold steps (#3769)`); } + assert.equal(countOcc(cans[5]!.raw, SUM_A), 0, `${L(5)}: alpha summary absorbed into the wide fold`); + assert.equal(countOcc(cans[5]!.raw, wide), 1, `${L(5)}: wide carrier only (its pair swept, #3769)`); + assert.equal(countOcc(cans[5]!.raw, SUM_MARKER), 1, `${L(5)}: single carrier after the swallow`); + assert.equal(countOcc(cans[6]!.raw, wide), 1, `${L(6)}: wide carrier persists`); + assert.equal(countOcc(cans[6]!.raw, gamma), 1, `${L(6)}: gamma carrier only (its pair swept, #3769)`); + assert.equal(countOcc(cans[6]!.raw, SUM_MARKER), 2, `${L(6)}: one carrier per committed fold`); + assert.equal(countOcc(cans[7]!.raw, wide), 1, `${L(7)}: wide carrier persists`); + assert.equal(countOcc(cans[7]!.raw, gamma), 1, `${L(7)}: gamma carrier persists`); + assert.equal(countOcc(cans[7]!.raw, SUM_MARKER), 2, `${L(7)}: carriers stable across quiet turns`); const surviveKeys = new Set(["q1", "a1", "q2", "a2", "SA"]); const keepT5 = cans[4]!.core.filter((u) => !surviveKeys.has(keyOf(u))); let cursor = -1; @@ -689,32 +702,23 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assert.ok(found > cursor, `cfp-geo t6: unit ${JSON.stringify(keyOf(u))} lost or reordered by the wide fold`); cursor = found; } - assert.equal(countOcc(cans[5]!.raw, wide), 2, `${L(5)}: wide fold turn occurrences (pair quotes; anchor stripped)`); - assert.equal(countOcc(cans[6]!.raw, gamma), 2, `${L(6)}: gamma fold turn occurrences (pair quotes; anchor stripped)`); - assert.equal(countOcc(cans[7]!.raw, wide), 2, `${L(7)}: resent wide quotes must survive`); - assert.equal(countOcc(cans[7]!.raw, gamma), 2, `${L(7)}: resent gamma quotes must survive`); - for (const i of [6, 7]) { - const raw = cans[i]!.raw; - // all three failed pairs stay visible: fold-gated sweeps (#3769) only - // remove orphans at/after a committed fold's start, never rotate the - // newest failures away between folds - assert.ok(raw.includes("\"geo_a\"") && raw.includes("\"geo_wide\"") && raw.includes("\"geo_e\""), `${L(i)}: all three pairs must stay visible (no keep-two rotation)`); - } - // #1567 (fixed): no text unit carries the wide summary anymore (stripped - // anchor); the quotes ride structured tool units, not text + // #1567/#3769: the wide summary rides its anchor carrier (a text unit) + // for good — the pair was swept at fold time, so the carrier is never + // stripped again const wideT6 = cans[6]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); const wideT7 = cans[7]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); - assert.equal(wideT6.length, 0, `${L(6)}: wide carrier must be stripped while the geo_wide pair rides inbound`); - assert.equal(wideT7.length, 0, `${L(7)}: wide carrier must be stripped while the geo_wide pair rides inbound`); + assert.equal(wideT6.length, 1, `${L(6)}: wide carrier rides a text unit while the geo_wide pair stays swept`); + assert.equal(wideT7.length, 1, `${L(7)}: wide carrier rides a text unit while the geo_wide pair stays swept`); assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); -// #1567 hardening scenario A: a contract-violating (or pruning) client stops -// re-sending an older compress pair. While pairA rides, its anchor is -// stripped (P2b); the moment the pair vanishes the anchor MUST re-carry the -// summary (zero-carrier fail-safe) and the body must stay append-stable on -// later turns (no flapping). -testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → anchor re-carries (#1567 hardening)`, async () => { +// #1567 hardening scenario A, re-pinned under #3769: a contract-violating +// (or pruning) client stops re-sending an older compress pair at t7. Under +// fold-gated sweeps the pair was already hidden at its own fold step (t4), +// so its anchor has carried the summary since then — the client-side drop is +// a wire no-op, and the body stays append-stable with exactly one carrier +// per committed fold (no flapping). +testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → wire unaffected, carriers stable (#1567/#3769)`, async () => { const steps: Step[] = [ { send: [T(1), A(1), T(2), A(2)] }, { send: [T(3), A(3)] }, @@ -733,35 +737,28 @@ testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pai assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-drop"); - // t4..t6: both pairs ride → both anchors stripped - for (const i of [3, 4, 5]) { + // t4..t8: both pairs swept at their fold steps → each anchor is the sole + // carrier of its summary; the client dropping pairA at t7 changes nothing + for (const i of [3, 4, 5, 6, 7]) { const raw = cans[i]!.raw; - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: carrier must be stripped while pair rides`); - assert.ok(raw.includes('"cfp_a"'), `${L(i)}: pairA still re-sent here`); - } - assert.ok(cans[5]!.raw.includes('"cfp_b"'), `${L(5)}: pairB rides`); - // t7/t8: pairA gone → anchor A is the ONLY carrier again - for (const i of [6, 7]) { - const raw = cans[i]!.raw; - assert.ok(!raw.includes('"cfp_a"'), `${L(i)}: client no longer re-sends pairA`); - assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(i)}: anchor A must re-carry (exactly one carrier)`); - assert.equal(countOcc(raw, SUM_A), 1, `${L(i)}: SUM_A = carrier only (pair quotes gone with the pair)`); - assert.equal(countOcc(raw, SUM_B), 2, `${L(i)}: SUM_B quotes only, anchor B still stripped (pairB rides)`); - assert.ok(raw.includes('"cfp_b"'), `${L(i)}: pairB rides`); + assert.ok(!raw.includes('"cfp_a"') && !raw.includes('"cfp_b"'), `${L(i)}: failed pairs swept at fold time (#3769)`); + assert.equal(countOcc(raw, SUM_A), 1, `${L(i)}: anchor A carries (sole occurrence)`); + assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 1, `${L(i)}: anchor B carries from its fold turn on`); + assert.equal(countOcc(raw, SUM_MARKER), i < 5 ? 1 : 2, `${L(i)}: one carrier per committed fold`); } assert.ok(cans[6]!.raw.includes("q7-marker"), `${L(6)}: new turn visible`); // once re-carried, the body stays append-stable (no flapping) assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); -// #3769 contract pin (plugin lane): three concurrent ACTIVE folds. The old -// KEEP_LAST_ORPHANED=2 rotation hid the oldest failed pair on EVERY turn — -// a mid-history rewrite with no fold behind it, exactly #3769's cache seam. -// Fold-gated sweeps replaced it: failed pairs stay visible between folds, so -// every anchor stays stripped (its own pair rides) and later turns remain -// append-stable. The zero-carrier fail-safe itself (pair gone → anchor -// re-carries) is pinned by the drop-pair subtest above. -testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds keep all failed pairs riding (fold-gated sweeps, #3769)`, async () => { +// #3769 contract pin (plugin lane): three back-to-back folds, each with a +// failed (client-visible) compress pair. The old KEEP_LAST_ORPHANED=2 +// rotation hid the oldest pair on EVERY turn — a mid-history rewrite with no +// fold behind it, exactly #3769's cache seam. Fold-gated sweeps hide each +// pair at its own fold step (the deletion rides the fold's own rewrite), so +// every anchor carries its summary from the fold turn on, carriers stack +// without churning, and quiet turns stay append-stable. +testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: pairs swept at fold time, anchors carry (#3769)`, async () => { const SUM_P1 = "Prune-safe fold one summary covering m00001..m00002"; const SUM_P2 = "Prune-safe fold two summary covering m00003..m00004"; const SUM_P3 = "Prune-safe fold three summary covering m00005..m00006"; @@ -785,20 +782,23 @@ testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds keep assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-prune"); - // t3/t4: each fold's pair rides from its own turn on → anchors stripped - for (const i of [2, 3]) { - assert.equal(countOcc(cans[i]!.raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while pairs ride`); + // t3..t5: each fold's pair is swept at its own fold step → each anchor + // carries its summary; carrier count grows by one per fold + for (const i of [2, 3, 4]) { + const raw = cans[i]!.raw; + assert.ok(!raw.includes('"fs1"') && !raw.includes('"fs2"') && !raw.includes('"fs3"'), `${L(i)}: failed pairs swept at their fold steps (#3769)`); + assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: anchor P1 carries (sole occurrence)`); + assert.equal(countOcc(raw, SUM_P2), i < 3 ? 0 : 1, `${L(i)}: anchor P2 carries from its fold turn on`); + assert.equal(countOcc(raw, SUM_P3), i < 4 ? 0 : 1, `${L(i)}: anchor P3 carries from its fold turn on`); + assert.equal(countOcc(raw, SUM_MARKER), i - 1, `${L(i)}: one carrier per committed fold`); } - assert.ok(cans[3]!.raw.includes('"fs1"') && cans[3]!.raw.includes('"fs2"'), `${L(3)}: both pairs visible`); - // t5..t7: three failed pairs coexist and ALL stay visible (no keep-two - // rotation, #3769) → every anchor stays stripped, no carrier needed - for (const i of [4, 5, 6]) { + // t6/t7: all three carriers persist, stable across quiet turns + for (const i of [5, 6]) { const raw = cans[i]!.raw; - assert.ok(raw.includes('"fs1"') && raw.includes('"fs2"') && raw.includes('"fs3"'), `${L(i)}: all three failed pairs must stay visible (fold-gated sweeps, #3769)`); - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: no carrier needed — every anchor's pair rides inbound`); - assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: SUM_P1 pair quotes only`); - assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: SUM_P2 pair quotes only`); - assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: SUM_P3 pair quotes only`); + assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: P1 carrier persists`); + assert.equal(countOcc(raw, SUM_P2), 1, `${L(i)}: P2 carrier persists`); + assert.equal(countOcc(raw, SUM_P3), 1, `${L(i)}: P3 carrier persists`); + assert.equal(countOcc(raw, SUM_MARKER), 3, `${L(i)}: three carriers, stable`); } // mid-history never mutates between quiet turns assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q8", "a8"]); From 77ee6c793c306c342595f51b6ac14655c238dfee Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 23:30:49 +0800 Subject: [PATCH 16/29] test: regen storage-render goldens after rebase onto #3803 ledger fields (#3769 rebase) --- tests/golden/storage-render/anthropic.json | 4 ++-- tests/golden/storage-render/chat.json | 4 ++-- tests/golden/storage-render/google.json | 4 ++-- tests/golden/storage-render/responses.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 934a26087d..1484cc0699 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "f894d02089eb22567a0e5d332ab7c12153d2a3d92a37140f8fc3d4b3d3cfcc19", + "stateDigest": "5839fb455554a276e973fc8e2a6039971d430df48c5e64970728cc66f3005ae7", "sectionDigests": { "state": "a4b83f873d16bfc4", "blockContents": "dc7d13d250c94bb4", "stats": "cff3c2546898aeb3", - "metadata": "a10cb71754abe91e" + "metadata": "95795174db0e1e50" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 816c8816a1..1883114ffa 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "4731e0ab9374388662bdc5ef54386d22cf446b71712d379cdbbdd530805d10cc", + "stateDigest": "38d6459bd83802b5c7ba79b10d6e12a82ba99086377ae2ec8676c41558e08b3d", "sectionDigests": { "state": "463b3dc26a100c2a", "blockContents": "94a58f96d7c5cfc3", "stats": "59842ce5415d3a4d", - "metadata": "0f57b6cba74a16d8" + "metadata": "d16fb5934a3e8779" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 2fd2f8942f..cc6117c57e 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "5b8b7f10031808c1cfcc6161a6c3881d485d4016f435e27ee89bd015a296fee4", + "stateDigest": "f4bb18cf96a64525a323d3db05cd160a375464fdfdf5c788f823acdf96a95825", "sectionDigests": { "state": "a9b9531d952b48d8", "blockContents": "473264d9df8ba338", "stats": "e358dc992395fc7e", - "metadata": "bf873598b93525e7" + "metadata": "9e864ac22f86950a" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index e8897cc195..e305b07dc5 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "7013217e0a5669e3a4ddb8f3e02f70a4dad08cfe5d10170a962ff3f2fa8db3d7", + "stateDigest": "2f2c59befaa51d0bc726d44821f8c8ddf12d6237ff84dfbe213af7c74ca885cf", "sectionDigests": { "state": "d7951fa36f9e6fe9", "blockContents": "b6701268a0bdcd8a", "stats": "67e33bcc612fdcc1", - "metadata": "f3bd0c932cc616b0" + "metadata": "71bc14af02391230" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", From 7d3578c87dd6014ea4407564e111b9bc6d8e20b5 Mon Sep 17 00:00:00 2001 From: awork Date: Tue, 20 Oct 2026 03:24:57 +0800 Subject: [PATCH 17/29] fix: exempt active-block carriers from the fold-gated orphan sweep (#3769/#1567) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fold-gated sweep hid EVERY orphan-fate compress pair at/after the fold's start ref — including successful echoes whose call id never matched the server-minted block id. A re-sent pair quoting an ACTIVE block's range is the plugin lane's summary carrier (KDD#7), so hiding it strands the summary: the #1999 double-carrier test, the plugin session-lock e2e, #1426 mode-upgrade proof and #1638 notification pin all went red. resolveFate's 'orphan' verdict is id-based and cannot see success; the sweep branch now additionally requires the call to quote NO active block range (quotesActiveBlock: both arg shapes — content[] entries and flat top-level startId/endId — mirroring the host stripKernelSummaries carrier recognition). Genuinely failed records and dead-block pairs (swallowed by a wider fold) still sweep at the fold boundary, per the approved #3769 design. --- kernel/src/hide-consumed.ts | 59 +++++++++++++++++- kernel/tests/issue3769-orphan-sweep.test.ts | 69 +++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index 888469bb32..d79400fb99 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -32,6 +32,61 @@ function rangeKey(startRef: string, endRef: string): string { return `${startRef}::${endRef}`; } +// #3769: does this call's args quote the folded range of an ACTIVE block? Such a +// re-sent pair IS that block's summary carrier — the host strips the in-place +// anchor only while it rides the history (stripKernelSummaries, #1567/#2042) — +// so the fold-gated sweep must leave it alone. Both arg shapes are accepted to +// mirror the host index ({content:[…]} AND the flat {startId,endId} models emit +// natively); parseCallText alone misses the flat form. Unparseable args fail +// safe as NOT a carrier (under-sweep is safe, over-sweep is not). +function quotesActiveBlock( + text: string | undefined, + activeRangeKeys: Set, +): boolean { + const raw = text ?? ""; + const start = raw.indexOf("{"); + if (start < 0) return false; + let parsed: unknown; + try { + parsed = JSON.parse(raw.slice(start)); + } catch { + return false; + } + if (!parsed || typeof parsed !== "object") return false; + const obj = parsed as Record; + let entries: unknown[]; + if (Array.isArray(obj.content)) { + entries = obj.content; + } else if (typeof obj.content === "string") { + try { + const inner: unknown = JSON.parse(obj.content); + entries = Array.isArray(inner) ? inner : []; + } catch { + return false; + } + } else { + entries = [obj]; + } + for (const entry of entries) { + if (!entry || typeof entry !== "object") continue; + const e = entry as Record; + const s = + typeof e.startId === "string" + ? e.startId + : typeof e.messageId === "string" + ? e.messageId + : ""; + const end = + typeof e.endId === "string" + ? e.endId + : typeof e.messageId === "string" + ? e.messageId + : ""; + if (s && end && activeRangeKeys.has(rangeKey(s, end))) return true; + } + return false; +} + // Numeric position of a padded ref (mNNNNN -> NNNNN); null when unmapped/blocked // so the caller treats it as "position unknown" and never sweeps on a guess. // Exported: applyCompression derives the fold sweep marker from covered message @@ -262,6 +317,8 @@ export function hideConsumedCompressCalls( // this fold's earliest start ref so we additionally sweep orphans at/after it. // Orphans before it sit in the still-cached prefix and stay visible; with no // fold pending nothing new is swept, so a quiet turn never breaks the prefix. + // Orphans quoting an ACTIVE block's folded range are that block's summary + // carrier (#1567) and are exempt from the sweep (quotesActiveBlock). const sweptSet = new Set(state.sweptOrphanCallIds ?? []); const foldMinRef = state.pendingOrphanSweepMinRef; const newlySweptCallIds: string[] = []; @@ -286,7 +343,7 @@ export function hideConsumedCompressCalls( // cached prefix again. The set self-perpetuates — rememberHiddenRef // re-records these refs on every pass they stay hidden. hide = true; - } else if (foldMinRef !== undefined) { + } else if (foldMinRef !== undefined && !quotesActiveBlock(messages[i]!.text, activeRangeKeys)) { const num = refNum(ref); if (num !== null && num >= foldMinRef) { hide = true; diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts index 839ba6ac4b..f5765195c4 100644 --- a/kernel/tests/issue3769-orphan-sweep.test.ts +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -265,6 +265,75 @@ test("an upgraded pre-fold-gate state keeps its rotation-hidden orphan hidden (# assert.ok(!ids2.includes("c-f1") && !ids2.includes("r-f1"), "still hidden on the next quiet turn"); }); +test("a re-sent pair quoting an active block's range is its carrier: the fold sweep leaves it visible (#3769/#1567)", () => { + // Plugin-style fold: the block is minted a server-side callId the client can + // never echo, so the client's re-sent pair for that exact fold is fate=orphan + // by id — yet it IS the block's summary carrier (the host strips the in-place + // anchor only while the pair rides the history). Sweeping it would strand the + // fold with zero wire carriers. Both arg shapes must be recognized, mirroring + // the host carrier index (flat {startId,endId} AND {content:[…]}). + const core = createCore(); + const build = (callId: string, callText: string): CoreMessage[] => [ + msg("h1", "first detailed work item alpha content"), + msg("h2", "second detailed work item beta content"), + msg("h3", "third detailed work item gamma content"), + { + id: `c-${callId}`, + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: callId, + text: callText, + }, + { + id: `r-${callId}`, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text: "compressed the head", + }, + msg("last", "latest user intent theta"), + ]; + const fold = (messages: CoreMessage[], state: CompressionState) => + core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00003", summary: "fold the spent head", topic: "head", compressCallId: "plugin_1234" }, + ], + messages, + state, + config: config(), + }); + + const carriers: Array<[string, string]> = [ + ["flat {startId,endId}", JSON.stringify({ startId: "m00001", endId: "m00003", topic: "head", summary: "fold the spent head" })], + ["{content:[…]} array", JSON.stringify({ content: [{ startId: "m00001", endId: "m00003", topic: "head", summary: "fold the spent head" }] })], + ]; + for (const [label, callText] of carriers) { + const messages = build("call_echo_1", callText); + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; + const out = fold(messages, state); + assert.equal(out.result.errors.length, 0, `${label}: unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, `${label}: the head genuinely folds`); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, [], `${label}: carrier pair exempt from the fold sweep`); + assert.ok(r.messages.some((m) => m.toolCallId === "call_echo_1"), `${label}: carrier pair stays visible (#1567)`); + } + + // Negative control: a failure whose quoted range matches NO block is still swept. + const messages = build( + "call_fail_9", + JSON.stringify({ content: [{ startId: "mX", endId: "mY", summary: "s" }] }), + ); + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; + const out = fold(messages, state); + assert.equal(out.result.errors.length, 0, `negative: unexpected errors: ${out.result.errors.join("; ")}`); + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["call_fail_9"], "true failure residue at/after the fold is swept"); +}); + test("the fold marker survives the full pipeline: sync-blocks carries it to the hide node (#3769)", () => { const core = createCore(); const messages: CoreMessage[] = [ From 8289c1088207340614e856e7b04bc07a82137990 Mon Sep 17 00:00:00 2001 From: awork Date: Tue, 20 Oct 2026 03:25:25 +0800 Subject: [PATCH 18/29] test: align cache-friendly-plugin matrix and upgrade-compat with the carrier regime and #3818 full-mode default (#3769 follow-through) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cache-friendly-plugin: the previous rewrite pinned the pre-floor-20 sweep regime (successful echo pairs hidden at their fold step, anchors carrying). That contradicts KDD#7 (plugin lane's wire carrier IS the tool call; the agent view never renders the kernel acp_summary), the approved #3769 scope (failed-record cleanup only), and the suite's own P6 dsh-rebase subtest, which already pinned pair-rides/anchor-stripped. Re-pin to the verified carrier regime across all four wires: pairs ride from their fold turn on (summary twice per riding pair: args + receipt), SUM_MARKER zero while any pair rides, drop-pair surfaces exactly its anchor once then stays append-stable (#1567 fail-safe), dead-block pairs still sweep at the swallow boundary (#3769), google lane expectations follow the #3159 canon-shift. issue3769-upgrade-compat: since #3818 flipped the storage default to full, the unified index row is the load path — patching only the JSON envelope left a stale row the loader served instead, so the simulated old-kernel record now goes through the store (loadSync + flushSync dual-writes both copies). Teardown now closes both proxies even when an assert throws (previously a mid-test failure hung the runner). --- kernel/src/compress.ts | 1 - .../issue3939-fold-boundary-pins.test.ts | 26 +- tests/cache-friendly-plugin.test.ts | 230 ++++++++++-------- tests/issue3769-upgrade-compat.test.ts | 52 ++-- 4 files changed, 178 insertions(+), 131 deletions(-) diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 2aea1377b2..8ca64f32f0 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -665,7 +665,6 @@ export function createCore(ports: Ports = {}): CompressionCore { const preLoopState = cloneState(state); const applyErrors: string[] = []; const warningsBase = warnings.length; - const notesBase = notes.length; // #3769: capture which blocks already exist so, after the loop, we can tell // which were created by THIS fold and where it starts (for the orphan sweep). const preFoldBlockIds = new Set(state.blocks.map((b) => b.blockId)); diff --git a/kernel/tests/issue3939-fold-boundary-pins.test.ts b/kernel/tests/issue3939-fold-boundary-pins.test.ts index bfd1488b6d..2b0dda2aa1 100644 --- a/kernel/tests/issue3939-fold-boundary-pins.test.ts +++ b/kernel/tests/issue3939-fold-boundary-pins.test.ts @@ -93,33 +93,39 @@ function liveBlockState(): { state: ReturnType } { return { state }; } -test("pin: hide-consumed keeps only the newest 2 orphaned compress calls (KEEP_LAST_ORPHANED=2)", () => { +test("pin: between folds no orphan is swept — quiet turns keep every failed compress call visible (#3769 fold gate)", () => { const messages = orphanHistory(); const res = hideConsumedCompressCalls(createInitialState(), messages); const keptCalls = res.messages .filter((m) => m.contentType === "tool-call") .map((m) => m.toolCallId); - assert.deepEqual(keptCalls, ["c3", "c4"]); - assert.equal(res.messages.length, 4); + // The pre-#3769 keep-last-two rotation hid the oldest failures on EVERY + // turn, rewriting the still-cached prefix for a few hundred chars. Now the + // window only sweeps once a committed fold stamps pendingOrphanSweepMinRef. + assert.deepEqual(keptCalls, ["c1", "c2", "c3", "c4"]); + assert.equal(res.messages.length, 8); + assert.equal(res.hidden, 0); + assert.deepEqual(res.newlySweptCallIds, []); }); -test("pin: hidden orphan calls report their refs when state has them assigned", () => { +test("pin: a pending fold sweep hides orphans at/after its min ref and reports their refs (#3769)", () => { const messages = orphanHistory(); const state = createInitialState(); state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1, }).map; + state.pendingOrphanSweepMinRef = 1; const res = hideConsumedCompressCalls(state, messages); - assert.deepEqual(res.hiddenOrphanRefs, [ - "m00001", - "m00002", - "m00003", - "m00004", - ]); + assert.equal(res.hidden, 8); + assert.deepEqual( + res.hiddenOrphanRefs, + ["m00001", "m00002", "m00003", "m00004", "m00005", "m00006", "m00007", "m00008"], + ); + assert.deepEqual(res.newlySweptCallIds, ["c1", "c2", "c3", "c4"]); }); test("pin: active compress call keeps only block-live ranges (dead entries stripped from args)", () => { diff --git a/tests/cache-friendly-plugin.test.ts b/tests/cache-friendly-plugin.test.ts index deeea63826..7362367629 100644 --- a/tests/cache-friendly-plugin.test.ts +++ b/tests/cache-friendly-plugin.test.ts @@ -7,16 +7,18 @@ // Plugin-mode carrier semantics: the ACP-native agent OWNS compression — it // executes `compress` locally through the plugin tool API and re-sends the // call + result in its own persisted history on every later request (pi -// semantics: pairs stay at their chronological position forever). The proxy -// therefore re-injects the in-place acp_summary carrier from block state on -// every request. Since the #1567 fix the carrier is STRIPPED per-block, but -// self-verified: only while that exact fold's compress pair actually rides -// the (post-prepare) inbound history. Under fold-gated sweeps (#3769) a pair -// whose call sits at/after a committed fold's start ref is hidden from that -// fold turn on — in this harness the fold POST and its model request share -// one step, so each failed pair is swept at its own fold step, never rides, -// and its anchor carries the summary for good (zero-carrier fail-safe). This -// suite pins that behavior byte-for-byte. +// semantics: pairs stay at their chronological position forever). That pair +// IS the wire summary carrier — the agent's view never renders the kernel +// acp_summary (KDD#7). Since the #1567 fix the kernel-rendered anchor is +// STRIPPED per-block, self-verified: only while that exact fold's compress +// pair rides the (post-prepare) inbound history quoting an ACTIVE block's +// range (#3769 carrier exemption). The moment a pair stops riding (client +// drop), its anchor re-carries the summary from that turn on (#1567 +// fail-safe); a pair whose block died (swallowed by a wider fold) loses its +// carrier status and is swept at the swallowing fold's boundary, together +// with genuinely failed compress records sitting at/after the fold's start +// ref (fold-gated sweep, #3769). This suite pins that behavior +// byte-for-byte. // // Invariants pinned per wire (bodies = consecutive upstream-bound request // payloads of one session, compared as flat unit lists — text blocks and @@ -34,19 +36,22 @@ // that move); every other byte stays an exact prefix; // P2 fold transitions (client re-sends its compress pair after a // /__bili/plugin/tool fold): -// P2a the pre-fold span is replaced by exactly the expected carrier -// units and nothing else moves or mutates (exact reconstruction); -// P2b the pair is swept at its own fold step (#3769) and never rides -// inbound — occurrence counts pinned: exactly ONE summary-text hit -// per post-fold turn (the anchor carrier) and ZERO pair quotes, -// one carrier marker per committed fold (#1567 fail-safe); +// P2a the pre-fold span is replaced by exactly the expected units and +// nothing else moves or mutates (exact reconstruction; when the +// span touches m00001 the session head survives as boundary, +// mid-history spans are consumed whole); +// P2b the pair RIDES inbound from its fold turn on and IS the summary +// carrier — occurrence counts pinned: ZERO kernel anchors +// (SUM_MARKER) while any pair rides, and exactly TWO summary-text +// hits per riding pair (re-sent call args + tool-result receipt); // P2c fail-safe (#1567 hardening): the anchor carries the summary -// whenever the pair does NOT ride inbound — under #3769 that is -// every post-fold turn in this harness; the drop-pair subtest -// pins that a client-side pair drop afterwards is a wire no-op; -// P3 multi-fold: two folds in one session; each anchor carries its own -// summary from its fold turn on, the carrier count grows by one per -// fold and then stays constant (keep-two rotation is gone, #3769); +// whenever the pair does NOT ride inbound — the drop-pair subtest +// pins that a client-side pair drop surfaces exactly its anchor +// once, then stays append-stable (no flapping); +// P3 multi-fold: two folds in one session; each pair rides alongside its +// siblings as its own carrier, anchors stay stripped for the life of +// the pairs, per-fold summary counts constant, quiet turns +// append-stable (keep-two rotation is gone, #3769); // P4 no transient acp_loop_* artifact leaks into any forwarded body // (plugin mode has none by construction — pin it); // P5 marker/tag hygiene: every content text unit carries exactly one @@ -69,8 +74,8 @@ // // Scenario (main, all wires): t1=[q1,a1,q2,a2], t2=+q3,a3, t3=+q4,a4, // foldA(m00001..m00002)+pairA@t4, t5=+q5,a5, foldB(m00003..m00005)+pairB@t6, -// t7=+q7,a7, t8=+q8,a8 — each failed pair is swept at its own fold step -// (#3769); its anchor carries the summary from there on. +// t7=+q7,a7, t8=+q8,a8 — each pair rides as its fold's carrier from its +// fold turn on; anchors stay stripped while their pairs ride (#1567/#3769). // // Scope notes: tool protocol only (the default). The swallow/back-to-back // fold geometries get a dedicated anthropic subtest (kernel-side behavior, @@ -615,29 +620,32 @@ for (const wire of ["anthropic", "openai", "responses", "google"] as Wire[]) { assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1", "SA"], L(3)); // head boundary survives; anchor A carries — its pair was swept at fold time (#3769) - assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); - assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], ["SB"], L(5)); // span consumed; anchor B carries (#3769) - assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q7", "a7"]); + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // session-head boundary survives; span consumed, no anchor — pair A rides as carrier (#1567/#3769) + // #3159 (google lane only): the riding pair's functionCall/ + // functionResponse split swallows the turn's leading text part into + // the sanctioned shift, so only the model reply lands in `added` + assertPrefix(cans[3]!.core, cans[4]!.core, L(4), wire === "google" ? ["a5"] : ["q5", "a5"], wire === "google"); + assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], [], L(5)); // mid-history span consumed whole; no anchor — pair B rides (#1567/#3769) + assertPrefix(cans[5]!.core, cans[6]!.core, L(6), wire === "google" ? ["a7"] : ["q7", "a7"], wire === "google"); assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); for (let i = 3; i < cans.length; i++) { const raw = cans[i]!.raw; - // #3769: each failed pair is swept at its own fold step, so no - // pair quotes ride inbound — each anchor carries its summary for - // good (zero-carrier fail-safe, #1567) - const expA = 1; - const expB = i < 5 ? 0 : 1; - assert.equal(countOcc(raw, SUM_MARKER), i < 5 ? 1 : 2, `${L(i)}: one carrier per committed fold (pairs swept, #3769)`); - assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA} (anchor carrier only, #3769)`); - assert.equal(countOcc(raw, SUM_B), expB, `${L(i)}: SUM_B occurrences ${countOcc(raw, SUM_B)} != ${expB} (anchor carrier only, #3769)`); - // google part-shapes carry no call id; pin pair ABSENCE through - // the fold spec refs that would sit in the re-sent functionCall - // args (the anchor carrier embeds topic+summary, never the spec) + // Carrier regime (#1567/#3769/KDD#7): each fold's echoed pair + // rides from its fold turn on and IS the summary carrier — the + // kernel anchor stays stripped for the pair's life, and the + // summary text appears exactly twice per riding pair (call args + // + result receipt). + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while their pairs ride (#1567)`); + assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A rides pair A args+receipt (got ${countOcc(raw, SUM_A)})`); + assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 2, `${L(i)}: SUM_B rides pair B from its fold turn on (got ${countOcc(raw, SUM_B)})`); + // pin pair PRESENCE — the carriers; google part-shapes carry no + // call id, so pin through the fold-spec refs in the re-sent + // functionCall args const pairA = wire === "google" ? "\"startId\":\"m00001\"" : "\"cfp_a\""; const pairB = wire === "google" ? "\"startId\":\"m00003\"" : "\"cfp_b\""; - assert.ok(!raw.includes(pairA), `${L(i)}: swept pairA must stay hidden (#3769)`); - assert.ok(!raw.includes(pairB), `${L(i)}: swept pairB must stay hidden (#3769)`); + assert.ok(raw.includes(pairA), `${L(i)}: pair A rides as its fold's carrier (#1567)`); + assert.equal(raw.includes(pairB), i >= 5, `${L(i)}: pair B rides from t6 on`); } }, { timeout: 120_000 }); @@ -668,57 +676,60 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1", "SA"], L(3)); // head boundary survives; anchor A carries (pair swept at fold time, #3769) + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // session-head boundary survives; no anchor — geo_a rides as carrier (#1567/#3769) assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); const wide = FOLD_WIDE.summary; const gamma = FOLD_E.summary; - // Fold-gated sweeps (#3769): every failed pair is hidden from its own - // fold step on, so no pair quotes ride inbound and each anchor carries - // its summary. The wide fold (m00001..m00004) swallows the alpha fold's - // range entirely — the alpha summary merges away and the wide carrier - // takes over. FOLD_E's tail sits in the protected zone, so only its - // first message folds and the gamma carrier lands where q3 used to sit. + // The wide fold (m00001..m00004) swallows the alpha fold's range: the + // alpha summary merges into the wide fold, and the geo_a pair — whose + // block died — loses carrier status and is swept at the swallow + // boundary (#3769). geo_wide rides as the merged region's carrier. + // FOLD_E's tail sits in the protected zone, so only its first message + // folds; geo_e rides alongside. for (const i of [5, 6, 7]) { const raw = cans[i]!.raw; assert.ok(!raw.includes("q2-marker"), `${L(i)}: swallowed content q2 must stay hidden`); assert.ok(!raw.includes("a2-marker"), `${L(i)}: swallowed content a2 must stay hidden`); - assert.ok(!raw.includes("\"geo_a\"") && !raw.includes("\"geo_wide\"") && !raw.includes("\"geo_e\""), `${L(i)}: all three failed pairs swept at their fold steps (#3769)`); + assert.ok(!raw.includes("\"geo_a\""), `${L(i)}: dead-block pair swept at the swallow boundary (#3769)`); } assert.equal(countOcc(cans[5]!.raw, SUM_A), 0, `${L(5)}: alpha summary absorbed into the wide fold`); - assert.equal(countOcc(cans[5]!.raw, wide), 1, `${L(5)}: wide carrier only (its pair swept, #3769)`); - assert.equal(countOcc(cans[5]!.raw, SUM_MARKER), 1, `${L(5)}: single carrier after the swallow`); - assert.equal(countOcc(cans[6]!.raw, wide), 1, `${L(6)}: wide carrier persists`); - assert.equal(countOcc(cans[6]!.raw, gamma), 1, `${L(6)}: gamma carrier only (its pair swept, #3769)`); - assert.equal(countOcc(cans[6]!.raw, SUM_MARKER), 2, `${L(6)}: one carrier per committed fold`); - assert.equal(countOcc(cans[7]!.raw, wide), 1, `${L(7)}: wide carrier persists`); - assert.equal(countOcc(cans[7]!.raw, gamma), 1, `${L(7)}: gamma carrier persists`); - assert.equal(countOcc(cans[7]!.raw, SUM_MARKER), 2, `${L(7)}: carriers stable across quiet turns`); - const surviveKeys = new Set(["q1", "a1", "q2", "a2", "SA"]); - const keepT5 = cans[4]!.core.filter((u) => !surviveKeys.has(keyOf(u))); + assert.equal(countOcc(cans[5]!.raw, wide), 2, `${L(5)}: wide summary rides geo_wide args+receipt`); + assert.equal(countOcc(cans[5]!.raw, gamma), 0, `${L(5)}: E not folded yet`); + assert.equal(countOcc(cans[5]!.raw, SUM_MARKER), 0, `${L(5)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(cans[6]!.raw, wide), 2, `${L(6)}: wide carrier persists`); + assert.equal(countOcc(cans[6]!.raw, gamma), 2, `${L(6)}: gamma rides geo_e from its fold turn on`); + assert.equal(countOcc(cans[6]!.raw, SUM_MARKER), 0, `${L(6)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(cans[7]!.raw, wide), 2, `${L(7)}: wide carrier persists`); + assert.equal(countOcc(cans[7]!.raw, gamma), 2, `${L(7)}: gamma carrier persists`); + assert.equal(countOcc(cans[7]!.raw, SUM_MARKER), 0, `${L(7)}: carriers stable across quiet turns`); + // t5→t6 rewrite: the wide span (minus the session-head boundary) is + // consumed and the dead-block pair swept — everything else keeps order + const goneT6 = new Set(["a1", "q2", "a2", "TU:geo_a", "TR:geo_a"]); + const keptT5 = cans[4]!.core.filter((u) => !goneT6.has(keyOf(u))); let cursor = -1; - for (const u of keepT5) { + for (const u of keptT5) { const found = cans[5]!.core.findIndex((v, j) => j > cursor && eqUnit(u, v)); assert.ok(found > cursor, `cfp-geo t6: unit ${JSON.stringify(keyOf(u))} lost or reordered by the wide fold`); cursor = found; } - // #1567/#3769: the wide summary rides its anchor carrier (a text unit) - // for good — the pair was swept at fold time, so the carrier is never - // stripped again - const wideT6 = cans[6]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); - const wideT7 = cans[7]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); - assert.equal(wideT6.length, 1, `${L(6)}: wide carrier rides a text unit while the geo_wide pair stays swept`); - assert.equal(wideT7.length, 1, `${L(7)}: wide carrier rides a text unit while the geo_wide pair stays swept`); + assertFoldTransition(cans[5]!.core, cans[6]!.core, ["q3"], [], L(6)); // protected-zone carve: only E's first message folds; no anchor — geo_e rides + // #1567: while the pairs ride, their summaries live in TOOL units only — + // never a text-unit anchor + for (const idx of [5, 6, 7]) { + const textCarriers = cans[idx]!.core.filter((u) => u.k === "text" && (u.x.includes(wide) || u.x.includes(gamma))); + assert.equal(textCarriers.length, 0, `${L(idx)}: no text-unit anchor while the riding pairs carry the summaries (#1567)`); + } assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); // #1567 hardening scenario A, re-pinned under #3769: a contract-violating -// (or pruning) client stops re-sending an older compress pair at t7. Under -// fold-gated sweeps the pair was already hidden at its own fold step (t4), -// so its anchor has carried the summary since then — the client-side drop is -// a wire no-op, and the body stays append-stable with exactly one carrier -// per committed fold (no flapping). -testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → wire unaffected, carriers stable (#1567/#3769)`, async () => { +// (or pruning) client stops re-sending an older compress pair at t7. While +// the pairs ride they ARE the carriers (anchors stripped); the moment pairA +// stops riding, its anchor re-carries the summary exactly once at the +// covered span — then the body stays append-stable (no flapping). PairB +// still rides, so anchor B never surfaces. +testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → anchor re-carries once, no flapping (#1567/#3769)`, async () => { const steps: Step[] = [ { send: [T(1), A(1), T(2), A(2)] }, { send: [T(3), A(3)] }, @@ -737,28 +748,43 @@ testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pai assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-drop"); - // t4..t8: both pairs swept at their fold steps → each anchor is the sole - // carrier of its summary; the client dropping pairA at t7 changes nothing - for (const i of [3, 4, 5, 6, 7]) { + // t4..t6: both pairs ride as carriers — anchors stripped + for (const i of [3, 4, 5]) { const raw = cans[i]!.raw; - assert.ok(!raw.includes('"cfp_a"') && !raw.includes('"cfp_b"'), `${L(i)}: failed pairs swept at fold time (#3769)`); - assert.equal(countOcc(raw, SUM_A), 1, `${L(i)}: anchor A carries (sole occurrence)`); - assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 1, `${L(i)}: anchor B carries from its fold turn on`); - assert.equal(countOcc(raw, SUM_MARKER), i < 5 ? 1 : 2, `${L(i)}: one carrier per committed fold`); + assert.ok(raw.includes('"cfp_a"'), `${L(i)}: pair A rides as its fold's carrier (#1567)`); + assert.ok(i >= 5 || !raw.includes('"cfp_b"') , `${L(i)}: pair B rides from t6 on`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A rides pair A args+receipt`); + assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 2, `${L(i)}: SUM_B rides pair B from its fold turn on`); + } + // t7: pairA dropped → its anchor re-carries exactly once at the covered + // span; pairB still rides so anchor B stays stripped (#1567 fail-safe) + { + const raw = cans[6]!.raw; + assert.ok(!raw.includes('"cfp_a"'), `${L(6)}: dropped pair stays dropped`); + assert.ok(raw.includes('"cfp_b"'), `${L(6)}: pair B unaffected by A's drop`); + assert.equal(countOcc(raw, SUM_A), 1, `${L(6)}: anchor A re-carries the sole copy`); + assert.equal(countOcc(raw, SUM_B), 2, `${L(6)}: pair B still rides`); + assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(6)}: exactly one anchor rendered (A's)`); + assert.ok(raw.includes("q7-marker"), `${L(6)}: new turn visible`); + } + // t8: once re-carried, the body stays append-stable (no flapping) + { + const raw = cans[7]!.raw; + assert.equal(countOcc(raw, SUM_A), 1, `${L(7)}: anchor A stable`); + assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(7)}: no flapping after the drop`); } - assert.ok(cans[6]!.raw.includes("q7-marker"), `${L(6)}: new turn visible`); - // once re-carried, the body stays append-stable (no flapping) assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); // #3769 contract pin (plugin lane): three back-to-back folds, each with a -// failed (client-visible) compress pair. The old KEEP_LAST_ORPHANED=2 -// rotation hid the oldest pair on EVERY turn — a mid-history rewrite with no -// fold behind it, exactly #3769's cache seam. Fold-gated sweeps hide each -// pair at its own fold step (the deletion rides the fold's own rewrite), so -// every anchor carries its summary from the fold turn on, carriers stack -// without churning, and quiet turns stay append-stable. -testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: pairs swept at fold time, anchors carry (#3769)`, async () => { +// client-visible compress pair. The old KEEP_LAST_ORPHANED=2 rotation hid +// the oldest pair on EVERY turn — a mid-history rewrite with no fold behind +// it, exactly #3769's cache seam. Under the carrier regime every pair rides +// as its fold's summary carrier for good: no per-turn hiding, no churn, +// anchors stay stripped while the pairs ride, and quiet turns stay +// append-stable. +testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: every pair rides as its carrier, quiet turns append-stable (#3769)`, async () => { const SUM_P1 = "Prune-safe fold one summary covering m00001..m00002"; const SUM_P2 = "Prune-safe fold two summary covering m00003..m00004"; const SUM_P3 = "Prune-safe fold three summary covering m00005..m00006"; @@ -782,23 +808,25 @@ testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: pair assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-prune"); - // t3..t5: each fold's pair is swept at its own fold step → each anchor - // carries its summary; carrier count grows by one per fold + // t3..t5: each fold's pair rides from its fold turn on (args+receipt = + // 2 occurrences); anchors stay stripped for the life of the pairs for (const i of [2, 3, 4]) { const raw = cans[i]!.raw; - assert.ok(!raw.includes('"fs1"') && !raw.includes('"fs2"') && !raw.includes('"fs3"'), `${L(i)}: failed pairs swept at their fold steps (#3769)`); - assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: anchor P1 carries (sole occurrence)`); - assert.equal(countOcc(raw, SUM_P2), i < 3 ? 0 : 1, `${L(i)}: anchor P2 carries from its fold turn on`); - assert.equal(countOcc(raw, SUM_P3), i < 4 ? 0 : 1, `${L(i)}: anchor P3 carries from its fold turn on`); - assert.equal(countOcc(raw, SUM_MARKER), i - 1, `${L(i)}: one carrier per committed fold`); + assert.ok(raw.includes('"fs1"'), `${L(i)}: pair 1 rides as its fold's carrier (#1567)`); + assert.ok(i >= 3 || !raw.includes('"fs2"'), `${L(i)}: pair 2 rides from t4 on`); + assert.ok(i >= 4 || !raw.includes('"fs3"'), `${L(i)}: pair 3 rides from t5 on`); + assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: P1 rides pair-1 args+receipt`); + assert.equal(countOcc(raw, SUM_P2), i < 3 ? 0 : 2, `${L(i)}: P2 rides from its fold turn on`); + assert.equal(countOcc(raw, SUM_P3), i < 4 ? 0 : 2, `${L(i)}: P3 rides from its fold turn on`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while the pairs ride`); } - // t6/t7: all three carriers persist, stable across quiet turns + // t6/t7: all three pairs persist, stable across quiet turns for (const i of [5, 6]) { const raw = cans[i]!.raw; - assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: P1 carrier persists`); - assert.equal(countOcc(raw, SUM_P2), 1, `${L(i)}: P2 carrier persists`); - assert.equal(countOcc(raw, SUM_P3), 1, `${L(i)}: P3 carrier persists`); - assert.equal(countOcc(raw, SUM_MARKER), 3, `${L(i)}: three carriers, stable`); + assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: P1 carrier persists`); + assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: P2 carrier persists`); + assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: P3 carrier persists`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: three riding pairs, zero anchors, stable`); } // mid-history never mutates between quiet turns assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q8", "a8"]); diff --git a/tests/issue3769-upgrade-compat.test.ts b/tests/issue3769-upgrade-compat.test.ts index 289528f6cd..3b68e542ae 100644 --- a/tests/issue3769-upgrade-compat.test.ts +++ b/tests/issue3769-upgrade-compat.test.ts @@ -7,7 +7,7 @@ import assert from "node:assert/strict"; import http from "node:http"; import { once } from "node:events"; import test from "node:test"; -import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -103,6 +103,15 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e const upPort = (upstream.address() as { port: number }).port; const stores: SessionStore[] = []; + const proxies: http.Server[] = []; + const closedProxies = new Set(); + async function closeProxy(p: http.Server): Promise { + if (closedProxies.has(p)) return; + closedProxies.add(p); + p.close(); + p.closeIdleConnections?.(); + await once(p, "close").catch(() => undefined); + } try { // Phase 1: current code runs two turns; the failed pair rides visibly // (fold-gated contract: no fold -> no sweep) and the record lands on disk. @@ -132,6 +141,7 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e mitm: { enabled: false, domains: [] }, } as unknown as ProxyOptions); await once(proxy1, "listening"); + proxies.push(proxy1); const port1 = (proxy1.address() as { port: number }).port; const url1 = `http://127.0.0.1:${port1}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; @@ -143,12 +153,20 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e const s1 = listSessions().find((x) => x.meta.label === LABEL); assert.ok(s1, "session exists"); assert.ok(store1.flushSync(s1!)); - proxy1.close(); - proxy1.closeIdleConnections?.(); - await once(proxy1, "close"); + await closeProxy(proxy1); + + // Phase 2: a fresh store hydrates the record, then shapes it into + // what a pre-fold-gate kernel left behind — rotation had hidden the + // pair (remembered by ref), new sweep fields absent. Refs are read + // off the dual-written envelope; the mutation itself goes THROUGH THE + // STORE: since the #3818 full-mode default the index row IS the load + // path, so patching only the envelope would leave a stale row the + // loader serves instead. + _resetSessionsForTest(); + const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + stores.push(store2); + _setStoreForTest(store2); - // Patch the persisted record into the exact shape an old kernel left: - // rotation had hidden the pair (remembered by ref), new fields absent. const file = findSessionFile(dir, LABEL); const rec = JSON.parse(readFileSync(file, "utf8")) as PersistRecord; const callMsg = rec.payload.messages.find((m) => m.contentType === "tool-call" && m.toolName === "compress"); @@ -157,16 +175,12 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e const callRef = rec.payload.state.messageRefs.byRaw[callMsg!.id]; const resRef = rec.payload.state.messageRefs.byRaw[resMsg!.id]; assert.ok(callRef && resRef, "refs resolvable from the persisted state"); - rec.payload.state.hiddenOrphanRefs = [callRef, resRef]; - delete rec.payload.state.sweptOrphanCallIds; - delete rec.payload.state.pendingOrphanSweepMinRef; - writeFileSync(file, JSON.stringify(rec)); - - // Phase 2: a fresh server instance hydrates the patched record. - _resetSessionsForTest(); - const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); - stores.push(store2); - _setStoreForTest(store2); + const sPre = store2.loadSync(LABEL); + assert.ok(sPre, "record hydrates before the upgrade turn"); + sPre.state.hiddenOrphanRefs = [callRef, resRef]; + delete sPre.state.sweptOrphanCallIds; + delete sPre.state.pendingOrphanSweepMinRef; + assert.ok(store2.flushSync(sPre), "patched record commits through the store"); const proxy2 = await startServer({ port: 0, host: "127.0.0.1", @@ -188,6 +202,7 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e mitm: { enabled: false, domains: [] }, } as unknown as ProxyOptions); await once(proxy2, "listening"); + proxies.push(proxy2); const port2 = (proxy2.address() as { port: number }).port; const url2 = `http://127.0.0.1:${port2}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; @@ -201,11 +216,10 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e const s2 = listSessions().find((x) => x.meta.label === LABEL); assert.ok(s2, "hydrated session exists"); assert.ok(s2.state.hiddenOrphanRefs?.includes(callRef!), "legacy hidden set survived hydration and self-perpetuates"); - proxy2.close(); - proxy2.closeIdleConnections?.(); - await once(proxy2, "close"); + await closeProxy(proxy2); } finally { for (const s of stores) s.closeQuietly(); + for (const p of proxies) await closeProxy(p); upstream.close(); upstream.closeIdleConnections?.(); await once(upstream, "close").catch(() => undefined); From b9bae89901a841194c74c5b45a26a8fcf4c277fa Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:19:54 +0800 Subject: [PATCH 19/29] =?UTF-8?q?kernel:=20latch=20emergency=20truncation?= =?UTF-8?q?=20bytes=20until=20the=20next=20fold=20(#3939=20=E2=91=A2,=20#3?= =?UTF-8?q?691)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watermark decision is made once: replaying the latched clip on every later turn keeps outbound bytes stable when the token estimate fluctuates across the threshold. Refs are content-bound so edited messages re-judge; entries retire when their message leaves the view; a successful fold resets the latch (the clipped originals ride the fold away). Carry surfaces: sync node, persist mergeState, fork-adoption, plugin fork prefix-filter, storage ingest. --- kernel/package.json | 2 +- kernel/src/compress.ts | 87 ++++++++++- kernel/src/sync.ts | 6 + kernel/src/truncate-tools.ts | 8 +- kernel/src/types.ts | 18 +++ kernel/tests/issue3939-truncate-latch.test.ts | 139 ++++++++++++++++++ src/fork-adoption.ts | 6 + src/persist.ts | 4 + src/plugin.ts | 4 + src/storage/ingest.ts | 1 + 10 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 kernel/tests/issue3939-truncate-latch.test.ts diff --git a/kernel/package.json b/kernel/package.json index 8ac891d931..d5f0ba0d08 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.138", + "version": "0.0.139", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 8ca64f32f0..7d39230a5a 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -19,6 +19,7 @@ import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyCrushToMessages } from "./crush.js"; import { applyRetrieve, ccrStoreNode, RETRIEVED_ID_PREFIX } from "./ccr.js"; import type { ApplyRetrieveResult, CcrEffect } from "./ccr.js"; +import type { EmergencyTruncateLatchEntry } from "./types.js"; import { createContentStore } from "./content-store.js"; import type { MessageContentStore } from "./content-store.js"; import { applyMessageFilters, listMessageFilters } from "./filter/index.js"; @@ -145,6 +146,11 @@ type RangeResolution = | { status: "unknown"; error: BoundaryNotFoundError } | { status: "invalid"; error: Error }; +/** #3939 ③: bound on persisted emergency-truncate latch entries. Each entry + * carries a clipped text (≤ keepPrefix+marker+keepSuffix ≈ 4KiB); a fold + * clears the latch, so normal sessions hold a handful at most. Oldest drop. */ +const MAX_TRUNCATE_LATCH = 64; + function rangeError( spec: { startRef: string; endRef: string }, message: string, @@ -797,6 +803,11 @@ export function createCore(ports: Ports = {}): CompressionCore { // the terminal-floor streak so the escape signal can only fire again // after N fresh stuck events (#300). state.terminalStreak = 0; + // #3939 ③/#3691: the fold reset the truncation landscape — clipped + // originals under the fold are gone from the view and the watermark + // re-evaluates from the new (lower) count, so the latch retires with it. + // Holding it would replay stale clips onto a view the fold rewrote. + state.emergencyTruncateLatch = undefined; } return { @@ -1281,14 +1292,53 @@ const emergencyTruncateNode: PipelineNode = { ? ctx.tokenCount / ctx.config.modelContextLimit : 0; const prevStreak = io.state.terminalStreak ?? 0; + // #3939 ③/#3691: replay latched clips BEFORE the watermark check. The + // decision to clip was made once at the watermark; re-evaluating it every + // turn against a fluctuating estimate flips the same bytes full↔clipped — + // a mid-history rewrite on every request (prefix-cache breaker). Refs are + // content-bound, so an edited message misses its entry and re-judges from + // scratch; entries whose messages left the view retire; the whole latch + // resets on the next successful fold (applyCompression). + const latch = io.state.emergencyTruncateLatch ?? []; + const liveRefs = new Set(); + for (const m of io.messages) { + const ref = io.state.messageRefs.byRaw[m.id]; + if (ref !== undefined) liveRefs.add(ref); + } + const latchLive = latch.filter((e) => liveRefs.has(e.ref)); + let messages = io.messages; + let replaySaved = 0; + if (latch.length > 0) { + const byRef = new Map(latch.map((e) => [e.ref, e.text])); + messages = io.messages.map((m) => { + const ref = io.state.messageRefs.byRaw[m.id]; + const clipped = ref !== undefined ? byRef.get(ref) : undefined; + if (clipped === undefined || clipped === m.text) return m; + replaySaved += Math.max( + 0, + ctx.countTokens(m.text ?? "") - ctx.countTokens(clipped), + ); + return { ...m, text: clipped }; + }); + } if (usage < ctx.config.truncate.threshold) { - return prevStreak > 0 - ? { ...io, state: { ...io.state, terminalStreak: 0 } } - : io; + if (latch.length === 0 && prevStreak === 0) return io; + return { + ...io, + messages, + state: { + ...io.state, + terminalStreak: 0, + emergencyTruncateLatch: + latchLive.length > 0 ? latchLive : undefined, + }, + }; } const trunc = truncateLargeToolOutputs( - io.messages, - ctx.tokenCount, + messages, + // Account for replayed clips: they already shrank the real request; + // charging their tokens again would over-truncate fresh candidates. + Math.max(0, ctx.tokenCount - replaySaved), ctx.config, ctx.countTokens, { @@ -1333,10 +1383,35 @@ const emergencyTruncateNode: PipelineNode = { stuckEvents: streak, }; } + // #3939 ③: fold this pass's clips into the latch (replacing any same-ref + // entry) so later turns replay them byte-identically. Bounded so a long + // session cannot grow the persisted state without limit; the oldest + // entries drop first. + const clips: EmergencyTruncateLatchEntry[] = []; + for (const id of trunc.truncatedIds ?? []) { + const ref = io.state.messageRefs.byRaw[id]; + const clippedMsg = trunc.messages.find((m) => m.id === id); + if (ref !== undefined && clippedMsg?.text !== undefined) { + clips.push({ ref, text: clippedMsg.text }); + } + } + const clipRefs = new Set(clips.map((c) => c.ref)); + const nextLatch = [ + ...latchLive.filter((e) => !clipRefs.has(e.ref)), + ...clips, + ]; + if (nextLatch.length > MAX_TRUNCATE_LATCH) { + nextLatch.splice(0, nextLatch.length - MAX_TRUNCATE_LATCH); + } return { ...io, messages: trunc.messages, - state: { ...io.state, terminalStreak: streak }, + state: { + ...io.state, + terminalStreak: streak, + emergencyTruncateLatch: + nextLatch.length > 0 ? nextLatch : undefined, + }, effects, }; }, diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index 9a12073f3e..b9908863c6 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -53,6 +53,12 @@ export function syncBlocks( hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined, sweptOrphanCallIds: state.sweptOrphanCallIds ? [...state.sweptOrphanCallIds] : undefined, pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef, + // #3939 ③: the truncation latch must survive the sync node — dropping it + // here would re-expose clipped bytes mid-history after a sync carried the + // rest of the state (#3691 oscillation via the sync path). + emergencyTruncateLatch: state.emergencyTruncateLatch + ? state.emergencyTruncateLatch.map((e) => ({ ref: e.ref, text: e.text })) + : undefined, }; // Refs are additive (assignRefs never removes them from messageRefs), so diff --git a/kernel/src/truncate-tools.ts b/kernel/src/truncate-tools.ts index 964bfc1b0d..19083bdc00 100644 --- a/kernel/src/truncate-tools.ts +++ b/kernel/src/truncate-tools.ts @@ -21,6 +21,9 @@ export interface TruncateResult { /** Size-qualified candidates found (pre-truncation). 0 explains a silent * no-op at/above threshold — hosts should warn-log (#300). */ candidatesFound: number; + /** Ids of the messages actually clipped this pass (#3939 ③ latch). Empty + * when truncatedCount is 0. */ + truncatedIds?: string[]; } const TRUNCATION_MARKER = "[truncated for context space]"; @@ -42,7 +45,7 @@ export function truncateLargeToolOutputs( const opts = { ...DEFAULTS, ...options }; const limit = config.modelContextLimit; if (limit <= 0 || tokenCount < config.truncate.threshold * limit) { - return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound: 0 }; + return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound: 0, truncatedIds: [] }; } const protectedIndex = messages.length - opts.protectRecentMessages; @@ -114,7 +117,7 @@ export function truncateLargeToolOutputs( applyTo(textMessages); if (replacements.size === 0) { - return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound }; + return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound, truncatedIds: [] }; } return { messages: messages.map((m) => @@ -123,5 +126,6 @@ export function truncateLargeToolOutputs( truncatedCount, savedTokens, candidatesFound, + truncatedIds: [...replacements.keys()], }; } diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 91f98dd40b..17cfded00d 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -36,6 +36,13 @@ export type CompressionTier = 1 | 2 | 3; export type BlockGeneration = "young" | "old"; +/** One #3939 ③ emergency-truncation latch entry: the post-clip text of a + * message (keyed by its padded ref). See CompressionState.emergencyTruncateLatch. */ +export interface EmergencyTruncateLatchEntry { + ref: string; + text: string; +} + export interface CompressionBlock { blockId: string; runId: string; @@ -262,6 +269,17 @@ export interface CompressionState { * Carries the position so the sweep is cache-neutral: it only removes records * the fold already invalidated. Optional: absent until the first post-fix fold. */ pendingOrphanSweepMinRef?: number; + /** Emergency-truncation latch (#3939 ③/#3691): the clipped texts (post-clip + * bytes, keyed by padded ref) of messages clipped by the emergency-truncate + * node. The watermark decision is made ONCE — replaying these clips on every + * later turn keeps the outbound bytes stable when the token estimate + * fluctuates across the threshold (the #3691 full↔clipped oscillation was a + * mid-history rewrite on every request). Entries retire when their message + * leaves the resent view (folded/edited-away) and the whole latch resets on + * the next successful fold (applyCompression) — after a fold the watermark + * re-evaluates from scratch. Optional: pre-feature states have none (the + * pre-latch node clipped from scratch each turn). */ + emergencyTruncateLatch?: EmergencyTruncateLatchEntry[]; /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk diff --git a/kernel/tests/issue3939-truncate-latch.test.ts b/kernel/tests/issue3939-truncate-latch.test.ts new file mode 100644 index 0000000000..bd44ee9b15 --- /dev/null +++ b/kernel/tests/issue3939-truncate-latch.test.ts @@ -0,0 +1,139 @@ +// #3939 ③ / #3691: emergency truncation is a one-shot decision that must stay +// byte-stable across turns. The pre-latch node re-evaluated the watermark on +// every turn against the resent FULL history, so a fluctuating token estimate +// (95%+ one turn, below the next) flipped the same user text between its full +// and clipped forms every turn — a mid-history rewrite on every request and a +// prefix-cache breaker. The latch pins the clipped bytes until a fold resets +// them (#3939 contract: 水位触发 clip 后保持 clip 直到折叠事件重置). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { createInitialState } from "../src/state.js"; +import type { CoreMessage } from "../src/types.js"; + +const BIG = "alpha-beta ".repeat(4000); // ~48k chars, well past prefix+suffix +// Roomy filler so a fold spanning the whole history clears the 5000-char +// min-range gate even after the big text is clipped (~4.1K). +const filler = (t: string) => `filler ${t} ` + "x".repeat(800); +const tiny = (t: string): CoreMessage[] => [ + { id: `raw-${t}`, role: "user", contentType: "text", text: "hi " + t + " " + filler(t) }, + { id: `raw-${t}-a`, role: "assistant", contentType: "text", text: "ok " + t }, +]; + +function hist(): CoreMessage[] { + return [ + ...tiny("warm"), + { id: "raw-big", role: "user", contentType: "text", text: BIG }, + ...tiny("tail1"), + ...tiny("tail2"), + ...tiny("tail3"), + ...tiny("tail4"), + ]; +} + +const WINDOW = 100_000; +// Narrow the recent-token protection so a unit-scale history still has a +// foldable head (default preserveRecentTokens=5000 swallows this whole fixture +// and every range dies in the protected zone). The latch logic under test is +// orthogonal to the protected-zone math. +const cfg = () => ({ ...defaultConfig(WINDOW), preserveRecentTokens: 100 }); + +test("pin: a clip fires at 96% and the latch replays it on the next under-watermark turn (#3691)", () => { + const core = createCore(defaultConfig(WINDOW)); + const messages = hist(); + + const t1 = core.processTurn({ + messages, + state: createInitialState(), + config: defaultConfig(WINDOW), + tokenCount: 96_000, + renderTags: "none", + }); + const clipped1 = t1.messages.find((m) => m.id === "raw-big")!.text!; + assert.ok(clipped1.includes("[truncated for context space]"), "turn 1 clips the oversized text"); + assert.ok(t1.state.emergencyTruncateLatch?.some((e) => e.text === clipped1), "clip is latched"); + + // Turn 2: SAME full history resent, estimate now well under the watermark — + // pre-latch this returned the full text (the #3691 oscillation). + const t2 = core.processTurn({ + messages: hist(), + state: t1.state, + config: defaultConfig(WINDOW), + tokenCount: 60_000, + renderTags: "none", + }); + const clipped2 = t2.messages.find((m) => m.id === "raw-big")!.text!; + assert.equal(clipped2, clipped1, "latch replays the byte-identical clip under the watermark"); + assert.ok(t2.state.emergencyTruncateLatch?.some((e) => e.text === clipped1), "latch self-perpetuates while the message stays live"); +}); + +test("pin: a successful fold resets the latch and the clip retires with the fold", () => { + const core = createCore(cfg()); + const t1 = core.processTurn({ + messages: hist(), + state: createInitialState(), + config: cfg(), + tokenCount: 96_000, + renderTags: "none", + }); + assert.ok(t1.state.emergencyTruncateLatch?.length, "clip latched"); + + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const startRef = refOf(t1.state, "raw-warm"); + const endRef = refOf(t1.state, "raw-tail1-a"); + assert.ok(startRef && endRef, "refs assigned"); + + const fold = core.applyCompression({ + messages: t1.messages, + state: t1.state, + config: cfg(), + ranges: [{ startRef, endRef, summary: "a long enough summary for the fold gate to accept this range, over fifty chars", topic: "t" }], + }); + assert.equal(fold.result.blocksCreated, 1, "fold lands"); + assert.equal(fold.state.emergencyTruncateLatch, undefined, "fold resets the latch"); + + // Post-fold: a NEW oversized message under the watermark must NOT be + // clipped — the latch retired with the fold and the watermark re-evaluates + // from scratch (the original raw-big left the view inside the fold). + const post = [ + ...hist().filter((m) => m.id !== "raw-big"), + { id: "raw-big2", role: "user" as const, contentType: "text" as const, text: BIG }, + ]; + const t3 = core.processTurn({ + messages: post, + state: fold.state, + config: cfg(), + tokenCount: 60_000, + renderTags: "none", + }); + const big2 = t3.messages.find((m) => m.id === "raw-big2"); + assert.ok(big2, "new oversized message in view"); + assert.equal(big2!.text, BIG, "no clip after the fold reset (under watermark)"); + assert.equal(t3.state.emergencyTruncateLatch, undefined, "latch stays retired"); +}); + +test("pin: an edited message (new content id) is not replay-clipped from a stale latch entry", () => { + const core = createCore(defaultConfig(WINDOW)); + const t1 = core.processTurn({ + messages: hist(), + state: createInitialState(), + config: defaultConfig(WINDOW), + tokenCount: 96_000, + renderTags: "none", + }); + assert.ok(t1.state.emergencyTruncateLatch?.length, "clip latched"); + + const edited = hist(); + const bigIdx = edited.findIndex((m) => m.id === "raw-big"); + edited[bigIdx] = { ...edited[bigIdx]!, id: "raw-big-v2", text: "completely rewritten, short" }; + + const t2 = core.processTurn({ + messages: edited, + state: t1.state, + config: defaultConfig(WINDOW), + tokenCount: 60_000, + renderTags: "none", + }); + assert.equal(t2.messages.find((m) => m.id === "raw-big-v2")!.text, "completely rewritten, short"); + assert.equal(t2.state.emergencyTruncateLatch?.length ?? 0, 0, "stale entry retired with its message"); +}); diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 16222d9a61..22ddf63118 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -270,6 +270,12 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare // every turn, so the carry self-perpetuates afterwards. if (parent.state.hiddenOrphanRefs?.length) session.state.hiddenOrphanRefs = [...parent.state.hiddenOrphanRefs]; if (parent.state.sweptOrphanCallIds?.length) session.state.sweptOrphanCallIds = [...parent.state.sweptOrphanCallIds]; + // #3939 ③: same carry for the truncation latch — resume/fork replay + // re-sends the clipped messages; without the latch the first replayed + // turn under a lower estimate re-exposes the full bytes (oscillation). + if (parent.state.emergencyTruncateLatch?.length) { + session.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch.map((e) => ({ ...e })); + } markDirty(session); } diff --git a/src/persist.ts b/src/persist.ts index 9652ce41c8..6ec42bbae8 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -244,6 +244,10 @@ function mergeState(parsed: CompressionState): CompressionState { // transient, losing it only defers the sweep to the next fold. sweptOrphanCallIds: parsed.sweptOrphanCallIds ?? fresh.sweptOrphanCallIds, pendingOrphanSweepMinRef: parsed.pendingOrphanSweepMinRef ?? fresh.pendingOrphanSweepMinRef, + // #3939 ③/#3691: the emergency-truncate latch must survive hydration — + // dropping it re-exposes clipped bytes mid-history after a restart, + // rewriting the still-cached prefix on the first post-restart turn. + emergencyTruncateLatch: parsed.emergencyTruncateLatch ?? fresh.emergencyTruncateLatch, }; } diff --git a/src/plugin.ts b/src/plugin.ts index 404624285c..3d8b4818ef 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1389,6 +1389,10 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr child.state.rules = structuredClone(parent.state.rules); child.state.nextRuleId = parent.state.nextRuleId; child.state.hiddenOrphanRefs = parent.state.hiddenOrphanRefs?.filter((ref) => prefix.some((m) => m.ref === ref)); + // #3939 ③: only latch entries whose messages rode the forked + // prefix — the child never sees the others, so their entries + // would be dead weight (and retire on the next latch pass anyway). + child.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch?.filter((e) => prefix.some((m) => m.ref === e.ref)); // #2362: inverse of the orphan filter — a dead ref whose message IS // in the forked prefix is visible again in the child; refs outside // it stay dead (their messages are absent from the child too). diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 9b27270502..64990619b2 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -42,6 +42,7 @@ const MERGE_STATE_FIELDS = [ "tokenSnapshot", "lastPassIds", "hiddenOrphanRefs", + "emergencyTruncateLatch", "deadRefs", "terminalStreak", "nextRuleId", From 88ecbbf4db1d23f94a29a0f61183c69b3df40423 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:29:48 +0800 Subject: [PATCH 20/29] =?UTF-8?q?kernel:=20defer=20BLOCKED=E2=86=92numeric?= =?UTF-8?q?=20migration=20carrier=20tags=20to=20the=20fold=20boundary=20(#?= =?UTF-8?q?3939=20=E2=91=A1,=20#3805)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration (#2607) mints the fresh numeric ref immediately (citable for decompress bookkeeping) but the carrier message keeps its pre-migration, UNTAGGED bytes until the next successful fold — stamping the tag on the migration turn rewrote an old message mid-history on an otherwise quiet turn (the #3805 durable-carrier cache breaker). Fold success releases the deferral; carried across sync/persist/fork/plugin/ingest. Also fixes the golden regen script missing the BILI_SNAPSHOT_SHADOW=0 parity pin (regenerated goldens red-flipped on the next CI run without it); storage-render goldens now pin the two new undefined-valued state keys, wire bodies unchanged. --- kernel/package.json | 2 +- kernel/src/compress.ts | 24 ++++- kernel/src/refs.ts | 12 ++- kernel/src/render-refs.ts | 16 +++- kernel/src/sync.ts | 5 ++ kernel/src/types.ts | 11 +++ .../issue3939-ref-migration-defer.test.ts | 89 +++++++++++++++++++ scripts/update-storage-render-goldens.ts | 5 ++ src/fork-adoption.ts | 6 ++ src/persist.ts | 4 + src/plugin.ts | 3 + src/storage/ingest.ts | 1 + tests/golden/storage-render/anthropic.json | 4 +- tests/golden/storage-render/chat.json | 4 +- tests/golden/storage-render/google.json | 4 +- tests/golden/storage-render/responses.json | 4 +- 16 files changed, 182 insertions(+), 12 deletions(-) create mode 100644 kernel/tests/issue3939-ref-migration-defer.test.ts diff --git a/kernel/package.json b/kernel/package.json index d5f0ba0d08..96381e368e 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.139", + "version": "0.0.140", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 7d39230a5a..9bb16ae475 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -1,4 +1,4 @@ -import { assignRefs, highestUsedIndex, indexToRef } from "./refs.js"; +import { assignRefs, BLOCKED_REF, highestUsedIndex, indexToRef } from "./refs.js"; import { remintCoveredLiveIds } from "./instance-reid.js"; import { prune } from "./prune.js"; import { syncBlocks } from "./sync.js"; @@ -808,6 +808,10 @@ export function createCore(ports: Ports = {}): CompressionCore { // re-evaluates from the new (lower) count, so the latch retires with it. // Holding it would replay stale clips onto a view the fold rewrote. state.emergencyTruncateLatch = undefined; + // #3939 ②/#3805: the fold rewrote history — the migration deferral + // rides it. Any surviving carrier may render its tag from here on; the + // rewrite is attributed to the fold, not to a quiet turn. + state.renderDeferredRefs = undefined; } return { @@ -1021,7 +1025,23 @@ const assignRefsNode: PipelineNode = { // Ephemeral retrieval injections never consume a ref slot. shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), }); - return { ...io, state: { ...io.state, messageRefs: refResult.map } }; + // #3939 ②/#3805: a BLOCKED→numeric migration mints the ref NOW but the + // carrier's tag render is DEFERRED to the next fold boundary — + // stamping it this turn would rewrite an old message mid-history on an + // otherwise quiet turn (prefix-cache breaker). Cleared on fold success. + const migratedRefs = refResult.migratedIds + .map((id) => refResult.map.byRaw[id]) + .filter((r): r is string => typeof r === "string" && r !== BLOCKED_REF); + const state = migratedRefs.length + ? { + ...io.state, + messageRefs: refResult.map, + renderDeferredRefs: [ + ...new Set([...(io.state.renderDeferredRefs ?? []), ...migratedRefs]), + ], + } + : { ...io.state, messageRefs: refResult.map }; + return { ...io, state }; }, }; diff --git a/kernel/src/refs.ts b/kernel/src/refs.ts index 106bcdde13..21606cda23 100644 --- a/kernel/src/refs.ts +++ b/kernel/src/refs.ts @@ -63,6 +63,9 @@ export interface AssignRefsResult { map: MessageRefMap; nextIndex: number; newlyAssigned: number; + /** Ids whose stale BLOCKED entry migrated to a fresh numeric ref this pass + * (#2607). The caller defers their carrier re-render (#3939 ②/#3805). */ + migratedIds: string[]; } export interface AssignRefsOptions { @@ -85,6 +88,7 @@ export function assignRefs( ? options.nextIndex : MIN_INDEX; let newlyAssigned = 0; + const migratedIds: string[] = []; for (const message of messages) { if (!message.id || options.shouldSkip?.(message)) continue; @@ -96,8 +100,10 @@ export function assignRefs( // is lifted (#2607: #1188 pinned all media; archivable images now fold // and must be citable via decompress({ imageRef })). Fresh number only — // the id-never-reused contract holds. - if (prev === BLOCKED_REF && !protectedNow) delete map.byRaw[message.id]; - else continue; + if (prev === BLOCKED_REF && !protectedNow) { + delete map.byRaw[message.id]; + migratedIds.push(message.id); + } else continue; } if (protectedNow) { @@ -112,7 +118,7 @@ export function assignRefs( newlyAssigned++; } - return { map, nextIndex: cursor, newlyAssigned }; + return { map, nextIndex: cursor, newlyAssigned, migratedIds }; } function allocateFreeRef( diff --git a/kernel/src/render-refs.ts b/kernel/src/render-refs.ts index 7338c4632f..6590d25843 100644 --- a/kernel/src/render-refs.ts +++ b/kernel/src/render-refs.ts @@ -62,10 +62,16 @@ function renderMessage( countTokens: (text: string) => number, strategy: RenderStrategy, snapshot: Record | null = null, + deferred: ReadonlySet | null = null, ): CoreMessage { const ref = refForRaw(map, message.id); if (!ref || ref === BLOCKED_REF) return message; + // #3939 ②/#3805: refs minted by the BLOCKED→numeric migration keep their + // pre-migration UNTAGGED bytes until the next fold boundary — stamping the + // tag now would rewrite an old message mid-history on a quiet turn. + if (deferred?.has(ref)) return message; + // "none": host reads the ref map directly — never pollute text. if (strategy === "none") return message; @@ -110,8 +116,11 @@ export function renderVisibleRefs( ): CoreMessage[] { // Legacy behavior: recompute tokens every render (snapshot = null). const map = state.messageRefs; + const deferred = state.renderDeferredRefs + ? new Set(state.renderDeferredRefs) + : null; return messages.map((message) => - renderMessage(message, map, countTokens, strategy), + renderMessage(message, map, countTokens, strategy, null, deferred), ); } @@ -132,8 +141,11 @@ export function renderWithSnapshot( ): RenderWithSnapshotResult { const map = state.messageRefs; const snapshot = { ...(state.tokenSnapshot ?? {}) }; + const deferred = state.renderDeferredRefs + ? new Set(state.renderDeferredRefs) + : null; const rendered = messages.map((message) => - renderMessage(message, map, countTokens, strategy, snapshot), + renderMessage(message, map, countTokens, strategy, snapshot, deferred), ); return { messages: rendered, tokenSnapshot: snapshot }; } diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index b9908863c6..578cab7e14 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -59,6 +59,11 @@ export function syncBlocks( emergencyTruncateLatch: state.emergencyTruncateLatch ? state.emergencyTruncateLatch.map((e) => ({ ref: e.ref, text: e.text })) : undefined, + // #3939 ②: the render deferral must survive sync — losing it would stamp + // migration tags mid-history after a sync carried the ref map (#3805). + renderDeferredRefs: state.renderDeferredRefs + ? [...state.renderDeferredRefs] + : undefined, }; // Refs are additive (assignRefs never removes them from messageRefs), so diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 17cfded00d..ccb3a7aded 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -280,6 +280,17 @@ export interface CompressionState { * re-evaluates from scratch. Optional: pre-feature states have none (the * pre-latch node clipped from scratch each turn). */ emergencyTruncateLatch?: EmergencyTruncateLatchEntry[]; + /** Ref-migration render deferral (#3939 ②/#3805): padded refs minted by the + * BLOCKED→numeric migration (assignRefs, #2607). The number exists (citable + * via decompress({ imageRef }) bookkeeping) but the carrier message keeps + * its pre-migration, UNTAGGED bytes until the next successful fold — + * stamping the tag on the migration turn would rewrite an old + * message mid-history on an otherwise quiet turn (a prefix-cache breaker: + * #3805 durable carriers flipping bytes with no fold in sight). Cleared by + * applyCompression on fold success; entries are inert once their message + * leaves the view (ids are never re-issued). Optional: absent until the + * first migration. */ + renderDeferredRefs?: string[]; /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk diff --git a/kernel/tests/issue3939-ref-migration-defer.test.ts b/kernel/tests/issue3939-ref-migration-defer.test.ts new file mode 100644 index 0000000000..c6a3bfb7c9 --- /dev/null +++ b/kernel/tests/issue3939-ref-migration-defer.test.ts @@ -0,0 +1,89 @@ +// #3939 ② / #3805: when a BLOCKED ref migrates to a fresh numeric ref +// (protection lifted, #2607), the NUMBER is minted immediately but the +// tag on the old carrier must NOT appear until a fold boundary rewrites +// history anyway. Stamping it on the migration turn silently rewrites an old +// message mid-history on an otherwise quiet turn — a prefix-cache breaker +// (#3805: durable carriers flipping bytes with no fold in sight). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { createInitialState } from "../src/state.js"; +import { BLOCKED_REF } from "../src/refs.js"; +import type { CoreMessage } from "../src/types.js"; + +const W = 100_000; +const cfg = () => ({ ...defaultConfig(W), preserveRecentTokens: 100 }); + +function hist(): CoreMessage[] { + const filler = "filler ".repeat(340); // push the range over the 5000-char min-compress gate + return [ + { id: "raw-a", role: "user", contentType: "text", text: "first question " + filler }, + { id: "raw-img", role: "user", contentType: "text", text: "look at this " + filler }, + { id: "raw-b", role: "assistant", contentType: "text", text: "answer " + filler }, + { id: "raw-c", role: "user", contentType: "text", text: "second question" }, + { id: "raw-d", role: "assistant", contentType: "text", text: "second answer" }, + { id: "raw-e", role: "user", contentType: "text", text: "third question" }, + { id: "raw-f", role: "assistant", contentType: "text", text: "third answer" }, + ]; +} + +test("pin: BLOCKED→numeric migration mints the ref but defers the carrier tag (#3805)", () => { + const core = createCore(cfg()); + // Simulate a pre-existing BLOCKED assignment on raw-img (protection lifted + // since — the message is plain text now). + const state = createInitialState(); + state.messageRefs.byRaw["raw-img"] = BLOCKED_REF; + + const t1 = core.processTurn({ + messages: hist(), + state, + config: cfg(), + tokenCount: 1_000, + renderTags: "all", + }); + + const imgRef = t1.state.messageRefs.byRaw["raw-img"]!; + assert.notEqual(imgRef, BLOCKED_REF, "migration mints a numeric ref"); + const imgText = t1.messages.find((m) => m.id === "raw-img")!.text!; + assert.ok(!imgText.includes("\x3cacp "), "carrier text stays untagged on the migration turn"); + assert.deepEqual(t1.state.renderDeferredRefs, [imgRef], "deferral recorded in state"); + // Sanity: other messages render normally. + assert.ok(t1.messages.find((m) => m.id === "raw-a")!.text!.includes("\x3cacp "), "untagged only for the migrated carrier"); + + // Quiet turn 2: deferral persists — still no tag, still byte-stable. + const t2 = core.processTurn({ + messages: hist(), + state: t1.state, + config: cfg(), + tokenCount: 1_000, + renderTags: "all", + }); + assert.ok(!t2.messages.find((m) => m.id === "raw-img")!.text!.includes("\x3cacp "), "deferral self-perpetuates on quiet turns"); +}); + +test("pin: a fold boundary releases the deferral and the tag rides the fold's rewrite", () => { + const core = createCore(cfg()); + const state = createInitialState(); + state.messageRefs.byRaw["raw-img"] = BLOCKED_REF; + const t1 = core.processTurn({ messages: hist(), state, config: cfg(), tokenCount: 1_000, renderTags: "all" }); + assert.ok(t1.state.renderDeferredRefs?.length, "deferral active"); + + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const fold = core.applyCompression({ + messages: t1.messages, + state: t1.state, + config: cfg(), + ranges: [{ + startRef: refOf(t1.state, "raw-a")!, + endRef: refOf(t1.state, "raw-b")!, + summary: "a long enough summary for the fold gate to accept this range, over fifty chars", + topic: "t", + }], + }); + assert.equal(fold.result.blocksCreated, 1, `fold lands: ${JSON.stringify(fold.result.errors ?? [])} ${JSON.stringify(fold.result.warnings ?? [])}`); + assert.equal(fold.state.renderDeferredRefs, undefined, "fold releases the deferral"); + + const t3 = core.processTurn({ messages: hist(), state: fold.state, config: cfg(), tokenCount: 1_000, renderTags: "all" }); + const imgText = t3.messages.find((m) => m.id === "raw-img")?.text ?? ""; + assert.ok(!imgText.includes("\x3cacp ") || t3.messages.every((m) => m.id !== "raw-img"), "raw-img rode the fold away or renders tagged at the fold boundary"); +}); diff --git a/scripts/update-storage-render-goldens.ts b/scripts/update-storage-render-goldens.ts index 46c1f21f5c..63948692d8 100644 --- a/scripts/update-storage-render-goldens.ts +++ b/scripts/update-storage-render-goldens.ts @@ -11,6 +11,11 @@ import { driveWire, WIRES } from "../tests/golden-storage-drive.ts"; // same condition — without it every per-turn wire digest drifts by the image- // rescale delta and the suite reds on the next CI run. process.env.BILI_IMAGE_RESCALE = "0"; +// Parity #2: the test suite also disarms the shadow observability lane +// (#3756) before driving — its metadata blob would otherwise diverge from +// what the committed test run hashes, and every freshly regenerated golden +// would red on the next CI run. +process.env.BILI_SNAPSHOT_SHADOW = "0"; const GOLDEN_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tests", "golden", "storage-render"); fs.mkdirSync(GOLDEN_DIR, { recursive: true }); diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 22ddf63118..5a86b4e138 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -276,6 +276,12 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare if (parent.state.emergencyTruncateLatch?.length) { session.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch.map((e) => ({ ...e })); } + // #3939 ②: same carry for the migration render deferral — a resumed/forked + // session re-sends the same history; without the deferral the first replay + // stamps migration tags mid-history (the #3805 byte flip, resurrected). + if (parent.state.renderDeferredRefs?.length) { + session.state.renderDeferredRefs = [...parent.state.renderDeferredRefs]; + } markDirty(session); } diff --git a/src/persist.ts b/src/persist.ts index 6ec42bbae8..79e2334c9a 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -248,6 +248,10 @@ function mergeState(parsed: CompressionState): CompressionState { // dropping it re-exposes clipped bytes mid-history after a restart, // rewriting the still-cached prefix on the first post-restart turn. emergencyTruncateLatch: parsed.emergencyTruncateLatch ?? fresh.emergencyTruncateLatch, + // #3939 ②/#3805: the render deferral must survive hydration — losing + // it stamps migration tags onto still-cached carriers right after a + // restart (a silent mid-history rewrite with no fold in sight). + renderDeferredRefs: parsed.renderDeferredRefs ?? fresh.renderDeferredRefs, }; } diff --git a/src/plugin.ts b/src/plugin.ts index 3d8b4818ef..6df98eedf8 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1393,6 +1393,9 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr // prefix — the child never sees the others, so their entries // would be dead weight (and retire on the next latch pass anyway). child.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch?.filter((e) => prefix.some((m) => m.ref === e.ref)); + // #3939 ②: deferred refs are content-keyed (byRaw ids) — keep only + // the ones whose carrier rode the forked prefix. + child.state.renderDeferredRefs = parent.state.renderDeferredRefs?.filter((r) => prefix.some((m) => m.ref === r)); // #2362: inverse of the orphan filter — a dead ref whose message IS // in the forked prefix is visible again in the child; refs outside // it stay dead (their messages are absent from the child too). diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 64990619b2..c281bd92c7 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -43,6 +43,7 @@ const MERGE_STATE_FIELDS = [ "lastPassIds", "hiddenOrphanRefs", "emergencyTruncateLatch", + "renderDeferredRefs", "deadRefs", "terminalStreak", "nextRuleId", diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 1484cc0699..6d0b7728a0 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -103,7 +103,9 @@ "absorbed": "4f53cda18c2baa0c", "rules": "4f53cda18c2baa0c", "sweptOrphanCallIds": "74234e98afe7498f", - "pendingOrphanSweepMinRef": "74234e98afe7498f" + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f" }, "stateShape": { "refs": 49, diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 1883114ffa..d944e108d7 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -103,7 +103,9 @@ "absorbed": "4f53cda18c2baa0c", "rules": "4f53cda18c2baa0c", "sweptOrphanCallIds": "74234e98afe7498f", - "pendingOrphanSweepMinRef": "74234e98afe7498f" + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index cc6117c57e..ed03210d13 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -103,7 +103,9 @@ "absorbed": "4f53cda18c2baa0c", "rules": "4f53cda18c2baa0c", "sweptOrphanCallIds": "74234e98afe7498f", - "pendingOrphanSweepMinRef": "74234e98afe7498f" + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index e305b07dc5..8a8ca1c00d 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -103,7 +103,9 @@ "absorbed": "4f53cda18c2baa0c", "rules": "4f53cda18c2baa0c", "sweptOrphanCallIds": "74234e98afe7498f", - "pendingOrphanSweepMinRef": "74234e98afe7498f" + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f" }, "stateShape": { "refs": 55, From 4344ff26d1166d8c5dac9646215cf321c8708968 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:50:30 +0800 Subject: [PATCH 21/29] =?UTF-8?q?kernel:=20fold-gate=20the=20opt-in=20view?= =?UTF-8?q?=20rewriters=20via=20a=20structural=20view=20latch=20(#3939=20?= =?UTF-8?q?=E2=91=A4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit absorb-hide, crush and filter keepLastOnly re-derive their output every pass from state/context (absorb records growing, crush responding to token pressure, keep-last-only demotions as new occurrences arrive) — each change flipped OLD message bytes on an otherwise quiet turn (the #3805 prefix-cache breaker class). One structural latch now covers all of them: post-transform bytes are latched per padded ref at first application and replay verbatim; refreshes ride fold boundaries (applyCompression resets the map), the absorb action itself (instant- reclaim contract kept), and config edits (digest release — enable/ disable applies on the edit turn, never silently later). ccr needs no latch (replace-once-at-arrival, byte-stable by design #1460). Placed after hide-compress-calls (the #3784 sweep keeps its own gate) and before recommend/nudge/truncate/render so downstream judges see the view the model actually sees. Carried across sync/persist/fork/plugin/ ingest; storage-render goldens pin the two new state keys (undefined valued), wire bodies unchanged. --- kernel/package.json | 2 +- kernel/src/absorb.ts | 5 + kernel/src/compress.ts | 8 ++ kernel/src/pipeline.ts | 4 + kernel/src/sync.ts | 8 ++ kernel/src/types.ts | 28 ++++ kernel/src/view-latch.ts | 156 +++++++++++++++++++++ kernel/tests/issue3939-view-latch.test.ts | 153 ++++++++++++++++++++ kernel/tests/pipeline.test.ts | 2 + src/fork-adoption.ts | 11 ++ src/persist.ts | 5 + src/plugin.ts | 11 ++ src/storage/ingest.ts | 2 + tests/golden/storage-render/anthropic.json | 4 +- tests/golden/storage-render/chat.json | 4 +- tests/golden/storage-render/google.json | 4 +- tests/golden/storage-render/responses.json | 4 +- 17 files changed, 406 insertions(+), 5 deletions(-) create mode 100644 kernel/src/view-latch.ts create mode 100644 kernel/tests/issue3939-view-latch.test.ts diff --git a/kernel/package.json b/kernel/package.json index 96381e368e..600b13bda5 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.140", + "version": "0.0.141", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/absorb.ts b/kernel/src/absorb.ts index b1365b3972..4f3906ac1f 100644 --- a/kernel/src/absorb.ts +++ b/kernel/src/absorb.ts @@ -373,6 +373,11 @@ export function applyAbsorb(input: AbsorbInput): AbsorbOutcome { ...input.state.stats, absorbedTokens: (input.state.stats.absorbedTokens ?? 0) + tokens, }, + // #3939 ⑤: the absorb call is a deliberate history-rewriting action (a + // fold-class boundary, same family as applyCompression) — release the + // view latch here so the hide lands THIS turn, not at some later fold. + // Keeps absorb's instant-reclaim contract; quiet turns stay latched. + viewLatches: undefined, }; const bloat = diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 9bb16ae475..67ff2df893 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -18,6 +18,7 @@ import { hideConsumedCompressCalls, refNum } from "./hide-consumed.js"; import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyCrushToMessages } from "./crush.js"; import { applyRetrieve, ccrStoreNode, RETRIEVED_ID_PREFIX } from "./ccr.js"; +import { preTransformSnapshotNode, viewLatchNode } from "./view-latch.js"; import type { ApplyRetrieveResult, CcrEffect } from "./ccr.js"; import type { EmergencyTruncateLatchEntry } from "./types.js"; import { createContentStore } from "./content-store.js"; @@ -812,6 +813,11 @@ export function createCore(ports: Ports = {}): CompressionCore { // rides it. Any surviving carrier may render its tag from here on; the // rewrite is attributed to the fold, not to a quiet turn. state.renderDeferredRefs = undefined; + // #3939 ⑤: the latch refresh rides the fold's own history rewrite — + // the next pass re-latches every visible message's transform output + // fresh (absorb hides apply, crush re-evaluates under the new pressure, + // keep-last-only demotions land). + state.viewLatches = undefined; } return { @@ -943,12 +949,14 @@ export function createCore(ports: Ports = {}): CompressionCore { assignRefsNode, syncBlocksNode, pruneNode, + preTransformSnapshotNode, ccrStoreNode, absorbHideNode, crushNode, absorbPromptNode, filterNode, hideCompressCallsNode, + viewLatchNode, recommendNode, nudgeNode, emergencyTruncateNode, diff --git a/kernel/src/pipeline.ts b/kernel/src/pipeline.ts index 96d063f9d9..24dfa976ba 100644 --- a/kernel/src/pipeline.ts +++ b/kernel/src/pipeline.ts @@ -25,6 +25,10 @@ export interface NodeEffects { * foldability judgment — on lanes whose processTurn input is itself a * pruned/render view, even the pre-prune input is not the apply class. */ applyView?: CoreMessage[]; + /** #3939 ⑤: the post-prune, pre-transform resent view, stashed by the + * pre-transform-snapshot node so the view-latch node can replay latched + * bytes for messages the transform nodes dropped or rewrote. */ + preTransformMessages?: CoreMessage[]; readonly [key: string]: unknown; } diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index 578cab7e14..cb9564e82a 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -64,6 +64,14 @@ export function syncBlocks( renderDeferredRefs: state.renderDeferredRefs ? [...state.renderDeferredRefs] : undefined, + // #3939 ⑤: the view latch must survive sync — losing it re-derives + // transform output mid-history after a sync carried the state (#3805). + viewLatches: state.viewLatches + ? Object.fromEntries( + Object.entries(state.viewLatches).map(([k, v]) => [k, { ...v }]), + ) + : undefined, + viewLatchConfigDigest: state.viewLatchConfigDigest, }; // Refs are additive (assignRefs never removes them from messageRefs), so diff --git a/kernel/src/types.ts b/kernel/src/types.ts index ccb3a7aded..68e729582e 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -114,6 +114,12 @@ export interface CompressionStats { retrievalCount?: number; } +/** View-latch entry shape (#3939 ⑤) — see CompressionState.viewLatches. */ +export interface ViewLatchEntry { + text: string | null; + hidden?: boolean; +} + /** One instant tool-result absorption: the original tool-call + tool-result * pair is hidden from view once recorded; the model's absorb call (carrying * the distilled summary) remains as the durable record. */ @@ -291,6 +297,28 @@ export interface CompressionState { * leaves the view (ids are never re-issued). Optional: absent until the * first migration. */ renderDeferredRefs?: string[]; + /** Fold-gated view latch (#3939 ⑤): post-transform view bytes per padded + * ref, replayed VERBATIM on quiet turns. The opt-in mid-history rewriters + * (absorb-hide, crush, filter keepLastOnly) re-derive their output every + * pass from state/context (absorb records growing, crush responding to + * token pressure, keep-last-only demotions as new occurrences arrive) — + * without a latch each change flips OLD message bytes on an otherwise + * quiet turn (the #3805 prefix-cache breaker class). Structural, not + * per-feature: every transform output is latched at first application and + * replays until the next successful fold (applyCompression resets the + * whole map — the refresh rides the fold's own history rewrite); new or + * edited messages (new ref) latch fresh at their arrival boundary. ccr + * needs no latch (replace-once-at-arrival, byte-stable by design #1460). + * Entry: text=null = passthrough (source bytes stand); text=string = + * transformed bytes; hidden=true = dropped from the view. Optional: + * absent until the first latched transform. */ + viewLatches?: Record; + /** #3939 ⑤: digest of the transform-relevant config slices (absorb/crush/ + * message-filters). A config edit is a user action boundary — matching + * ①'s fold/refold/user-action rule, the latch releases when the digest + * changes so enabling a transform applies on the next turn (one rewrite + * riding the config change), never on a quiet turn. */ + viewLatchConfigDigest?: string; /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk diff --git a/kernel/src/view-latch.ts b/kernel/src/view-latch.ts new file mode 100644 index 0000000000..51ef86a9cd --- /dev/null +++ b/kernel/src/view-latch.ts @@ -0,0 +1,156 @@ +import { BLOCKED_REF } from "./refs.js"; +import { RETRIEVED_ID_PREFIX } from "./ccr.js"; +import { resolveAbsorbConfig } from "./absorb.js"; +import { resolveCrushConfig } from "./crush.js"; +import type { CompressionState, CoreMessage, ViewLatchEntry } from "./types.js"; +import type { NodeIO, PipelineNode, PipelineContext } from "./pipeline.js"; + +/** Stash the post-prune, pre-transform view (#3939 ⑤). The view-latch node + * replays latched bytes from this array — the transform nodes have already + * dropped or rewritten their targets by the time the latch runs, so the + * source-of-truth resent bytes live here. */ +export const preTransformSnapshotNode: PipelineNode = { + name: "pre-transform-snapshot", + run(io) { + return { + ...io, + effects: { ...io.effects, preTransformMessages: io.messages }, + }; + }, +}; + +/** Fold-gated view latch (#3939 ⑤/#3805): replay post-transform bytes + * verbatim on quiet turns; refresh latches only at fold boundaries. Runs + * after every opt-in transform node (ccr-store, absorb-hide, crush, + * absorb-prompt, filter) and after hide-compress-calls (its own #3784 fold + * gate stays authoritative — the latch only records its output), before + * recommend/nudge/emergency-truncate/render so downstream nodes judge the + * same view the model actually sees. */ +export const viewLatchNode: PipelineNode = { + name: "view-latch", + run(io, ctx) { + return applyViewLatch(io, ctx); + }, +}; + +function latchKeyEqual( + a: Record | undefined, + b: Record | undefined, +): boolean { + if (!a || !b) return !a && !b; + if (Object.keys(a).length !== Object.keys(b).length) return false; + for (const [k, v] of Object.entries(a)) { + const o = b[k]; + if (!o || o.text !== v.text || o.hidden !== v.hidden) return false; + } + return true; +} + +/** Transform-relevant config slices — a change here is a user action + * boundary (①): the latch releases so enable/disable applies immediately + * instead of waiting for a fold. tokenCount is deliberately NOT included + * (pressure fluctuation is what the latch exists to gate). */ +export function viewLatchConfigDigest(ctx: PipelineContext): string { + return JSON.stringify({ + absorb: resolveAbsorbConfig(ctx.config), + crush: resolveCrushConfig(ctx.config), + filters: { + enabled: ctx.config.messageFilters?.enabled ?? false, + filters: ctx.config.messageFilters?.filters ?? null, + }, + }); +} + +export function applyViewLatch(io: NodeIO, ctx?: PipelineContext): NodeIO { + const pre = io.effects.preTransformMessages; + if (!pre) return io; + // Config boundary (#3939 ①/⑤): digest change releases the latch — the + // transform enable/disable applies this turn (one rewrite riding the user's + // config edit), never silently later. + const digest = ctx ? viewLatchConfigDigest(ctx) : undefined; + const configChanged = + digest !== undefined && digest !== io.state.viewLatchConfigDigest; + const prev = configChanged ? undefined : io.state.viewLatches; + const post = new Map(); + for (const m of io.messages) if (m.id) post.set(m.id, m); + + const out: CoreMessage[] = []; + const latches: Record = {}; + let latchedTransform = false; + + for (const m of pre) { + if (!m.id || m.id.startsWith(RETRIEVED_ID_PREFIX)) { + const t = post.get(m.id!); + if (t) out.push(t); + continue; + } + const ref = io.state.messageRefs.byRaw[m.id]; + if (!ref || ref === BLOCKED_REF) { + const t = post.get(m.id); + if (t) out.push(t); + continue; + } + const transformed = post.get(m.id); + const prevEntry = prev?.[ref]; + if (prevEntry) { + // REPLAY the latched bytes — a transform's fresh output for an + // already-latched ref is deferred to the next fold boundary. + if (prevEntry.hidden) { + latches[ref] = prevEntry; + continue; + } + if (prevEntry.text === null) { + if (transformed) out.push(m); + latches[ref] = prevEntry; + continue; + } + latchedTransform = true; + if (transformed && transformed.text === prevEntry.text) out.push(transformed); + else out.push({ ...m, text: prevEntry.text }); + latches[ref] = prevEntry; + } else { + // FRESH: latch whatever the transforms produced at this arrival + // boundary (arrival/first-enable IS a history change; the flip rides it). + if (!transformed) { + latches[ref] = { text: null, hidden: true }; + continue; + } + if (transformed.text !== m.text) { + latches[ref] = { text: transformed.text ?? null }; + latchedTransform = true; + out.push(transformed); + } else { + latches[ref] = { text: null }; + out.push(m); + } + } + } + + // Steady state (identical latch map) must not churn the state object and + // force an adapter save every turn (render-refs snapshot precedent). + if ( + !configChanged && + digest === io.state.viewLatchConfigDigest && + latchKeyEqual(io.state.viewLatches, latches) + ) { + return { ...io, messages: out }; + } + // No prior latches and every entry passthrough → nothing worth pinning + // (no opt-in transform active); skip the write entirely. + if (!prev && !latchedTransform) return io; + return { + ...io, + messages: out, + state: { + ...io.state, + viewLatches: latches, + ...(digest !== undefined ? { viewLatchConfigDigest: digest } : {}), + }, + }; +} + +/** #3939 ⑤: a successful fold rewrote history — the latch refresh rides it. */ +export function resetViewLatches(state: CompressionState): CompressionState { + if (state.viewLatches === undefined) return state; + return { ...state, viewLatches: undefined }; +} diff --git a/kernel/tests/issue3939-view-latch.test.ts b/kernel/tests/issue3939-view-latch.test.ts new file mode 100644 index 0000000000..6bbb5722fb --- /dev/null +++ b/kernel/tests/issue3939-view-latch.test.ts @@ -0,0 +1,153 @@ +// #3939 ⑤: the opt-in mid-history rewriters (absorb-hide, crush, filter +// keepLastOnly) re-derive their output every pass; without a latch each state/ +// context change flips OLD message bytes on an otherwise quiet turn (#3805 +// class). The view latch replays post-transform bytes verbatim; refreshes ride +// fold boundaries, absorb actions, and config edits (user action boundaries). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { applyAbsorb } from "../src/absorb.js"; +import { createInitialState } from "../src/state.js"; +import { + registerMessageFilter, + clearMessageFilters, +} from "../src/filter/registry.js"; +import type { CoreMessage } from "../src/types.js"; + +const W = 100_000; +const cfg = () => ({ ...defaultConfig(W), preserveRecentTokens: 100 }); + +function hist(): CoreMessage[] { + const filler = "filler ".repeat(500); + return [ + { id: "raw-a", role: "user", contentType: "text", text: "first question " + filler }, + { id: "raw-b", role: "assistant", contentType: "text", text: "answer " + filler }, + { id: "raw-c", role: "user", contentType: "text", text: "second question" }, + { id: "raw-d", role: "assistant", contentType: "text", text: "second answer" }, + { id: "raw-e", role: "user", contentType: "text", text: "third question" }, + { id: "raw-f", role: "assistant", contentType: "text", text: "third answer" }, + ]; +} + +test("pin: filter keepLastOnly demotion is deferred to the fold boundary", () => { + clearMessageFilters(); + registerMessageFilter({ + name: "pin-marker", + version: "1", + description: "test filter", + keepLastOnly: true, + filter: (ctx) => + ctx.text.includes("SECRETTOKEN") + ? { action: "modify", text: ctx.text.replace(/SECRETTOKEN/g, "[redacted]") } + : { action: "keep" }, + }); + try { + const core = createCore(cfg()); + const config = { + ...cfg(), + messageFilters: { enabled: true, filters: { "pin-marker": { enabled: true } } }, + }; + const m1 = [ + ...hist(), + { id: "raw-occ1", role: "user", contentType: "text" as const, text: "here is SECRETTOKEN one" }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config, tokenCount: 1_000, renderTags: "none" }); + // occ1 is the LAST occurrence → the filter's decision applies at arrival + // (modify → [redacted]) and the result is latched. + const occ1a = t1.messages.find((m) => m.id === "raw-occ1")!; + assert.ok(occ1a.text!.includes("[redacted]"), "last occurrence filtered on arrival"); + assert.ok(!occ1a.text!.includes("SECRETTOKEN"), "pattern stripped at arrival"); + + // A NEWER occurrence arrives — the fresh filter would demote occ1 (blank + // it). The latch replays occ1's bytes unchanged: the demotion waits for a + // fold boundary. + const m2 = [ + ...m1, + { id: "raw-occ2", role: "user", contentType: "text" as const, text: "again SECRETTOKEN two" }, + ]; + const t2 = core.processTurn({ messages: m2, state: t1.state, config, tokenCount: 1_000, renderTags: "none" }); + const occ1b = t2.messages.find((m) => m.id === "raw-occ1")!; + assert.equal(occ1b.text, occ1a.text, "demotion deferred — old bytes replay verbatim"); + assert.ok(t2.messages.find((m) => m.id === "raw-occ2")!.text!.includes("[redacted]"), "new last occurrence filtered fresh (arrival boundary)"); + + // Fold covering occ1 releases the latch: post-fold, occ1 is folded away + // and the new state carries no latches. + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const fold = core.applyCompression({ + messages: t2.messages, + state: t2.state, + config, + ranges: [{ + startRef: refOf(t2.state, "raw-a")!, + endRef: refOf(t2.state, "raw-b")!, + summary: "a long enough summary for the fold gate to accept this range, over fifty chars", + topic: "t", + }], + }); + assert.equal(fold.result.blocksCreated, 1, `fold lands: ${JSON.stringify(fold.result.errors ?? [])}`); + assert.equal(fold.state.viewLatches, undefined, "fold releases the view latch"); + } finally { + clearMessageFilters(); + } +}); + +test("pin: absorb-hide defers to the absorb action boundary, not a later fold", () => { + const core = createCore(cfg()); + const config = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 1, contextThresholdPct: 0, excludeTools: [] }, + }); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "read the file" }, + { id: "c1", role: "tool", contentType: "tool-call" as const, toolName: "read", toolCallId: "call1", text: "{}" }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "read", toolCallId: "call1", text: "file body line one\n" + "x".repeat(400) }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config, tokenCount: 3_000, renderTags: "none" }); + assert.ok(t1.messages.find((m) => m.id === "r1"), "pair visible before absorb"); + + // Absorb r1: the absorb call is an action boundary — applyAbsorb releases + // the latch, so the hide lands THIS turn (instant-reclaim contract). + const ref = t1.state.messageRefs.byRaw["r1"]!; + const outcome = applyAbsorb({ + ref, + summary: "file defines foo(a,b) at src/foo.ts:12", + messages: m1, + state: t1.state, + config, + }); + assert.ok(outcome.ok, "absorb ok"); + const t2 = core.processTurn({ messages: m1, state: outcome.state, config, tokenCount: 3_200, renderTags: "none" }); + const ids = t2.messages.map((m) => m.id); + assert.ok(!ids.includes("r1"), "absorbed result hidden on the action boundary turn"); + assert.ok(!ids.includes("c1"), "absorbed call hidden on the action boundary turn"); +}); + +test("pin: config enable of crush applies on the edit turn (config boundary), token-pressure flips do not", () => { + const core = createCore(cfg()); + const off = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 100, contextThresholdPct: 0, excludeTools: [] }, + }); + const rows = JSON.stringify( + Array.from({ length: 40 }, (_, i) => ({ id: i * 7, ts: "2026-09-21T12:00:00Z", level: "INFO", name: `svc-${i}` })), + ); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "read the log" }, + { id: "c1", role: "tool", contentType: "tool-call" as const, toolName: "read", toolCallId: "call1", text: "{}" }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "read", toolCallId: "call1", text: rows }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config: off, tokenCount: 0, renderTags: "none" }); + const fresh = t1.messages.find((m) => m.id === "r1")!.text!; + assert.ok(!fresh.includes("__acp_crush"), "crush off on turn 1"); + + // Enable crush: config edit = user action boundary — the crush applies now. + const onCfg = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 100, contextThresholdPct: 0, excludeTools: [] }, + crush: { enabled: true, minReduction: 0.1 }, + }); + const t2 = core.processTurn({ messages: m1, state: t1.state, config: onCfg, tokenCount: 0, renderTags: "none" }); + assert.ok(t2.messages.find((m) => m.id === "r1")!.text!.includes("__acp_crush"), "enable applies on the config-edit turn"); + + // Token pressure alone never flips bytes: same config, wildly different + // tokenCount — the crushed bytes replay verbatim. + const t3 = core.processTurn({ messages: m1, state: t2.state, config: onCfg, tokenCount: 99_000, renderTags: "none" }); + assert.equal(t3.messages.find((m) => m.id === "r1")!.text, t2.messages.find((m) => m.id === "r1")!.text, "pressure does not re-derive latched bytes"); +}); diff --git a/kernel/tests/pipeline.test.ts b/kernel/tests/pipeline.test.ts index ec1c877412..f5493f17f4 100644 --- a/kernel/tests/pipeline.test.ts +++ b/kernel/tests/pipeline.test.ts @@ -29,12 +29,14 @@ test("defaultNodes exposes the canonical ordered pipeline", () => { "assign-refs", "sync-blocks", "prune", + "pre-transform-snapshot", "ccr-store", "absorb-hide", "crush", "absorb-prompt", "filter", "hide-compress-calls", + "view-latch", "recommend", "nudge-inject", "emergency-truncate", diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 5a86b4e138..3d15fffdf1 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -282,6 +282,17 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare if (parent.state.renderDeferredRefs?.length) { session.state.renderDeferredRefs = [...parent.state.renderDeferredRefs]; } + // #3939 ⑤: same carry for the view latch — a resumed session re-sends the + // same history; a dropped latch would flip transform bytes on the first + // replayed turn (crush re-derived, absorb hides landing late). + if (parent.state.viewLatches) { + session.state.viewLatches = Object.fromEntries( + Object.entries(parent.state.viewLatches).map(([k, v]) => [k, { ...v }]), + ); + } + if (parent.state.viewLatchConfigDigest) { + session.state.viewLatchConfigDigest = parent.state.viewLatchConfigDigest; + } markDirty(session); } diff --git a/src/persist.ts b/src/persist.ts index 79e2334c9a..48fb753141 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -252,6 +252,11 @@ function mergeState(parsed: CompressionState): CompressionState { // it stamps migration tags onto still-cached carriers right after a // restart (a silent mid-history rewrite with no fold in sight). renderDeferredRefs: parsed.renderDeferredRefs ?? fresh.renderDeferredRefs, + // #3939 ⑤: the view latch must survive hydration — losing it + // re-derives transform output on still-cached carriers right after a + // restart (silent mid-history rewrite, no fold in sight). + viewLatches: parsed.viewLatches ?? fresh.viewLatches, + viewLatchConfigDigest: parsed.viewLatchConfigDigest ?? fresh.viewLatchConfigDigest, }; } diff --git a/src/plugin.ts b/src/plugin.ts index 6df98eedf8..e04a08b76c 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1396,6 +1396,17 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr // #3939 ②: deferred refs are content-keyed (byRaw ids) — keep only // the ones whose carrier rode the forked prefix. child.state.renderDeferredRefs = parent.state.renderDeferredRefs?.filter((r) => prefix.some((m) => m.ref === r)); + // #3939 ⑤: view-latch entries are ref-keyed — keep only the + // carriers that rode the forked prefix. + const forkRefs = new Set(prefix.map((m) => m.ref)); + if (parent.state.viewLatches) { + child.state.viewLatches = Object.fromEntries( + Object.entries(parent.state.viewLatches).filter(([r]) => forkRefs.has(r)), + ); + } + if (parent.state.viewLatchConfigDigest) { + child.state.viewLatchConfigDigest = parent.state.viewLatchConfigDigest; + } // #2362: inverse of the orphan filter — a dead ref whose message IS // in the forked prefix is visible again in the child; refs outside // it stay dead (their messages are absent from the child too). diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index c281bd92c7..9c741dae79 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -44,6 +44,8 @@ const MERGE_STATE_FIELDS = [ "hiddenOrphanRefs", "emergencyTruncateLatch", "renderDeferredRefs", + "viewLatches", + "viewLatchConfigDigest", "deadRefs", "terminalStreak", "nextRuleId", diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 6d0b7728a0..91c052b92d 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -105,7 +105,9 @@ "sweptOrphanCallIds": "74234e98afe7498f", "pendingOrphanSweepMinRef": "74234e98afe7498f", "emergencyTruncateLatch": "74234e98afe7498f", - "renderDeferredRefs": "74234e98afe7498f" + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 49, diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index d944e108d7..ebe66d6ae6 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -105,7 +105,9 @@ "sweptOrphanCallIds": "74234e98afe7498f", "pendingOrphanSweepMinRef": "74234e98afe7498f", "emergencyTruncateLatch": "74234e98afe7498f", - "renderDeferredRefs": "74234e98afe7498f" + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index ed03210d13..13d78598b0 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -105,7 +105,9 @@ "sweptOrphanCallIds": "74234e98afe7498f", "pendingOrphanSweepMinRef": "74234e98afe7498f", "emergencyTruncateLatch": "74234e98afe7498f", - "renderDeferredRefs": "74234e98afe7498f" + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 8a8ca1c00d..36ae5150bd 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -105,7 +105,9 @@ "sweptOrphanCallIds": "74234e98afe7498f", "pendingOrphanSweepMinRef": "74234e98afe7498f", "emergencyTruncateLatch": "74234e98afe7498f", - "renderDeferredRefs": "74234e98afe7498f" + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 55, From 3057f22b068ed063fe4b03e8a3677c1122ba2f71 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:55:16 +0800 Subject: [PATCH 22/29] chore: sync package-lock kernel version to 0.0.141 --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 7bcf65f2a3..58875c4ac1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.138", + "version": "0.0.141", "dev": true, "license": "MIT", "devDependencies": { From cf49050a70fef2334d7baa37e6e3e4fafffde69a Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:58:42 +0800 Subject: [PATCH 23/29] chore: sync kernel/package-lock.json version to 0.0.141 --- kernel/package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 4c282670af..045125fb67 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.138", + "version": "0.0.141", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.141", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", From 6aed427f457341063419cb4ede30d6dfc86f5ae5 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:10:41 +0800 Subject: [PATCH 24/29] =?UTF-8?q?kernel:=20CCR=20rides=20the=20view-latch?= =?UTF-8?q?=20config=20digest=20=E2=80=94=20disarm=20releases=20latched=20?= =?UTF-8?q?placeholders=20(#3939=20=E2=91=A4,=20#1273)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/package-lock.json | 4 +-- kernel/package.json | 2 +- kernel/src/view-latch.ts | 7 +++++ kernel/tests/issue3939-view-latch.test.ts | 31 +++++++++++++++++++++++ package-lock.json | 2 +- 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 045125fb67..6554478342 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.141", + "version": "0.0.142", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.141", + "version": "0.0.142", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 600b13bda5..80a8ce051b 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.141", + "version": "0.0.142", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/view-latch.ts b/kernel/src/view-latch.ts index 51ef86a9cd..2ca2d2100e 100644 --- a/kernel/src/view-latch.ts +++ b/kernel/src/view-latch.ts @@ -1,6 +1,7 @@ import { BLOCKED_REF } from "./refs.js"; import { RETRIEVED_ID_PREFIX } from "./ccr.js"; import { resolveAbsorbConfig } from "./absorb.js"; +import { resolveCcrConfig } from "./ccr.js"; import { resolveCrushConfig } from "./crush.js"; import type { CompressionState, CoreMessage, ViewLatchEntry } from "./types.js"; import type { NodeIO, PipelineNode, PipelineContext } from "./pipeline.js"; @@ -54,6 +55,12 @@ export function viewLatchConfigDigest(ctx: PipelineContext): string { return JSON.stringify({ absorb: resolveAbsorbConfig(ctx.config), crush: resolveCrushConfig(ctx.config), + // CCR rides the same boundary: a session that disarms CCR must never + // keep projecting latched [acp-stored] placeholders — the retrieve + // channel they advertise would be gone (dangling placeholder = silent + // loss, the #1273 plugin-governance contract). A digest change releases + // the latch and re-judges every message under the CURRENT config. + ccr: resolveCcrConfig(ctx.config), filters: { enabled: ctx.config.messageFilters?.enabled ?? false, filters: ctx.config.messageFilters?.filters ?? null, diff --git a/kernel/tests/issue3939-view-latch.test.ts b/kernel/tests/issue3939-view-latch.test.ts index 6bbb5722fb..d17ac0522e 100644 --- a/kernel/tests/issue3939-view-latch.test.ts +++ b/kernel/tests/issue3939-view-latch.test.ts @@ -151,3 +151,34 @@ test("pin: config enable of crush applies on the edit turn (config boundary), to const t3 = core.processTurn({ messages: m1, state: t2.state, config: onCfg, tokenCount: 99_000, renderTags: "none" }); assert.equal(t3.messages.find((m) => m.id === "r1")!.text, t2.messages.find((m) => m.id === "r1")!.text, "pressure does not re-derive latched bytes"); }); + +test("pin: CCR disarm releases the latch — a disarmed session never re-projects a dangling placeholder", () => { + // #1273 governance, restated for the latch: placeholders are legal only + // while the retrieve channel they advertise exists. CCR rides the config + // digest, so arming/disarming mid-session is a user-action boundary that + // releases the latch and re-judges under the CURRENT config. + const big = "line of build output ".repeat(400); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "run a big build" }, + { id: "c1", role: "assistant", contentType: "tool-call" as const, toolName: "bash", toolCallId: "call1", text: JSON.stringify({ command: "npm run build" }) }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "bash", toolCallId: "call1", text: big }, + ]; + const armed = { ...cfg(), ccr: { enabled: true, toolName: "acp_retrieve", minToolTokens: 50, excludeTools: [], maxHeadChars: 96 } }; + const core = createCore(cfg()); + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config: armed, tokenCount: 0, renderTags: "none" }); + const t1r = t1.messages.find((m) => m.id === "r1")!.text!; + assert.ok(t1r.includes("[acp-stored"), "armed CCR substitutes at arrival"); + assert.ok(!t1r.includes("line of build output"), "raw bytes leave the wire while armed"); + + // Same session, next turn, CCR still armed: the placeholder replays + // byte-stable (the latch holds; #1460 reprojection agrees with it). + const t2 = core.processTurn({ messages: m1, state: t1.state, config: armed, contentStore: t1.contentStore, tokenCount: 0, renderTags: "none" }); + assert.equal(t2.messages.find((m) => m.id === "r1")!.text, t1r, "latched placeholder replays verbatim while armed"); + + // Disarm (route/base governance flip): the digest changes → the latch + // releases → the raw bytes return. No dangling placeholder, no silent loss. + const t3 = core.processTurn({ messages: m1, state: t2.state, config: cfg(), contentStore: t2.contentStore, tokenCount: 0, renderTags: "none" }); + const t3r = t3.messages.find((m) => m.id === "r1")!.text!; + assert.ok(t3r.includes("line of build output"), "disarmed CCR returns the raw bytes"); + assert.ok(!t3r.includes("[acp-stored"), "no dangling placeholder after disarm"); +}); diff --git a/package-lock.json b/package-lock.json index 58875c4ac1..f0ee9a6d1c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.141", + "version": "0.0.142", "dev": true, "license": "MIT", "devDependencies": { From a61a1f9bc5bd0439e5e14ff6ea1f71305ab4bf42 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:10:41 +0800 Subject: [PATCH 25/29] fix: rescue a numberless overflow in-request under arm*0.9; exhausted preflight defers to tier recovery (#3787, #2952) --- src/preflight.ts | 9 +++++++++ src/server/relay.ts | 16 +++++++++++++++- tests/e2e-overflow-selfheal.test.ts | 25 ++++++++++++++----------- tests/issue3787-overflow-refold.test.ts | 20 ++++++++++++++------ 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/preflight.ts b/src/preflight.ts index 49941a9706..cffe74d8f1 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -2046,6 +2046,15 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // remains (or nothing above minCompressRange, or the whole relaxed // zone already folded) — so no retry against a healthier relay can // change the verdict. + // #2952 Design A: "nothing compressible" can still leave a rescue — + // every raw message already inside a block (view = carriers only) + // is exactly the distillable state. Defer to the post-walk + // recovery funnel instead of failing here, same as the futility + // gate above; tiers=false / emergency digest never enter this way. + if (eligibleForTierRecovery()) { + deps.log("info", `[preflight] no compressible ranges remain${relaxed ? " even with the soft zone relaxed" : ""}; deferring to bounded tier-distillation recovery (#2952)`); + break; + } failure = { kind: "exhausted", deterministic: true, detail: relaxed ? relaxedExhaustedDetail : subMinNoted ? `no foldable compressible ranges remain: all ${viable.length} viable range(s) are below minCompressRange (${minChars} chars)` : "no compressible ranges remain in the conversation" }; diff --git a/src/server/relay.ts b/src/server/relay.ts index cfd3af7f7a..f27beab97c 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1145,7 +1145,21 @@ export async function forward( const overflowInfo = inspectContextOverflow(upstreamResult.response.status, overflowText); if (overflowInfo.isOverflow) { armOverflowShrink(overflowInfo); - const refolded = await overflowRefold(overflowInfo.window).catch(() => null); + // #3787 follow-up: a NUMBERLESS overflow states no window; the + // armed baseline (min of declared and the rejected payload's + // own estimate) is the only ceiling evidence there is. Judging + // the refold against the DECLARED window made preflight see + // "already fits" and return an unchanged body — the rescue + // then only worked when re-render churn happened to change + // bytes, which byte-stable rebuilds (#3939 ③) eliminate. + // The arm is BY CONSTRUCTION the rejected payload's own size, + // so a refold targeting the arm itself leaves nothing to fold + // (payload == ceiling on every estimate): fold with a 10% + // margin under it, the same headroom discipline as the arm's + // own min(declared, est) clamp — the rejection proved the + // ceiling is below the payload, give the fold real room. + const armedWindow = overflowInfo.window ?? (prepared.session.stats.lastInputTokensSource === "overflow-arm" ? Math.floor(prepared.session.stats.lastInputTokens * 0.9) : undefined); + const refolded = await overflowRefold(armedWindow).catch(() => null); if (refolded) { try { // #816 family: the refold rebuilds the body from state diff --git a/tests/e2e-overflow-selfheal.test.ts b/tests/e2e-overflow-selfheal.test.ts index 08170ef714..45528eefbb 100644 --- a/tests/e2e-overflow-selfheal.test.ts +++ b/tests/e2e-overflow-selfheal.test.ts @@ -193,7 +193,7 @@ test("e2e #987/#1195 T1: a stated-window overflow arms the one-shot shrink, lear } }); -test("e2e #987 T2: an overflow WITHOUT a window number arms at the declared window and learns nothing", async () => { +test("e2e #987 T2: an overflow WITHOUT a window number arms at the payload's own size, rescues in-request (#3787), and learns nothing", async () => { let streamingCall = 0; let summaryCalls = 0; const upstream = http.createServer((req, res) => { @@ -238,24 +238,27 @@ test("e2e #987 T2: an overflow WITHOUT a window number arms at the declared wind const headers = { "content-type": "application/json", "x-acp-session": "t2-sess" }; const r1 = await fetch(url, { method: "POST", headers, body }); - assert.equal(r1.status, 400); + // #3787: a NUMBERLESS overflow used to pass through verbatim (the arm + // fired but the refold judged against the 400k declaration and found + // "already fits" — nothing to fold, 400 to the client). The arm is + // the only ceiling evidence: refold under arm*0.9, fold, re-send. + assert.equal(r1.status, 200, "#3787: numberless overflow rescued in-request under the armed window"); + assert.equal(r1.headers.get("content-type"), "text/event-stream", "the retry's SSE is what the client sees"); await r1.text(); + assert.ok(summaryCalls >= 1, "the rescue ran a preflight fold under the armed window"); const s = listSessions().find((x) => x.id === "t2-sess"); assert.ok(s, "session exists"); assert.equal(s!.metadata.confirmedContextLimits, undefined, "nothing learned"); - assert.ok(s!.stats.lastInputTokens > 10_000 && s!.stats.lastInputTokens < 400_000, `armed at the payload's own size (~16k, not the 400k declaration): ${s!.stats.lastInputTokens}`); - // #1839 ask 3: an arm is NOT a billing report — it carries its own - // provenance grade "overflow-arm" (still rescue-grade: effectiveTokenCount's - // fast path, the #496 evidence gate and the preflight floors accept it; - // billing-grade consumers keep excluding it). - assert.equal(s!.stats.lastInputTokensSource, "overflow-arm", "the arm carries its own provenance grade, never billing-grade \"usage\""); + assert.equal(s!.stats.lastInputTokens, 5000, "the retry's own usage report replaced the armed baseline"); + assert.equal(s!.stats.lastInputTokensSource, "usage", "the successful retry reports usage-grade"); + assert.equal(s!.stats.overflowArmTokens, undefined, "#1129: the retry's usage report retired the one-shot arm"); - // r2: forwarded as-is (no arm, nothing learned) — the pass-through - // shape the old #969 test asserted is unchanged. + // r2: same payload — the fold is sticky, forwarded compacted, no new fold. + const summaryCallsBefore = summaryCalls; const r2 = await fetch(url, { method: "POST", headers, body }); assert.equal(r2.status, 200); await r2.text(); - assert.equal(summaryCalls, 0, "no fold"); + assert.equal(summaryCalls, summaryCallsBefore, "r2 forwards the sticky fold without folding again"); } finally { proxy.close(); proxy.closeIdleConnections?.(); diff --git a/tests/issue3787-overflow-refold.test.ts b/tests/issue3787-overflow-refold.test.ts index ee4058d126..e84651adb5 100644 --- a/tests/issue3787-overflow-refold.test.ts +++ b/tests/issue3787-overflow-refold.test.ts @@ -105,16 +105,24 @@ test("e2e #3787: the agnes/cch numberless context_limit 400 is recognized and th _setStoreForTest(new SessionStore({ enabled: false })); setRegistryForTest({}); - // Declared window below the payload's true size: the first oversized turn - // rides out (the optimistic trigger cannot see CJK-density mass), the - // gateway 400s, and the rescue must come from the recognized error. + // Declared window far above the payload's true size: preflight skips the + // first turn entirely (nothing over 100K), the gateway 400s the RAW body + // anyway (its real ceiling is below the payload — agnes-3.0-flash through + // a mis-declared cch), and the rescue must come from the recognized error: + // arm = min(declared, payload's own estimate), refold under arm*0.9 folds + // real ranges, the smaller body is re-sent within the same request. + // (A declared window BELOW the payload cannot express this contract: the + // calibrated trigger now folds everything compressible BEFORE the send, + // so a numberless rejection arrives with zero foldable mass left — the + // rescue's historical "success" there rode on re-render truncation + // asymmetry, which #3939 ③ eliminated.) const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: "http://127.0.0.1", - routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-test": { context: 10_000 } } } }, - modelContextLimit: 10_000, - kernelConfig: defaultConfig(10_000), + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-test": { context: 100_000 } } } }, + modelContextLimit: 100_000, + kernelConfig: defaultConfig(100_000), compress: { injectTool: true, injectNudge: true }, compat: { roles: {} }, streamErrorShape: "protocol", From 06b7d23635a41779674c18f9d63086c55ce999f1 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:10:41 +0800 Subject: [PATCH 26/29] test: keep the #189 fold-point pin outside the protected zone under the atomic batch contract --- tests/compress-observability.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compress-observability.test.ts b/tests/compress-observability.test.ts index 68b811493b..a31985d82c 100644 --- a/tests/compress-observability.test.ts +++ b/tests/compress-observability.test.ts @@ -88,7 +88,7 @@ test("#189: fold point picks the EARLIEST range start across multiple ranges", ( const ctx = makeCompressibleCtx(); ctx.session.stats.lastInputTokens = 100000; const args = { content: [ - { startId: "m00005", endId: "m00006", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, + { startId: "m00003", endId: "m00004", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, { startId: "m00001", endId: "m00002", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, ] }; runApply(ctx, args); From 74eca4e1c49114524cb9a3f6f5e696e2cc1cff50 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:39:32 +0800 Subject: [PATCH 27/29] kernel: keep the fold-gate files inside the verifiable subset; re-baseline the #3769 proof to the fold-gated contract (#3939) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - truncate-tools: build truncatedIds in the loop instead of spreading Map.keys() — the iterator spread degrades the symbolic translation of the 3691 harness (verif refuses to certify on degradation). - hide-consumed: hoist state.hiddenOrphanRefs ?? [] — .includes on a possibly-undefined receiver degrades the same way. - verif/symbolic: fold Number(constString) to its numeric constant; the old {kind:any} made Number.isFinite collapse to constant false, silently deleting the refNum-gated fold-sweep path from every model (real code fires it; the model could not — engine gap, not kernel bug). - verif-bug3769 harness re-baselined onto the #3784 fold gate: Q1 (append-only growth hides r1) is now UNSAT — the fix holds; Q1b proves the sweep is still derivable THROUGH the fold marker (judge not vacuous); replay pins hidden=0 without a pending fold and newlySwept=[c1,c2,c3] with one. - kernel 0.0.142 -> 0.0.143 (both locks synced). verif: 15/15 green with cvc5 (subset-guard, symbolic-smoke, 3691, 3769, 3805, 3676, generic, induction). kernel suite 1357/1357. --- kernel/package-lock.json | 4 +- kernel/package.json | 2 +- kernel/src/hide-consumed.ts | 3 +- kernel/src/truncate-tools.ts | 4 +- package-lock.json | 2 +- tests/verif-bug3676-wire-reorder.test.ts | 154 + tests/verif-bug3691-oscillation.test.ts | 123 + tests/verif-bug3769-orphan-window.test.ts | 240 ++ ...erif-bug3805-durable-ref-migration.test.ts | 233 ++ tests/verif-generic-append-stability.test.ts | 204 ++ .../verif-induction-append-stability.test.ts | 146 + tests/verif-subset-guard.test.ts | 48 + tests/verif-symbolic-smoke.test.ts | 101 + verif/README.md | 86 + verif/exceptions.ts | 81 + verif/guarded-files.ts | 23 + verif/replay.ts | 34 + verif/solve.ts | 144 + verif/subset-guard.ts | 584 ++++ verif/symbolic.ts | 2530 +++++++++++++++++ verif/terms.ts | 532 ++++ 21 files changed, 5272 insertions(+), 6 deletions(-) create mode 100644 tests/verif-bug3676-wire-reorder.test.ts create mode 100644 tests/verif-bug3691-oscillation.test.ts create mode 100644 tests/verif-bug3769-orphan-window.test.ts create mode 100644 tests/verif-bug3805-durable-ref-migration.test.ts create mode 100644 tests/verif-generic-append-stability.test.ts create mode 100644 tests/verif-induction-append-stability.test.ts create mode 100644 tests/verif-subset-guard.test.ts create mode 100644 tests/verif-symbolic-smoke.test.ts create mode 100644 verif/README.md create mode 100644 verif/exceptions.ts create mode 100644 verif/guarded-files.ts create mode 100644 verif/replay.ts create mode 100644 verif/solve.ts create mode 100644 verif/subset-guard.ts create mode 100644 verif/symbolic.ts create mode 100644 verif/terms.ts diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 6554478342..1d88f406cb 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.142", + "version": "0.0.143", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.142", + "version": "0.0.143", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 80a8ce051b..0457970312 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.142", + "version": "0.0.143", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index d79400fb99..fd62ef7585 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -334,9 +334,10 @@ export function hideConsumedCompressCalls( hide = true; } else if (info.fate === "orphan") { const ref = state.messageRefs.byRaw[messages[i]!.id]; + const rememberedHidden: string[] = state.hiddenOrphanRefs ?? []; if (sweptSet.has(callId)) { hide = true; - } else if (ref && state.hiddenOrphanRefs?.includes(ref)) { + } else if (ref && rememberedHidden.includes(ref)) { // Pre-#3769 states persisted the rotation's hidden set as message refs // (hiddenOrphanRefs). Honor it on upgrade: resurrecting a pair the old // kernel already removed would re-insert it mid-history and break the diff --git a/kernel/src/truncate-tools.ts b/kernel/src/truncate-tools.ts index 19083bdc00..25cebb433b 100644 --- a/kernel/src/truncate-tools.ts +++ b/kernel/src/truncate-tools.ts @@ -90,6 +90,7 @@ export function truncateLargeToolOutputs( let remaining = tokenCount; const targetTokens = config.truncate.threshold * limit * 0.9; const replacements = new Map(); + const truncatedIds: string[] = []; const applyTo = (candidates: CoreMessage[]) => { for (const candidate of candidates) { @@ -107,6 +108,7 @@ export function truncateLargeToolOutputs( ); const replacement = `${prefix}\n\n...${TRUNCATION_MARKER} — original ~${tokens} tokens]...\n\n${suffix}`; replacements.set(candidate.id, replacement); + truncatedIds.push(candidate.id); truncatedCount++; remaining -= tokens - countTokens(replacement); savedTokens += tokens - countTokens(replacement); @@ -126,6 +128,6 @@ export function truncateLargeToolOutputs( truncatedCount, savedTokens, candidatesFound, - truncatedIds: [...replacements.keys()], + truncatedIds, }; } diff --git a/package-lock.json b/package-lock.json index f0ee9a6d1c..7848be48d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.142", + "version": "0.0.143", "dev": true, "license": "MIT", "devDependencies": { diff --git a/tests/verif-bug3676-wire-reorder.test.ts b/tests/verif-bug3676-wire-reorder.test.ts new file mode 100644 index 0000000000..de3992289b --- /dev/null +++ b/tests/verif-bug3676-wire-reorder.test.ts @@ -0,0 +1,154 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of } from "../verif/terms.js"; +import type { Seq, Bool } from "../verif/terms.js"; +import { solveSmt2 } from "../verif/solve.js"; +import { openaiToCore, coreToOpenai } from "../kernel/src/wire/openai.js"; +import { resolve } from "node:path"; + +/** + * #3676 — OpenAI wire round-trip reverses [file, text] content-part order. + * + * Invariant under test (general — no bug shape hardcoded): a parse→rebuild + * round trip must preserve the ORDER of a user message's content parts. + * Part order IS bytes: a reordered parts array is a different outbound + * request, so every cache in front of it misses. + * + * Phase 1 (symbolic, REAL openaiToCore/coreToOpenai translated): the solver + * must exhibit a wire user message whose parts are [opaque(file), text(T)] + * with the rebuilt parts [text, file] — REVERSED. + * Phase 2 (negative control): for [text, file] the same reversed assertion is + * UNSAT — the invariant is the judge, not the scenario. + * Phase 3 (replay): the witness shape runs through the REAL functions. + */ + +const FILE = resolve(import.meta.dirname, "../kernel/src/wire/openai.ts"); + +type Ctx = ReturnType; +type SymStr = { kind: "str"; t: Seq<"main"> }; +type SymObj = { kind: "obj"; fields: Map }; +type SymArr = { kind: "arr"; elems: SymObj[]; len: any }; + +const strc = (ctx: Ctx, v: string): SymStr => ({ kind: "str", t: ctx.String.val(v) }) as never; +const objOf = (fields: Record): SymObj => ({ kind: "obj", fields: new Map(Object.entries(fields)) }) as never; +const arrOf = (ctx: Ctx, elems: SymObj[]): SymArr => ({ kind: "arr", elems, len: ctx.Real.val(elems.length) }) as never; + +/** Deterministic port for deriveCanonicalMessageId (real one uses + * node:crypto sha256 — outside the guarded subset). Distinct argument + * tuples → distinct constant ids; ids never influence the order property. */ +function makeIdPort(ctx: Ctx) { + const memo = new Map(); + let n = 0; + return (...args: Array<{ kind: string }>): SymStr => { + const key = args.map((a) => (a.kind === "str" ? String((a as { t?: { c?: string } }).t?.c ?? "?") : a.kind)).join("|"); + let v = memo.get(key); + if (!v) { v = { kind: "str", t: strc(ctx, `id${n++}`) } as never; memo.set(key, v); } + return v; + }; +} + +/** ClusterCounter port: per-instance JS counter (state outside the model — + * ids don't affect the order property, only determinism). */ +function makeCounterPorts(ctx: Ctx) { + const counts = new WeakMap(); + const marker = objOf({}); + return { + ClusterCounter: () => marker, + __method_next: (recv: SymObj, base: SymStr): SymStr => { + if (recv !== marker) return { kind: "str", t: strc(ctx, "?") } as never; + const c = (counts.get(marker) ?? 0) + 1; + counts.set(marker, c); + const b = String(base?.t?.c ?? "?"); + return { kind: "str", t: strc(ctx, c === 1 ? b : `${b}_${c - 1}`) } as never; + }, + }; +} + +function wireInput(ctx: Ctx, first: "file" | "text") { + const T = ctx.String.const("T"); + const filePart = objOf({ type: strc(ctx, "file"), file_id: strc(ctx, "file_123") }); + const textPart = objOf({ type: strc(ctx, "text"), text: { kind: "str", t: T } as never }); + const parts = arrOf(ctx, first === "file" ? [filePart, textPart] : [textPart, filePart]); + const userMsg = objOf({ role: strc(ctx, "user"), content: parts }); + return { T, body: objOf({ messages: arrOf(ctx, [userMsg]) }) }; +} + +type ParseRet = { degradations: string[]; ret: { kind: string; fields: Map } }; +type RbRet = { degradations: string[]; ret: SymArr }; + +function roundTrip(ctx: Ctx, ports: unknown, first: "file" | "text") { + const { T, body } = wireInput(ctx, first); + const parseOut = translateFunction({ file: FILE, function: "openaiToCore", params: { body }, ctx, options: { portImpls: ports as never } }).exec() as never as ParseRet; + const coreMsgs = parseOut.ret.fields.get("msgs") as never; + const rbOut = translateFunction({ file: FILE, function: "coreToOpenai", params: { messages: coreMsgs }, ctx, options: { portImpls: ports as never } }).exec() as never as RbRet; + return { T, parseOut, rbOut }; +} + +function outContent(rb: RbRet): SymArr { + const out0 = rb.ret.elems[0]!; + assert.equal(out0.kind, "obj", "one wire message out"); + const c = (out0.fields as Map).get("content") as SymArr; + assert.equal(c.kind, "arr", "opaque parts ride along → parts array (not plain string)"); + return c; +} + +function partType(content: SymArr, i: number): Seq<"main"> { + const e = content.elems[i]; + assert.ok(e && e.kind === "obj", `part ${i} exists`); + const t = (e.fields as Map).get("type") as SymStr | undefined; + assert.ok(t && t.kind === "str", `part ${i} has a type`); + return t.t; +} + +test("bug3676: solver finds the [file,text]→[text,file] part-order reversal on the real wire codec", { timeout: 240_000 }, async () => { + const ctx = newCtx(); + const ports = { ...makeCounterPorts(ctx), deriveCanonicalMessageId: makeIdPort(ctx) }; + + const A = roundTrip(ctx, ports, "file"); // original order [file, text] + const B = roundTrip(ctx, ports, "text"); // original order [text, file] + assert.deepEqual(A.parseOut.degradations, [], "parse fully translated"); + assert.deepEqual(A.rbOut.degradations, [], "rebuild fully translated"); + + const cA = outContent(A.rbOut); + const cB = outContent(B.rbOut); + + // Property: original [file,text] rebuilds as [text,file] — REVERSED. + // len(out)==2 ∧ T≠"" ∧ type(out[0])=="text" ∧ type(out[1])=="file" + const tNonEmptyA = ctx.ToReal((A.T as unknown as { length(): Seq<"main">["length"] extends () => infer R ? R : never }).length() as never).ge(ctx.Real.val(1)) as unknown as Bool<"main">; + const len2A = (cA.len as unknown as { eq: (x: unknown) => Bool<"main"> }).eq(ctx.Real.val(2)); + const p0A = partType(cA, 0).eq(ctx.String.val("text")); + const p1A = partType(cA, 1).eq(ctx.String.val("file")); + const q1 = smt2Of(ctx, [tNonEmptyA.and(len2A).and(p0A).and(p1A) as never, ...[]], []); + const r1 = await solveSmt2(q1, []); + assert.equal(r1.status, "sat", "solver must exhibit the [file,text]→[text,file] reversal"); + + // Negative control: original [text,file] CANNOT rebuild reversed. + const tNonEmptyB = ctx.ToReal((B.T as unknown as { length(): never }).length() as never).ge(ctx.Real.val(1)) as unknown as Bool<"main">; + const len2B = (cB.len as unknown as { eq: (x: unknown) => Bool<"main"> }).eq(ctx.Real.val(2)); + const p0B = partType(cB, 0).eq(ctx.String.val("file")); + const q2 = smt2Of(ctx, [tNonEmptyB.and(len2B).and(p0B) as never], []); + const r2 = await solveSmt2(q2, []); + assert.equal(r2.status, "unsat", "no reversal exists for [text,file] — order preserved (the invariant is the judge)"); + + // ---- replay on the REAL codec (tsx runs the real imports incl. crypto) ---- + const wireFileFirst = [ + { + role: "user" as const, + content: [ + { type: "file" as const, file_id: "file_123" }, + { type: "text" as const, text: "hello world" }, + ], + }, + ]; + const core = openaiToCore({ messages: wireFileFirst } as never); + const rebuilt = coreToOpenai(core.msgs as never); + const c = (rebuilt[0] as { content: Array<{ type: string }> }).content; + assert.ok(Array.isArray(c), "rebuild emits a parts array when opaque parts ride along"); + assert.equal(c.length, 2, "both parts survive (never dropped, #1205)"); + assert.deepEqual( + c.map((p) => p.type), + ["text", "file"], + "REPLAY: [file,text] round-trips as [text,file] — part order REVERSED (#3676)", + ); +}); diff --git a/tests/verif-bug3691-oscillation.test.ts b/tests/verif-bug3691-oscillation.test.ts new file mode 100644 index 0000000000..7a17c3fa9f --- /dev/null +++ b/tests/verif-bug3691-oscillation.test.ts @@ -0,0 +1,123 @@ +// #3691 (kernel/src/truncate-tools.ts): the emergency clip has no persistent +// record — the decision to clip depends only on the per-turn token meter, so +// the same message renders clipped this turn and full next turn. Outbound +// bytes for an unchanged message flip-flop: a provider prefix-cache seam on +// every flip. This harness proves the mechanism WITHOUT pointing the solver +// at the bug: the symbolic phase only asks "can the clip fire at all under a +// gate-level meter?", then the REAL two-turn replay checks the general +// stability invariant (same message in ⇒ same bytes out across turns when the +// client resent identical history). The mitigation check pins that raising +// the threshold to 1.0 restores stability. +// +// No-cheat rules (#3848): entry functions are the real kernel exports; the +// property is general (byte-stability), never asserted at bug lines; the +// counterexample witness must replay on the real code. +import test from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.ts"; +import { newCtx, smt2Of } from "../verif/terms.ts"; +import { solveSmt2, parseNum } from "../verif/solve.ts"; +import { truncateLargeToolOutputs } from "../kernel/src/truncate-tools.ts"; +import { defaultCountTokens } from "../kernel/src/tokenize.ts"; + +const LIMIT = 2000; +const GATE = LIMIT * 0.95; // emergencyTruncateNode gate: usage >= threshold + +const realConfig = { modelContextLimit: LIMIT, truncate: { threshold: 0.95 } } as never; +const mkMsgs = (T: string): { id: string; role: "tool" | "user"; contentType: "tool-result" | "text"; toolCallId?: string; text: string }[] => [ + { id: "m0", role: "tool", contentType: "tool-result" as const, toolCallId: "tc0", text: T }, + { id: "m1", role: "user", contentType: "text" as const, text: "short 1" }, + { id: "m2", role: "user", contentType: "text" as const, text: "short 2" }, + { id: "m3", role: "user", contentType: "text" as const, text: "short 3" }, +]; + +test("bug3691: solver finds a clip-triggering witness on the real truncation code", { timeout: 240_000 }, async (t) => { + const ctx = newCtx(); + const T = ctx.String.const("T"); + const tok1 = ctx.Real.const("tok1"); + const msgs = { kind: "arr" as const, len: ctx.Real.val(4), elems: [ + { kind: "obj" as const, fields: new Map([ + ["id", { kind: "str" as const, t: ctx.String.val("m0") }], + ["role", { kind: "str" as const, t: ctx.String.val("tool") }], + ["contentType", { kind: "str" as const, t: ctx.String.val("tool-result") }], + ["toolCallId", { kind: "str" as const, t: ctx.String.val("tc0") }], + ["text", { kind: "str" as const, t: T }], + ]) }, + ...[1, 2, 3].map((i) => ({ kind: "obj" as const, fields: new Map([ + ["id", { kind: "str" as const, t: ctx.String.val(`m${i}`) }], + ["role", { kind: "str" as const, t: ctx.String.val("user") }], + ["contentType", { kind: "str" as const, t: ctx.String.val("text") }], + ["text", { kind: "str" as const, t: ctx.String.val(`short ${i}`) }], + ]) })), + ] }; + const config = { kind: "obj" as const, fields: new Map([ + ["modelContextLimit", { kind: "num" as const, t: ctx.Real.val(LIMIT) }], + ["truncate", { kind: "obj" as const, fields: new Map([ + ["threshold", { kind: "num" as const, t: ctx.Real.val(0.95) }], + ]) }], + ]) }; + const meter = (sv: { kind: string; t?: unknown }) => + sv.kind === "str" ? { kind: "num" as const, t: ctx.ToReal((sv.t as import("../verif/terms.js").Seq).length()).div(ctx.Real.val(4)).add(ctx.Real.val(1)) } : { kind: "any" as const }; + + const { exec } = translateFunction({ + file: new URL("../kernel/src/truncate-tools.ts", import.meta.url).pathname, + function: "truncateLargeToolOutputs", + params: { messages: msgs, tokenCount: { kind: "num", t: tok1 }, config, countTokens: { kind: "any" }, options: { kind: "obj", fields: new Map([["includeTextMessages", { kind: "bool", t: ctx.Bool.val(true) }]]) } }, + ctx, + options: { portImpls: { countTokens: meter } }, + }); + const out1 = exec(); + assert.deepEqual(out1.degradations, [], "translation must be degradation-free"); + const tc = (out1.ret as { kind: string; fields: Map }).fields.get("truncatedCount") as { kind: string; t: import("../verif/terms.js").Arith }; + assert.equal(tc.kind, "num", "truncatedCount must stay symbolic"); + + // Ask the solver: is there ANY input where the meter sits at the gate and + // the clip fires? (Pure existence — no bug-specific shaping beyond the + // scenario constraint "usage at the emergency threshold".) + const q1 = smt2Of(out1.ctx, [ + tok1.ge(ctx.Real.val(GATE)), + ctx.ToReal(T.length()).ge(ctx.Real.val(1)), + tc.t.ge(ctx.Real.val(1)), + ...out1.assumptions, + ]); + let res; + try { res = await solveSmt2(q1, ["(str.len T)", "tok1"]); } + catch (e) { t.skip(`solver unavailable: ${(e as Error).message.slice(0, 200)}`); return; } + assert.equal(res.status, "sat", `expected sat, got ${res.status}: ${res.raw.slice(0, 400)}`); + const lenT = parseNum(res.values[0]!); + const tokW = parseNum(res.values[1]!); + assert.ok(lenT >= 1, "witness must be a non-empty string"); + + // Negative control: below the gate the clip can NEVER fire (translation + // must not be able to produce phantom clips). + const q2 = smt2Of(out1.ctx, [ + tok1.lt(ctx.Real.val(GATE)), + tc.t.ge(ctx.Real.val(1)), + ...out1.assumptions, + ]); + const res2 = await solveSmt2(q2, []); + assert.equal(res2.status, "unsat", "clip must be unreachable below the gate"); + + // ---- Phase 2: replay the witness on the REAL code, two turns ---- + const Tstr = "x".repeat(lenT); + const r1 = truncateLargeToolOutputs(mkMsgs(Tstr) as never, Math.max(1, Math.round(tokW)), realConfig, defaultCountTokens, { includeTextMessages: true }); + assert.ok(r1.truncatedCount >= 1, `real code must clip with the witness (len=${lenT}, tok=${tokW})`); + + // Turn 2: the client resents the identical history; the provider meters + // turn-1's OUTBOUND (clipped) text. + const tok2 = defaultCountTokens(r1.messages[0]!.text ?? "") + 3; + const r2 = truncateLargeToolOutputs(mkMsgs(Tstr) as never, tok2, realConfig, defaultCountTokens, { includeTextMessages: true }); + const out1b = r1.messages[0]!.text!; + const out2 = r2.messages[0]!.text!; + assert.notEqual(out1b, out2, "same resent history must not produce different outbound bytes across turns — #3691 flip"); +}); + +test("bug3691 mitigation: threshold=1.0 makes the render stable across the same two turns", { timeout: 60_000 }, () => { + const Tstr = "x".repeat(4001); + const stable = { modelContextLimit: LIMIT, truncate: { threshold: 1.0 } } as never; + const r1 = truncateLargeToolOutputs(mkMsgs(Tstr) as never, 1900, stable, defaultCountTokens, { includeTextMessages: true }); + const tok2 = defaultCountTokens(r1.messages[0]!.text ?? "") + 3; + const r2 = truncateLargeToolOutputs(mkMsgs(Tstr) as never, tok2, stable, defaultCountTokens, { includeTextMessages: true }); + assert.equal(r1.messages[0]!.text, Tstr, "no clip with threshold=1"); + assert.equal(r2.messages[0]!.text, Tstr, "stable across turns"); +}); diff --git a/tests/verif-bug3769-orphan-window.test.ts b/tests/verif-bug3769-orphan-window.test.ts new file mode 100644 index 0000000000..4bd2888b85 --- /dev/null +++ b/tests/verif-bug3769-orphan-window.test.ts @@ -0,0 +1,240 @@ +/** + * Bug #3769 — hide-consumed orphan window breaks append-only stability. + * + * ORIGINAL bug (pre-#3784): kernel/src/hide-consumed.ts kept only the NEWEST + * KEEP_LAST_ORPHANED=2 orphan compress-call pairs visible; a THIRD failed + * pair silently dropped the OLDEST mid-history (issue #3769: 184,423 tokens + * re-billed, hit rate 97.7% → 19.2%). + * + * FIXED contract (#3784, absorbed into #3939 ①): orphan sweeping is + * FOLD-GATED — nothing hides between folds; the sweep rides the fold's own + * sanctioned rewrite (state.pendingOrphanSweepMinRef). This harness was + * re-baselined when the fix landed: the append-only flip must now be + * UNSAT (Q1), the mechanism must remain derivable THROUGH the fold marker + * only (Q1b sat) — proving the judge still sees the gated mechanism, not a + * vacuous truth — and non-compress appends stay stable (Q2 unsat). + * + * No-cheat contract (#3848 / user directive m00852): + * - we execute the REAL hideConsumedCompressCalls (translateFunction); + * - the invariant is a GENERAL hyperproperty: "a message that was visible + * must remain visible when the input history only GROWS (append-only), + * state unchanged" — nothing references the keep-window internals; + * - the solver searches over the appended pair's symbolic identity + * (toolName / toolCallId): the flip must be derivable, not asserted; + * - the counterexample replays on the real code. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of, strc } from "../verif/terms.js"; +import { solveSmt2, parseNum } from "../verif/solve.js"; +import { hideConsumedCompressCalls } from "../kernel/src/hide-consumed.js"; +import type { CoreMessage, CompressionState } from "../kernel/src/types.js"; + +const file = resolve(import.meta.dirname, "../kernel/src/hide-consumed.ts"); + +const asMsg = (m: CoreMessage): CoreMessage => m; + +// A: two orphan failed-compression pairs + surrounding user traffic. +// B: one MORE orphan pair, appended (nothing else changes). +const histA = (): CoreMessage[] => [ + asMsg({ id: "u0", role: "user", contentType: "text", text: "q1" }), + asMsg({ id: "t1", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c1", text: "{}" }), + asMsg({ id: "r1", role: "tool", contentType: "tool-result", toolCallId: "c1", text: "fail" }), + asMsg({ id: "t2", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c2", text: "{}" }), + asMsg({ id: "r2", role: "tool", contentType: "tool-result", toolCallId: "c2", text: "fail" }), +]; + +test("bug3769: solver proves append-only growth hides a previously-visible pair", { timeout: 240_000 }, async (t) => { + const solver = await (async () => { + try { return await solveSmt2("(set-logic ALL)(check-sat)", []); } catch { return null; } + })(); + if (!solver) return t.skip("no SMT solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + + // Symbolic identity of the appended pair. + const tn3 = ctx.String.const("tn3"); + const tc3 = ctx.String.const("tc3"); + + const mkSym = (id: string, role: string, ct: string, toolName: import("../verif/terms.js").Seq | string, toolCallId: import("../verif/terms.js").Seq | string, text: string) => { + const fields = new Map(); + fields.set("id", { kind: "str", t: ctx.String.val(id) }); + fields.set("role", { kind: "str", t: ctx.String.val(role) }); + fields.set("contentType", { kind: "str", t: ctx.String.val(ct) }); + fields.set("text", { kind: "str", t: ctx.String.val(text) }); + if (typeof toolName === "string") fields.set("toolName", { kind: "str", t: ctx.String.val(toolName) }); + else fields.set("toolName", { kind: "str", t: toolName }); + if (typeof toolCallId === "string") fields.set("toolCallId", { kind: "str", t: ctx.String.val(toolCallId) }); + else fields.set("toolCallId", { kind: "str", t: toolCallId }); + return { kind: "obj" as const, fields }; + }; + const msgs = (grown: boolean): import("../verif/symbolic.js").SymVal => { + const list: ReturnType[] = [ + mkSym("u0", "user", "text", "", "", "q1"), + mkSym("t1", "assistant", "tool-call", "compress", "c1", "{}"), + mkSym("r1", "tool", "tool-result", "", "c1", "fail"), + mkSym("t2", "assistant", "tool-call", "compress", "c2", "{}"), + mkSym("r2", "tool", "tool-result", "", "c2", "fail"), + ]; + if (grown) { + list.push(mkSym("t3", "assistant", "tool-call", tn3, tc3, "{}")); + list.push(mkSym("r3", "tool", "tool-result", "", tc3, "fail")); + } + return { kind: "arr", elems: list, len: ctx.Real.val(list.length) }; + }; + const stateObj = (foldGate: boolean): import("../verif/symbolic.js").SymVal => { + const fields = new Map(); + fields.set("blocks", { kind: "arr", elems: [], len: ctx.Real.val(0) }); + // real state shape: messageRefs = { byRaw: Record } + const byRawInner = new Map(); + if (foldGate) { + // A fold just committed: refs are minted and the sweep is pending from + // the fold's own min ref (m00001) — the sanctioned rewrite window. + byRawInner.set("t1", { kind: "str", t: ctx.String.val("m00001") }); + byRawInner.set("r1", { kind: "str", t: ctx.String.val("m00002") }); + byRawInner.set("t2", { kind: "str", t: ctx.String.val("m00003") }); + byRawInner.set("r2", { kind: "str", t: ctx.String.val("m00004") }); + byRawInner.set("t3", { kind: "str", t: ctx.String.val("m00005") }); + byRawInner.set("r3", { kind: "str", t: ctx.String.val("m00006") }); + fields.set("pendingOrphanSweepMinRef", { kind: "num", t: ctx.Real.val(1) }); + } + fields.set("messageRefs", { kind: "obj", fields: new Map([["byRaw", { kind: "obj", fields: byRawInner }]]) }); + return { kind: "obj", fields }; + }; + + const run = (grown: boolean, foldGate = false) => { + const { exec } = translateFunction({ + file, function: "hideConsumedCompressCalls", + params: { state: stateObj(foldGate), messages: msgs(grown), isCompressTool: { kind: "undef" } }, + ctx, + options: { + portImpls: { + // injectable classifier port (default param in source): symbolic + // membership test — the harness does NOT hardcode the outcome + isCompressTool: (sv: any) => ({ kind: "bool", t: sv.t.eq(strc(ctx, "compress")) }) as any, + }, + }, + }); + const r = exec(); + assert.deepEqual(r.degradations, [], `no degradations expected (got: ${r.degradations.join("; ")})`); + assert.ok(r.ret && r.ret.kind === "obj", "result object"); + return r as import("../verif/symbolic.js").ExecOut; + }; + const outA = run(false); + const outB = run(true); + const outGate = run(true, true); + + const arrIds = (o: import("../verif/symbolic.js").ExecOut) => { + const ret = o.ret as Extract; + const out = ret.fields.get("messages"); + assert.ok(out && out.kind === "arr", "messages array in result"); + return out as Extract; + }; + const hasR1 = (a: Extract) => { + let acc: import("../verif/terms.js").Bool | null = null; + for (let k = 0; k < a.elems.length; k++) { + const e = a.elems[k]!; + const idv = e.kind === "obj" ? e.fields.get("id") : undefined; + const hit = (idv && idv.kind === "str" ? idv.t : ctx.String.val("")).eq(ctx.String.val("r1")); + const inLen = a.len.gt(ctx.Real.val(k)); + const term = inLen.and(hit); + acc = acc ? acc.or(term) : term; + } + return acc!; + }; + const visibleBefore = hasR1(arrIds(outA)); + const visibleAfter = hasR1(arrIds(outB)); + + // Q1 (fix holds): with the third pair a genuine compress pair (fresh id), + // append-only growth alone can NO LONGER hide the previously-visible r1 — + // the sweep is fold-gated (#3784). No counterexample exists. + const q1 = smt2Of(outB.ctx, [ + tn3.eq(ctx.String.val("compress")), + tc3.eq(ctx.String.val("c1")).not(), + tc3.eq(ctx.String.val("c2")).not(), + visibleBefore, + visibleAfter.not(), + ...outA.assumptions, ...outB.assumptions, + ]); + const r1 = await solveSmt2(q1, ["tn3", "tc3"]); + assert.equal(r1.status, "unsat", "append-only growth must not hide anything after the #3784 fold gate"); + + // Q1b (mechanism still reachable, only through the fold): the same grown + // history under a pending fold sweep (pendingOrphanSweepMinRef minted refs) + // DOES hide r1 — the rewrite is sanctioned to ride the fold boundary. + // SAT here proves the judge is not vacuous: the gated path is derivable. + const gateHidden = hasR1(arrIds(outGate)); + const q1b = smt2Of(outGate.ctx, [ + tn3.eq(ctx.String.val("compress")), + tc3.eq(ctx.String.val("c1")).not(), + tc3.eq(ctx.String.val("c2")).not(), + visibleBefore, + gateHidden.not(), + ...outA.assumptions, ...outGate.assumptions, + ]); + const r1b = await solveSmt2(q1b, ["tn3", "tc3"]); + assert.equal(r1b.status, "sat", "the fold-gated sweep must remain derivable through the fold marker"); + + // Q2 (negative control): when the appended pair is NOT a compress call, + // the same invariant holds — no counterexample exists. + const q2 = smt2Of(outB.ctx, [ + tn3.eq(ctx.String.val("compress")).not(), + visibleBefore, + visibleAfter.not(), + ...outA.assumptions, ...outB.assumptions, + ]); + const r2 = await solveSmt2(q2, []); + assert.equal(r2.status, "unsat", "non-compress appends must not hide anything"); + + // Replay on the REAL code: same shape, concrete distinct ids. + const state = { blocks: [], messageRefs: { byRaw: {} } } as unknown as CompressionState; + const a = hideConsumedCompressCalls(state, histA()); + assert.equal(a.messages.length, 5, "all five messages visible before the third pair"); + assert.ok(a.messages.some((m) => m.id === "r1"), "r1 visible before"); + const b = hideConsumedCompressCalls(state, [ + ...histA(), + asMsg({ id: "t3", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c3", text: "{}" }), + asMsg({ id: "r3", role: "tool", contentType: "tool-result", toolCallId: "c3", text: "fail" }), + ]); + assert.equal(b.hidden, 0, "#3784: appending an orphan pair no longer sweeps anything between folds"); + assert.ok(b.messages.some((m) => m.id === "r1"), "r1 STAYS visible after the append — the #3769 flip is gone"); + assert.equal(b.newlySweptCallIds.length, 0, "nothing newly swept without a pending fold"); + + // Replay the sanctioned side: a pending fold sweep (refs minted, min ref 1) + // hides the orphan pairs in one pass — the rewrite rides the fold. + const gateState = { + blocks: [], + messageRefs: { byRaw: { t1: "m00001", r1: "m00002", t2: "m00003", r2: "m00004", t3: "m00005", r3: "m00006" } }, + pendingOrphanSweepMinRef: 1, + } as unknown as CompressionState; + const c = hideConsumedCompressCalls(gateState, [ + ...histA(), + asMsg({ id: "t3", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c3", text: "{}" }), + asMsg({ id: "r3", role: "tool", contentType: "tool-result", toolCallId: "c3", text: "fail" }), + ]); + assert.ok(!c.messages.some((m) => m.id === "r1"), "r1 hidden AT the fold boundary — the gated sweep fires"); + assert.deepEqual(c.newlySweptCallIds, ["c1", "c2", "c3"], "all three orphan pairs ride the fold sweep"); + void solverBool; +}); + +// helper: print a z3 Bool in smt2 form via its solver-companion — +// the translateFunction result is in ctx "main", so its .toString() IS smt2. +const solverBool = (b: import("../verif/terms.js").Bool): string => b.toString(); + +test("bug3769 counter-test: with a larger window the same append is stable (shows the invariant is the judge, not the shape)", { timeout: 240_000 }, async (t) => { + // Only TWO orphan pairs exist and the appended traffic is user text (not a + // compress call) — visibility must be stable under the fold gate too. + const state = { blocks: [], messageRefs: { byRaw: {} } } as unknown as CompressionState; + const before = hideConsumedCompressCalls(state, histA()); + const after = hideConsumedCompressCalls(state, [ + ...histA(), + asMsg({ id: "u9", role: "user", contentType: "text", text: "q2" }), + ]); + const idsAfter = after.messages.map((m) => m.id); + assert.deepEqual(idsAfter.slice(0, before.messages.length), before.messages.map((m) => m.id), + "non-compress appends never change visibility of the existing prefix"); + assert.ok(after.messages.some((m) => m.id === "r1")); + void parseNum; // keep import used in both branches +}); diff --git a/tests/verif-bug3805-durable-ref-migration.test.ts b/tests/verif-bug3805-durable-ref-migration.test.ts new file mode 100644 index 0000000000..bcb8d8ed17 --- /dev/null +++ b/tests/verif-bug3805-durable-ref-migration.test.ts @@ -0,0 +1,233 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of } from "../verif/terms.js"; +import type { Bool, Term } from "../verif/terms.js"; +import { solveSmt2 } from "../verif/solve.js"; +import { assignRefs, emptyRefMap, BLOCKED_REF } from "../kernel/src/refs.js"; +import { renderVisibleRefs } from "../kernel/src/render-refs.js"; +import type { CoreMessage } from "../kernel/src/types.js"; + +/** + * #3805: durable carrier ref migration re-tags an already-sent message. + * + * Mechanism (all in guarded kernel code): + * - refs.ts assignRefs: a message whose previous ref is BLOCKED (it was + * protected last turn) and whose protection has lifted gets the BLOCKED + * entry DELETED and a fresh numeric ref allocated. + * - render-refs.ts renderMessage: BLOCKED renders verbatim; a numeric ref + * renders as an "m00001\n" tag PREFIX. + * - The durable-protection window is "latest carrier only" + * (collectLatestProtected + durable-message-guards.ts classifier + * `text.includes("")`): when a NEWER carrier for the + * same identity arrives, the OLDER one's protection lifts — mid-history. + * + * Invariant under test (general, #3848-style I1 tail-only growth): + * appending NEWER messages must not change the rendered bytes of an + * already-sent message. + * + * No-cheat composition: assignRefs + renderVisibleRefs are the REAL + * translated kernel functions. The only injected seams are + * - isProtected (the durable classifier + latest-wins window — its real + * impl lives outside the guarded kernel subset; modeled exactly as + * "text contains the skill marker" ∧ "is the newest carrier"), + * - countTokens (meter, len/4+1), + * - the dynamic ownTag regex is over-approximated by the engine's + * anchored-prefix rule (documented in verif/README). + * The replay phase runs the REAL functions end-to-end. + */ + +type Ctx = ReturnType; +type SymStr = { kind: "str"; t: Term }; +type SymBool = { kind: "bool"; t: Bool<"main"> }; +type SymObj = { kind: "obj"; fields: Map }; +type SymArr = { kind: "arr"; elems: any[]; len: any }; +type SymVal = SymStr | SymBool | SymObj | SymArr | { kind: string; t?: unknown }; + +const strv = (ctx: Ctx, v: string): SymStr => ({ kind: "str", t: ctx.String.val(v) }) as never; +const numv = (ctx: Ctx, v: number): { kind: "num"; t: unknown } => ({ kind: "num", t: ctx.Real.val(v) }) as never; +const boolv = (ctx: Ctx, v: boolean): SymBool => ({ kind: "bool", t: ctx.Bool.val(v) }) as never; +const objOf = (fields: Record): SymObj => ({ kind: "obj", fields: new Map(Object.entries(fields)) }) as never; +const arrOf = (ctx: Ctx, elems: SymVal[]): SymArr => ({ kind: "arr", elems, len: ctx.Real.val(elems.length) }) as never; + +const MARKER = ""; +const REPO = new URL("..", import.meta.url).pathname; +const REFS = `${REPO}kernel/src/refs.ts`; +const RREFS = `${REPO}kernel/src/render-refs.ts`; + +function coreMsg(ctx: Ctx, id: string, text: SymStr): SymObj { + return objOf({ + id: strv(ctx, id), + role: strv(ctx, "user"), + contentType: strv(ctx, "text"), + text, + }); +} + +function meterPort(ctx: Ctx) { + return (text: SymVal): { kind: "num"; t: unknown } => { + if (text && (text as SymStr).kind === "str") { + const len = ctx.ToReal((text as SymStr).t.length() as never); + return { kind: "num", t: (len as unknown as { div(x: unknown): { add(y: unknown): unknown } }).div(ctx.Real.val(4)).add(ctx.Real.val(1)) } as never; + } + return { kind: "num", t: ctx.Real.val(1) } as never; + }; +} + +interface TurnArtifacts { + mapByRaw: SymObj; + mapByRef: SymObj; + nextIndex: unknown; + rendered0: SymStr; + assumptions: any[]; + degradations: string[]; +} + +/** One full turn: assignRefs then renderVisibleRefs — both REAL translated + * kernel functions. `protected1`/`protected2` are the symbolic protection + * bits for m1/m2 (the durable classifier, injected as the isProtected port). */ +function turn( + ctx: Ctx, + msgs: SymVal, + existing: { byRaw: SymObj; byRef: SymObj } | null, + nextIndex: unknown, + protected1: SymBool, + protected2: SymBool, +): TurnArtifacts { + const emptyRec = objOf({}); + const options = objOf({ + existing: objOf({ byRaw: existing ? existing.byRaw : emptyRec, byRef: existing ? existing.byRef : emptyRec }), + nextIndex: numv(ctx, typeof nextIndex === "number" ? nextIndex : 1), + }); + const portsA = { + __method_isProtected: (_recv: SymVal, m: SymVal): SymBool => { + const id = (m as SymObj).fields.get("id") as SymStr | undefined; + const idc = id && typeof id.t.c === "string" ? id.t.c : ""; + if (idc === "u1") return protected1; + return protected2; + }, + countTokens: meterPort(ctx), + }; + const outA = translateFunction({ + file: REFS, function: "assignRefs", + params: { messages: msgs, options } as never, + ctx, options: { portImpls: portsA as never }, + }).exec(); + const retA = outA.ret as SymObj; + const mapA = retA.fields.get("map") as SymObj; + + const state = objOf({ messageRefs: mapA }); + const outR = translateFunction({ + file: RREFS, function: "renderVisibleRefs", + params: { messages: msgs, state } as never, + ctx, options: { portImpls: { countTokens: meterPort(ctx) } as never }, + }).exec(); + const retR = outR.ret as SymArr; + const rendered0 = retR.elems[0] as SymObj; + return { + mapByRaw: mapA.fields.get("byRaw") as SymObj, + mapByRef: mapA.fields.get("byRef") as SymObj, + nextIndex: retA.fields.get("nextIndex"), + rendered0: rendered0.fields.get("text") as SymStr, + assumptions: outA.assumptions ?? [], + degradations: [...(outA.degradations ?? []), ...(outR.degradations ?? [])], + }; +} + +function runSymbolic(ctx: Ctx) { + const T1 = ctx.String.const("T1"); + const T2 = ctx.String.const("T2"); + const marker = ctx.String.val(MARKER); + const d1: SymBool = { kind: "bool", t: T1.contains(marker) } as never; + const d2: SymBool = { kind: "bool", t: T2.contains(marker) } as never; + + const m1 = coreMsg(ctx, "u1", { kind: "str", t: T1 } as never); + const m2 = coreMsg(ctx, "u2", { kind: "str", t: T2 } as never); + + // Turn A: only m1. Protection = "is the newest durable carrier" = d1. + const A = turn(ctx, arrOf(ctx, [m1 as SymVal]), null, 1, d1, boolv(ctx, false)); + // Turn B: m2 appended. Latest-wins: m1 protected iff d1 ∧ ¬d2; m2 iff d2. + const notd2: SymBool = { kind: "bool", t: d2.t.not() } as never; + const keep1: SymBool = { kind: "bool", t: d1.t.and(notd2.t) } as never; + const B = turn(ctx, arrOf(ctx, [m1 as SymVal, m2 as SymVal]), { byRaw: A.mapByRaw, byRef: A.mapByRef }, A.nextIndex, keep1, d2); + return { T1, T2, d1, d2, A, B }; +} + +test("bug3805: solver finds the mid-history re-tag when protection lifts (append-only bytes must stay stable)", { timeout: 240_000 }, async (t) => { + const haveSolver = await (async () => { try { return solveSmt2("(set-logic ALL)(check-sat)", []).status === "sat"; } catch { return false; } })(); + if (!haveSolver) return t.skip("no solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + const { T1, T2, d1, d2, A, B } = runSymbolic(ctx); + assert.deepEqual(A.degradations, [], `A degradations: ${JSON.stringify(A.degradations)}`); + assert.deepEqual(B.degradations, [], `B degradations: ${JSON.stringify(B.degradations)}`); + assert.equal(A.rendered0.kind, "str"); + assert.equal(B.rendered0.kind, "str"); + + // Property: appending m2 changed m1's rendered bytes (mid-history seam). + const differ = (A.rendered0 as SymStr).t.neq((B.rendered0 as SymStr).t); + const q1 = smt2Of(ctx, [...(A.assumptions as never[]), ...(B.assumptions as never[]), d1.t, d2.t, differ] as never[], ["str.len T1", "str.len T2"]); + const r1 = await solveSmt2(q1, []); + assert.equal(r1.status, "sat", "solver must find the re-tag witness (d1 ∧ d2 → flip)"); + + // Negative control: m2 is NOT a durable carrier → m1 keeps its protection + // (or never had one) → bytes stay stable. + const q2 = smt2Of(ctx, [...(A.assumptions as never[]), ...(B.assumptions as never[]), d1.t, d2.t.not(), differ] as never[], []); + const r2 = await solveSmt2(q2, []); + assert.notEqual(r2.status, "sat", "no byte change may occur when the appended message is not a durable carrier"); + + // ---- replay on the REAL code (witness semantics: both carry the marker) ---- + const realCount = (text: string) => Math.ceil(text.length / 4) + 1; + const t1r = MARKER + "\nskill catalog v1 (some body)"; + const t2r = MARKER + "\nskill catalog v2 (some body)"; + const mk = (id: string, text: string): CoreMessage => + ({ id, role: "user", contentType: "text", text }) as unknown as CoreMessage; + + // Turn A: m1 is the newest (only) durable carrier → protected → BLOCKED. + const rA = assignRefs([mk("u1", t1r)], { + existing: emptyRefMap(), nextIndex: 1, + isProtected: () => true, + }); + assert.equal(rA.map.byRaw.u1, BLOCKED_REF, "protected carrier renders verbatim (BLOCKED)"); + const renderA = renderVisibleRefs([mk("u1", t1r)], { messageRefs: rA.map } as never, realCount, "all"); + assert.equal(renderA[0]!.text, t1r, "turn A: no tag (BLOCKED → verbatim)"); + + // Turn B: a NEWER carrier for the same identity arrives → m1's protection + // lifts → BLOCKED deleted → fresh numeric ref → tag prefix mid-history. + const rB = assignRefs([mk("u1", t1r), mk("u2", t2r)], { + existing: rA.map, nextIndex: rA.nextIndex, + isProtected: (m) => m.id === "u2", + }); + assert.notEqual(rB.map.byRaw.u1, BLOCKED_REF, "older carrier lost protection (latest-wins window)"); + const renderB = renderVisibleRefs([mk("u1", t1r), mk("u2", t2r)], { messageRefs: rB.map } as never, realCount, "all"); + const t1After: string = renderB[0]!.text ?? ""; + assert.notEqual(t1After, t1r, "#3805 seam: already-sent message re-tagged mid-history"); + assert.ok(t1After.startsWith(" { + const haveSolver = await (async () => { try { return solveSmt2("(set-logic ALL)(check-sat)", []).status === "sat"; } catch { return false; } })(); + if (!haveSolver) return t.skip("no solver available"); + + const ctx = newCtx(); + const { T1, T2, d1, A, B } = runSymbolic(ctx); + void T2; + // m2 without the marker: m1's protection never lifts. + const same = (A.rendered0 as SymStr).t.eq((B.rendered0 as SymStr).t); + const q = smt2Of(ctx, [...(A.assumptions as never[]), ...(B.assumptions as never[]), d1.t, same] as never[], []); + const r = await solveSmt2(q, []); + assert.equal(r.status, "sat", "stable model exists (d1 ∧ ¬d2 → identical bytes)"); + + // replay: plain user append + const realCount = (text: string) => Math.ceil(text.length / 4) + 1; + const t1r = MARKER + "\nskill catalog v1"; + const t2r = "just a plain user message"; + const mk = (id: string, text: string): CoreMessage => + ({ id, role: "user", contentType: "text", text }) as unknown as CoreMessage; + const rA = assignRefs([mk("u1", t1r)], { existing: emptyRefMap(), nextIndex: 1, isProtected: () => true }); + const rB = assignRefs([mk("u1", t1r), mk("u2", t2r)], { existing: rA.map, nextIndex: rA.nextIndex, isProtected: (m) => m.id === "u1" }); + assert.equal(rB.map.byRaw.u1, BLOCKED_REF, "protection persists for a non-durable append"); + const a = renderVisibleRefs([mk("u1", t1r)], { messageRefs: rA.map } as never, realCount, "all")[0]!.text; + const b = renderVisibleRefs([mk("u1", t1r), mk("u2", t2r)], { messageRefs: rB.map } as never, realCount, "all")[0]!.text; + assert.equal(a, b, "bytes stable"); +}); diff --git a/tests/verif-generic-append-stability.test.ts b/tests/verif-generic-append-stability.test.ts new file mode 100644 index 0000000000..22c36a01b2 --- /dev/null +++ b/tests/verif-generic-append-stability.test.ts @@ -0,0 +1,204 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of } from "../verif/terms.js"; +import type { Seq } from "../verif/terms.js"; +import { solveSmt2 } from "../verif/solve.js"; +import { assignRefs } from "../kernel/src/refs.js"; +import { renderVisibleRefs } from "../kernel/src/render-refs.js"; +import { defaultCountTokens } from "../kernel/src/tokenize.js"; +import type { CoreMessage } from "../kernel/src/types.js"; + +/** + * Generic arbitrary-input discovery (issue #3848 acceptance, directive: find + * bugs under ARBITRARY inputs — not scenario-shaped harnesses). + * + * Input model: EVERYTHING symbolic. Message ids, roles, contentTypes and + * texts are free string constants; protection flags are free booleans per + * turn. Nothing encodes a known bug shape. + * + * Invariant under test (I1, append-only stability): when the host resends + * the same history with one more message appended (turn A → turn B), the + * rendered bytes of every message that was already visible in turn A must + * be IDENTICAL in turn B. + * + * Q1: violation reachable? (expect SAT — at minimum the known #3805 + * durable-ref migration re-emerges from the general model) + * Q2: violation reachable WITHOUT any protection dropping? + * (p_i → q_i for all i). Expect UNSAT: no NEW violation shapes within + * bounds (≤3 messages, text length ≤ 200, unroll bounds). SAT here = + * a candidate NEW bug → must replay on real code. + */ + +const REPO = new URL("..", import.meta.url).pathname; +const REFS = `${REPO}kernel/src/refs.ts`; +const RENDER_REFS = `${REPO}kernel/src/render-refs.ts`; + +type SymVal = Record; + +interface TurnOut { + rendered: SymVal[]; + assumptions: unknown[]; + ctx: ReturnType; + degradations: string[]; +} + +test("generic append-only stability: arbitrary inputs over assignRefs + renderVisibleRefs", { timeout: 300_000 }, async (t) => { + const haveSolver = (() => { try { return solveSmt2("(set-logic ALL)(check-sat)", []).status === "sat"; } catch { return false; } })(); + if (!haveSolver) t.skip("no SMT solver available (set VERIF_PYTHON or VERIF_SOLVER_CMD)"); + + const ctx = newCtx(); + const strv = (v: string) => ({ kind: "str", t: ctx.String.val(v) }) as never; + const numv = (n: number) => ({ kind: "num", t: ctx.Real.val(n) }) as never; + const boolv = (b: boolean) => ({ kind: "bool", t: ctx.Bool.val(b) }) as never; + const objOf = (f: Record) => ({ kind: "obj", fields: new Map(Object.entries(f)) }) as never; + const arrOf = (elems: unknown[]) => ({ kind: "arr", elems, len: ctx.Real.val(elems.length) }) as never; + + // Arbitrary messages: every field a free symbolic constant. + const mkMsg = (tag: string) => objOf({ + id: { kind: "str", t: ctx.String.const(`${tag}_id`) }, + role: { kind: "str", t: ctx.String.const(`${tag}_role`) }, + contentType: { kind: "str", t: ctx.String.const(`${tag}_ct`) }, + text: { kind: "str", t: ctx.String.const(`${tag}_text`) }, + }); + const m1 = mkMsg("m1"), m2 = mkMsg("m2"), m3 = mkMsg("m3"); + // Texts non-empty and reasonably short (solver bound, not a shape). + const textLenBound = (name: string): unknown => { + const x = ctx.String.const(name) as Seq<"main">; + const lenR = ctx.ToReal(x.length() as never) as unknown as { ge(a: unknown): { and(b: unknown): unknown }; le(a: unknown): unknown }; + return lenR.ge(ctx.Real.val(1)).and(lenR.le(ctx.Real.val(200))); + }; + const p1 = ctx.Bool.const("p1") as { implies(x: unknown): { and(b: unknown): unknown } }, p2 = ctx.Bool.const("p2") as { implies(x: unknown): unknown }; + const q1 = ctx.Bool.const("q1"), q2 = ctx.Bool.const("q2"), q3 = ctx.Bool.const("q3"); + void q3; + + const meterPort = (a: unknown) => { + const t = (a as { t: { length(): unknown } }).t; + const r = ctx.ToReal(t.length() as never) as unknown as { div(x: unknown): { add(y: unknown): unknown } }; + return { kind: "num", t: r.div(ctx.Real.val(4)).add(ctx.Real.val(1)) } as never; + }; + const protFor = (table: [SymVal, unknown][]) => (_recv: unknown, m: unknown) => { + // The port receives the MESSAGE object; match by SymVal identity — obj + // values carry no .t, an earlier .t-based compare never matched and + // silently pinned every protection to false (Q1 went UNSAT). + for (const [want, val] of table) if (want === (m as SymVal)) return { kind: "bool", t: val }; + return { kind: "bool", t: ctx.Bool.val(false) }; + }; + + const emptyMap = objOf({ byRaw: objOf({}), byRef: objOf({}) }); + + const runAssign = (msgs: unknown[], existing: unknown, nextIndex: unknown, prot: unknown) => + translateFunction({ + file: REFS, function: "assignRefs", + params: { messages: arrOf(msgs), options: objOf({ existing, nextIndex }) } as never, + ctx, + options: { portImpls: { __method_isProtected: prot as never, countTokens: meterPort } } as never, + }).exec(); + + const runRender = (msgs: unknown[], state: unknown) => + translateFunction({ + file: RENDER_REFS, function: "renderVisibleRefs", + params: { messages: arrOf(msgs), state, countTokens: numv(0), strategy: strv("text-only") } as never, + ctx, + options: { portImpls: { countTokens: meterPort } } as never, + }).exec(); + + // ---- turn A: [m1, m2] ---- + const outA = runAssign([m1, m2], emptyMap, numv(1), protFor([[m1, p1], [m2, p2]])); + assert.deepEqual(outA.degradations, [], `turn-A assignRefs degradations: ${JSON.stringify(outA.degradations)}`); + const mapA = (outA.ret as { fields: Map }).fields.get("map"); + const nextA = (outA.ret as { fields: Map }).fields.get("nextIndex"); + const stateA = objOf({ messageRefs: mapA }); + const rendA = runRender([m1, m2], stateA as never); + assert.deepEqual(rendA.degradations, [], `turn-A render degradations: ${JSON.stringify(rendA.degradations)}`); + const rendAArr = (rendA.ret as { elems: SymVal[] }).elems; + + // ---- turn B: same history + m3 appended ---- + const outB = runAssign([m1, m2, m3], mapA, nextA, protFor([[m1, q1], [m2, q2], [m3, q3]])); + assert.deepEqual(outB.degradations, [], `turn-B assignRefs degradations: ${JSON.stringify(outB.degradations)}`); + const mapB = (outB.ret as { fields: Map }).fields.get("map"); + const stateB = objOf({ messageRefs: mapB }); + const rendB = runRender([m1, m2, m3], stateB as never); + assert.deepEqual(rendB.degradations, [], `turn-B render degradations: ${JSON.stringify(rendB.degradations)}`); + const rendBArr = (rendB.ret as { elems: SymVal[] }).elems; + + // ---- I1 property: rendered[i] turn A == rendered[i] turn B, i ∈ {0,1} ---- + const textOf = (m: SymVal): unknown => (m.fields as Map).get("text"); + const valEqStr = (a: unknown, b: unknown): unknown => (a as { t: { eq(x: unknown): unknown } }).t.eq((b as { t: unknown }).t); + const stable0 = valEqStr(textOf(rendAArr[0]!), textOf(rendBArr[0]!)) as { not(): { or(b: unknown): unknown } }; + const stable1 = valEqStr(textOf(rendAArr[1]!), textOf(rendBArr[1]!)) as { not(): unknown }; + const violation = stable0.not().or(stable1.not()) as unknown; + + const baseAsserts = [ + textLenBound("m1_text"), textLenBound("m2_text"), textLenBound("m3_text"), + ...outA.assumptions, ...rendA.assumptions, ...outB.assumptions, ...rendB.assumptions, + ] as never[]; + + // Q1: can ANY arbitrary input violate append-only stability? + const names = ["p1", "p2", "q1", "q2", "q3"]; + const q1res = solveSmt2(smt2Of(ctx, [...baseAsserts, violation] as never[], names as never), []); + assert.equal(q1res.status, "sat", "Q1 expected SAT — append-only violation must be reachable under arbitrary inputs"); + const w: Record = {}; + (q1res.values ?? []).forEach((v, i) => { w[names[i]!] = v; }); + + // Q2: violation without ANY protection drop (p_i → q_i)? SAT here found a + // NEW shape the scenario harnesses never covered: m3 reusing m1's raw id + // while being unprotected — the per-id BLOCKED binding is shared but + // protection is per-message, so the colliding message steals the binding + // (refs.ts:99 delete + fresh alloc re-tags m1's already-sent bytes). + const noDrop = p1.implies(q1).and(p2.implies(q2)); + const q2res = solveSmt2(smt2Of(ctx, [...baseAsserts, noDrop, violation] as never[], [] as never), []); + assert.equal(q2res.status, "sat", "Q2 expected SAT — the id-collision re-tag must be reachable without protection drops"); + // Q3: with pairwise-distinct ids and no protection drop, NO violation may + // remain within bounds (bounded absence — the known shapes are gone). + const idt = (n: string) => ctx.String.const(n); + const distinct = idt("m1_id").neq(idt("m2_id")).and(idt("m1_id").neq(idt("m3_id"))).and(idt("m2_id").neq(idt("m3_id"))); + const q3res = solveSmt2(smt2Of(ctx, [...baseAsserts, noDrop, distinct, violation] as never[], [] as never), []); + assert.equal(q3res.status, "unsat", "Q3 expected UNSAT — nothing beyond protection-drop and id-collision shapes within bounds"); + + // ---- replay on REAL code with a witness-shaped input ---- + // Interpret witness booleans through the real durable classifier semantics + // (latest marker-carrier wins): p=true ⇒ carries "". + const marker = ""; + const textFor = (tag: string, prot: boolean | undefined) => + (prot ? `${marker} catalog ${tag}` : `plain ${tag} body`); + const bool = (v: unknown): boolean => { + if (v === "true") return true; + if (v === "false") return false; + if (typeof v === "string" && (v === "1" || v === "0")) return v === "1"; + return false; + }; + const p1w = bool(w.p1), p2w = bool(w.p2), q1w = bool(w.q1), q2w = bool(w.q2), q3w = bool(w.q3); + const rmsg = (tag: string, prot: boolean): CoreMessage => ({ + id: `${tag}-real`, role: "user", contentType: "text", text: textFor(tag, prot), + } as CoreMessage); + const turnProt = (table: Record) => (m: { id: string }) => table[m.id] === true; + // Real code, turn A then turn B, threading map/nextIndex like the proxy does. + const fresh = { byRaw: {} as Record, byRef: {} as Record }; + const a1 = assignRefs([rmsg("m1", p1w), rmsg("m2", p2w)], { existing: fresh, nextIndex: 1, isProtected: turnProt({ "m1-real": p1w, "m2-real": p2w }) }); + const ra = renderVisibleRefs([rmsg("m1", p1w), rmsg("m2", p2w)], { messageRefs: a1.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + const a2 = assignRefs([rmsg("m1", q1w), rmsg("m2", q2w), rmsg("m3", q3w)], { existing: a1.map, nextIndex: a1.nextIndex, isProtected: turnProt({ "m1-real": q1w, "m2-real": q2w, "m3-real": q3w }) }); + const rb = renderVisibleRefs([rmsg("m1", q1w), rmsg("m2", q2w), rmsg("m3", q3w)], { messageRefs: a2.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + // The witness must ACTUALLY violate on real code (replay lock) when its + // protection pattern is realizable; protection-drop lifts (#3805 shape) + // are the expected realization. + const realFlip = ra[0]!.text !== rb[0]!.text || ra[1]!.text !== rb[1]!.text; + const lifted = (p1w && !q1w) || (p2w && !q2w); + assert.ok(realFlip === true || lifted === false, + `witness p=${p1w},${p2w} q=${q1w},${q2w},${q3w} replayFlip=${realFlip} lifted=${lifted}`); + // Q2 shape replayed for real: duplicate raw id, protection NOT dropped. + const mkDup = (id: string, text: string, prot: boolean): CoreMessage => + ({ id, role: "user", contentType: "text", text: prot ? `${marker} catalog ${id}` : text } as CoreMessage); + const d1 = mkDup("C", "marker body catalog", true); + const d2 = mkDup("B", "plain tool payload", false); + const d3 = mkDup("C", "A", false); + const dupProt = (m: { id: string; text?: string }) => m.text !== undefined && m.text.includes(marker) && m.id === "C"; + const dupA = assignRefs([d1, d2], { existing: { byRaw: {}, byRef: {} }, nextIndex: 1, isProtected: dupProt as never }); + const dupRA = renderVisibleRefs([d1, d2], { messageRefs: dupA.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + const dupB = assignRefs([d1, d2, d3], { existing: dupA.map, nextIndex: dupA.nextIndex, isProtected: dupProt as never }); + const dupRB = renderVisibleRefs([d1, d2, d3], { messageRefs: dupB.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + assert.ok(dupRA[0]!.text !== dupRB[0]!.text, + "id-collision witness must flip real code: protected m1 verbatim in A, re-tagged in B"); + assert.equal(dupA.map.byRaw.C, "BLOCKED"); + assert.notEqual(dupB.map.byRaw.C, "BLOCKED"); +}); diff --git a/tests/verif-induction-append-stability.test.ts b/tests/verif-induction-append-stability.test.ts new file mode 100644 index 0000000000..8ffebecc42 --- /dev/null +++ b/tests/verif-induction-append-stability.test.ts @@ -0,0 +1,146 @@ +/** + * k-induction judge for append-only send-byte stability (#3848, #3945 follow-up). + * + * Claim (for ARBITRARY message count n): + * If turn k's outbound bytes for messages [1..n] were derived from map M_n, + * and the host only APPENDS message n+1 (prefix byte-identical, same ids), + * then turn k+1's rendered bytes for [1..n] are unchanged — PROVIDED the + * appended message does not REUSE a raw id already present in M_n + * (the #3945 id-collision shape is the one real counterexample). + * + * Why this is induction, not another bounded scan: + * - The prefix is ABSTRACTED to its effect: a symbolic map M with one + * observed binding (id_p -> r_p). No 1..n loop, no per-message unroll — + * formula size is independent of n. + * - Step: run the REAL translated assignRefs([m], M) and prove the observed + * binding is untouched when m.id != id_p. UNSAT = step holds for every M, + * reachable or not (weaker premise than reachability — the abstraction + * characterization is the part we deliberately skip; it cannot weaken the + * conclusion, only ever strengthen it). + * - Render side: renderVisibleRefs renders each message independently + * (messages.map(renderMessage) over the legacy path) and rendered bytes + * are a pure function of (message fields, byRaw binding, countTokens). + * Same message object + same binding + deterministic token meter => same + * bytes. The binding is the only channel through which append can act — + * and the step closes it. + * + * Side condition (documented, not hidden): id collisions. The step formula + * WITHOUT the distinctness constraint must be SAT — that witness is exactly + * issue #3945, replayed here on real code as the induction's honest failure + * case. A real-code randomized spot-check pins the locality argument on the + * implementation itself. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.ts"; +import { newCtx, smt2Of } from "../verif/terms.ts"; +import { solveSmt2 } from "../verif/solve.ts"; +import { assignRefs } from "../kernel/src/refs.ts"; +import { renderVisibleRefs } from "../kernel/src/render-refs.ts"; +import { defaultCountTokens } from "../kernel/src/tokenize.ts"; +import type { SymVal } from "../verif/symbolic.ts"; + +const REFS_FILE = "kernel/src/refs.ts"; + +test("induction step: appending a distinct-id message cannot touch any observed binding (arbitrary n)", { timeout: 240_000 }, () => { + const ctx = newCtx(); + // Abstract prefix state: one observed binding under a FREE key/value. + const idP = ctx.String.const("id_p"); + const rP = ctx.String.const("r_p"); + const binding: SymVal = { kind: "obj", fields: new Map(), symFields: new Map([[idP, { g: ctx.Bool.val(true), v: { kind: "str", t: rP } }]]) } as never; + const byRawA: SymVal = { kind: "obj", fields: new Map(), symFields: new Map([[idP, { g: ctx.Bool.val(true), v: binding }]]) } as never; + // Wait: byRaw is Record; fix shape — entries are STRING refs. + const byRaw: SymVal = { kind: "obj", fields: new Map(), symFields: new Map([[idP, { g: ctx.Bool.val(true), v: { kind: "str", t: rP } }]]) } as never; + void byRawA; void binding; + const mapA: SymVal = { kind: "obj", fields: new Map([ + ["byRaw", byRaw], + ["byRef", { kind: "obj", fields: new Map(), symFields: new Map() } as SymVal], + ]) } as never; + const nextIdx = ctx.Real.const("next_idx"); + // The appended message: fully free shape. + const mId = ctx.String.const("m_id"); + const mk = (role: string, ct: string, text: unknown): SymVal => ({ kind: "obj", fields: new Map([ + ["id", { kind: "str", t: mId }], + ["role", { kind: "str", t: ctx.String.const("m_role_" + role) }], + ["contentType", { kind: "str", t: ctx.String.const("m_ct_" + ct) }], + ["text", text as SymVal], + ]) } as never); + const mText = ctx.String.const("m_text"); + const m1 = mk("r", "c", { kind: "str", t: mText }); + const prot = ctx.Bool.const("m_protected"); + const run = translateFunction({ + file: REFS_FILE, function: "assignRefs", + params: { + messages: { kind: "arr", elems: [m1], len: ctx.Real.val(1) } as never, + options: { kind: "obj", fields: new Map([ + ["existing", mapA], + ["nextIndex", { kind: "num", t: nextIdx }], + ["isProtected", { kind: "undef" } as SymVal], + ]) } as never, + }, + ctx, + options: { + portImpls: { + __method_isProtected: (_recv: SymVal, m: SymVal) => (m === m1 ? { kind: "bool", t: prot } : { kind: "bool", t: ctx.Bool.val(false) }), + }, + }, + }).exec(); + assert.deepEqual(run.degradations, [], "translation must be faithful"); + const mapB = (run.ret as { fields: Map }).fields.get("map") as SymVal; + const byRawB = (mapB as { fields: Map }).fields.get("byRaw") as { symFields?: Map }; + assert.ok(byRawB?.symFields?.size, "map.byRaw must carry symbolic entries"); + // Read like the engine does — a key-match FAN over every symbolic entry + // (looking the entry up by Term identity would ignore entries written + // under a different Term that the solver may bind equal to ours). + type B = { g: unknown; v: SymVal }; + const eqT = (a: unknown, b: unknown) => (a as { eq: (x: unknown) => unknown }).eq(b); + const andT = (a: unknown, b: unknown) => (a as { and: (x: unknown) => unknown }).and(b); + const iteT = (c: unknown, t: unknown, e: unknown) => ((ctx as unknown as { If: (c: unknown, t: unknown, e: unknown) => unknown }).If(c, t, e)); + let refB: unknown = ctx.String.val(""); + for (const [kt, en] of byRawB.symFields!.entries() as Iterable<[unknown, B]>) { + const hit = andT(eqT(idP, kt), en.g); + refB = iteT(hit, (en.v as { t: unknown }).t, refB); + } + refB = refB as { eq: (x: unknown) => { not: () => unknown } }; + // Violation: binding value differs from r_p (and is not our never-value). + const changedReal = (rP as { eq: (x: unknown) => { not: () => unknown } }).eq(refB).not(); + const base = [ + ...run.assumptions as never[], + nextIdx.ge(ctx.Real.val(1)), + ] as never[]; + // Q-step: with distinct ids, NO change to the observed binding is possible. + const step = solveSmt2(smt2Of(ctx, [...base, mId.neq(idP), changedReal] as never[], [] as never), []); + assert.equal(step.status, "unsat", + "induction step must hold: appending a distinct-id message cannot alter the observed binding, for ANY prefix state"); + // Q-collision: WITHOUT distinctness, the change IS reachable — that is #3945. + const col = solveSmt2(smt2Of(ctx, [...base, mId.eq(idP), changedReal] as never[], [] as never), []); + assert.equal(col.status, "sat", + "id-collision shape must break the step — this SAT is issue #3945 talking"); +}); + +test("induction anchor: real code keeps append local (randomized spot-check, 60 runs)", () => { + let seed = 0x2f6e2b1; + const rnd = (n: number) => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed % n; }; + for (let i = 0; i < 60; i++) { + const ids = ["p1", "p2", "p3"].slice(0, 1 + rnd(3)); + const byRaw: Record = {}; + const byRef: Record = {}; + for (const id of ids) { + if (rnd(2)) { byRaw[id] = "BLOCKED"; } + else { const r = "m" + String(1000 + rnd(9000)).padStart(5, "0"); byRaw[id] = r; byRef[r] = id; } + } + const nextIndex = 1 + rnd(50); + const prot = rnd(2) === 1; + const mk = (id: string, text: string) => ({ id, role: "user", contentType: "text", text }) as never; + const prefix = ids.map((id, k) => mk(id, "prefix body " + k)); + const rendA = renderVisibleRefs(prefix, { messageRefs: { byRaw, byRef } } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + const nid = "new_" + i; + const mNew = mk(nid, prot ? "marker " : "appended text " + i); + const out = assignRefs([mNew], { existing: { byRaw, byRef }, nextIndex, isProtected: ((m: { text?: string }) => !!m.text?.includes("")) as never }); + const rendB = renderVisibleRefs([...prefix, mNew], { messageRefs: out.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + for (let k = 0; k < prefix.length; k++) { + assert.equal(rendB[k]!.text, rendA[k]!.text, `run ${i}: prefix message ${k} must render identically after append`); + assert.equal((out.map.byRaw as Record)[ids[k]!], byRaw[ids[k]!] ?? "BLOCKED", `run ${i}: binding for ${ids[k]} must not change`); + } + } +}); diff --git a/tests/verif-subset-guard.test.ts b/tests/verif-subset-guard.test.ts new file mode 100644 index 0000000000..62be031a2b --- /dev/null +++ b/tests/verif-subset-guard.test.ts @@ -0,0 +1,48 @@ +/** + * CI guard: every file in verif/guarded-files.ts must stay inside the + * translatable subset (verif/subset-guard.ts). Violations and stale + * exceptions both fail — there is no silent escape hatch. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { runGuard } from "../verif/subset-guard.js"; +import { resolve } from "node:path"; + +const repoRoot = resolve(import.meta.dirname, ".."); + +test("guarded kernel files stay inside the verifiable subset", () => { + const { violations, stale } = runGuard(repoRoot); + const summary = + violations.length === 0 + ? "" + : "\\n" + + violations + .map((v) => ` ${v.file}:${v.line} [${v.code}] ${v.detail} (fn ${v.fn ?? ""})`) + .join("\\n"); + assert.deepEqual( + violations, + [], + `guarded files contain constructs outside the verifiable subset.\\n` + + `Fix the code, or add a reviewed exception in verif/exceptions.ts:\\n${summary}`, + ); + assert.deepEqual( + stale, + [], + "exceptions/entries that no longer match anything must be deleted " + + "(they are an audited inventory):\\n" + + stale.map((s) => ` ${s.kind}: ${s.label}`).join("\\n"), + ); +}); + +test("guard actually detects a violation (negative control)", async () => { + const { checkGuardedFile } = await import("../verif/subset-guard.js"); + const bad = `import fs from "node:fs";\nasync function f(){ return await fs.readFile("x"); }\n`; + const out = checkGuardedFile({ + file: "kernel/src/refs.ts", + source: bad, + excludedFunctions: new Set(), + }); + const codes = new Set(out.map((v) => v.code)); + assert.ok(codes.has("NODE_BUILTIN_IMPORT"), `expected NODE_BUILTIN_IMPORT, got ${[...codes]}`); + assert.ok(codes.has("ASYNC"), `expected ASYNC, got ${[...codes]}`); +}); diff --git a/tests/verif-symbolic-smoke.test.ts b/tests/verif-symbolic-smoke.test.ts new file mode 100644 index 0000000000..ee625d25a3 --- /dev/null +++ b/tests/verif-symbolic-smoke.test.ts @@ -0,0 +1,101 @@ +/** + * Symbolic smoke test — first end-to-end proof on REAL kernel code. + * + * Target: kernel/src/refs.ts indexToRef bounds behavior (issue #3848 + * acceptance step "首个目标函数出界内无反例结论,且注入已知 bug 形状时必红"). + * + * V1 for all n in [1, 9_999_999]: indexToRef(n) does NOT throw → UNSAT expected + * V2 for n outside that range: indexToRef(n) throws → SAT, witness replayed on real code + * V3 drift control: a deliberately wrong spec bound produces NO + * witness (the solver refuses a false claim). + * + * All solving goes through the out-of-process runner (verif/solve.ts) — + * terms are built with the string-backed facade (verif/terms.ts). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of } from "../verif/terms.js"; +import { solveSmt2, parseNum } from "../verif/solve.js"; +import { replayThrows, parseIntWitness } from "../verif/replay.js"; +import { indexToRef } from "../kernel/src/refs.js"; + +const file = resolve(import.meta.dirname, "../kernel/src/refs.ts"); + +function translateIndexToRef(ctx: import("../verif/terms.js").Context, n: import("../verif/terms.js").Arith) { + const { exec } = translateFunction({ + file, + function: "indexToRef", + params: { index: { kind: "num", t: n } }, + ctx, + }); + const r = exec(); + assert.ok(r.thrown && r.thrown.kind === "bool", "translated indexToRef exposes a boolean throw flag"); + assert.deepEqual(r.degradations.filter((d) => !d.startsWith("template")), [], + `no degradations expected (got: ${r.degradations.join("; ")})`); + return r; +} + +test("symbolic V1: indexToRef never throws in [1, 9_999_999] (real code, solver-checked)", async (t) => { + const solver = await (async () => { try { return await solveSmt2("(set-logic ALL)(check-sat)", []); } catch { return null; } })(); + if (!solver) return t.skip("no SMT solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + const n = ctx.Real.const("n"); + const r = translateIndexToRef(ctx, n); + + const smt2 = smt2Of(r.ctx, [ + n.ge(ctx.Real.val(1)), n.le(ctx.Real.val(9_999_999)), n.eq(ctx.ToInt(n)), + r.thrown!.t, + ]); + const out = await solveSmt2(smt2, []); + assert.equal(out.status, "unsat", "solver must find NO in-range n that throws"); +}); + +test("symbolic V2: indexToRef throws outside [1, 9_999_999] — witness replayed on real code", async (t) => { + const solver = await (async () => { try { return await solveSmt2("(set-logic ALL)(check-sat)", []); } catch { return null; } })(); + if (!solver) return t.skip("no SMT solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + const n = ctx.Real.const("n"); + const r = translateIndexToRef(ctx, n); + + const smt2 = smt2Of(r.ctx, [ + n.ge(ctx.Real.val(-100_000_000)), n.le(ctx.Real.val(100_000_000)), n.eq(ctx.ToInt(n)), + n.lt(ctx.Real.val(1)).or(n.gt(ctx.Real.val(9_999_999))), + r.thrown!.t, + ]); + const out = await solveSmt2(smt2, ["n"]); + assert.equal(out.status, "sat"); + assert.ok(out.values.length >= 1, "witness value for n"); + const witness = parseIntWitness(out.values[0]!); + assert.ok(Number.isFinite(witness), `witness must be an integer, got ${witness}`); + + const reallyThrows = await replayThrows(indexToRef, witness); + assert.ok(reallyThrows, `witness n=${witness} must throw on real indexToRef`); +}); + +test("symbolic V3: the solver refuses a wrong spec — 9_999_999 is in range, no counterexample exists", async (t) => { + const solver = await (async () => { try { return await solveSmt2("(set-logic ALL)(check-sat)", []); } catch { return null; } })(); + if (!solver) return t.skip("no SMT solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + const n = ctx.Real.const("n"); + const r = translateIndexToRef(ctx, n); + + // WRONG spec: claim n=9_999_999 (the real max) throws. A faithful + // translation must produce NO counterexample for a false claim. + const smt2 = smt2Of(r.ctx, [ + n.gt(ctx.Real.val(9_999_998)), n.le(ctx.Real.val(9_999_999)), n.eq(ctx.ToInt(n)), + r.thrown!.t, + ]); + const out = await solveSmt2(smt2, []); + assert.equal(out.status, "unsat", "no witness may exist for a false claim — translation is faithful"); + + // And the drift control still bites: a synthetic phantom witness the REAL + // code does not reproduce must be detected as a mismatch. + const phantom = parseNum("12345.0"); + const reallyThrows = await replayThrows(indexToRef, phantom); + assert.ok(!reallyThrows, "phantom in-range witness must NOT throw on real code (mismatch detectable)"); +}); diff --git a/verif/README.md b/verif/README.md new file mode 100644 index 0000000000..943bb9014e --- /dev/null +++ b/verif/README.md @@ -0,0 +1,86 @@ +# verif — 对 kernel 决策核做"真代码求解器验证"的小模块 + +Issue #3848。目标:把受保护文件里的真实 TypeScript 逻辑翻译成 SMT 约束, +用求解器找违反缓存稳定不变量的反例;反例必须回放真代码复现才算数。 + +这不是"根据代码写证明",也不是"自己建模":被翻译的就是仓库里的源文件, +翻译器只是镜头,镜头歪了会被回放当场抓住(见下)。 + +## 部件 + +| 部件 | 文件 | 作用 | +|---|---|---| +| 子语言守卫 | `subset-guard.ts` + `guarded-files.ts` + `exceptions.ts` | 守卫清单内的文件只许写可翻译的语法;白名单外 → `tests/verif-subset-guard.test.ts` 红。例外必须显式登记,过期例外也红 | +| 符号执行器 | `symbolic.ts` | 真 TS AST → 符号值;分支用 ITE 值合并不枚举路径;本地函数内联(不换算子);跨文件 import 图解析 | +| SMT 项门面 | `terms.ts` | 自研 DAG Term(不走 z3-solver WASM——它 2GB 堆上限扛不住两轮公式);常量折叠;`smt2Of` 发射带 define-fun 共享的 SMT-LIB2 | +| 求解器桥 | `solve.ts` | 进程外求解,优先 cvc5(经 `VERIF_PYTHON` 指定的 python),回退 z3 二进制;可用 `VERIF_SOLVER_CMD` 覆盖 | +| 反例回放 | `replay.ts` | 求解器 witness 必须在真实 kernel 代码上复现;不复现 = 翻译器漂移/规格写错,是警报不是通过 | + +## 运行 + +```bash +# 守卫 + 冒烟不需要求解器 +node --import tsx --test tests/verif-subset-guard.test.ts tests/verif-symbolic-smoke.test.ts + +# 求解器测试需要 cvc5(cvc5 优先,z3 WASM 只够冒烟级公式) +VERIF_PYTHON=/path/to/python-with-cvc5 node --import tsx --test tests/verif-bug*.test.ts tests/verif-generic-*.test.ts tests/verif-induction-*.test.ts +# 没有求解器的环境自动 skip,CI 不会红 +``` + +## 测试布局(每一层证明什么) + +| 测试 | 层 | 证明内容 | +|---|---|---| +| `verif-subset-guard` | 守卫 | 守护文件仍在可翻译子语言内 + 负控制(塞进 async/node:fs 会红) | +| `verif-symbolic-smoke` | 引擎保真 | indexToRef 界内 UNSAT / 界外 witness 回放 / 错误规格被拒(翻译保真) | +| `verif-bug3691-oscillation` | 定向不变量 | 紧急截断在门限处可触发(SAT+witness)且门限下不可触发(UNSAT);两轮回放展示同一重发历史出站字节翻转 | +| `verif-bug3769-orphan-window` | 定向不变量 | 只增不删的历史里"可见的必须保持可见"被孤儿窗口破坏;窗口调大后稳定 | +| `verif-bug3676-wire-reorder` | 定向不变量 | openai wire 重建 [file,text]→[text,file] 部件重排(SAT + 真代码回放) | +| `verif-bug3805-durable-ref-migration` | 定向不变量 | 保护解除时 BLOCKED→数字引用迁移给已发送字节中途加标签;非 durable 追加稳定(UNSAT) | +| `verif-generic-append-stability` | **任意输入** | 消息全符号化、唯一不变量"只增不删的字节稳定":Q1 SAT(#3805 形状泛化重现)、Q2 SAT(**求解器自己发现的新 bug #3945**:id 撞车重打标,零保护解除)、Q3 UNSAT(排除两个已知形状后界内无第三种) | +| `verif-induction-append-stability` | **归纳判官** | 任意消息数的 append-only 稳定性:前缀抽象成一个符号绑定,归纳步 UNSAT(id 不重复时任何前缀状态都动不了);撤掉约束 SAT = #3945;真代码随机抽查 60 轮 | + +## 无作弊规则(写新 harness 前必读) + +1. **只调真入口函数**:harness 通过 `translateFunction` 编译仓库里的真实 + 文件,不复制逻辑、不复述算法。 +2. **不变量必须是通用性质**(幂等/单调/append-only 稳定…),绝不在 bug + 行号附近写断言。允许手工挑选的只有"场景脚本"(输入的形状),因为宿主 + 行为是开放集。 +3. **反例必须回放真代码**:witness 代入真实函数,行为不符 = 引擎 bug, + 修引擎,绝不放宽断言。 +4. **port 是公开接缝**:token 计数器、isProtected 这类真实可注入参数用 + `portImpls` 提供符号实现;但本地函数名和 import 的真实函数**永远优先于 + port**,防止用 port 偷换真实逻辑。 +5. **负控制必须有**:每个 SAT 证明配一个"已知修好/排除后 UNSAT"的对照, + 证明判官不是永真式。 + +## 建模决定(诚实清单) + +- 数字 = Real(有理数);整数性用 `IsInt` 约束,不静默取整。 +- 求解不可判定的角落**降级并登记**,不假装:降级项会出现在 + `exec().degradations`,harness 断言空列表才能立案。 +- 动态正则(ownTagRe)按"锚定前缀"过近似:前缀不匹配 → 精确 false; + 匹配 → 未解释函数 + 降级标记(仅可达时)。 +- `String(n)` / `toFixed` 等确定性渲染 = 未解释函数(同名同参同值), + 保证跨轮稳定;witness 回放仍然落在真代码上。 +- 循环展开 K(默认 8)、数组槽位 N(默认 8):**有界模型检测**。要"任意 + 消息数"不用加 N —— 用归纳判官(`verif-induction-*`)。 +- 模块级常量从源文件真实收集,不手抄。 + +## 已发现的 bug(求解器自动 + 回放确认) + +- #3691 紧急截断震荡(harness 定向找到;回放顺带发现截断结果比原文长 → #3935) +- #3769 失败压缩轮换的孤儿窗口 +- #3676 openai wire 重建重排 +- #3805 durable 引用迁移重打标 +- **#3945 重复 id 重打标 —— 通用输入 harness 在任意输入下自己发现的新形状** + +## 扩展 + +- 加守护文件:改 `guarded-files.ts`,先确认翻译器支持其语法(守卫会告诉你 + 缺什么),例外登记 `exceptions.ts`。 +- 加不变量:新 harness 文件,遵守上面的无作弊规则;SAT → 回放 → 真 bug + 开 issue + 回归钉;UNSAT → 界内证无,写清楚界。 +- 从有界到无界:归纳判官模板 = 前缀抽象 + 单步 UNSAT + 已知例外 SAT + + 真代码随机锚点。 diff --git a/verif/exceptions.ts b/verif/exceptions.ts new file mode 100644 index 0000000000..34c13ed91e --- /dev/null +++ b/verif/exceptions.ts @@ -0,0 +1,81 @@ +/** + * Explicit exceptions to the verif subset rules. + * + * Every entry must carry a human reason. Entries that no longer match + * anything in the guarded files are reported as stale by the guard test + * and must be deleted — this list is an audited inventory, not a dumping + * ground. + */ + +/** A whole function scoped out of symbolic verification. Calls to it are + * treated as uninterpreted; the function body is not checked. Use only for + * code that is provably off every verified path (e.g. optional ports). */ +interface ExcludedFunction { + file: string; + fn: string; + reason: string; +} + +export interface SubsetException { + file: string; + code: string; + /** Restrict the exception to one function (by local name). */ + fn?: string; + /** Restrict to violations whose detail matches exactly. */ + detail?: string; + reason: string; +} + +export const EXCLUDED_FUNCTIONS: readonly ExcludedFunction[] = [ + { + file: "kernel/src/tokenize.ts", + fn: "createBpeTokenizer", + reason: + "Optional BPE tokenizer probe: node require + returned closures. Never on a verified path — CompressionCore takes countTokens as an injectable port and the verified entry points use defaultCountTokens (pure arithmetic).", + }, +]; + +export const EXCEPTIONS: readonly SubsetException[] = [ + { + file: "kernel/src/tokenize.ts", + code: "NODE_BUILTIN_IMPORT", + reason: + "createRequire from node:module at module top feeds only createBpeTokenizer (an excluded function); the verified path (defaultCountTokens) is pure.", + }, + { + file: "kernel/src/compress.ts", + code: "ENV_ACCESS", + fn: "applySingleRange", + reason: + "createdAt: Date.now() — wall-clock stamp on new blocks; orthogonal to every verified invariant, modeled as an unconstrained symbolic integer.", + }, + { + file: "kernel/src/compress.ts", + code: "ENV_ACCESS", + fn: "processTurn", + detail: "console.warn", + reason: + "Log-only console.warn on a degraded path; modeled as a no-op observation point — never influences verified state.", + }, + { + file: "kernel/src/render-refs.ts", + code: "DYNAMIC_REGEX", + reason: + "ownTagRe = new RegExp(escapeRegex(TAG_OPEN)+...+escapeRegex(ref)+...) — pattern is a concatenation of literal fragments with the message ref escaped in; representable as a symbolic regex concatenation once the translator grows dynamic-pattern support. Tracked until then.", + }, + { + file: "kernel/src/tokenize.ts", + code: "UNKNOWN_GLOBAL", + detail: "createRequire", + reason: + "Module-scope const require = createRequire(import.meta.url); only createBpeTokenizer (excluded function) uses it. Verified paths never call it.", + }, + { + file: "kernel/src/wire/responses.ts", + code: "ENV_ACCESS", + fn: "shouldDropAllReasoning", + detail: "read process.env", + reason: + "process.env.ACP_REASONING_KEEP read — process-wide config constant, modeled as a fixed symbolic boolean, never mutated inside the guarded scope.", + }, +]; diff --git a/verif/guarded-files.ts b/verif/guarded-files.ts new file mode 100644 index 0000000000..7d4830201c --- /dev/null +++ b/verif/guarded-files.ts @@ -0,0 +1,23 @@ +/** + * Guarded file list for the symbolic-verification subset ("verif-subset"). + * + * Everything listed here must stay inside the translatable sublanguage + * (see verif/subset-guard.ts). Anything the symbolic executor cannot + * compile needs an explicit, reviewed exception in verif/exceptions.ts — + * there is no silent escape hatch. + * + * Grow this list only together with translator coverage, never ahead of it. + */ +export const GUARDED_FILES: readonly string[] = [ + // Tier A — current translator targets + "kernel/src/compress.ts", + "kernel/src/hide-consumed.ts", + "kernel/src/refs.ts", + "kernel/src/render-refs.ts", + "kernel/src/truncate.ts", + "kernel/src/truncate-tools.ts", + "kernel/src/tokenize.ts", + "kernel/src/rebuild.ts", + "kernel/src/wire/openai.ts", + "kernel/src/wire/responses.ts", +]; diff --git a/verif/replay.ts b/verif/replay.ts new file mode 100644 index 0000000000..b33d32b46c --- /dev/null +++ b/verif/replay.ts @@ -0,0 +1,34 @@ +/** + * Counterexample replay — the anti-drift lock of the verification pipeline. + * + * A solver witness is only a FINDING if the real compiled kernel code + * reproduces it. If the translated model and the real code disagree, the + * replay fails and the translator is wrong — that failure is the signal, + * never "verification passed anyway". + */ + +/** Parse z3's s-expression numeric output: "5", "(- 5808901)", "96.0", + * "10000000.0", "(/ 1 3)". Reals that are not integral return NaN — the + * caller must constrain the witness variable to integers. */ +export function parseIntWitness(raw: string): number { + const t = raw.trim(); + if (/^-?\d+$/.test(t)) return Number(t); + // Real numerals print with a decimal tail ("96.0", "-3.0") + const real = /^(-?\d+)\.0+$/.exec(t); + if (real) return Number(real[1]); + const m = /^\(\s*(-|\+)\s+(\d+)\)$/.exec(t); + if (m) return m[1] === "-" ? -Number(m[2]) : Number(m[2]); + return Number.NaN; +} + +export async function replayThrows( + fn: (x: number) => unknown, + arg: number, +): Promise { + try { + fn(arg); + return false; + } catch { + return true; + } +} diff --git a/verif/solve.ts b/verif/solve.ts new file mode 100644 index 0000000000..3cd9c34eae --- /dev/null +++ b/verif/solve.ts @@ -0,0 +1,144 @@ +// Out-of-process SMT solving. The in-process z3-solver npm package is WASM +// (2GB heap cap) and OOMs on two-turn cache-stability formulas; native z3 +// spins on the seq/LRA mix. cvc5 solves the same formulas instantly, so the +// runner prefers it when available. Override with VERIF_SOLVER_CMD (a command +// prefix that receives the smt2 file path as its last argument and must print +// `sat`/`unsat` plus `(get-value ...)` s-expr replies on stdout). +import { spawnSync } from "node:child_process"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +interface SolveResult { + status: "sat" | "unsat" | "unknown"; + /** Parsed `(get-value ...)` replies, in declaration order: raw strings. */ + values: string[]; + raw: string; +} + +/** Parse the printed form of a get-value reply: `((name value) (name value))` + * — one outer list of pairs; handles nested parens in name and value. */ +const parseValues = (raw: string): string[] => { + const out: string[] = []; + const start = raw.indexOf("(("); + if (start < 0) return out; + const s = raw.slice(start + 1); // s starts at the first PAIR's `(` + let i = 0; + const n = s.length; + const readSexp = (): string => { + while (i < n && /\s/.test(s[i]!)) i++; + if (s[i] === "\"") { // quoted SMT string: until the real closing quote ("" is an escaped quote) + const b = i; i++; + while (i < n) { if (s[i] === "\"") { if (s[i + 1] === "\"") { i += 2; continue; } i++; break; } i++; } + return s.slice(b, i); + } + if (s[i] !== "(") { const j = i; while (i < n && !/[\s)]/.test(s[i]!)) i++; return s.slice(j, i); } + let depth = 0; const b = i; + while (i < n) { + if (s[i] === "(") depth++; + else if (s[i] === ")") { depth--; if (depth === 0) { i++; return s.slice(b, i); } } + i++; + } + return s.slice(b); + }; + while (i < n) { + while (i < n && /\s/.test(s[i]!)) i++; + if (s[i] !== "(") break; // end of the pair list + i++; // past the pair's opening paren + const name = readSexp(); void name; + const value = readSexp(); + out.push(value); + while (i < n && /\s/.test(s[i]!)) i++; + if (s[i] === ")") i++; // pair close — keep going; next pair or list end follows + } + return out; +}; + +export const solveSmt2 = (input: string | { smt2: string }, getValue: string[]): SolveResult => { + const smt2 = typeof input === "string" ? input : input.smt2; + const dir = mkdtempSync(join(tmpdir(), "verif-solve-")); + const file = join(dir, "query.smt2"); + try { + const header = smt2.startsWith("(set-logic") ? "" : "(set-logic ALL)\n"; + const tail = getValue.length + ? `(check-sat)\n(get-value (${getValue.join(" ")}))\n` + : "(check-sat)\n"; + writeFileSync(file, header + smt2.replace(/\(check-sat\)[\s\S]*$/, "") + tail); + + // Try, in order: explicit override, cvc5 via python, z3 binaries. + const attempts: { cmd: string; args: string[]; parse: (raw: string) => SolveResult }[] = []; + if (process.env.VERIF_SOLVER_CMD) { + const parts = process.env.VERIF_SOLVER_CMD.split(/\s+/); + attempts.push({ + cmd: parts[0]!, + args: [...parts.slice(1), file], + parse: (raw) => ({ status: /(^|\n)sat/.test(raw) && !/unsat/.test(raw) ? "sat" : /unsat/.test(raw) ? "unsat" : "unknown", values: parseValues(raw), raw }), + }); + } + const py = process.env.VERIF_PYTHON ?? "python3"; + const cvc5Py = `${process.env.CVC5_HOME ?? process.env.HOME + "/tmp/cvc5"}`; + const script = [ + "import sys", + "import cvc5", + "tm = cvc5.TermManager(); s = cvc5.Solver(tm)", + "s.setOption('produce-models','true')", + "sm = cvc5.SymbolManager(tm)", + "p = cvc5.InputParser(s, sm)", + `p.setFileInput(cvc5.InputLanguage.SMT_LIB_2_6, ${JSON.stringify(file)})`, + "while True:", + " cmd = p.nextCommand()", + " if cmd.isNull(): break", + " out = cmd.invoke(s, sm)", + " if out: print(out.strip())", + ].join("\n"); + attempts.push({ + cmd: py, + args: ["-c", script], + parse: (raw) => ({ status: /(^|\n)sat\b/.test(raw) && !/unsat/.test(raw) ? "sat" : /unsat/.test(raw) ? "unsat" : "unknown", values: parseValues(raw), raw }), + // note: PYTHONPATH handled via env below + }); + const z3Bin = process.env.Z3_BIN ?? "z3"; + attempts.push({ + cmd: z3Bin, + args: [file], + parse: (raw) => ({ status: /(^|\n)sat\b/.test(raw) ? "sat" : /unsat/.test(raw) ? "unsat" : "unknown", values: parseValues(raw), raw }), + }); + + let lastErr = ""; + for (const a of attempts) { + const r = spawnSync(a.cmd, a.args, { + encoding: "utf8", + timeout: 120_000, + maxBuffer: 64 * 1024 * 1024, + env: a.cmd === py || a.cmd.endsWith("python3") || a.cmd.includes("python") + ? { ...process.env, PYTHONPATH: `${cvc5Py}:${process.env.PYTHONPATH ?? ""}` } + : process.env, + }); + if (r.error) { lastErr += `${a.cmd}: ${r.error.message}\n`; continue; } + // A verdict printed to stdout wins over the exit code: native z3 + // prints `unsat` and THEN exits 1 because the trailing (get-value) + // fails with "model is not available" — discarding stdout there would + // turn every unsat into "no solver available". + const parsed = a.parse(r.stdout ?? ""); + if (parsed.status !== "unknown") return parsed; + lastErr += r.status !== 0 + ? `${a.cmd}: exit ${r.status}: ${r.stderr.slice(0, 500)}\n` + : `${a.cmd}: unknown\n`; + } + throw new Error(`no solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON):\n${lastErr}`); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}; + +/** Parse a model value like `4001`, `4001.0`, `(- 3)` into a JS number. */ +export const parseNum = (v: string): number => { + const t = v.trim(); + if (t.startsWith("(-")) { + const inner = t.replace(/^\(-\s*/, "").replace(/\)$/, "").trim(); + return -parseNum(inner); + } + const n = Number(t.replace(/\.0+$/, "")); + if (!Number.isFinite(n)) throw new Error(`unparseable model value: ${v}`); + return n; +}; diff --git a/verif/subset-guard.ts b/verif/subset-guard.ts new file mode 100644 index 0000000000..106c643bf3 --- /dev/null +++ b/verif/subset-guard.ts @@ -0,0 +1,584 @@ +/** + * verif subset guard — the sublanguage enforcement engine. + * + * Purpose: the files in verif/guarded-files.ts must stay inside the subset + * that verif/symbolic.ts can translate to SMT. This module walks the real + * TypeScript AST and reports every construct outside the subset. Anything + * reported must either be fixed or be covered by an explicit, reviewed + * exception in verif/exceptions.ts — there is no third option. + * + * Design rules: + * - The engine is driven by ALLOWLISTS, not denylists: an unknown builtin, + * method, or import is a violation until it is taught to the translator + * and added here. That is what keeps "cannot be verified" code out. + * - Matching an exception requires file + code (+ function); unused + * exceptions are reported so stale entries get deleted. + */ +import ts from "typescript"; + +type ViolationCode = + | "ASYNC" + | "GENERATOR" + | "CLASS_DECL" + | "INHERITANCE" + | "ACCESSOR" + | "DECORATOR" + | "TAGGED_TEMPLATE" + | "BIGINT" + | "DYNAMIC_CODE" + | "REFLECTION" + | "ENV_ACCESS" + | "NODE_BUILTIN_IMPORT" + | "EXTERNAL_IMPORT" + | "LABEL" + | "WITH_STMT" + | "DELETE_EXPR" + | "VOID_EXPR" + | "FOR_IN" + | "FREE_CLOSURE" + | "UNKNOWN_GLOBAL" + | "UNKNOWN_METHOD" + | "DYNAMIC_REGEX" + | "REGEX_STATEFUL" + | "EXCLUDED_FUNCTION"; + +interface Violation { + file: string; + line: number; + code: ViolationCode; + detail: string; + fn: string | undefined; +} + +interface GuardResult { + violations: Violation[]; + /** Exceptions/entries that matched nothing — must be deleted. */ + stale: Array<{ kind: "exception" | "excluded"; label: string }>; +} + +/* ---------------------------------------------------------------- allowlists */ + +const MATH_FNS = new Set(["abs", "min", "max", "round", "floor", "ceil", "trunc", "sign", "pow", "sqrt"]); + +const JSON_FNS = new Set(["parse", "stringify"]); + +const OBJECT_FNS = new Set(["keys", "values", "entries", "hasOwn"]); + +const ARRAY_FNS = new Set(["isArray", "from", "of"]); + +const NUMBER_FNS = new Set(["isFinite", "isNaN", "isInteger", "isSafeInteger", "parseInt", "parseFloat"]); + +const STRING_CTOR_FNS = new Set(["fromCharCode", "fromCodePoint"]); + +const NAMESPACE_MEMBERS: Record> = { + Math: MATH_FNS, + JSON: JSON_FNS, + Object: OBJECT_FNS, + Array: ARRAY_FNS, + Number: NUMBER_FNS, + String: STRING_CTOR_FNS, +}; + +/** Higher-order array methods whose callback arguments are the only legal + * closure shape. The translator inlines these combinators. */ +const HOF_ARRAY_METHODS = new Set([ + "map", "filter", "find", "findIndex", "findLast", "findLastIndex", + "every", "some", "reduce", "reduceRight", "flatMap", "forEach", "sort", +]); + +const PLAIN_ARRAY_METHODS = new Set([ + "includes", "indexOf", "lastIndexOf", "slice", "concat", "join", + "push", "pop", "shift", "unshift", "splice", "at", "flat", "reverse", + "entries", "keys", "values", +]); + +const STRING_METHODS = new Set([ + "slice", "substring", "indexOf", "lastIndexOf", "includes", "startsWith", + "endsWith", "replace", "replaceAll", "split", "trim", "trimStart", + "trimEnd", "toLowerCase", "toUpperCase", "normalize", "padStart", + "padEnd", "repeat", "concat", "charAt", "charCodeAt", "codePointAt", + "at", "match", "search", +]); + +const MAP_SET_METHODS = new Set([ + "get", "set", "has", "delete", "clear", "add", "keys", "values", "entries", "forEach", +]); + +/** Methods callable on regex literals (stateless flags only; the stateful + * exec/test-with-g combination is rejected separately). */ +const REGEX_METHODS = new Set(["test", "exec"]); + +/** Function-typed fields on options/state objects (injected ports). Calls to + * these are modeled as uninterpreted functions — they are extension seams, + * not logic the verifier reasons about. */ +const PORT_METHODS = new Set(["countTokens", "shouldSkip", "isProtected"]); + +/** Deterministic number formatting. */ +const NUMBER_INSTANCE_METHODS = new Set(["toFixed"]); + +/** Methods on instances/records produced by kernel-internal code. */ +const KERNEL_INSTANCE_METHODS = new Set([ + "next", "namespaceFor", + // createCore() entry-point record: + "applyCompression", "processTurn", +]); + +/** new X is allowed only for these. */ +const NEWABLE = new Set(["Map", "Set", "Error", "RangeError", "TypeError", "RegExp"]); +/** Kernel classes (imported from other guarded/Tier-B modules) that may be + * constructed; their instance methods are registered below. */ +const IMPORTED_NEWABLE = new Set(["ClusterCounter"]); + +/** Bare callables allowed as plain functions. */ +const GLOBAL_FNS = new Set([ + "isNaN", "isFinite", "parseFloat", "parseInt", + "String", "Number", "Boolean", "Object", "Array", + // function-typed parameters (injected ports, uninterpreted): + "countTokens", "predicate", "isCompressTool", +]); + +/** Namespace roots (Math.x etc.) handled via NAMESPACE_MEMBERS. */ +const NAMESPACE_ROOTS = new Set(Object.keys(NAMESPACE_MEMBERS)); + +/** Namespace roots also usable as bare coercion calls: String(x), Number(x)… */ +const COERCION_ROOTS = new Set(["String", "Number", "Boolean", "Object", "Array"]); + +/** Reads that reach the environment / nondeterminism. */ +const ENV_ROOTS = new Set([ + "Date", "Math.random", "crypto", "performance", "process", "console", + "fetch", "setTimeout", "setInterval", "setImmediate", "queueMicrotask", + "require", "Buffer", "globalThis", "global", "WebAssembly", +]); + +const REFLECTION_ROOTS = new Set(["Proxy", "Reflect", "WeakMap", "WeakSet", "Symbol", "Function", "eval"]); + +const STATEFUL_REGEX_FLAGS = new Set(["g", "y"]); + +const ALL_VALUE_METHODS = new Set([ + ...HOF_ARRAY_METHODS, ...PLAIN_ARRAY_METHODS, ...STRING_METHODS, + ...MAP_SET_METHODS, ...REGEX_METHODS, ...PORT_METHODS, + ...NUMBER_INSTANCE_METHODS, ...KERNEL_INSTANCE_METHODS, +]); + +/* ------------------------------------------------------------------ helpers */ + +function unwrap(node: ts.Node): ts.Node { + let cur = node; + while (ts.isParenthesizedExpression(cur) || ts.isAsExpression(cur) || + ts.isNonNullExpression(cur) || ts.isSatisfiesExpression(cur)) { + cur = cur.expression; + } + return cur; +} + +function dottedName(expr: ts.Expression): string | undefined { + let node = unwrap(expr); + const parts: string[] = []; + while (ts.isPropertyAccessExpression(node)) { + parts.unshift(node.name.text); + node = unwrap(node.expression); + } + if (ts.isIdentifier(node)) { + parts.unshift(node.text); + return parts.join("."); + } + return undefined; +} + +function nearestFunctionName(node: ts.Node): string | undefined { + // Walk up to the nearest enclosing function; report its binding name. + // Module-level constructs report undefined (no enclosing function). + let cur: ts.Node | undefined = node.parent; + while (cur && !ts.isSourceFile(cur)) { + if (ts.isFunctionDeclaration(cur) && cur.name) return cur.name.text; + if (ts.isMethodDeclaration(cur) && cur.name && ts.isIdentifier(cur.name)) return cur.name.text; + if (ts.isFunctionExpression(cur) || ts.isArrowFunction(cur)) { + const parent = cur.parent; + if (ts.isVariableDeclaration(parent) && ts.isIdentifier(parent.name)) return parent.name.text; + if (ts.isPropertyAssignment(parent) && ts.isIdentifier(parent.name)) return parent.name.text; + if (ts.isFunctionDeclaration(parent)) return parent.name?.text; // IIFE-ish, rare + } + cur = cur.parent; + } + return undefined; +} + +/* ------------------------------------------------------------------- engine */ + +interface CheckInput { + /** Repo-relative path, e.g. kernel/src/refs.ts */ + file: string; + source: string; + excludedFunctions: ReadonlySet; +} + +export function checkGuardedFile({ file, source, excludedFunctions }: CheckInput): Violation[] { + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.ES2022, true); + const violations: Violation[] = []; + const add = (node: ts.Node, code: ViolationCode, detail: string) => { + violations.push({ + file, + line: sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1, + code, + detail, + fn: nearestFunctionName(node), + }); + }; + + const kernelImports = new Set(); + const localFunctions = new Set(); + + // Pre-pass: collect function declarations and named const functions/calls + // (calls may appear textually before the declaration — hoisting). + const moduleConsts = new Set(); + const prepass = (node: ts.Node, top: boolean): void => { + if (ts.isFunctionDeclaration(node) && node.name) localFunctions.add(node.name.text); + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name)) { + const init = node.initializer; + if (init && (ts.isArrowFunction(init) || ts.isFunctionExpression(init))) { + localFunctions.add(node.name.text); + } else if (top) { + moduleConsts.add(node.name.text); + } + } + ts.forEachChild(node, (c) => prepass(c, false)); + }; + prepass(sf, true); + + /** Resolve a call's callee into either a dotted path (root is an + * identifier) or the trailing method name on an arbitrary receiver. */ + const resolveCallee = ( + callee: ts.Expression, + ): { dotted?: string; methodName?: string } => { + const dotted = dottedName(callee); + if (dotted !== undefined) return { dotted }; + const c = unwrap(callee); + if (ts.isPropertyAccessExpression(c)) { + return { methodName: c.name.text }; + } + return {}; + }; + + const checkCall = (node: ts.CallExpression): void => { + const { dotted, methodName } = resolveCallee(node.expression); + + if (dotted !== undefined) { + const segs = dotted.split("."); + const root = segs[0]; + const last = segs[segs.length - 1]; + + if (REFLECTION_ROOTS.has(root) || REFLECTION_ROOTS.has(dotted)) { + add(node, "REFLECTION", dotted); + return; + } + if (ENV_ROOTS.has(dotted) || ENV_ROOTS.has(root)) { + add(node, "ENV_ACCESS", dotted); + return; + } + if (NAMESPACE_ROOTS.has(root)) { + if (segs.length === 1 && COERCION_ROOTS.has(root)) return; // String(x)/Number(x)/... + // Namespace calls must be exactly Root.member with member allowed. + const allowed = NAMESPACE_MEMBERS[root]; + if (segs.length !== 2 || !allowed.has(segs[1])) { + add(node, "UNKNOWN_METHOD", dotted); + } + return; + } + if (segs.length === 1) { + // Bare call of a local/imported/whitelisted global function. + if (!kernelImports.has(root) && !localFunctions.has(root) && !GLOBAL_FNS.has(root)) { + add(node, "UNKNOWN_GLOBAL", dotted); + } + return; + } + // Method call on a local/imported/module-const value: check the + // trailing method against the registered method families. + if (!ALL_VALUE_METHODS.has(last)) { + add(node, "UNKNOWN_METHOD", `.${last} on local value (${dotted})`); + return; + } + return; + } + + if (methodName !== undefined) { + // Receiver is a computed expression (call result, literal, ...). + if (!ALL_VALUE_METHODS.has(methodName)) { + add(node, "UNKNOWN_METHOD", `.${methodName} on computed receiver`); + } + return; + } + + add(node, "UNKNOWN_GLOBAL", `indirect call (${node.expression.getText(sf)})`); + }; + + /** Legal closure = argument of a whitelisted HOF / kernel-internal call, + * or part of a record returned from a local function (entry-point table). */ + const isLegalCallback = (fn: ts.ArrowFunction | ts.FunctionExpression): boolean => { + // Default value in a parameter list: pure deterministic fallback. + if (ts.isParameter(fn.parent)) return true; + // Right side of a ?? fallback: const x = provided ?? (() => default). + { + let holder: ts.Node = fn; + while (ts.isParenthesizedExpression(holder.parent)) holder = holder.parent; + const bp = holder.parent; + if ( + ts.isBinaryExpression(bp) && + bp.operatorToken.kind === ts.SyntaxKind.QuestionQuestionToken && + bp.right === holder + ) { + return true; + } + } + let cur: ts.Node | undefined = fn.parent; + while ( + cur && + !ts.isCallExpression(cur) && + !ts.isStatement(cur) && + !ts.isVariableDeclaration(cur) && + !ts.isBinaryExpression(cur) + ) { + cur = cur.parent; + } + if (cur && ts.isCallExpression(cur)) { + const { dotted, methodName } = resolveCallee(cur.expression); + const m = methodName ?? (dotted !== undefined && dotted.includes(".") ? dotted.split(".").pop() : undefined); + if (m !== undefined && (HOF_ARRAY_METHODS.has(m) || MAP_SET_METHODS.has(m))) return true; + const root = dotted?.split(".")[0]; + if (root !== undefined && (kernelImports.has(root) || localFunctions.has(root))) return true; + return false; + } + if (cur && ts.isReturnStatement(cur)) { + // Arrow inside a record returned by a local function — dispatch table. + return true; + } + if (cur && ts.isVariableDeclaration(cur)) { + // Named local helper (const f = (x) => ...). Statically call-graphable; + // every use is checked as a call/argument elsewhere in this file. + return true; + } + return false; + }; + + const visit = (node: ts.Node): void => { + /* ---- exclusions: skip whole excluded function bodies ---- */ + if ( + ts.isFunctionDeclaration(node) && + node.name && + excludedFunctions.has(`${file}#${node.name.text}`) + ) { + add(node, "EXCLUDED_FUNCTION", `function ${node.name.text} is excluded from the verified subset`); + violations[violations.length - 1].fn = node.name.text; + return; + } + + switch (node.kind) { + case ts.SyntaxKind.AwaitExpression: + add(node, "ASYNC", "await — the verified subset is fully synchronous"); + return; + case ts.SyntaxKind.YieldExpression: + add(node, "GENERATOR", "yield"); + return; + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.ClassExpression: + add(node, "CLASS_DECL", "class declarations are outside the subset"); + return; + case ts.SyntaxKind.HeritageClause: + add(node, "INHERITANCE", "extends/implements"); + return; + case ts.SyntaxKind.GetAccessor: + case ts.SyntaxKind.SetAccessor: + add(node, "ACCESSOR", "get/set accessor"); + return; + case ts.SyntaxKind.Decorator: + add(node, "DECORATOR", "decorator"); + return; + case ts.SyntaxKind.TaggedTemplateExpression: + add(node, "TAGGED_TEMPLATE", "tagged template"); + return; + case ts.SyntaxKind.BigIntLiteral: + add(node, "BIGINT", "bigint literal"); + return; + case ts.SyntaxKind.LabeledStatement: + add(node, "LABEL", "labeled statement"); + return; + case ts.SyntaxKind.WithStatement: + add(node, "WITH_STMT", "with statement"); + return; + case ts.SyntaxKind.DeleteExpression: { + const operand = unwrap((node as ts.DeleteExpression).expression); + if (ts.isIdentifier(operand)) { + add(node, "DELETE_EXPR", "delete of a binding"); + return; + } + break; // delete obj.k / delete obj[k]: bounded-map removal, allowed + } + case ts.SyntaxKind.VoidExpression: + add(node, "VOID_EXPR", "void expression"); + return; + case ts.SyntaxKind.ForInStatement: + add(node, "FOR_IN", "for-in enumeration"); + return; + default: + break; + } + + if (ts.isImportDeclaration(node)) { + const spec = node.moduleSpecifier.getText(sf).replace(/^['"]|['"]$/g, ""); + if (spec.startsWith("node:")) { + add(node, "NODE_BUILTIN_IMPORT", `import from "${spec}"`); + } else if (!spec.startsWith(".")) { + add(node, "EXTERNAL_IMPORT", `import from "${spec}"`); + } else if (node.importClause?.namedBindings && ts.isNamedImports(node.importClause.namedBindings)) { + for (const el of node.importClause.namedBindings.elements) { + kernelImports.add((el.propertyName ?? el.name).text); + } + } + return; + } + + if (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword) { + add(node, "DYNAMIC_CODE", "dynamic import()"); + return; + } + + if (ts.isNewExpression(node)) { + const name = dottedName(node.expression); + const root = name?.split(".")[0]; + const okNew = + name !== undefined && + root !== undefined && + (NEWABLE.has(root) || (IMPORTED_NEWABLE.has(root) && kernelImports.has(root))); + if (!okNew) { + add(node, "UNKNOWN_GLOBAL", `new ${name ?? ""}`); + return; + } + if (name === "RegExp") { + const first = node.arguments?.[0]; + const literal = first && (ts.isStringLiteral(first) || ts.isNoSubstitutionTemplateLiteral(first)); + if (!literal) { + add(node, "DYNAMIC_REGEX", "new RegExp with non-literal pattern"); + return; + } + const flags = node.arguments?.[1]; + if (flags && ts.isStringLiteral(flags)) { + for (const ch of flags.text) { + if (STATEFUL_REGEX_FLAGS.has(ch)) { + add(node, "REGEX_STATEFUL", `RegExp flags "${flags.text}" carry lastIndex state`); + return; + } + } + } + } + } + + /* Stateful regex use: /re/g.exec(...) or /re/g.test(...) on the SAME + * literal — string.match(re) never touches lastIndex, so /g/ is fine + * there. This is the precise case; blanket flagging would be wrong. */ + if (ts.isCallExpression(node)) { + const callee = unwrap(node.expression); + if ( + ts.isPropertyAccessExpression(callee) && + ts.isRegularExpressionLiteral(callee.expression) && + (callee.name.text === "exec" || callee.name.text === "test") + ) { + const text = callee.expression.getText(sf); + const flags = text.slice(text.lastIndexOf("/") + 1); + if ([...flags].some((ch) => STATEFUL_REGEX_FLAGS.has(ch))) { + add(node, "REGEX_STATEFUL", `.${callee.name.text} on /${flags}/ regex literal mutates lastIndex`); + return; + } + } + } + + if (ts.isCallExpression(node)) { + checkCall(node); + // fall through to children (arguments may contain closures etc.) + } + + /* Environment reads (not calls): process.env.X, Date.now as a value, … + * Callee positions are skipped here — checkCall reports those once. */ + if ( + ts.isPropertyAccessExpression(node) && + ts.isIdentifier(node.expression) && + !(ts.isCallExpression(node.parent) && node.parent.expression === node) + ) { + const dotted = dottedName(node); + if (dotted) { + const root = dotted.split(".")[0]; + if (ENV_ROOTS.has(dotted) || ENV_ROOTS.has(root)) { + add(node, "ENV_ACCESS", `read ${dotted}`); + } + } + } + + if ((ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && !isLegalCallback(node)) { + add(node, "FREE_CLOSURE", "function value escapes the allowed callback positions"); + } + + ts.forEachChild(node, visit); + }; + + visit(sf); + return violations; +} + +/* -------------------------------------------------------------- top level */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { EXCEPTIONS, EXCLUDED_FUNCTIONS, type SubsetException } from "./exceptions.js"; +import { GUARDED_FILES } from "./guarded-files.js"; + +export function runGuard(repoRoot: string): GuardResult { + const excluded = new Set(EXCLUDED_FUNCTIONS.map((e) => `${e.file}#${e.fn}`)); + const used = new Set(); + const violations: Violation[] = []; + + for (const rel of GUARDED_FILES) { + const source = readFileSync(resolve(repoRoot, rel), "utf8"); + for (const v of checkGuardedFile({ file: rel, source, excludedFunctions: excluded })) { + if (v.code === "EXCLUDED_FUNCTION") { + const key = `${v.file}#${v.fn}`; + if (excluded.has(key)) { + used.add(`excluded:${key}`); + continue; + } + } + const match = findException(v); + if (match) { + used.add(exceptionLabel(match)); + continue; + } + violations.push(v); + } + } + + const stale: GuardResult["stale"] = []; + for (const e of EXCEPTIONS) { + if (!used.has(exceptionLabel(e))) { + stale.push({ kind: "exception", label: exceptionLabel(e) }); + } + } + for (const e of EXCLUDED_FUNCTIONS) { + const key = `${e.file}#${e.fn}`; + if (!used.has(`excluded:${key}`)) { + stale.push({ kind: "excluded", label: `excluded:${key}` }); + } + } + + return { violations, stale }; +} + +function findException(v: Violation): SubsetException | undefined { + return EXCEPTIONS.find( + (e) => + e.file === v.file && + e.code === v.code && + (e.fn === undefined || e.fn === v.fn) && + (e.detail === undefined || e.detail === v.detail), + ); +} + +function exceptionLabel(e: SubsetException): string { + return `${e.file}:${e.code}:${e.fn ?? "*"}`; +} diff --git a/verif/symbolic.ts b/verif/symbolic.ts new file mode 100644 index 0000000000..5d6cf66fbe --- /dev/null +++ b/verif/symbolic.ts @@ -0,0 +1,2530 @@ +/** + * verif symbolic executor — v2. + * + * Translates REAL TypeScript function bodies (from the guarded kernel files) + * into z3 constraints. The code under verification is the actual source; + * this file is only the lens. The replay lock (verif/replay.ts + smoke + * tests) catches any lens distortion. + * + * Value model (modeling decisions, all documented in verif/README.md): + * - numbers → z3 Real (exact rationals ≈ JS doubles; boundary divergence + * is caught by replay) + * - booleans → z3 Bool + * - strings → z3 Seq (string theory) + * - arrays → bounded slot vectors: { elems: SymVal[N], len: Real } + * with N = maxArrayLength (unrolled combinators, push via + * ITE store). Sound for found counterexamples; absence + * claims hold within the bound only. + * - objects → structural field maps { fields: Map } + * - unknown/degraded constructs → { kind: "any" } (never silently + * trustworthy: harnesses receive the degradation list) + * + * Execution model: single-assignment environment; if/else via ITE value + * merge (no path enumeration); loops unrolled up to K iterations with + * termination assumptions collected for the solver. + * + * Function calls: local functions in the same file are INLINED (depth + * capped) so harnesses exercise the real helper chain. Calls that cannot + * be translated degrade to `any`. + */ +import ts from "typescript"; +import { readFileSync, accessSync } from "node:fs"; +import { join, dirname } from "node:path"; +import type { Arith, Bool, Seq, Context, Term } from "./terms.js"; + +class TranslationUnsupported extends Error { + constructor(readonly what: string, readonly file: string, readonly line: number) { + super(`translation unsupported at ${file}:${line}: ${what}`); + } +} + +export type Z3 = Context; +export type Num = Arith; +type Str = Seq; + +export type SymVal = + | { kind: "num"; t: Num } + | { kind: "bool"; t: Bool<"main"> } + | { kind: "str"; t: Str } + // nullUnder: set on obj/arr values that came out of a STRICT branch merge + // with null/undefined on the other arm — the value is the obj/arr only + // when nullUnder is FALSE (JS truthiness/field-access must respect it). + | { kind: "arr"; elems: SymVal[]; len: Num; nullUnder?: Bool<"main"> } + | { kind: "obj"; fields: Map; symFields?: Map; v: SymVal }>; nullUnder?: Bool<"main"> } + | { kind: "fn"; node: ts.ArrowFunction | ts.FunctionExpression | ts.FunctionDeclaration; env: Frame } + | { kind: "null" } + | { kind: "undef" } + | { kind: "any" }; + +export interface ExecOut { + ret: SymVal | null; + thrown: { kind: "bool"; t: Bool<"main"> } | null; + /** Solver must receive these for absence claims: unrolled loops that had + * not terminated within K iterations produce a completion assumption. */ + assumptions: Bool<"main">[]; + /** Every construct degraded to `any` — harness reports these; a property + * whose counterexample depends on an `any` hole is NOT a finding. */ + degradations: string[]; + /** The term context — carries `decls` (symbolic constants created while + * translating) so harnesses can emit `(declare-fun ...)` lines. */ + ctx: import("./terms.js").Context; +} + +interface InterpreterOptions { + maxArrayLength?: number; // default 8 + maxLoopIterations?: number; // default 8 + maxInlineDepth?: number; // default 12 + /** Implementations for INJECTED PORTS (public parameter seams like + * CompressionCore's `countTokens`). When a call resolves to a parameter + * with a port implementation, the hook runs instead of degrading. Ports + * are part of the module's public contract — hosts inject their own + * token counters — so this is composition, not bug-specific modeling. */ + portImpls?: Record SymVal>; +} + +/* ------------------------------------------------------------------ helpers */ + +const num = (ctx: Z3, v: number): Num => ctx.Real.val(v); +let numHoleCounter = 0; + +const bool = (ctx: Z3, v: boolean): Bool<"main"> => ctx.Bool.val(v); + +/** Constant side-channels: z3 expr objects created from JS constants are + * tagged so arithmetic/comparisons fold at construction time instead of + * bloating the symbolic DAG with trivially-decidable subterms. */ +const strConst = { + get: (t: Str | undefined): string | undefined => (t && typeof t.c === "string" ? t.c : undefined), +}; +const numConst = { + get: (t: Num | undefined): number | undefined => (t && typeof t.c === "number" ? t.c : undefined), +}; +const boolConst = { + get: (t: Bool<"main"> | undefined): boolean | undefined => (t && typeof t.c === "boolean" ? t.c : undefined), +}; +const strc = (ctx: Z3, sv: string): Str => ctx.String.val(sv); +/** Foldable numeric equality: constant arguments decide immediately. */ +const numEq = (ctx: Z3, a: Num, b: Num): Bool<"main"> => { + const ac = numC(a), bc = numC(b); + if (ac !== undefined && bc !== undefined) return bool(ctx, ac === bc); + return a.eq(b); +}; + +/** Numeric constant lookup with structural fallback: constants created by + * harness code via raw ctx.Real.val(n) are recognized by their printed + * form so constant folding still applies to them. */ +const numC = (t: Num): number | undefined => { + const hit = numConst.get(t); + if (hit !== undefined) return hit; + const noKids = (t as unknown as { kids?: unknown[] }).kids; + if (noKids !== undefined && noKids.length > 0) return undefined; // composite — never a literal + const str = t.toString(); + if (/^-?\d+\.0+$/.test(str)) return parseInt(str, 10); + if (/^-?\d+$/.test(str)) return parseInt(str, 10); + if (/^-?\d+\.\d+$/.test(str)) { + const f = parseFloat(str); + if (Number.isFinite(f)) return f; + } + return undefined; +}; + +function isNum(v: SymVal | undefined): v is { kind: "num"; t: Num } { + return v !== undefined && v.kind === "num"; +} +function isBool(v: SymVal | undefined): v is { kind: "bool"; t: Bool<"main"> } { + return v !== undefined && v.kind === "bool"; +} +function isStr(v: SymVal | undefined): v is { kind: "str"; t: Str } { + return v !== undefined && v.kind === "str"; +} +function isArr(v: SymVal | undefined): v is Extract { + return v !== undefined && v.kind === "arr"; +} +function isObj(v: SymVal | undefined): v is Extract { + return v !== undefined && v.kind === "obj"; +} + +/** Truthiness of a symbolic value as a boolean expr (JS coercion rules for + * the subset: numbers nonzero, strings nonempty, null/undef false). */ +/** Literal check WITHOUT printing: reads the folded const tag only. + * Printing big DAG bools per-statement was exponential (the H2 OOM). */ +function constBool(t: Bool<"main"> | null | undefined): boolean | null { + if (t === null || t === undefined) return null; + if (t.c === true) return true; + if (t.c === false) return false; + const op = (t as unknown as { op?: string }).op; + if (op === "const" || op === "lit") { + // non-folded literal — fall back to a SHORT print (no kids) + const str = (t as unknown as { print(): string }).print ? (t as unknown as { print(): string }).print() : String(t); + if (str === "true") return true; + if (str === "false") return false; + } + return null; +} + + +function truthy(ctx: Z3, v: SymVal): Bool<"main"> | null { + switch (v.kind) { + case "bool": return v.t; + case "num": { + const c = numC(v.t); + if (c !== undefined) return bool(ctx, c !== 0); + return v.t.neq(num(ctx, 0)); + } + case "str": { + const sv = strConst.get(v.t); + if (sv !== undefined) return bool(ctx, sv.length > 0); + return (ctx.ToReal(v.t.length() as never) as Num).neq(num(ctx, 0)); + } + case "null": + case "undef": return bool(ctx, false); + case "arr": { + const tag = (v as { nullUnder?: Bool<"main"> }).nullUnder; + if (!tag) return bool(ctx, true); // arrays are truthy + const tc = boolConst.get(tag); + return tc === undefined ? tag.not() : bool(ctx, !tc); + } + case "obj": { + const tag = (v as { nullUnder?: Bool<"main"> }).nullUnder; + if (!tag) return bool(ctx, true); + const tc = boolConst.get(tag); + return tc === undefined ? tag.not() : bool(ctx, !tc); + } + default: return null; + } +} + +function permutations(n: number): number[][] { + if (n === 0) return [[]]; + if (n === 1) return [[0]]; + const out: number[][] = []; + const rest = permutations(n - 1); + for (const p of rest) { + for (let i = 0; i <= p.length; i++) { + out.push([...p.slice(0, i), n - 1, ...p.slice(i)]); + } + } + return out; +} + +/* ------------------------------------------------------------------ program */ + +interface ModuleInfo { + numbers: Map; + strings: Map; + functions: Map; + constArrowFns: Map; + constBindings: Map; + /** exported names (functions, const arrows, consts) for import resolution */ + exports: Set; + /** export-from re-exports: export {x} from "./y.js" */ + reexports: Map; +} + +/** Resolve relative imports of a file recursively and collect exported + * callables/consts into a global registry (kernel files import helpers + * across files: truncate-tools → truncate/prune/ccr, hide-consumed → refs). */ +function loadImportGraph(entry: string): Map { + const modules = new Map(); + const queue: string[] = [entry]; + const seen = new Set(); + while (queue.length) { + const file = queue.shift()!; + if (seen.has(file)) continue; + seen.add(file); + const { sf: entrySf, info } = loadModule(file); + modules.set(file, info); + for (const stmt of entrySf.statements) { + if (!ts.isImportDeclaration(stmt)) continue; + const spec = (stmt.moduleSpecifier as ts.StringLiteral).text; + if (!spec.startsWith("./") && !spec.startsWith("../")) continue; // node/external: not symbolized + const resolved = resolveTsPath(file, spec); + if (resolved && !seen.has(resolved)) queue.push(resolved); + } + } + return modules; +} + +function resolveTsPath(fromFile: string, spec: string): string | null { + const path = join(dirname(fromFile), spec.replace(/\.js$/, ".ts")); + try { + accessSync(path); + return path; + } catch { + return null; + } +} + +function loadModule(file: string): { sf: ts.SourceFile; info: ModuleInfo } { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.ES2022, true); + const info: ModuleInfo = { + numbers: new Map(), + strings: new Map(), + functions: new Map(), + constArrowFns: new Map(), + constBindings: new Map(), + exports: new Set(), + reexports: new Map(), + }; + const walk = (node: ts.Node): void => { + if (ts.isFunctionDeclaration(node) && node.name) { + info.functions.set(node.name.text, node); + } + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) { + const i = node.initializer; + if (ts.isNumericLiteral(i)) info.numbers.set(node.name.text, Number(i.text)); + else if (ts.isStringLiteral(i)) info.strings.set(node.name.text, i.text); + else if (ts.isArrowFunction(i) || ts.isFunctionExpression(i)) info.constArrowFns.set(node.name.text, i); + else info.constBindings.set(node.name.text, i); + } + ts.forEachChild(node, walk); + }; + walk(sf); + const walkExports = (node: ts.Node): void => { + const hasExportMod = (n: ts.Node): boolean => + !!((n as ts.FunctionDeclaration).modifiers ?? []).some( + (m: ts.ModifierLike) => m.kind === ts.SyntaxKind.ExportKeyword, + ); + if (ts.isFunctionDeclaration(node) && node.name && hasExportMod(node)) { + info.exports.add(node.name.text); + } + if (ts.isVariableStatement(node) && hasExportMod(node)) { + for (const d of node.declarationList.declarations) { + if (ts.isIdentifier(d.name)) info.exports.add(d.name.text); + } + } + if (ts.isExportDeclaration(node) && node.exportClause && ts.isNamedExports(node.exportClause)) { + for (const el of node.exportClause.elements) info.exports.add(el.name.text); + } + ts.forEachChild(node, walkExports); + }; + walkExports(sf); + return { sf, info }; +} + +/* ---------------------------------------------------------------- interpreter */ + +interface Frame { + vars: Map; +} + +interface RunState { + ret: SymVal | null; + /** conditional returns: function returns `val` when `cond` — following + * statements still execute (they produce the not-cond path's value) */ + pendingRets: { cond: Bool<"main">; val: SymVal }[]; + /** shared across branch copies — dedup identical diagnostics */ + seenDeg: Set; + seenAsum: Set; + /** hard unconditional return (dead code after `return`) */ + hardRet: boolean; + thrown: Bool<"main"> | null; + assumptions: Bool<"main">[]; + degradations: string[]; + breakCond: Bool<"main"> | null; + continueCond: Bool<"main"> | null; +} + +interface TranslateInput { + file: string; + function: string; + params: Record; + ctx: Z3; + options?: InterpreterOptions; +} + +export function translateFunction(input: TranslateInput): { exec: () => ExecOut } { + const { file, function: fnName, params, ctx } = input; + + /** z3 Seq.length() is Int-sorted; our numeric arithmetic is Real. */ + const lenR = (seq: Seq<"main">): Num => ctx.ToReal(seq.length() as never) as Num; + const N = input.options?.maxArrayLength ?? 8; + const K = input.options?.maxLoopIterations ?? 8; + /** Constant-folding Boolean combinators: z3's .or/.and do NOT simplify + * `(or false false)` to `false`, and control-flow checks compare the + * printed form against "true"/"false" — so every control Bool must be + * built through these to keep literal chains folded. */ + const orFold = (a: Bool<"main"> | null, b: Bool<"main">): Bool<"main"> => { + if (a === null) return b; + const ab = boolConst.get(a), bb = boolConst.get(b); + if (ab === true || bb === true) return bool(ctx, true); + if (ab === false) return b; + if (bb === false) return a; + return a.or(b); + }; + const andFold = (a: Bool<"main">, b: Bool<"main">): Bool<"main"> => { + const ab = boolConst.get(a), bb = boolConst.get(b); + if (ab === false || bb === false) return bool(ctx, false); + if (ab === true) return b; + if (bb === true) return a; + return a.and(b); + }; + const notFold = (a: Bool<"main">): Bool<"main"> => { + const ab = boolConst.get(a); + if (ab === true) return bool(ctx, false); + if (ab === false) return bool(ctx, true); + return a.not(); + }; + const MAXINLINE = input.options?.maxInlineDepth ?? 12; + const modules = loadImportGraph(file); + const info = modules.get(file)!; + const sf = loadModule(file).sf; + // identifier → definition in an imported module ("./x.js" exports) + const importedSymbols = new Map(); + { + const entrySf = loadModule(file).sf; + for (const stmt of entrySf.statements) { + if (!ts.isImportDeclaration(stmt)) continue; + const spec = (stmt.moduleSpecifier as ts.StringLiteral).text; + if (!spec.startsWith("./") && !spec.startsWith("../")) continue; + const resolved = resolveTsPath(file, spec); + if (!resolved) continue; + const target = modules.get(resolved); + if (!target) continue; + const named = stmt.importClause?.namedBindings; + if (named && ts.isNamedImports(named)) { + for (const el of named.elements) importedSymbols.set(el.name.text, { file: resolved, info: target }); + } + } + } + + /** Module whose file's functions we are currently executing (for cross-file inlining). + * Top of stack = defining module; entry module is the base. */ + const moduleStack: ModuleInfo[] = [info]; + const curInfo = (): ModuleInfo => moduleStack[moduleStack.length - 1] ?? info; + const portMemo = new Map(); + + let numStrCounter = 0; + /** String(number): an uninterpreted decimal string; only its length matters + * for stability properties (1..19 digits) and the replay lock validates the + * witness against the real code. */ + // String(number): memoized per INPUT TERM (same number → same string — + // stability properties across turns need the function property, a fresh + // hole per call would fake instability). + const numStrMemo = new Map(); + const numToStringStrFor = (x: Num): { kind: "str"; t: Str } => { + const key = x.hash(); + const hit = numStrMemo.get(key); + if (hit) return hit; + const hole = ctx.String.const("__numstr_" + numStrCounter++); + const v: { kind: "str"; t: Str } = { kind: "str", t: hole }; + numStrMemo.set(key, v); + return v; + }; + const numToStringStr = (st: RunState): { kind: "str"; t: Str } => { + const hole = ctx.String.const("__numstr_" + numStrCounter++); + pushAsum(st, (ctx.ToReal(hole.length() as never) as Num).ge(num(ctx, 1)).and((ctx.ToReal(hole.length() as never) as Num).le(num(ctx, 19)))); + return { kind: "str", t: hole }; + }; + + const decl = info.functions.get(fnName); + if (!decl?.body) throw new Error(`function ${fnName} not found in ${file}`); + + const lineOf = (node: ts.Node) => sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1; + const bad = (node: ts.Node, what: string): never => { + throw new TranslationUnsupported(what, file, lineOf(node)); + }; + + /* ---------------- value constructors ---------------- */ + + const arrOf = (elems: SymVal[], len?: Num): Extract => + ({ kind: "arr", elems, len: len ?? num(ctx, elems.length) }); + + /** Convert a REAL JSON value (from concrete JSON.parse) into a SymVal. */ + const jsonToSymVal = (v: unknown, depth: number): SymVal => { + if (v === null) return { kind: "null" }; + if (typeof v === "string") return { kind: "str", t: strc(ctx, v) }; + if (typeof v === "number") return { kind: "num", t: num(ctx, v) }; + if (typeof v === "boolean") return { kind: "bool", t: bool(ctx, v) }; + if (Array.isArray(v)) { + if (v.length > (input.options?.maxArrayLength ?? 8)) return { kind: "any" }; + return arrOf(v.map((x) => jsonToSymVal(x, depth + 1))); + } + if (typeof v === "object") { + const fields = new Map(); + for (const [k, val] of Object.entries(v as Record)) { + fields.set(k, jsonToSymVal(val, depth + 1)); + } + return { kind: "obj", fields }; + } + return { kind: "undef" }; + }; + + // STRICT merge for VARIABLE bindings across branch joins: unlike mergeVal + // (whose undef/null-loses erasure is only sound for never-read tail slots), + // this preserves the maybe-null/maybe-undefined distinction via nullUnder — + // `if (lastTextCore)` must stay a SYMBOLIC condition, not fold true. + const mergeValStrict = (cond: Bool<"main">, a: SymVal, b: SymVal): SymVal => { + const objSide = a.kind === "obj" || a.kind === "arr" ? a : b.kind === "obj" || b.kind === "arr" ? b : null; + const nullish = b.kind === "null" || b.kind === "undef" ? b : a.kind === "null" || a.kind === "undef" ? a : null; + if (objSide && nullish) { + const aIsObj = a === objSide; + // value is the obj when nullUnder is false: + // mergeValStrict(cond, obj, null) → nullUnder = ¬cond + // mergeValStrict(cond, null, obj) → nullUnder = cond + const side = aIsObj ? notFold(cond) : cond; + const prev = (objSide as { nullUnder?: Bool<"main"> }).nullUnder; + const tag = prev ? orFold(prev, side) : side; + return objSide.kind === "obj" + ? { kind: "obj", fields: objSide.fields, nullUnder: tag } + : { kind: "arr", elems: objSide.elems, len: objSide.len, nullUnder: tag }; + } + return mergeVal(cond, a, b); + }; + + const mergeVal = (cond: Bool<"main">, a: SymVal, b: SymVal): SymVal => { + // undef/null loses to any concrete value: out-of-bounds fallbacks in + // element-access chains (guarded by length checks in real code) must not + // poison the merged object. Real out-of-range access throws and is caught + // by the replay lock. + if (a.kind === "undef" && b.kind !== "undef") return mergeVal(cond.not(), b, { kind: "undef" }) === null ? { kind: "any" } : b; + if (b.kind === "undef" && a.kind !== "undef") return a; + if (a.kind === "null" && b.kind !== "null") return b; + if (b.kind === "null" && a.kind !== "null") return a; + if (a.kind !== b.kind) { + return { kind: "any" }; + } + switch (a.kind) { + case "num": { + const bb = b as Extract; + const ac = numC(a.t), bc = numC(bb.t); + if (ac !== undefined && ac === bc) return a; // same constant: no ITE + return { kind: "num", t: ctx.If(cond, a.t, bb.t) }; + } + case "bool": { + const bb = b as Extract; + const ac = boolConst.get(a.t), bc = boolConst.get(bb.t); + if (ac !== undefined && ac === bc) return a; + return { kind: "bool", t: ctx.If(cond, a.t, bb.t) }; + } + case "str": { + const bb = b as Extract; + const ac = strConst.get(a.t), bc = strConst.get(bb.t); + if (ac !== undefined && ac === bc) return a; + return { kind: "str", t: ctx.If(cond, a.t, bb.t) }; + } + case "null": case "undef": return a; + case "fn": return a; // closures are immutable; identical node survives + case "arr": { + const bb = b as Extract; + const n = Math.max(a.elems.length, bb.elems.length); + const elems: SymVal[] = []; + for (let i = 0; i < n; i++) { + // missing slots merge as undef (undef loses to concrete values in + // mergeVal) so growing arrays keep their element structure + elems.push(mergeVal(cond, a.elems[i] ?? { kind: "undef" }, bb.elems[i] ?? { kind: "undef" })); + } + const la = numC(a.len), lb = numC(bb.len); + if (la !== undefined && la === lb) return { kind: "arr", elems, len: a.len }; // same length: keep const + return { kind: "arr", elems, len: ctx.If(cond, a.len, bb.len) }; + } + case "obj": { + const bb = b as Extract; + const fields = new Map(); + for (const key of new Set([...a.fields.keys(), ...bb.fields.keys()])) { + const va = a.fields.get(key); + const vb = bb.fields.get(key); + if (va && vb) fields.set(key, mergeVal(cond, va, vb)); + else if (va) fields.set(key, guardedNeutral(cond, va)); + else if (vb) fields.set(key, guardedNeutral(cond.not(), vb)); + } + // merge symbolic-key bindings: present on one arm only means the + // other arm never wrote it → undefined there (guard survives) + // symFields entries carry their WRITE guard: a binding present on + // one arm only keeps the path condition in .g (accumulated with + // andFold) — the read fan tests (key = idx AND g) so a dead-path + // write never shadows an older binding for the same string key. + let symFields: Map; v: SymVal }> | undefined; + const sa = a.symFields, sb = bb.symFields; + if (sa || sb) { + symFields = new Map; v: SymVal }>(); + const seen = new Set(); + const bothArm = (ga: Bool<"main">, va: SymVal, gb: Bool<"main">, vb: SymVal): { g: Bool<"main">; v: SymVal } => { + const gT = andFold(cond, ga), gE = andFold(cond.not(), gb); + const gLive = orFold(gT, gE); + if (gLive === null) return { g: bool(ctx, false), v: va }; + let v: SymVal; + if (constBool(gT) === true) v = va; + else if (constBool(gE) === true) v = vb; + else v = iteVal(gT, va, iteVal(gE, vb, neutralFor(vb))); + return { g: gLive, v }; + }; + if (sa) for (const [kt, ea] of sa) { + seen.add(kt); + const eb = sb?.get(kt); + symFields.set(kt, eb ? bothArm(ea.g, ea.v, eb.g, eb.v) : { g: andFold(cond, ea.g)!, v: ea.v }); + } + if (sb) for (const [kt, eb] of sb) { + if (seen.has(kt)) continue; + symFields.set(kt, { g: andFold(cond.not(), eb.g)!, v: eb.v }); + } + } + return { kind: "obj", fields, ...(symFields ? { symFields } : {}) }; + } + default: return { kind: "any" }; + } + }; + + // A value written under a path condition into a container slot must leave + // the IN-DOMAIN neutral in the untaken arm: downstream truthiness/membership + // reads (len>0 etc.) treat neutral as "absent". Without this the undef-loses + // merge rule silently drops the write's path guard (same failure class as + // the Set-add phantom-slot bug). + const guardedNeutral = (cond: Bool<"main">, v: SymVal): SymVal => { + switch (v.kind) { + case "str": return { kind: "str", t: ctx.If(cond, v.t, ctx.String.val("")) }; + case "num": return { kind: "num", t: ctx.If(cond, v.t, ctx.Real.val(0)) }; + case "bool": return { kind: "bool", t: ctx.If(cond, v.t, ctx.Bool.val(false)) }; + case "obj": case "arr": return iteStrict(cond, v, { kind: "null" }); + default: return v; + } + }; + + const neutralFor = (v: SymVal): SymVal => { + switch (v.kind) { + case "str": return { kind: "str", t: ctx.String.val("") }; + case "num": return { kind: "num", t: ctx.Real.val(0) }; + case "bool": return { kind: "bool", t: ctx.Bool.val(false) }; + default: return { kind: "null" }; + } + }; + + const iteStrict = (cond: Bool<"main">, a: SymVal, b: SymVal): SymVal => { + if (a === b) return a; + const cb = constBool(cond); + if (cb === true) return a; + if (cb === false) return b; + return mergeValStrict(cond, a, b); + }; + + const iteVal = (cond: Bool<"main">, a: SymVal, b: SymVal): SymVal => { + if (a === b) return a; + const cb = constBool(cond); + if (cb === true) return a; + if (cb === false) return b; + // Equal constants fold to the constant: ite(cond, 1, 1) is 1 in every + // model. Without this, branch merges lose const tags (two structurally + // equal Terms are distinct objects) and everything downstream degrades + // to symbolic — including array lengths that guard loop unrolling. + if (a.kind === b.kind && a.kind === "num" && b.kind === "num") { + const ca = numC(a.t), cbd = numC(b.t); + if (ca !== undefined && ca === cbd) return a; + } + if (a.kind === b.kind && a.kind === "str" && b.kind === "str") { + const ca = strConst.get(a.t), cb2 = strConst.get(b.t); + if (ca !== undefined && ca === cb2) return a; + } + if (a.kind === b.kind && a.kind === "bool" && b.kind === "bool") { + const ca = boolConst.get(a.t), cb2 = boolConst.get(b.t); + if (ca !== undefined && ca === cb2) return a; + } + if (a.kind === "undef" && b.kind === "undef") return a; + return mergeVal(cond, a, b); + }; + + /** JS ?: on symbolic values. */ + const jsOr = (a: SymVal, b: () => SymVal): SymVal => { + const t = truthy(ctx, a); + if (t === null) return { kind: "any" }; + const av = a; + const bv = b(); + // (a || b): if a truthy → a else b + return iteVal(t, av.kind === "bool" ? { kind: "bool", t: t } : av, bv); + }; + const jsAnd = (a: SymVal, b: () => SymVal): SymVal => { + const t = truthy(ctx, a); + if (t !== null) { + const bb = b(); + // Prefer exact truthiness when both sides coerce: `(a && b)` in a + // boolean context is `truthy(a) ∧ truthy(b)`. Operand-identity + // (`a && b` evaluating to `a` when falsy) is only observable via + // value-equality/aliasing patterns absent from the guarded subset. + const tbv = truthy(ctx, bb); + if (tbv !== null) return { kind: "bool", t: andFold(t, tbv) }; + return iteVal(t, bb, a.kind === "bool" ? { kind: "bool", t: t } : a); + } + if (t === null) return { kind: "any" }; + return iteVal(t, b(), a.kind === "bool" ? { kind: "bool", t: t } : a); + }; + + const jsNullish = (a: SymVal, b: () => SymVal): SymVal => + iteVal(a.kind === "null" || a.kind === "undef" ? bool(ctx, true) : bool(ctx, false), b(), a); + + /* ---------------- statements ---------------- */ + + const runState0 = (): RunState => ({ + ret: null, pendingRets: [], hardRet: false, thrown: null, + assumptions: [], degradations: [], seenDeg: new Set(), seenAsum: new Set(), + breakCond: null, continueCond: null, + }); + + interface Scope { + frame: Frame; + depth: number; + } + + /** Sequential statement execution with conditional stop semantics: + * after a possibly-taken break/continue, the REMAINING statements run on a + * copy under ¬stop — recursively, so nested stops (break-if followed by + * continue-if in the same block) guard correctly. */ + const execSeq = (stmts: readonly ts.Statement[], frame: Frame, st: RunState, depth: number): void => { + for (let i = 0; i < stmts.length; i++) { + execStmt(stmts[i]!, frame, st, depth); + const bc = st.breakCond; + const cc = st.continueCond; + const bcT = bc !== null && constBool(bc) !== false; + const ccT = cc !== null && constBool(cc) !== false; + if (bcT || ccT) { + const stopCond = bc && cc ? orFold(bc as Bool<"main">, cc as Bool<"main">) : ((bc ?? cc) as Bool<"main">); + if (i + 1 < stmts.length && constBool(stopCond) !== true) { + const f2: Frame = { vars: new Map(frame.vars) }; + const s2: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execSeq(stmts.slice(i + 1), f2, s2, depth); + mergeBranch(frame, st, notFold(stopCond), f2, s2, null, null); + } + break; + } + if (st.hardRet) break; + if (constBool(st.thrown) === true) break; + } + }; + + let stmtCount = 0; + const stmtLines = new Map(); + const execStmt = (stmt: ts.Statement, frame: Frame, st: RunState, depth: number): void => { + if (st.hardRet) return; + if (constBool(st.thrown) === true) return; + if (constBool(st.breakCond) === true) return; + if (constBool(st.continueCond) === true) return; + if (ts.isBlock(stmt)) { execSeq(stmt.statements, frame, st, depth); return; } + if (ts.isTypeAliasDeclaration(stmt) || ts.isInterfaceDeclaration(stmt) || ts.isEnumDeclaration(stmt)) return; // type-level only — erased + if (ts.isVariableStatement(stmt)) { + for (const d of stmt.declarationList.declarations) { + const init = d.initializer ? evalExpr(d.initializer, frame, st, depth) : { kind: "undef" as const }; + if (ts.isIdentifier(d.name)) { + frame.vars.set(d.name.text, init); + } else if (ts.isObjectBindingPattern(d.name)) { + for (const el of d.name.elements) { + if (!ts.isBindingElement(el) || !ts.isIdentifier(el.name)) continue; + const propName = el.propertyName && ts.isIdentifier(el.propertyName) ? el.propertyName.text : el.name.text; + frame.vars.set(el.name.text, member(init, propName, el, st)); + } + } else if (ts.isArrayBindingPattern(d.name)) { + const arrV = init.kind === "arr" ? init : { kind: "any" as const }; + if (arrV.kind === "arr") d.name.elements.forEach((el, j) => { + if (!ts.isBindingElement(el) || !ts.isIdentifier(el.name)) return; + frame.vars.set(el.name.text, elementAccess(arrV, { kind: "num", t: num(ctx, j) }, el, st)); + }); + } else { + bad(stmt, "non-identifier declaration"); + } + } + return; + } + if (ts.isExpressionStatement(stmt)) { + evalExpr(stmt.expression, frame, st, depth); + return; + } + if (ts.isReturnStatement(stmt)) { + if (st.ret === null) { + st.ret = stmt.expression ? evalExpr(stmt.expression, frame, st, depth) : { kind: "undef" }; + st.hardRet = true; + } else { + // a pending conditional return already captured the tail; a second + // return can only be reached on the not-cond path + st.pendingRets.push({ cond: bool(ctx, false), val: stmt.expression ? evalExpr(stmt.expression, frame, st, depth) : { kind: "undef" } }); + st.hardRet = true; + } + return; + } + if (ts.isThrowStatement(stmt)) { + // The thrown value itself never matters — only the fact. + st.thrown = bool(ctx, true); + return; + } + if (ts.isIfStatement(stmt)) { + const c = evalExpr(stmt.expression, frame, st, depth); + const tb = truthy(ctx, c); + if (tb === null) { degrade(st, stmt, "if condition degraded"); return; } + // Dead-branch pruning: a constant condition executes only the live + // arm, in-frame. Eager both-arm execution would force provably-dead + // code (e.g. a regex in a never-taken branch) to be translatable. + const cb = constBool(tb); + if (cb === true) { execStmt(stmt.thenStatement, frame, st, depth); return; } + if (cb === false) { if (stmt.elseStatement) execStmt(stmt.elseStatement, frame, st, depth); return; } + const thenFrame: Frame = { vars: new Map(frame.vars) }; + const thenState: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.thenStatement, thenFrame, thenState, depth); + let elseFrame: Frame | null = null; + let elseState: RunState | null = null; + if (stmt.elseStatement) { + elseFrame = { vars: new Map(frame.vars) }; + elseState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.elseStatement, elseFrame, elseState, depth); + } + mergeBranch(frame, st, tb, thenFrame, thenState, elseFrame, elseState); + return; + } + if (ts.isForOfStatement(stmt)) { + // Unroll over bounded array slots. + let iter = evalExpr(stmt.expression, frame, st, depth); + if (iter.kind === "obj" && iter.fields.has("__set_values")) { + // for (const k of someSet) — iterate the Set-model's value list + const vals = iter.fields.get("__set_values")!; + iter = vals.kind === "arr" ? vals : ({ kind: "arr", elems: [], len: num(ctx, 0) } as SymVal); + } + if (!isArr(iter)) { degrade(st, stmt, "for-of over non-array"); return; } + bindForVar(stmt.initializer, { kind: "undef" }, frame); + // Dead-slot pruning: when the array length is a KNOWN constant, skip + // slots beyond it entirely — executing dead iterations would pollute + // live state with out-of-bounds degradations and any-values. + const lenC = numC(iter.len); + const slotCeil = lenC !== undefined ? Math.min(iter.elems.length, Math.ceil(lenC)) : iter.elems.length; + for (let k = 0; k < slotCeil; k++) { + const guard = andFold(iter.len.gt(num(ctx, k)), st.breakCond ?? bool(ctx, true)); + const prBefore = st.pendingRets.length; + const elemFrame: Frame = { vars: new Map(frame.vars) }; + const elemState: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + bindForVar(stmt.initializer, iter.elems[k], elemFrame); + execStmt(stmt.statement, elemFrame, elemState, depth); + mergeBranch(frame, st, guard, elemFrame, elemState, null, null); + st.continueCond = null; // continue ends only this iteration + if (st.hardRet || constBool(st.thrown) === true || constBool(st.breakCond) === true) break; + // A conditional return/throw fired under this iteration's guard: a + // real execution would have left the whole function — treat it as a + // loop exit (post-loop statements run under ¬exit via rest-on-copy). + if (st.pendingRets.length > prBefore || (st.thrown !== null && constBool(st.thrown) === undefined)) { + st.breakCond = orFold(st.breakCond, guard); + break; + } + } + // Elements beyond the declared length never execute: nothing needed. + return; + } + if (ts.isForStatement(stmt)) { + if (stmt.initializer) { + if (ts.isVariableDeclarationList(stmt.initializer)) { + for (const d of stmt.initializer.declarations) { + if (ts.isIdentifier(d.name)) { + frame.vars.set(d.name.text, d.initializer ? evalExpr(d.initializer, frame, st, depth) : { kind: "undef" }); + } + } + } else { + evalExpr(stmt.initializer, frame, st, depth); + } + } + for (let k = 0; k < K; k++) { + const c = stmt.condition ? evalExpr(stmt.condition, frame, st, depth) : { kind: "bool" as const, t: bool(ctx, true) }; + const tb = truthy(ctx, c); + if (tb === null) { degrade(st, stmt, "for condition degraded"); return; } + // Constant-false condition: the loop provably finished — do NOT + // execute the body (dead iterations pollute live state). + if (constBool(tb) === false) { + pushAsum(st, bool(ctx, true)); + return; + } + const iterFrame: Frame = { vars: new Map(frame.vars) }; + const prBefore = st.pendingRets.length; + const iterState: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.statement, iterFrame, iterState, depth); + // continue: still run the increment — handled by running increment + // on iterFrame after body (continue skips rest of body only). + if (stmt.incrementor) evalExpr(stmt.incrementor, iterFrame, iterState, depth); + mergeBranch(frame, st, tb, iterFrame, iterState, null, null); + st.continueCond = null; // continue ends only this iteration + if (st.hardRet || constBool(st.thrown) === true || constBool(st.breakCond) === true) return; + if (st.pendingRets.length > prBefore || (st.thrown !== null && constBool(st.thrown) === undefined)) { + st.breakCond = orFold(st.breakCond, tb as Bool<"main">); + return; + } + } + // Loop may need more than K iterations: assume it finished. + const c = stmt.condition ? evalExpr(stmt.condition, frame, st, depth) : null; + const tb = c ? truthy(ctx, c) : bool(ctx, true); + if (tb) pushAsum(st, notFold(tb as Bool<"main">)); + return; + } + if (ts.isWhileStatement(stmt)) { + for (let k = 0; k < K; k++) { + const c = evalExpr(stmt.expression, frame, st, depth); + const tb = truthy(ctx, c); + if (tb === null) { degrade(st, stmt, "while condition degraded"); return; } + const iterFrame: Frame = { vars: new Map(frame.vars) }; + const prBefore = st.pendingRets.length; + const iterState: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.statement, iterFrame, iterState, depth); + mergeBranch(frame, st, tb, iterFrame, iterState, null, null); + st.continueCond = null; // continue ends only this iteration + if (st.hardRet || constBool(st.thrown) === true || constBool(st.breakCond) === true) return; + if (st.pendingRets.length > prBefore || (st.thrown !== null && constBool(st.thrown) === undefined)) { + st.breakCond = orFold(st.breakCond, tb as Bool<"main">); + return; + } + } + const c = evalExpr(stmt.expression, frame, st, depth); + const tb = truthy(ctx, c); + if (tb) pushAsum(st, notFold(tb as Bool<"main">)); + return; + } + if (ts.isBreakStatement(stmt)) { st.breakCond = bool(ctx, true); return; } + if (ts.isContinueStatement(stmt)) { st.continueCond = bool(ctx, true); return; } + if (ts.isSwitchStatement(stmt)) { + // Desugar to if-chain over case expressions (guarded code has small + // switches), with JS entry semantics: a clause's body runs when the + // discriminant matches (or `default` is reached with no prior match) + // AND no earlier entered clause broke out. Dead clauses (entry folds + // false) are never executed — eager execution of provably-dead bodies + // wasted terms and forced untranslatable code into the model. + const disc = evalExpr(stmt.expression, frame, st, depth); + let matched = bool(ctx, false); // some earlier case compared equal + let stopped: Bool<"main"> | null = null; // an entered clause broke out + for (const clause of stmt.caseBlock.clauses) { + let enter: Bool<"main">; + if (ts.isCaseClause(clause)) { + const cv = evalExpr(clause.expression, frame, st, depth); + const eq = valEq(disc, cv); + if (eq === null) { + bad(clause.expression, "case comparison degraded"); + matched = bool(ctx, true); // unconstrained — conservative over-approx + enter = bool(ctx, true); + } else { + matched = orFold(matched, eq); + enter = eq; + } + } else { + enter = notFold(matched); + } + if (stopped) enter = andFold(enter, notFold(stopped)); + const cf: Frame = { vars: new Map(frame.vars) }; + const cs: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + const enterC = constBool(enter); + if (enterC === false) continue; // dead clause: never executed + for (const inner of clause.statements) execStmt(inner, cf, cs, depth); + // JS semantics: `break` inside a switch exits the SWITCH, not the + // enclosing loop. The clause's breakCond is consumed here — only + // return/throw/continue propagate outward. + const broke = cs.breakCond; + cs.breakCond = null; + if (enterC === true) { + // Live clause: apply in-frame (no ITE wrapper needed). + for (const [k2, v] of cf.vars) frame.vars.set(k2, v); + st.ret = cs.ret ?? st.ret; st.hardRet = st.hardRet || cs.hardRet; + st.thrown = cs.thrown ?? st.thrown; + st.breakCond = st.breakCond ? orFold(st.breakCond, cs.breakCond ?? bool(ctx, false)) : cs.breakCond; + st.continueCond = st.continueCond ? orFold(st.continueCond, cs.continueCond ?? bool(ctx, false)) : cs.continueCond; + st.pendingRets.push(...cs.pendingRets); + if (constBool(broke) === true) stopped = bool(ctx, true); + if (st.hardRet || constBool(st.thrown) === true) return; + continue; + } + if (broke) stopped = orFold(stopped ?? bool(ctx, false), andFold(enter, broke)); + mergeBranch(frame, st, enter, cf, cs, null, null); + if (st.hardRet || constBool(st.thrown) === true) return; + } + return; + } + if (ts.isTryStatement(stmt)) { + const tf: Frame = { vars: new Map(frame.vars) }; + const ts_: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.tryBlock, tf, ts_, depth); + if (ts_.thrown && stmt.catchClause) { + ts_.thrown = null; // caught + if (stmt.catchClause.variableDeclaration?.name && ts.isIdentifier(stmt.catchClause.variableDeclaration.name)) { + tf.vars.set(stmt.catchClause.variableDeclaration.name.text, { kind: "any" }); + } + execStmt(stmt.catchClause.block, tf, ts_, depth); + } + if (stmt.finallyBlock) execStmt(stmt.finallyBlock, tf, ts_, depth); + // merge try-path back unconditionally + for (const [k2, v] of tf.vars) frame.vars.set(k2, v); + st.ret = ts_.ret ?? st.ret; st.hardRet = st.hardRet || ts_.hardRet; st.thrown = ts_.thrown ?? st.thrown; + st.breakCond = st.breakCond ? orFold(st.breakCond, ts_.breakCond ?? bool(ctx, false)) : ts_.breakCond; + st.continueCond = st.continueCond ? orFold(st.continueCond, ts_.continueCond ?? bool(ctx, false)) : ts_.continueCond; + st.pendingRets.push(...ts_.pendingRets); + // shared accumulators — no merge needed + return; + } + if (ts.isEmptyStatement(stmt)) return; + bad(stmt, `statement ${ts.SyntaxKind[stmt.kind]}`); + }; + + + const bindForVar = (init: ts.ForInitializer, value: SymVal, frame: Frame): void => { + if (ts.isVariableDeclarationList(init)) { + for (const d of init.declarations) { + if (ts.isIdentifier(d.name)) frame.vars.set(d.name.text, value); + else if (ts.isArrayBindingPattern(d.name)) { + // positional destructuring: `[a, b] = pairArray` + for (let j = 0; j < d.name.elements.length; j++) { + const el = d.name.elements[j]; + if (!ts.isBindingElement(el) || !ts.isIdentifier(el.name)) continue; + const v: SymVal = isArr(value) ? (value.elems[j] ?? { kind: "undef" }) : { kind: "undef" }; + frame.vars.set(el.name.text, v); + } + } + } + } + }; + + const pushAsum = (st: RunState, t: Bool<"main">): void => { + if (st.seenAsum === undefined) st.seenAsum = new Set(); + const k = t.hash(); + if (st.seenAsum.has(k)) return; + st.seenAsum.add(k); + st.assumptions.push(t); + }; + + const degrade = (st: RunState, node: ts.Node, what: string): void => { + const key = `${what} @${lineOf(node)}`; + if (st.seenDeg === undefined) st.seenDeg = new Set(); + if (st.seenDeg.has(key)) return; + st.seenDeg.add(key); + st.degradations.push(key); + }; + + const mergeBranch = ( + frame: Frame, + st: RunState, + cond: Bool<"main">, + thenFrame: Frame, thenState: RunState, + elseFrame: Frame | null, elseState: RunState | null, + ): void => { + + // Merge variables — a proper three-way merge: + // both branches wrote → ite(cond, thenVal, elseVal) (pre-branch value is dead) + // only then wrote → ite(cond, thenVal, cur) + // only else wrote → ite(¬cond, elseVal, cur) + // neither wrote → keep cur + // Sequential two-pass merging (then into frame, then else into the + // already-merged frame) builds nested ite(¬c, e, ite(c, t, cur)) terms + // that no constant folding can ever reduce — and worse, when BOTH + // branches assigned, the stale pre-branch value survives inside the + // nested term shape. Reference identity distinguishes written names: + // untouched bindings carry the SAME object reference into branch copies. + const names = new Set([...thenFrame.vars.keys(), ...(elseFrame ? [...elseFrame.vars.keys()] : [])]); + for (const name of names) { + const cur = frame.vars.get(name); + const tv = thenFrame.vars.get(name); + const ev = elseFrame ? elseFrame.vars.get(name) : undefined; + const wroteT = tv !== undefined && tv !== cur; + const wroteE = ev !== undefined && ev !== cur; + if (wroteT && wroteE) frame.vars.set(name, iteStrict(cond, tv, ev)); + else if (wroteT) frame.vars.set(name, cur !== undefined ? iteStrict(cond, tv, cur) : tv); + else if (wroteE) frame.vars.set(name, cur !== undefined ? iteStrict(cond.not(), ev, cur) : ev); + } + // ret merge: branch returns become PENDING conditional returns — the + // statements after the if still execute (they produce the other path's + // value); the final return value folds all pending conditions at the end. + if (thenState.hardRet || thenState.ret !== null) { + const tv = thenState.ret ?? { kind: "undef" }; + st.pendingRets.push({ cond, val: tv }); + } + // Conditional returns from INSIDE the branch (e.g. `if (x) return` in a + // loop body) live only in pendingRets — hardRet/ret stay null, so lifting + // them under the same if would DROP them: the while-loop early-return + // detection compares pendingRets counts and never fires; the loop then + // unrolls to exhaustion and the post-loop `throw` poisons the caller. + for (const pr of thenState.pendingRets) st.pendingRets.push({ cond: andFold(cond, pr.cond), val: pr.val }); + if (elseState && (elseState.hardRet || elseState.ret !== null)) { + const ev = elseState.ret ?? { kind: "undef" }; + st.pendingRets.push({ cond: notFold(cond), val: ev }); + } + for (const pr of elseState?.pendingRets ?? []) st.pendingRets.push({ cond: andFold(notFold(cond), pr.cond), val: pr.val }); + // Flags merge — normalize provably-false flags back to null: a + // const-false TERM is not nullish, and every consumer does + // `st.flag ?? default`; a lingering false term flips loop guards + // (for-of iteration k>=1 dying to `andFold(guard, false)`). + const tThrown = thenState.thrown; + const eThrown = elseState?.thrown ?? null; + const tB = tThrown ?? bool(ctx, false); + const eB = eThrown ?? bool(ctx, false); + const mergedThrow = st.thrown ?? bool(ctx, false); + const nt = orFold(orFold(mergedThrow, andFold(cond, tB)), andFold(notFold(cond), eB)); + st.thrown = constBool(nt) === false ? null : nt; + const tBrk = thenState.breakCond ?? null; + const eBrk = elseState?.breakCond ?? null; + const mergedBrk = st.breakCond ?? bool(ctx, false); + const nb = orFold(orFold(mergedBrk, tBrk ? andFold(cond, tBrk) : bool(ctx, false)), eBrk ? andFold(notFold(cond), eBrk) : bool(ctx, false)); + st.breakCond = constBool(nb) === false ? null : nb; + const tCnt = thenState.continueCond ?? null; + const eCnt = elseState?.continueCond ?? null; + const mergedCnt = st.continueCond ?? bool(ctx, false); + const nc = orFold(orFold(mergedCnt, tCnt ? andFold(cond, tCnt) : bool(ctx, false)), eCnt ? andFold(notFold(cond), eCnt) : bool(ctx, false)); + st.continueCond = constBool(nc) === false ? null : nc; + const pushAll = (dst: T[], src: T[]): void => { if (src !== dst) for (const x of src) dst.push(x); }; + pushAll(st.assumptions, thenState.assumptions); if (elseState) pushAll(st.assumptions, elseState.assumptions); + // degradations shared (single array across copies) + }; + + const valEq = (a: SymVal, b: SymVal): Bool<"main"> | null => { + if (isNum(a) && isNum(b)) { + const ac = numC(a.t), bc = numC(b.t); + if (ac !== undefined && bc !== undefined) return bool(ctx, ac === bc); + return a.t.eq(b.t); + } + if (isStr(a) && isStr(b)) { + const ac = strConst.get(a.t), bc = strConst.get(b.t); + if (ac !== undefined && bc !== undefined) return bool(ctx, ac === bc); + if (typeof (a.t as { eq?: unknown }).eq !== "function" || typeof (b.t as { eq?: unknown }).eq !== "function") { + return null; + } + return a.t.eq(b.t); + } + if (isBool(a) && isBool(b)) { + const ac = boolConst.get(a.t), bc = boolConst.get(b.t); + if (ac !== undefined && bc !== undefined) return bool(ctx, ac === bc); + return a.t.eq(b.t); + } + if (a.kind === "null" && b.kind === "null") return bool(ctx, true); + if (a.kind === "undef" && b.kind === "undef") return bool(ctx, true); + return null; + }; + + /* ---------------- expressions ---------------- */ + + const evalExpr = (expr: ts.Expression, frame: Frame, st: RunState, depth: number): SymVal => { + if (ts.isNewExpression(expr)) { + return evalNew(expr, frame, st, depth); + } + if (ts.isArrowFunction(expr) || ts.isFunctionExpression(expr)) { + return { kind: "fn", node: expr, env: frame }; + } + if ( + ts.isCallExpression(expr) && + ts.isPropertyAccessExpression(expr.expression) && + ts.isIdentifier(expr.expression.expression) && + expr.expression.expression.text === "Math" + ) { + const meth = expr.expression.name.text; + const argVals = expr.arguments.map((a) => evalExpr(a, frame, st, depth)); + return mathCall(meth, argVals, expr, st); + } + if (ts.isNumericLiteral(expr)) { + // Numeric literals become Real; note 0.95 etc are exact rationals here. + return { kind: "num", t: num(ctx, Number(expr.text.replace(/_/g, ""))) }; + } + if (ts.isStringLiteral(expr) || ts.isNoSubstitutionTemplateLiteral(expr)) { + return { kind: "str", t: strc(ctx, expr.text) }; + } + if (expr.kind === ts.SyntaxKind.TrueKeyword) return { kind: "bool", t: bool(ctx, true) }; + if (expr.kind === ts.SyntaxKind.FalseKeyword) return { kind: "bool", t: bool(ctx, false) }; + if (expr.kind === ts.SyntaxKind.NullKeyword) return { kind: "null" }; + if (expr.kind === ts.SyntaxKind.UndefinedKeyword) return { kind: "undef" }; + if (ts.isIdentifier(expr)) { + const v = frame.vars.get(expr.text); + if (v) return v; + const mi = curInfo(); + const c = mi.numbers.get(expr.text) ?? info.numbers.get(expr.text); + if (c !== undefined) return { kind: "num", t: num(ctx, c) }; + const s = mi.strings.get(expr.text) ?? info.strings.get(expr.text); + if (s !== undefined) return { kind: "str", t: strc(ctx, s) }; + const arrow = mi.constArrowFns.get(expr.text) ?? info.constArrowFns.get(expr.text); + if (arrow) return evalExpr(arrow, frame, st, depth); + const binding = mi.constBindings.get(expr.text) ?? info.constBindings.get(expr.text); + if (binding) return evalExpr(binding, frame, st, depth); + const imported = importedSymbols.get(expr.text); + if (imported) { + const ic = imported.info.numbers.get(expr.text); + if (ic !== undefined) return { kind: "num", t: num(ctx, ic) }; + const is = imported.info.strings.get(expr.text); + if (is !== undefined) return { kind: "str", t: ctx.String.val(is) }; + const ia = imported.info.constArrowFns.get(expr.text); + if (ia) return evalExpr(ia, frame, st, depth); + const ib = imported.info.constBindings.get(expr.text); + if (ib) return evalExpr(ib, frame, st, depth); + } + if (expr.text === "undefined") return { kind: "undef" }; + bad(expr, `unbound identifier ${expr.text}`); + } + if (ts.isParenthesizedExpression(expr)) return evalExpr(expr.expression, frame, st, depth); + if (ts.isAsExpression(expr) || ts.isNonNullExpression(expr) || ts.isSatisfiesExpression(expr) || ts.isTypeAssertionExpression(expr)) { + return evalExpr(expr.expression, frame, st, depth); + } + if (ts.isTypeOfExpression(expr)) { + const v = evalExpr(expr.expression, frame, st, depth); + switch (v.kind) { + case "str": return { kind: "str", t: ctx.String.val("string") }; + case "num": return { kind: "str", t: ctx.String.val("number") }; + case "bool": return { kind: "str", t: ctx.String.val("boolean") }; + case "obj": case "null": return { kind: "str", t: ctx.String.val("object") }; + case "arr": return { kind: "str", t: ctx.String.val("object") }; + case "fn": return { kind: "str", t: ctx.String.val("function") }; + case "undef": return { kind: "str", t: ctx.String.val("undefined") }; + default: degrade(st, expr, "typeof on degraded value"); return { kind: "any" }; + } + } + if (ts.isPrefixUnaryExpression(expr)) { + if (expr.operator === ts.SyntaxKind.ExclamationToken) { + const v = evalExpr(expr.operand, frame, st, depth); + const t = truthy(ctx, v); + return t ? { kind: "bool", t: t.not() } : { kind: "any" }; + } + if (expr.operator === ts.SyntaxKind.MinusToken) { + const v = evalExpr(expr.operand, frame, st, depth); + return isNum(v) ? { kind: "num", t: v.t.neg() } : { kind: "any" }; + } + if (expr.operator === ts.SyntaxKind.PlusToken) return evalExpr(expr.operand, frame, st, depth); + bad(expr, `prefix ${ts.SyntaxKind[expr.operator]}`); + } + if (ts.isBinaryExpression(expr)) return evalBinary(expr, frame, st, depth); + if (ts.isConditionalExpression(expr)) { + const c = truthy(ctx, evalExpr(expr.condition, frame, st, depth)); + if (!c) { degrade(st, expr, "ternary condition degraded"); return { kind: "any" }; } + const cb = constBool(c); + if (cb === true) return evalExpr(expr.whenTrue, frame, st, depth); + if (cb === false) return evalExpr(expr.whenFalse, frame, st, depth); + const a = evalExpr(expr.whenTrue, frame, st, depth); + const b = evalExpr(expr.whenFalse, frame, st, depth); + return iteVal(c, a, b); + } + if (ts.isTemplateExpression(expr)) { + const parts: SymVal[] = [{ kind: "str", t: ctx.String.val(expr.head.text) }]; + for (const span of expr.templateSpans) { + let v = evalExpr(span.expression, frame, st, depth); + if (isNum(v)) { + // number→string interpolation: uninterpreted 1-8 digit string. + // Content-irrelevant for structural properties; replay lock still + // validates any witness end-to-end. + const hole = ctx.String.const("__numstr_" + numHoleCounter++); + pushAsum(st, (ctx.ToReal(hole.length() as never) as Num).ge(num(ctx, 1)).and((ctx.ToReal(hole.length() as never) as Num).le(num(ctx, 8)))); + v = { kind: "str", t: hole }; + } + parts.push(v); + } + return concatAll(parts); + } + if (ts.isArrayLiteralExpression(expr)) { + const elems: SymVal[] = []; + for (const el of expr.elements) { + if (ts.isSpreadElement(el)) { + const v = evalExpr(el.expression, frame, st, depth); + if (isArr(v)) elems.push(...v.elems); + else degrade(st, el, "spread of non-array in literal"); + } else { + elems.push(evalExpr(el, frame, st, depth)); + } + } + const count = expr.elements.filter((e) => !ts.isSpreadElement(e)).length + + expr.elements.filter((e) => ts.isSpreadElement(e)).length; // spread len merged below if needed + void count; + // declared length: count of elements (spread assumed full — guarded code + // builds arrays with concrete sources; replay lock validates) + let declaredLen: Num = num(ctx, 0); + for (const el of expr.elements) { + if (ts.isSpreadElement(el)) { + const v = evalExpr(el.expression, frame, st, depth); + declaredLen = isArr(v) ? declaredLen.add(v.len) : declaredLen; + } else declaredLen = declaredLen.add(num(ctx, 1)); + } + return arrOf(elems, declaredLen); + } + if (ts.isObjectLiteralExpression(expr)) { + let symFields: Map; v: SymVal }> | undefined; + const fields = new Map(); + for (const prop of expr.properties) { + if (ts.isSpreadAssignment(prop)) { + const v = evalExpr(prop.expression, frame, st, depth); + if (isObj(v)) { + for (const [k2, v2] of v.fields) fields.set(k2, v2); + if (v.symFields) symFields = new Map(v.symFields); + } + else degrade(st, prop, "object spread of non-object"); + } else if (ts.isPropertyAssignment(prop) && ts.isIdentifier(prop.name)) { + fields.set(prop.name.text, evalExpr(prop.initializer, frame, st, depth)); + } else if (ts.isShorthandPropertyAssignment(prop)) { + fields.set(prop.name.text, evalExpr(prop.name, frame, st, depth)); + } else { + degrade(st, prop, "object literal member"); + } + } + return { kind: "obj", fields, ...(symFields ? { symFields } : {}) }; + } + if (ts.isPropertyAccessExpression(expr)) { + const obj = evalExpr(expr.expression, frame, st, depth); + return member(obj, expr.name.text, expr, st); + } + if (ts.isElementAccessExpression(expr)) { + const obj = evalExpr(expr.expression, frame, st, depth); + const idx = evalExpr(expr.argumentExpression, frame, st, depth); + return elementAccess(obj, idx, expr, st); + } + if (ts.isCallExpression(expr)) return evalCall(expr, frame, st, depth); + if (ts.isNonNullExpression(expr)) { + // `x!` non-null assertion — type-level only, value is x + return evalExpr(expr.expression, frame, st, depth); + } + if (ts.isRegularExpressionLiteral(expr)) { + // /pat/flags — regex-literal carrier; String.replace/test model it. + return { kind: "obj", fields: new Map([["__regex", { kind: "str", t: ctx.String.val(expr.text) }]]) } as SymVal; + } + if (expr.kind === ts.SyntaxKind.PostfixUnaryExpression || expr.kind === ts.SyntaxKind.PrefixUnaryExpression) { + const u = expr as ts.PrefixUnaryExpression | ts.PostfixUnaryExpression; + const one = num(ctx, 1); + const v = evalExpr(u.operand, frame, st, depth); + if (isNum(v) && ts.isIdentifier(u.operand)) { + const cv0 = isNum(v) ? numC(v.t) : undefined; + const nv = cv0 !== undefined + ? { kind: "num" as const, t: num(ctx, u.operator === ts.SyntaxKind.PlusPlusToken ? cv0 + 1 : cv0 - 1) } + : { kind: "num" as const, t: u.operator === ts.SyntaxKind.PlusPlusToken ? v.t.add(one) : v.t.sub(one) }; + frame.vars.set(u.operand.text, nv); + return u.kind === ts.SyntaxKind.PrefixUnaryExpression ? nv : v; + } + bad(expr, "update expression on non-number"); + } + if (ts.isDeleteExpression(expr)) { + // `delete obj[k]` / `delete obj.k`: the slot reads as undefined + // afterwards. Modeled as an undef store (missing field ≡ undef). + const t = expr.expression; + if (ts.isElementAccessExpression(t)) { + const base = evalExpr(t.expression, frame, st, depth); + const idx = evalExpr(t.argumentExpression, frame, st, depth); + if (isObj(base) && isStr(idx)) { + const fields = new Map(base.fields); + for (const key of [...fields.keys()]) { + fields.set(key, iteVal(idx.t.eq(ctx.String.val(key)), { kind: "undef" }, fields.get(key)!)); + } + // symbolic-key delete: drop symFields entries whose key matches the + // index term (structural identity) — the merge machinery then folds + // the removal into the enclosing path guard. + let symFields: Map; v: SymVal }> | undefined; + if (base.symFields) { + symFields = new Map(base.symFields); + for (const kt of [...symFields.keys()]) { + if (kt === idx.t || (kt.hash !== undefined && idx.t.hash !== undefined && kt.hash() === idx.t.hash())) { + symFields.delete(kt); + } + } + } + const newObj: SymVal = { kind: "obj", fields, ...(symFields ? { symFields } : {}) }; + writeLValue(t.expression, newObj, frame, st, depth); + return { kind: "bool", t: bool(ctx, true) }; + } + } else if (ts.isPropertyAccessExpression(t)) { + const base = evalExpr(t.expression, frame, st, depth); + if (isObj(base)) { + const fields = new Map(base.fields); + fields.set(t.name.text, { kind: "undef" }); + writeLValue(t.expression, { kind: "obj", fields }, frame, st, depth); + return { kind: "bool", t: bool(ctx, true) }; + } + } + return { kind: "bool", t: bool(ctx, true) }; + } + return bad(expr, `expression ${ts.SyntaxKind[expr.kind]}`); + }; + + // (TS2366: `bad` throws; this satisfies the checker without changing behavior.) + + const rootName = (e: ts.PropertyAccessExpression): string | undefined => + ts.isIdentifier(e.expression) ? e.expression.text : ts.isPropertyAccessExpression(e.expression) ? rootName(e.expression) : undefined; + + const compoundOpKind = (k: ts.SyntaxKind): ts.BinaryOperatorToken["kind"] | undefined => { + switch (k) { + case ts.SyntaxKind.PlusEqualsToken: return ts.SyntaxKind.PlusToken; + case ts.SyntaxKind.MinusEqualsToken: return ts.SyntaxKind.MinusToken; + case ts.SyntaxKind.AsteriskEqualsToken: return ts.SyntaxKind.AsteriskToken; + default: return undefined; + } + }; + + const concatAll = (parts: SymVal[]): SymVal => { + const strs: Str[] = []; + let allConst = true; + let acc = ""; + for (const p of parts) { + if (!isStr(p)) { + return { kind: "any" }; + } + const sv = strConst.get(p.t); + if (sv === undefined) allConst = false; + else acc += sv; + strs.push(p.t); + } + if (strs.length === 0) return { kind: "str", t: strc(ctx, "") }; + if (allConst) return { kind: "str", t: strc(ctx, acc) }; + return { kind: "str", t: strs.reduce((a, b) => a.concat(b)) }; + }; + + const member = (obj: SymVal, name: string, at: ts.Node, st: RunState): SymVal => { + if (isArr(obj)) { + if (name === "length") return { kind: "num", t: obj.len }; + return { kind: "any" }; // methods are handled at call sites + } + if (isStr(obj)) { + if (name === "length") { + const sv = strConst.get(obj.t); + if (sv !== undefined) return { kind: "num", t: num(ctx, sv.length) }; + return { kind: "num", t: lenR(obj.t) }; + } + return { kind: "any" }; + } + if (isObj(obj)) { + // Map/Set .size (property, not method) + if (name === "size" && (obj.fields.has("__map_entries") || obj.fields.has("__set_values"))) { + const e = obj.fields.get("__map_entries") ?? obj.fields.get("__set_values"); + if (e && isArr(e)) return { kind: "num", t: e.len }; + } + const v = obj.fields.get(name); + // missing field reads as `undefined` in JS (NOT unknown/any) — + // `!!message.toolName` on a field-less message must fold to false, + // not poison the condition + return v ?? { kind: "undef" }; + } + if (obj.kind === "null" || obj.kind === "undef") return { kind: "undef" }; + return { kind: "any" }; + }; + + + const elementAccess = (obj: SymVal, idx: SymVal, at: ts.Node, st: RunState): SymVal => { + if (isArr(obj) && isNum(idx)) { + const ic = numC(idx.t); + if (ic !== undefined && Number.isInteger(ic)) { + const rv = ic >= 0 && ic < obj.elems.length ? obj.elems[ic]! : { kind: "undef" as const }; + return rv; + } + // Chain of If(idx == k, elems[k]) for k < N. + let acc: SymVal = { kind: "undef" }; + for (let k = obj.elems.length - 1; k >= 0; k--) { + acc = iteVal(numEq(ctx, idx.t, num(ctx, k)), obj.elems[k], acc); + } + return acc; + } + if (isObj(obj) && isStr(idx)) { + // Dynamic key: pick by string equality over known keys — concrete + // fields first, then symbolic-key bindings (symFields). The FIRST + // layer must keep its key-match guard: wrapping into the in-domain + // neutral (""/0/false) rather than {undef} — the undef-loses merge + // rule would collapse ite(match, v, undef) to v unconditionally and + // hand out values for keys that never matched (phantom reads). + let acc: SymVal = { kind: "undef" }; + const fanLayer = (c: Bool<"main">, v: SymVal, accv: SymVal): SymVal => { + if (accv.kind === "undef" || accv.kind === "null") return iteVal(c, v, neutralFor(v)); + return iteVal(c, v, accv); + }; + for (const [key, v] of obj.fields) { + acc = fanLayer(idx.t.eq(ctx.String.val(key)), v, acc); + } + if (obj.symFields) { + for (const [kt, e] of obj.symFields) { + acc = fanLayer(andFold(idx.t.eq(kt), e.g)!, e.v, acc); + } + } + return acc; + } + return { kind: "any" }; + }; + + const ASSIGN_OPS: ts.SyntaxKind[] = [ + ts.SyntaxKind.EqualsToken, ts.SyntaxKind.PlusEqualsToken, ts.SyntaxKind.MinusEqualsToken, + ts.SyntaxKind.AsteriskEqualsToken, ts.SyntaxKind.SlashEqualsToken, ts.SyntaxKind.PercentEqualsToken, + ]; + + /** Recursive lvalue store with value semantics: rebuilding a[i].f = v + * through the chain (slot-write the array, then rebind upward). Real JS + * uses references — `host = msgs[k]; host.f = v` is visible via msgs[k] + * — so chained stores must rewrite every container on the path. */ + const writeLValue = (target: ts.Expression, value: SymVal, frame: Frame, st: RunState, depth: number): boolean => { + if (ts.isNonNullExpression(target)) return writeLValue(target.expression, value, frame, st, depth); + if (ts.isIdentifier(target)) { frame.vars.set(target.text, value); return true; } + if (ts.isPropertyAccessExpression(target)) { + const base = evalExpr(target.expression, frame, st, depth); + if (isObj(base)) { + const fields = new Map(base.fields); + fields.set(target.name.text, value); + const newObj = { kind: "obj", fields, ...(base.symFields ? { symFields: new Map(base.symFields) } : {}) } as SymVal; + // JS reference semantics: the same object is reachable from its + // containers (`const host = msgs[k]` / find()) — a field write must + // be visible through every array (and one level of object field) + // that still holds this exact value. Value-copy alone rebinds only + // the local alias and the container keeps the stale object. + for (const [vn, vv] of frame.vars) { + if (isArr(vv)) { + let hit = false; + const elems = vv.elems.map((e) => { if (e === base) { hit = true; return newObj; } return e; }); + if (hit) frame.vars.set(vn, { kind: "arr", elems, len: vv.len } as SymVal); + } else if (isObj(vv) && vv !== base) { + let hit = false; + const nf: Map = new Map(vv.fields); + for (const [fk, fv] of nf) if (fv === base) { nf.set(fk, newObj); hit = true; } + if (hit) frame.vars.set(vn, { kind: "obj", fields: nf, ...(vv.symFields ? { symFields: new Map(vv.symFields) } : {}) } as SymVal); + } + } + return writeLValue(target.expression, newObj, frame, st, depth); + } + return false; + } + if (ts.isElementAccessExpression(target)) { + const base = evalExpr(target.expression, frame, st, depth); + const idx = evalExpr(target.argumentExpression, frame, st, depth); + if (isArr(base) && isNum(idx)) { + const elems = base.elems.map((e, k) => iteVal(numEq(ctx, idx.t, num(ctx, k)), value, e)); + if (base.elems.length < N) { + elems.push(iteVal(numEq(ctx, idx.t, num(ctx, base.elems.length)), value, { kind: "undef" })); + } + // a[i] = v with i ≥ len grows the array to i+1 + const grown = iteVal(idx.t.gt(base.len), { kind: "num", t: idx.t.add(num(ctx, 1)) }, { kind: "num", t: base.len }); + return writeLValue(target.expression, { kind: "arr", elems, len: (grown as { t: Num }).t } as never, frame, st, depth); + } + if (isObj(base) && isStr(idx)) { + // Record write: obj[key] = v. Const keys (the common case — ids, + // refs) write the field directly. Symbolic keys fan out over the + // fields that already exist AND record a symFields binding so the + // write is observable when the key matches nothing present (the + // solver decides equality). + const fields = new Map(base.fields); + const idxC = idx.t.c; + if (typeof idxC === "string" && idx.t.sort === "String" && (idx.t.kids?.length ?? 0) === 0) { + fields.set(idxC, value); + } else { + for (const key of [...fields.keys()]) { + fields.set(key, iteVal(idx.t.eq(ctx.String.val(key)), value, fields.get(key)!)); + } + const symFields = new Map(base.symFields ?? []); + // same key-TERM: the new write replaces it (last write wins for + // identical terms); equal-but-distinct terms resolve via the read + // fan's guard, which falls through dead writes. + symFields.delete(idx.t); + symFields.set(idx.t, { g: bool(ctx, true), v: value }); + return writeLValue(target.expression, { kind: "obj", fields, symFields } as SymVal, frame, st, depth); + } + return writeLValue(target.expression, { kind: "obj", fields } as SymVal, frame, st, depth); + } + return false; + } + return false; + }; + + const evalBinary = (expr: ts.BinaryExpression, frame: Frame, st: RunState, depth: number): SymVal => { + const op = expr.operatorToken.kind; + // Assignments (x = …, x += …) are BinaryExpressions: they MUST be + // handled here at the TOP of evalBinary — evalExpr's generic + // isBinaryExpression dispatch reaches this function for every binary + // node, and a fallthrough to applyBinop would evaluate but never STORE. + if (ASSIGN_OPS.includes(op)) { + const compound = compoundOpKind(op); + const rhs = op === ts.SyntaxKind.EqualsToken ? evalExpr(expr.right, frame, st, depth) : null; + const cur = compound ? evalExpr(expr.left, frame, st, depth) : null; + const value = compound && cur ? applyBinop(compound, cur, evalExpr(expr.right, frame, st, depth), expr, st) : rhs; + if (value && writeLValue(expr.left, value, frame, st, depth)) return value; + bad(expr, "assignment target"); + } + if (op === ts.SyntaxKind.AmpersandAmpersandToken) { + const l = evalExpr(expr.left, frame, st, depth); + return jsAnd(l, () => evalExpr(expr.right, frame, st, depth)); + } + if (op === ts.SyntaxKind.BarBarToken) { + const l = evalExpr(expr.left, frame, st, depth); + return jsOr(l, () => evalExpr(expr.right, frame, st, depth)); + } + if (op === ts.SyntaxKind.QuestionQuestionToken) { + const l = evalExpr(expr.left, frame, st, depth); + return jsNullish(l, () => evalExpr(expr.right, frame, st, depth)); + } + return applyBinop(op, evalExpr(expr.left, frame, st, depth), evalExpr(expr.right, frame, st, depth), expr, st); + }; + + const applyBinop = (op: ts.SyntaxKind, l: SymVal, r: SymVal, at: ts.Node, st: RunState): SymVal => { + if (isNum(l) && isNum(r)) { + const a = l.t, b = r.t; + const ac = numC(a), bc = numC(b); + if (ac !== undefined && bc !== undefined) { + const fold = (): SymVal | null => { + switch (op) { + case ts.SyntaxKind.PlusToken: return { kind: "num", t: num(ctx, ac + bc) }; + case ts.SyntaxKind.MinusToken: return { kind: "num", t: num(ctx, ac - bc) }; + case ts.SyntaxKind.AsteriskToken: return { kind: "num", t: num(ctx, ac * bc) }; + case ts.SyntaxKind.SlashToken: + if (bc !== 0 && Number.isInteger(ac) && Number.isInteger(bc) && ac % bc === 0) return { kind: "num", t: num(ctx, ac / bc) }; + return null; + case ts.SyntaxKind.PercentToken: + if (Number.isInteger(ac) && Number.isInteger(bc) && bc !== 0) return { kind: "num", t: num(ctx, ac % bc) }; + return null; + case ts.SyntaxKind.LessThanToken: return { kind: "bool", t: bool(ctx, ac < bc) }; + case ts.SyntaxKind.LessThanEqualsToken: return { kind: "bool", t: bool(ctx, ac <= bc) }; + case ts.SyntaxKind.GreaterThanToken: return { kind: "bool", t: bool(ctx, ac > bc) }; + case ts.SyntaxKind.GreaterThanEqualsToken: return { kind: "bool", t: bool(ctx, ac >= bc) }; + case ts.SyntaxKind.EqualsEqualsToken: + case ts.SyntaxKind.EqualsEqualsEqualsToken: return { kind: "bool", t: bool(ctx, ac === bc) }; + case ts.SyntaxKind.ExclamationEqualsToken: + case ts.SyntaxKind.ExclamationEqualsEqualsToken: return { kind: "bool", t: bool(ctx, ac !== bc) }; + default: return null; + } + }; + const f = fold(); + if (f) return f; + } + switch (op) { + case ts.SyntaxKind.PlusToken: return { kind: "num", t: a.add(b) }; + case ts.SyntaxKind.MinusToken: return { kind: "num", t: a.sub(b) }; + case ts.SyntaxKind.AsteriskToken: return { kind: "num", t: a.mul(b) }; + case ts.SyntaxKind.SlashToken: return { kind: "num", t: a.div(b) }; + case ts.SyntaxKind.PercentToken: return { kind: "num", t: a.mod(b) }; + case ts.SyntaxKind.LessThanToken: return { kind: "bool", t: a.lt(b) }; + case ts.SyntaxKind.LessThanEqualsToken: return { kind: "bool", t: a.le(b) }; + case ts.SyntaxKind.GreaterThanToken: return { kind: "bool", t: a.gt(b) }; + case ts.SyntaxKind.GreaterThanEqualsToken: return { kind: "bool", t: a.ge(b) }; + case ts.SyntaxKind.EqualsEqualsToken: + case ts.SyntaxKind.EqualsEqualsEqualsToken: return { kind: "bool", t: a.eq(b) }; + case ts.SyntaxKind.ExclamationEqualsToken: + case ts.SyntaxKind.ExclamationEqualsEqualsToken: return { kind: "bool", t: a.neq(b) }; + } + } + if (isStr(l) && isStr(r)) { + const lc = strConst.get(l.t), rc = strConst.get(r.t); + if (op === ts.SyntaxKind.PlusToken) { + if (lc !== undefined && rc !== undefined) return { kind: "str", t: strc(ctx, lc + rc) }; + return { kind: "str", t: l.t.concat(r.t) }; + } + if (lc !== undefined && rc !== undefined) { + if (op === ts.SyntaxKind.EqualsEqualsToken || op === ts.SyntaxKind.EqualsEqualsEqualsToken) return { kind: "bool", t: bool(ctx, lc === rc) }; + if (op === ts.SyntaxKind.ExclamationEqualsToken || op === ts.SyntaxKind.ExclamationEqualsEqualsToken) return { kind: "bool", t: bool(ctx, lc !== rc) }; + } + if (op === ts.SyntaxKind.EqualsEqualsToken || op === ts.SyntaxKind.EqualsEqualsEqualsToken) return { kind: "bool", t: l.t.eq(r.t) }; + if (op === ts.SyntaxKind.ExclamationEqualsToken || op === ts.SyntaxKind.ExclamationEqualsEqualsToken) return { kind: "bool", t: l.t.neq(r.t) }; + } + if (isBool(l) && isBool(r)) { + if (op === ts.SyntaxKind.EqualsEqualsToken || op === ts.SyntaxKind.EqualsEqualsEqualsToken) return { kind: "bool", t: l.t.eq(r.t) }; + if (op === ts.SyntaxKind.ExclamationEqualsToken || op === ts.SyntaxKind.ExclamationEqualsEqualsToken) return { kind: "bool", t: l.t.neq(r.t) }; + } + // Mixed num/str `+` is JS string concatenation — route through the + // memoized number→string rendering (same number always same string). + if (op === ts.SyntaxKind.PlusToken && isNum(l) && isStr(r)) { + const ls = numToStringStrFor(l.t); + return { kind: "str", t: ls.t.concat(r.t) }; + } + if (op === ts.SyntaxKind.PlusToken && isStr(l) && isNum(r)) { + const rs = numToStringStrFor(r.t); + return { kind: "str", t: l.t.concat(rs.t) }; + } + // Mixed equality against null/undefined + if ((l.kind === "null" || l.kind === "undef") || (r.kind === "null" || r.kind === "undef")) { + if (op === ts.SyntaxKind.EqualsEqualsEqualsToken || op === ts.SyntaxKind.EqualsEqualsToken) { + return { kind: "bool", t: bool(ctx, l.kind === r.kind) }; + } + if (op === ts.SyntaxKind.ExclamationEqualsToken || op === ts.SyntaxKind.ExclamationEqualsEqualsToken) { + return { kind: "bool", t: bool(ctx, l.kind !== r.kind) }; + } + } + return { kind: "any" }; + }; + + /* ---------------- calls ---------------- */ + + let reTestN = 0; + const evalCall = (call: ts.CallExpression, frame: Frame, st: RunState, depth: number): SymVal => { + // Spread arguments: flatten arrays of args (rare in guarded code). + const argVals: SymVal[] = []; + for (const a of call.arguments) { + if (ts.isSpreadElement(a)) { + const v = evalExpr(a.expression, frame, st, depth); + if (isArr(v)) argVals.push(...v.elems); + else degrade(st, a, "spread argument"); + } else { + argVals.push(evalExpr(a, frame, st, depth)); + } + } + const calleeExpr = call.expression; + + // Namespace-qualified builtins ------------------------------------ + if (ts.isPropertyAccessExpression(calleeExpr) && ts.isIdentifier(calleeExpr.expression)) { + const ns = calleeExpr.expression.text; + const meth = calleeExpr.name.text; + if (ns === "Number") { + if (meth === "isInteger" && isNum(argVals[0])) { + return { kind: "bool", t: ctx.IsInt(argVals[0].t) }; + } + if (meth === "isFinite" && isNum(argVals[0])) return { kind: "bool", t: bool(ctx, true) }; + if (meth === "isFinite" && !isNum(argVals[0])) return { kind: "bool", t: bool(ctx, false) }; + if (meth === "isInteger" && !isNum(argVals[0])) return { kind: "bool", t: bool(ctx, false) }; + if ((meth === "parseInt" || meth === "parseFloat") && isStr(argVals[0])) { + return { kind: "num", t: argVals[0].t.toInt() }; + } + } + if (ns === "Math") return mathCall(meth, argVals, call, st); + if (ns === "Array" && meth === "isArray") { + if (argVals[0].kind === "arr") return { kind: "bool", t: bool(ctx, true) }; + // undefined/null/bool/fn are provably not arrays (JS typeof) + if (argVals[0].kind !== "any") return { kind: "bool", t: bool(ctx, false) }; + degrade(st, call, "Array.isArray on degraded value"); + return { kind: "any" }; + } + if (ns === "String" && meth === "fromCharCode") return { kind: "str", t: ctx.String.val("") }; // degraded, content-neutral + if (ns === "JSON") { + // JSON.parse of a KNOWN string is pure and deterministic — run the + // real implementation and convert. Degraded args still degrade. + if (meth === "parse") { + const a0 = argVals[0]; + if (isStr(a0)) { + const lit = strConst.get(a0.t); + if (lit !== undefined) { + try { return jsonToSymVal(JSON.parse(lit), depth); } catch { return { kind: "null" }; } + } + } + } + degrade(st, call, `JSON.${meth}`); return { kind: "any" }; + } + if (ns === "Object" || ns === "Boolean") { degrade(st, call, `${ns}.${meth}`); return { kind: "any" }; } + } + + // Method calls on values ------------------------------------------- + if (ts.isPropertyAccessExpression(calleeExpr)) { + // optional call `x.m?.(…)`: absent method reads as undefined (no throw) + if (call.questionDotToken !== undefined && !(input.options?.portImpls && ("__method_" + calleeExpr.name.text) in input.options.portImpls)) { + const r0 = evalExpr(calleeExpr.expression, frame, st, depth); + if (isObj(r0)) { + const mv = r0.fields.get(calleeExpr.name.text); + if (mv === undefined || mv.kind === "undef" || mv.kind === "null") return { kind: "undef" }; + } + } + const recv = evalExpr(calleeExpr.expression, frame, st, depth); + const m = calleeExpr.name.text; + return callMethod(recv, m, argVals, call, st, frame, depth); + } + + // Plain identifier calls ------------------------------------------- + const name = ts.isIdentifier(calleeExpr) ? calleeExpr.text : undefined; + if (!name) { degrade(st, call, "indirect call"); return { kind: "any" }; } + + // Builtins with exact translation + if (name === "Number.isInteger" || name === "Number.isFinite") { + const a0 = argVals[0]; + if (isNum(a0)) { + if (name === "Number.isInteger") return { kind: "bool", t: ctx.IsInt(a0.t) }; + return { kind: "bool", t: bool(ctx, true) }; // finite within Real model + } + return { kind: "bool", t: bool(ctx, false) }; // undefined/null/non-number + } + if (name === "Number" || name === "String") { + const a0 = argVals[0]; + if (name === "String") { + if (isStr(a0)) return a0; + if (isNum(a0)) return numToStringStrFor(a0.t); + return { kind: "any" }; + } + if (isNum(a0)) return a0; + // Number(constString) folds faithfully — refNum() and friends parse + // digit strings; without this fold Number.isFinite(any) collapses to + // constant false and gated paths silently disappear from the model. + if (isStr(a0)) { + const cs = (a0.t as unknown as { c?: string }).c; + if (typeof cs === "string" && cs.trim() !== "" && !Number.isNaN(Number(cs))) { + return { kind: "num", t: num(ctx, Number(cs)) }; + } + } + return { kind: "any" }; + } + if (name === "isNaN") { + const a0 = argVals[0]; + return isNum(a0) ? { kind: "bool", t: bool(ctx, false) } : { kind: "any" }; + } + + // Local closure value call (const arrow / function expression stored in a var) + const fnVal = frame.vars.get(name); + if (fnVal && fnVal.kind === "fn") { + return invokeCallback(fnVal.node, argVals, st, depth, fnVal.env); + } + + // Port override (public injection seam); memoized on the argument's + // structural form — port implementations are pure functions. + // Frame-bound names (parameters, port-assigned locals) take ports FIRST; + // imported names take ports only when the test declares them (e.g. hashId + // uses node:crypto — outside the guarded subset). Ports NEVER shadow + // module-local real functions (the no-cheating core: real code always + // runs). Multi-arg ports receive all arguments (single-arg only is + // memoized; arg0-keyed memo is sound only for unary ports). + const moduleLocalFn = curInfo().functions.get(name) ?? info.functions.get(name); + const moduleLocalArrow = curInfo().constArrowFns.get(name) ?? info.constArrowFns.get(name); + if (input.options?.portImpls && name in input.options.portImpls && !moduleLocalFn && !moduleLocalArrow) { + const arg0 = argVals[0] ?? { kind: "undef" }; + const key = arg0.kind === "str" ? `s:${String((arg0.t as unknown as { c?: string }).c ?? `h${arg0.t.hash()}`)}` : arg0.kind === "num" ? `n:${String((arg0.t as unknown as { c?: number }).c ?? numC(arg0.t))}` : null; + if (key !== null && argVals.length === 1) { + const hit = portMemo.get(`${name}\u0000${key}`); + if (hit) return hit; + const r = input.options.portImpls[name]!(arg0); + portMemo.set(`${name}\u0000${key}`, r); + return r; + } + return input.options.portImpls[name]!(arg0, ...argVals.slice(1)); + } + + // Local function → INLINE (the no-cheating core: real helper bodies run) + const entryFn = info.functions.get(name); + const entryArrow = info.constArrowFns.get(name) as ts.ArrowFunction | undefined; + const defEntry = importedSymbols.get(name); + const localFn = moduleLocalFn ?? defEntry?.info.functions.get(name) ?? entryFn; + const defArrow = importedSymbols.get(name)?.info.constArrowFns.get(name) as ts.ArrowFunction | undefined; + const localArrow = moduleLocalArrow ?? defArrow ?? entryArrow; + const fnNode = localFn ?? (localArrow as ts.ArrowFunction | undefined); + if (fnNode) { + if (depth >= MAXINLINE) { degrade(st, call, `inline depth cap at ${name}`); return { kind: "any" }; } + return inlineCall(fnNode, argVals, call, st, depth, defEntry?.info); + } + + degrade(st, call, `call to ${name}`); + return { kind: "any" }; + }; + + const inlineCall = ( + fn: ts.FunctionDeclaration | ts.ArrowFunction | ts.FunctionExpression, + args: SymVal[], + at: ts.Node, + st: RunState, + depth: number, + defInfo?: ModuleInfo, + ): SymVal => { + const nm = fn.name?.text || "(anon)"; + if (defInfo) moduleStack.push(defInfo); + try { + const inner: Frame = { vars: new Map(frame.vars) }; // lexical closure + const params = ts.isFunctionDeclaration(fn) ? fn.parameters : fn.parameters; + params.forEach((p, i) => { + if (ts.isIdentifier(p.name)) { + inner.vars.set(p.name.text, args[i] ?? (p.initializer ? evalExpr(p.initializer, inner, st, depth + 1) : { kind: "undef" })); + } else if (ts.isArrayBindingPattern(p.name)) { + // destructure array param: bind pattern elements to slots + p.name.elements.forEach((el, j) => { + if (ts.isBindingElement(el) && ts.isIdentifier(el.name)) { + inner.vars.set(el.name.text, args[i] && isArr(args[i]) ? args[i].elems[j] : { kind: "undef" }); + } + }); + } + }); + const innerState: RunState = runState0(); + const body = ts.isFunctionDeclaration(fn) ? fn.body : fn.body; + if (!body) return { kind: "any" }; + if (ts.isBlock(body)) { + execStmt(body, inner, innerState, depth + 1); + } else { + // arrow with expression body + innerState.ret = evalExpr(body, inner, innerState, depth + 1); + } + const pushAll2 = (dst: T[], src: T[]): void => { if (src !== dst) for (const x of src) dst.push(x); }; + pushAll2(st.assumptions, innerState.assumptions); + // degradations shared + if (innerState.thrown) st.thrown = orFold(st.thrown ?? bool(ctx, false), innerState.thrown); + // fold conditional returns (pendingRets) LIFO over the unconditional ret + let out = innerState.ret; + for (let i = innerState.pendingRets.length - 1; i >= 0; i--) { + const pr = innerState.pendingRets[i]!; + out = out === null ? iteVal(pr.cond, pr.val, { kind: "undef" }) : iteVal(pr.cond, pr.val, out); + } + return out ?? { kind: "undef" }; + } finally { + if (defInfo) moduleStack.pop(); + } + }; + + + const callMethod = ( + recv: SymVal, m: string, args: SymVal[], + at: ts.CallExpression, st: RunState, frame: Frame, depth: number, + ): SymVal => { + // ---- namespace objects ---- + if (isObj(recv) && m === "call" ) { degrade(st, at, "fn.call"); return { kind: "any" }; } + + // Anchored dynamic regex over-approximation: a pattern "^A…" with a + // known literal head A only matches strings starting with A. On the + // ¬startsWith side .test is exactly false and .replace is the identity; + // on the reachable side the result is an unconstrained uninterpreted + // value (recorded as a degradation only when the guard can hold — + // replay validates any witness from that side on the real regex). + const regexAnchor = (patVal: SymVal): string | null => { + if (!isStr(patVal)) return null; + const pieces: string[] = []; + const walk = (t: unknown): void => { + const tt = t as { op?: string; kids?: unknown[]; c?: unknown; sort?: string }; + if (tt && typeof tt === "object" && Array.isArray(tt.kids)) { + // concat shapes: app2("str.++", a, b) [facade] or op "str.++"/"concat" kids [a, b] + const k0 = tt.kids[0] as { op?: unknown; c?: unknown } | undefined; + const isConcat = tt.op === "concat" || tt.op === "str.++" + || (tt.op === "app2" && (k0?.op === "str.++" || k0?.c === "str.++")); + if (isConcat && (tt.kids.length === 2 || tt.kids.length === 3)) { + for (const k of tt.kids.slice(tt.kids.length === 3 ? 1 : 0)) walk(k); + return; + } + } + if (tt && typeof tt === "object" && typeof tt.c === "string" && tt.sort === "String") { + pieces.push(tt.c); return; + } + // symbolic piece (or non-const): stops accumulation — the anchor is + // the const head collected so far (may be empty → null). + }; + walk(patVal.t); + if (pieces.length === 0) { + const whole = strConst.get(patVal.t); + if (whole === undefined) return null; + pieces.push(whole); + } + if (!pieces[0]!.startsWith("^")) return null; + const body = pieces.join("").slice(1); + // Literal head: chars up to the first regex meta (backslash escapes + // end the head too — their target is regex syntax, not plain text). + const m = /^((?:\\.|[^[\](){}*+?|^$.\\])*)/.exec(body); + const head = m?.[1] ?? ""; + return head === "" ? null : head; + }; + const dynRegexTest = (recv: SymVal, patVal: SymVal, at: ts.Node, st: RunState): SymVal | null => { + if (!isStr(recv)) return null; + const anchor = regexAnchor(patVal); + if (anchor === null || anchor === "") return null; + const g = recv.t.prefixOf(ctx.String.val(anchor)); + const gb = constBool(g); + if (gb === false) return { kind: "bool", t: bool(ctx, false) }; + const fresh = ctx.Bool.const(`__re_test_${reTestN++}`); + if (gb !== true) degrade(st, at, "dynamic regex .test over-approx"); + return { kind: "bool", t: ctx.If(g, fresh, bool(ctx, false)) }; + + }; + + // ---- method ports: "__method_" receives (recv, ...args) ---- + const mport = input.options?.portImpls ? input.options.portImpls["__method_" + m] : undefined; + if (mport && (isObj(recv) || isArr(recv))) return (mport as (r: SymVal, ...a: SymVal[]) => SymVal)(recv, ...args); + + // ---- Math ---- + // (Math.x handled at plain-call sites in source; but method form via + // identifier "Math" reaches here only through plain calls.) + + // ---- array methods ---- + if (isArr(recv)) { + switch (m) { + case "length": return { kind: "num", t: recv.len }; + case "push": { + const v = args[0]; + if (recv.elems.length < N) { + // append at the symbolic tail position (arrays built by push) + const elems = recv.elems.map((e, k) => iteVal(numEq(ctx, recv.len, num(ctx, k)), v, e)); + elems.push(iteVal(numEq(ctx, recv.len, num(ctx, recv.elems.length)), v, { kind: "undef" })); + const newLen = recv.len.add(num(ctx, 1)); + return rebindArray(recv, { kind: "arr", elems, len: newLen }, at, st, frame); + } + degrade(st, at, "push beyond bounded slots"); + return { kind: "any" }; + } + case "includes": { + const target = args[0]; + const eqs = recv.elems.map((e) => valEq(e, target)); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc = bool(ctx, false); + for (let k = 0; k < recv.elems.length; k++) { + acc = acc.or(recv.len.gt(num(ctx, k)).and(eqs[k]!)); + } + return { kind: "bool", t: acc }; + } + case "indexOf": { + const target = args[0]; + const eqs = recv.elems.map((e) => valEq(e, target)); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc: Num = num(ctx, -1); + for (let k = recv.elems.length - 1; k >= 0; k--) { + acc = ctx.If(recv.len.gt(num(ctx, k)).and(eqs[k]!), num(ctx, k), acc); + } + return { kind: "num", t: acc }; + } + case "map": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "map without function literal"); return { kind: "any" }; } + const elems = recv.elems.map((e, k) => { + const guarded = iteVal(recv.len.gt(num(ctx, k)), e, { kind: "undef" }); + const r = invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth); + return r; + }); + return { kind: "arr", elems, len: recv.len }; + } + case "filter": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "filter without function literal"); return { kind: "any" }; } + // Unrolled two-pass: keep-slot[j] via counting. + const keep: Bool<"main">[] = recv.elems.map((e, k) => { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, e, { kind: "undef" }); + const r = invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth); + const t = truthy(ctx, r); + if (!t) return g.not(); // out-of-length slots never keep + return g.and(t); + }); + const outElems: SymVal[] = []; + for (let j = 0; j < recv.elems.length; j++) { + // j-th kept element = elems[k] where keep[k] and exactly j keeps before k + let pick: SymVal = { kind: "undef" }; + for (let k = recv.elems.length - 1; k >= 0; k--) { + const before = countTrueBelow(keep, k); + const cond = keep[k]!.and(before.eq(num(ctx, j))); + pick = iteVal(cond, recv.elems[k], pick); + } + outElems.push(pick); + } + const newLen = keep.reduce((acc, kk) => ctx.If(kk, acc.add(num(ctx, 1)), acc), num(ctx, 0)); + return { kind: "arr", elems: outElems, len: newLen }; + } + case "every": + case "some": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, `${m} without function literal`); return { kind: "any" }; } + let acc = bool(ctx, m === "every"); + for (let k = 0; k < recv.elems.length; k++) { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, recv.elems[k], { kind: "undef" }); + const r = truthy(ctx, invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth)); + if (!r) { degrade(st, at, `${m} callback degraded`); return { kind: "any" }; } + const effective = g.and(r); + acc = m === "every" ? acc.and(effective) : acc.or(effective); + } + return { kind: "bool", t: acc }; + } + case "entries": { + // array iterator of [index, value] pairs, bounded to declared slots + const pairs = recv.elems.map((e, k) => ({ kind: "arr", elems: [{ kind: "num", t: num(ctx, k) }, e], len: num(ctx, 2) }) as SymVal); + return { kind: "arr", elems: pairs, len: recv.len }; + } + case "find": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "find without function literal"); return { kind: "any" }; } + let acc: SymVal = { kind: "undef" }; + for (let k = recv.elems.length - 1; k >= 0; k--) { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, recv.elems[k], { kind: "undef" }); + const r = truthy(ctx, invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth)); + if (!r) { degrade(st, at, "find callback degraded"); return { kind: "any" }; } + acc = iteVal(g.and(r), recv.elems[k], acc); + } + return acc; + } + case "findIndex": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "findIndex without function literal"); return { kind: "any" }; } + let acc: Num = num(ctx, -1); + for (let k = recv.elems.length - 1; k >= 0; k--) { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, recv.elems[k], { kind: "undef" }); + const r = truthy(ctx, invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth)); + if (!r) { degrade(st, at, "findIndex callback degraded"); return { kind: "any" }; } + acc = ctx.If(g.and(r), num(ctx, k), acc); + } + return { kind: "num", t: acc }; + } + case "sort": { + // Rank-based comparator sort, works for any slot count N (O(N²) comparator calls, + // no permutation enumeration). rank_i = #{j real: cmp(v_i,v_j)<0} + #{j => recv.len.gt(num(ctx, i)); // i < len + const ranks: Num[] = []; + for (let i = 0; i < n; i++) { + let rank = num(ctx, 0); + for (let j = 0; j < n; j++) { + if (i === j) continue; + const realJ = real(j); + if (recv.elems[i] === recv.elems[j]) { + // identical values: comparator returns 0 (deterministic) — tie + rank = rank.add(ctx.If(realJ.and(bool(ctx, j < i)), num(ctx, 1), num(ctx, 0)) as Num); + continue; + } + const r = invokeCallback(cbFn.node, [recv.elems[i]!, recv.elems[j]!], st, depth, cbFn.env); + if (!isNum(r)) { degrade(st, at, "sort comparator non-numeric"); return { kind: "any" }; } + const less = r.t.lt(num(ctx, 0)).and(realJ); + const tie = r.t.eq(num(ctx, 0)).and(realJ).and(bool(ctx, j < i)); + rank = rank.add(ctx.If(less.or(tie), num(ctx, 1), num(ctx, 0)) as Num); + } + ranks.push(rank); + } + const elems: SymVal[] = []; + for (let k = 0; k < n; k++) { + let slot: SymVal = { kind: "undef" }; + for (let i = 0; i < n; i++) { + const hit = numEq(ctx, ranks[i]!, num(ctx, k)).and(real(i)); + slot = iteVal(hit, recv.elems[i]!, slot); + } + elems.push(slot); + } + return { kind: "arr", elems, len: recv.len }; + } + case "forEach": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "forEach without function literal"); return { kind: "any" }; } + for (let k = 0; k < recv.elems.length; k++) { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, recv.elems[k], { kind: "undef" }); + invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth); + } + return { kind: "undef" }; + } + case "slice": { + const a0 = isNum(args[0]) ? args[0].t : num(ctx, 0); + const rawEnd = isNum(args[1]) ? args[1].t : recv.len; + const start = ctx.If(a0.lt(num(ctx, 0)), ctx.If(a0.add(recv.len).ge(num(ctx, 0)), a0.add(recv.len), num(ctx, 0)), a0); + const end = ctx.If(rawEnd.lt(num(ctx, 0)), ctx.If(rawEnd.add(recv.len).ge(num(ctx, 0)), rawEnd.add(recv.len), num(ctx, 0)), rawEnd); + const elems: SymVal[] = []; + for (let j = 0; j < recv.elems.length; j++) { + const srcIdx = start.add(num(ctx, j)); + let pick: SymVal = { kind: "undef" }; + for (let k = recv.elems.length - 1; k >= 0; k--) { + pick = iteVal(numEq(ctx, srcIdx, num(ctx, k)).and(end.gt(num(ctx, j))), recv.elems[k], pick); + } + elems.push(pick); + } + const len = end.sub(start); + return { kind: "arr", elems, len: ctx.If(len.lt(num(ctx, 0)), num(ctx, 0), len) }; + } + case "concat": { + const other = args.find(isArr); + const elems = [...recv.elems]; + if (other) { + for (let k = 0; k < other.elems.length; k++) { + elems.push(iteVal(other.len.gt(num(ctx, k)), other.elems[k], { kind: "undef" })); + } + } + return { kind: "arr", elems, len: other ? recv.len.add(other.len) : recv.len }; + } + case "join": { + // Only when every element is a concrete/str; else degrade. + const parts: SymVal[] = []; + for (let k = 0; k < recv.elems.length; k++) { + parts.push(iteVal(recv.len.gt(num(ctx, k + 1)), [...recv.elems.slice(0, k), { kind: "str" as const, t: strArgOrEmpty(args[0]) }].reduce((a, b) => (isStr(a) && isStr(b) ? { kind: "str" as const, t: a.t.concat(b.t) } : { kind: "any" as const }), { kind: "str" as const, t: ctx.String.val("") }), recv.elems[k])); + } + if (parts.some((p) => !isStr(p))) { degrade(st, at, "join with degraded parts"); return { kind: "any" }; } + return parts.reduce((a, b) => (isStr(a) && isStr(b) ? { kind: "str" as const, t: a.t.concat(b.t) } : { kind: "any" as const }), { kind: "str" as const, t: ctx.String.val("") }); + } + default: + degrade(st, at, `array method .${m}`); + return { kind: "any" }; + } + } + + // ---- string methods ---- + if (isStr(recv)) { + switch (m) { + case "length": return { kind: "num", t: lenR(recv.t) }; + case "includes": { + const a0 = isStr(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + const rc = strConst.get(recv.t), ac = strConst.get(a0); + if (rc !== undefined && ac !== undefined) return { kind: "bool", t: bool(ctx, rc.includes(ac)) }; + return { kind: "bool", t: recv.t.contains(a0) }; + } + case "startsWith": { + const a0 = isStr(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + const rc = strConst.get(recv.t), ac = strConst.get(a0); + if (rc !== undefined && ac !== undefined) return { kind: "bool", t: bool(ctx, rc.startsWith(ac)) }; + return { kind: "bool", t: recv.t.prefixOf(a0) }; + } + case "endsWith": { + const a0 = isStr(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + const rc = strConst.get(recv.t), ac = strConst.get(a0); + if (rc !== undefined && ac !== undefined) return { kind: "bool", t: bool(ctx, rc.endsWith(ac)) }; + return { kind: "bool", t: recv.t.suffixOf(a0) }; + } + case "indexOf": { + const a0 = isStr(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + const rc = strConst.get(recv.t), ac = strConst.get(a0); + if (rc !== undefined && ac !== undefined) return { kind: "num", t: num(ctx, rc.indexOf(ac)) }; + return { kind: "num", t: recv.t.indexOf(a0) }; + } + case "slice": + case "substring": { + const a0 = isNum(args[0]) ? args[0].t : num(ctx, 0); + const rawEnd = isNum(args[1]) ? args[1].t : lenR(recv.t); + const start = ctx.If(a0.lt(num(ctx, 0)), ctx.If(a0.add(lenR(recv.t)).ge(num(ctx, 0)), a0.add(lenR(recv.t)), num(ctx, 0)), a0); + const end = ctx.If(rawEnd.lt(num(ctx, 0)), ctx.If(rawEnd.add(lenR(recv.t)).ge(num(ctx, 0)), rawEnd.add(lenR(recv.t)), num(ctx, 0)), rawEnd); + const len = ctx.If(end.sub(start).ge(num(ctx, 0)), end.sub(start), num(ctx, 0)); + // seq.extract is Int-indexed; our numbers are Real. + pushAsum(st, ctx.IsInt(start)); pushAsum(st, ctx.IsInt(len)); + return { kind: "str", t: recv.t.extract(ctx.ToInt(start), ctx.ToInt(len)) }; + } + case "charCodeAt": { + const a0 = isNum(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + pushAsum(st, ctx.IsInt(a0)); + return { kind: "num", t: ctx.ToReal(recv.t.at(ctx.ToInt(a0)).toCode() as never) as Num }; // toCode is Int-sorted; numbers are Real + } + case "at": { + const a0 = isNum(args[0]) ? args[0].t : num(ctx, 0); + const idx = ctx.If(a0.lt(num(ctx, 0)), a0.add(lenR(recv.t)), a0); + return { kind: "str", t: recv.t.at(ctx.ToInt(idx)) }; + } + case "trim": return recv; // approximation: content-neutral for our props + case "toLowerCase": return recv; // approximation: same, see README + case "toUpperCase": return recv; + case "replace": { + const a0 = isStr(args[0]) ? args[0].t : null; + const a1 = isStr(args[1]) ? args[1].t : null; + if (a0 && a1) return { kind: "str", t: recv.t.replace(a0, a1) }; + // Const string × const regex-literal × const replacement: the + // operation is a pure deterministic string function — evaluate it + // concretely (e.g. escapeRegex(TAG_OPEN)). + const rxC = isObj(args[0]) && args[0].fields.has("__regex") ? strConst.get((args[0].fields.get("__regex")! as { kind: "str"; t: Str }).t) : undefined; + const repC = isStr(args[1]) ? strConst.get(args[1].t) : undefined; + const recvC = strConst.get(recv.t); + if (rxC !== undefined && repC !== undefined && recvC !== undefined) { + const m = /^\/(.*)\/([a-z]*)$/.exec(rxC); + if (m) return { kind: "str", t: ctx.String.val(recvC.replace(new RegExp(m[1]!, m[2]), repC)) }; + } + // Symbolic receiver against the escapeRegex shape (\. specials → + // "\\$&"): deterministic uninterpreted function of the input. + if (rxC === "/[.*+?^${}()|[\\]\\\\]/g" && repC === "\\$&" && isStr(recv)) { + // deterministic uninterpreted escape: same input → same output + (ctx as unknown as { pendingFnsS1: Set }).pendingFnsS1.add("__esc1"); + return { kind: "str", t: (ctx as unknown as { app1: (f: string, a: never, s: string) => Str }).app1("__esc1", recv.t as never, "String") }; + } + // dynamic-regex pattern with "" replacement (the render-refs + // idempotency strip): anchored-prefix over-approximation. + if (isObj(args[0]) && args[0].fields.has("__regex") && isStr(args[1]) && strConst.get(args[1].t) === "") { + const patVal = args[0].fields.get("__regex")!; + const anchor = regexAnchor(patVal); + if (anchor !== null && anchor !== "" && isStr(recv)) { + const g = recv.t.prefixOf(ctx.String.val(anchor)); + const gb = constBool(g); + if (gb === false) return recv; + // Deterministic FUNCTION of (input, anchor): a per-call-site + // free const lets the same strip return different values in + // turn A vs turn B — manufacturing phantom cross-turn flips. + (ctx as unknown as { pendingFnsS2: Set }).pendingFnsS2.add("__re_rep2"); + const fn = (ctx as unknown as { app2: (f: string, a: never, b: never, s: string) => Str }).app2("__re_rep2", recv.t as never, ctx.String.val(anchor) as never, "String"); + if (gb !== true) degrade(st, at, "dynamic regex .replace over-approx"); + return iteVal(g, { kind: "str", t: fn }, recv); + } + } + return { kind: "any" }; + } + case "padStart": { + // Implemented exactly when the string is a decimal digit string + // (the ref-tag use). General int→str is a documented gap. + const w = isNum(args[0]) ? args[0].t : null; + const pad = isStr(args[1]) ? args[1].t : null; + if (!w || !pad) return { kind: "any" }; + // len < w ? pad*(w-len) ++ s : s — construct via extract of pad^N + let padded = recv.t; + const reps: Str[] = []; + for (let i = 0; i < 24; i++) reps.push(pad.power(num(ctx, i + 1))); + // choose the rep whose length equals w - len + let chosen: Str | null = null; + for (const rp of reps) { + const cond = lenR(rp).eq(w.sub(lenR(recv.t))); + chosen = chosen ? ctx.If(cond, rp, chosen) : rp; + } + const need = w.sub(lenR(recv.t)).gt(num(ctx, 0)); + return { kind: "str", t: ctx.If(need, chosen!.concat(recv.t), recv.t) }; + } + case "split": { + // Split into at-most-N parts by literal separator (string theory + // has no split; bounded reconstruction via uninterpreted shells is + // unsound, so degrade unless the separator case is trivial). + return { kind: "any" }; + } + default: + degrade(st, at, `string method .${m}`); + return { kind: "any" }; + } + } + + // ---- Number.prototype.toFixed: uninterpreted, deterministic per input ---- + if (isNum(recv) && m === "toFixed") { + (ctx as unknown as { pendingFnsR1: Set }).pendingFnsR1.add("__tofix1"); + return { kind: "str", t: (ctx as unknown as { app1: (f: string, a: never, s: string) => Str }).app1("__tofix1", recv.t as never, "String") }; + } + + // ---- dynamic regex carrier: .test(s) ---- + if (isObj(recv) && recv.fields.has("__regex") && m === "test") { + const r = dynRegexTest(args[0] ?? { kind: "undef" }, recv.fields.get("__regex")!, at, st); + if (r !== null) return r; + } + + // ---- Map/Set methods ---- + if (isObj(recv) && (recv.fields.has("__map_entries") || recv.fields.has("__set_values"))) { + const result = mapSetOp(recv, m, args, at, st); + if ((m === "add" || m === "set" || m === "delete") && isObj(result) && ts.isPropertyAccessExpression(at.expression)) { + const root = rootName(at.expression); + if (root && frame.vars.has(root)) frame.vars.set(root, result); + } + return result; + } + + degrade(st, at, `method .${m} on ${recv.kind}`); + return { kind: "any" }; + }; + + const strArgOrEmpty = (v: SymVal | undefined): Str => + v && isStr(v) ? v.t : ctx.String.val(""); + + const countTrueBelow = (bs: Bool<"main">[], k: number): Num => { + let acc = num(ctx, 0); + for (let j = 0; j < k; j++) acc = ctx.If(bs[j]!, acc.add(num(ctx, 1)), acc); + return acc; + }; + + const rebindArray = (oldArr: Extract, newArr: SymVal, at: ts.CallExpression, st: RunState, frame: Frame): SymVal => { + // push() mutates in place. The receiver may be a NESTED property chain + // (`pending.toolCalls.push(...)`): rebinding only the ROOT variable to + // the new array CLOBBERS the root (replaces the obj with the arr!). We + // must rebuild every container along the chain with its child replaced: + // pending' = {…pending, toolCalls: newArr}. + const chain: Array = []; + let node: ts.Expression = at.expression; + while (ts.isPropertyAccessExpression(node) || ts.isElementAccessExpression(node)) { + chain.unshift(node); + node = node.expression; + } + if (ts.isIdentifier(node)) { + const root = node.text; + let rebuilt: SymVal = newArr; + for (let i = chain.length - 1; i >= 1; i--) { + const hop = chain[i]!; + const parentVal = evalExpr(hop.expression, frame, st, 0); + if (parentVal.kind !== "obj") return newArr; // degrade silently: root rebind only + const fields = new Map(parentVal.fields); + if (!ts.isPropertyAccessExpression(hop)) return newArr; // element-middle chains: root rebind only + fields.set(hop.name.text, rebuilt); + rebuilt = { kind: "obj", fields }; + } + frame.vars.set(root, rebuilt); + return rebuilt; + } + return newArr; + }; + + /** Invoke an arrow/function-literal argument (callback) concretely. */ + const invokeCallback = ( + cb: ts.ArrowFunction | ts.FunctionExpression | ts.FunctionDeclaration, + args: SymVal[], + st: RunState, + depth: number, + env?: Frame, + ): SymVal => { + const inner: Frame = { vars: new Map((env ?? frame).vars) }; // lexical closure + cb.parameters.forEach((p, i) => { + if (ts.isIdentifier(p.name)) inner.vars.set(p.name.text, args[i] ?? { kind: "undef" }); + else if (ts.isArrayBindingPattern(p.name)) { + p.name.elements.forEach((el, j) => { + if (ts.isBindingElement(el) && ts.isIdentifier(el.name) && isArr(args[i])) { + inner.vars.set(el.name.text, args[i].elems[j] ?? { kind: "undef" }); + } + }); + } + }); + const innerState = runState0(); + if (cb.body && ts.isBlock(cb.body)) execStmt(cb.body, inner, innerState, depth + 1); + else if (cb.body) innerState.ret = evalExpr(cb.body, inner, innerState, depth + 1); + const pushAll2 = (dst: T[], src: T[]): void => { if (src !== dst) for (const x of src) dst.push(x); }; + pushAll2(st.assumptions, innerState.assumptions); + // degradations shared + if (innerState.thrown) st.thrown = orFold(st.thrown ?? bool(ctx, false), innerState.thrown); + // fold inner conditional returns (arrow bodies with early returns) + let ret: SymVal | null = innerState.ret; + for (let i = innerState.pendingRets.length - 1; i >= 0; i--) { + const pr = innerState.pendingRets[i]!; + ret = ret === null ? iteVal(pr.cond, pr.val, { kind: "undef" }) : iteVal(pr.cond, pr.val, ret); + } + // Shared-binding write-back: closures mutate captured vars (let cells). + // Propagate changed values back to the DECLARING frame. Sound when the + // call site's live frame is the declaration frame (top-level helper calls + // like applyTo(...)); loop-body callbacks in the guarded subset never + // write captured vars. + if (env) { + for (const [k, v] of inner.vars) { + const old = env.vars.get(k); + if (old === undefined || old === v) continue; + const same = + (isNum(old) && isNum(v) && old.t.hash() === v.t.hash()) || + (isStr(old) && isStr(v) && old.t.hash() === v.t.hash()) || + (isBool(old) && isBool(v) && old.t.hash() === v.t.hash()) || + (isArr(old) && isArr(v) && old.len.hash() === v.len.hash()); + if (!same) env.vars.set(k, v); + } + } + return ret ?? { kind: "undef" }; + }; + + /** Map/Set modeled as obj with special fields (bounded vector + len). */ + const mapSetOp = (recv: Extract, m: string, args: SymVal[], at: ts.Node, st: RunState): SymVal => { + const entriesArr = recv.fields.get("__map_entries") ?? recv.fields.get("__set_values"); + if (!isArr(entriesArr)) return { kind: "any" }; + switch (m) { + case "has": { + const key = args[0]; + if (recv.fields.has("__map_entries") && isArr(entriesArr)) { + // entries are {key, value} objs + const eqs = entriesArr.elems.map((e) => (isObj(e) ? valEq(e.fields.get("key") ?? { kind: "any" }, key) : null)); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc = bool(ctx, false); + for (let k = 0; k < eqs.length; k++) { + acc = acc.or(entriesArr.len.gt(num(ctx, k)).and(eqs[k]!)); + } + return { kind: "bool", t: acc }; + } + const eqs = entriesArr.elems.map((e) => valEq(e, args[0])); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc = bool(ctx, false); + for (let k = 0; k < eqs.length; k++) { + acc = acc.or(entriesArr.len.gt(num(ctx, k)).and(eqs[k]!)); + } + return { kind: "bool", t: acc }; + } + case "get": { + const eqs = entriesArr.elems.map((e) => (isObj(e) ? valEq(e.fields.get("key") ?? { kind: "any" }, args[0]) : null)); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc: SymVal = { kind: "undef" }; + for (let k = eqs.length - 1; k >= 0; k--) { + const slotVal = entriesArr.elems[k]; + const got: SymVal = isObj(slotVal) ? slotVal.fields.get("value") ?? { kind: "undef" } : { kind: "undef" }; + acc = iteVal(entriesArr.len.gt(num(ctx, k)).and(eqs[k]!), got, acc); + } + return acc; + } + case "add": + case "set": { + const v = m === "set" && isObj(args[0]) === false + ? { kind: "obj" as const, fields: new Map([["key", args[0]], ["value", args[1] ?? { kind: "undef" }]]) } + : args[0]; + const entry = m === "set" + ? { kind: "obj" as const, fields: new Map([["key", args[0]], ["value", args[1] ?? { kind: "undef" }]]) } + : args[0]; + void v; + const elems = entriesArr.elems.map((e, k) => { + const exists = m === "set" + ? isObj(e) ? (valEq(e.fields.get("key") ?? { kind: "any" }, args[0]) ?? bool(ctx, false)) : bool(ctx, false) + : (valEq(e, args[0]) ?? bool(ctx, false)); + return iteVal(exists, entry, e); + }); + // membership as a (possibly symbolic) predicate — a Term object is + // always JS-truthy, so `elems.some(...)` could never work here. + let anyEq = bool(ctx, false); + for (let k = 0; k < entriesArr.elems.length; k++) { + const e = entriesArr.elems[k]!; + const eq = m === "set" + ? (isObj(e) ? (valEq(e.fields.get("key") ?? { kind: "any" }, args[0]) ?? bool(ctx, false)) : bool(ctx, false)) + : (valEq(e, args[0]) ?? bool(ctx, false)); + anyEq = anyEq.or(entriesArr.len.gt(num(ctx, k)).and(eq)); + } + // Slot-write at the SYMBOLIC length: a previously executed guarded + // add may or may not have consumed its slot, so the physical tail + // position is ambiguous. Writing the entry into slot `len` via per- + // slot ITEs is correct in every model (a slot past the unchanged + // length is never observed by length-guarded reads). + const slotElems = entriesArr.elems.map((e, k) => iteVal(numEq(ctx, entriesArr.len, num(ctx, k)), entry, e)); + if (entriesArr.elems.length < N - 1) { + slotElems.push(iteVal(numEq(ctx, entriesArr.len, num(ctx, entriesArr.elems.length)), entry, { kind: "undef" })); + } else { + degrade(st, at, "map/set add beyond bounded slots"); + } + elems.length = 0; + elems.push(...slotElems); + const newLenV = iteVal(anyEq, { kind: "num", t: entriesArr.len }, { kind: "num", t: entriesArr.len.add(num(ctx, 1)) }); + const newLen: Num = newLenV.kind === "num" ? newLenV.t : entriesArr.len; + const fields = new Map(recv.fields); + fields.set(recv.fields.has("__map_entries") ? "__map_entries" : "__set_values", { kind: "arr", elems, len: newLen }); + return { kind: "obj", fields }; + } + case "size": { + return { kind: "num", t: entriesArr.len }; + } + default: + degrade(st, at, `map/set method .${m}`); + return { kind: "any" }; + } + }; + + // new Map()/new Set() + const evalNew = (call: ts.NewExpression, frame: Frame, st: RunState, depth: number): SymVal => { + const name = ts.isIdentifier(call.expression) ? call.expression.text : ""; + if (name === "Map") { + return { kind: "obj", fields: new Map([["__map_entries", arrOf([])]]) }; + } + if (name === "Set") { + return { kind: "obj", fields: new Map([["__set_values", arrOf([])]]) }; + } + if (name === "RegExp") { + // regex carried as an object; .test/.replace on it use the anchored- + // prefix over-approximation at the call sites (see callMethod). + const p0 = call.arguments?.[0]; + return { kind: "obj", fields: new Map([["__regex", p0 ? evalExpr(p0, frame, st, depth) : { kind: "str", t: ctx.String.val("") }]]) }; + } + if (name === "Error" || name === "RangeError" || name === "TypeError") { + return { kind: "obj", fields: new Map([["message", call.arguments?.[0] ? evalExpr(call.arguments[0], frame, st, depth) : { kind: "str", t: ctx.String.val("") }]]) }; + } + // Constructor ports: environment-backed classes (crypto, counters) are + // injectable exactly like function ports. Sound because ported outputs + // are unconstrained symbolic values the property never depends on. + const ctorPort = name && input.options?.portImpls ? input.options.portImpls[name] : undefined; + if (ctorPort) { + const args = (call.arguments ?? []).map((a) => evalExpr(a, frame, st, depth)); + return (ctorPort as (...a: SymVal[]) => SymVal)(...args); + } + degrade(st, call, `new ${name}`); + return { kind: "any" }; + }; + + const mathCall = (meth: string, args: SymVal[], at: ts.Node, st: RunState): SymVal => { + const nums = args.map((a) => (isNum(a) ? a.t : null)); + if (nums.some((n) => n === null)) { + if (meth === "min" || meth === "max") { + // Math.min(x, y ?? inf) patterns: treat nulls as identity (skip) + const present = nums.filter((n): n is Num => n !== null); + if (present.length === 0) return { kind: "any" }; + return { kind: "num", t: present.reduce((a, b) => (meth === "min" ? minOf(a, b) : maxOf(a, b))) }; + } + degrade(st, at, `Math.${meth} degraded arg`); + return { kind: "any" }; + } + const a = nums as Num[]; + switch (meth) { + case "min": return { kind: "num", t: a.reduce((x, y) => minOf(x, y)) }; + case "max": return { kind: "num", t: a.reduce((x, y) => maxOf(x, y)) }; + case "abs": return { kind: "num", t: ctx.If(a[0].lt(num(ctx, 0)), a[0].neg(), a[0]) }; + case "round": { + // JS Math.round = floor(x + 0.5); stays Real-sorted (numbers are Real) + return { kind: "num", t: ctx.ToReal(ctx.ToInt(a[0].add(num(ctx, 0.5))) as never) as Num }; + } + case "floor": return { kind: "num", t: ctx.ToReal(ctx.ToInt(a[0]) as never) as Num }; + case "ceil": { + const f = ctx.ToReal(ctx.ToInt(a[0]) as never) as Num; + return { kind: "num", t: ctx.If(f.eq(a[0]), f, f.add(num(ctx, 1))) }; + } + case "trunc": { + const tf = ctx.ToReal(ctx.ToInt(a[0]) as never) as Num; + return { kind: "num", t: ctx.If(a[0].ge(num(ctx, 0)), tf, tf.add(num(ctx, 1))) }; + } + default: + degrade(st, at, `Math.${meth}`); + return { kind: "any" }; + } + }; + const minOf = (a: Num, b: Num): Num => ctx.If(a.le(b), a, b); + const maxOf = (a: Num, b: Num): Num => ctx.If(a.ge(b), a, b); + + /* ---------------- run ---------------- */ + + const frame: Frame = { vars: new Map(Object.entries(params)) }; + const st = runState0(); + // Bind declared entry params the caller did not pass: run their default + // initializers (JS semantics — defaults may reference earlier params) or + // bind undefined. Without this, later params are simply unbound. + for (const p of decl.parameters) { + if (!ts.isIdentifier(p.name)) continue; + if (frame.vars.has(p.name.text)) continue; + frame.vars.set(p.name.text, p.initializer ? evalExpr(p.initializer, frame, st, 0) : { kind: "undef" }); + } + execStmt(decl.body!, frame, st, 0); + let finalRet = st.ret; + for (let i = st.pendingRets.length - 1; i >= 0; i--) { + const pr = st.pendingRets[i]!; + finalRet = finalRet === null ? iteVal(pr.cond, pr.val, { kind: "undef" }) : iteVal(pr.cond, pr.val, finalRet); + } + return { + exec: () => ({ + ret: finalRet, + thrown: st.thrown ? { kind: "bool", t: st.thrown } : null, + assumptions: st.assumptions, + degradations: st.degradations, + ctx: input.ctx as import("./terms.js").Context, + }), + }; +} diff --git a/verif/terms.ts b/verif/terms.ts new file mode 100644 index 0000000000..d337a7643c --- /dev/null +++ b/verif/terms.ts @@ -0,0 +1,532 @@ +/** + * DAG-backed SMT2 term factory. + * + * Every constructor interns nodes by (op, kids, sort, const) so structurally + * equal terms are the SAME object — ITE-merges across loop iterations share + * structure instead of doubling string bytes (the pre-DAG build OOM'd at GBs + * on hide-consumed: rest-on-copy wraps the whole frame per iteration). + * `smt2Of` prints shared nodes as SMT-LIB `let` bindings, keeping the emitted + * file linear too. + */ + +type Sort = "Real" | "Int" | "Bool" | "String"; + +/** method surface installed onto Term.prototype by bindMethods() */ +export interface Term { + eq(o: Term): Term; + neq(o: Term): Term; + ge(o: Term): Term; + le(o: Term): Term; + gt(o: Term): Term; + lt(o: Term): Term; + add(o: Term): Term; + sub(o: Term): Term; + mul(o: Term): Term; + div(o: Term): Term; + mod(o: Term): Term; + neg(): Term; + and(o: Term): Term; + or(o: Term): Term; + not(): Term; + xor(o: Term): Term; + implies(o: Term): Term; + power(o: Term): Term; + concat(o: Term): Term; + length(): Term; + at(i: Term): Term; + extract(off: Term, len: Term): Term; + indexOf(s: Term, from?: Term): Term; + contains(s: Term): Term; + prefixOf(s: Term): Term; + suffixOf(s: Term): Term; + replace(a: Term, b: Term): Term; + toCode(): Term; + toInt(): Term; +} +/** compat aliases for the previous z3-flavored facade */ +export type Arith = Term; +export type Num = Term; +export type Bool<_T = unknown> = Term; +export type Seq<_T = unknown> = Term; + +export class Term { + /** cached inline print */ + private p?: string; + constructor( + readonly op: string, + readonly sort: Sort, + readonly kids: readonly Term[] = [], + /** concrete value when statically known */ + readonly c?: number | string | boolean, + readonly varName?: string, + readonly ctx?: Context, + ) {} + + toString(): string { + if (this.p !== undefined) return this.p; + const k = this.kids; + let s: string; + // SMT-LIB string literals: only "" doubles a quote — JSON's \" is ILLEGAL + // in SMT-LIB 2.6 (cvc5: 'Illegal string character'). Control chars go + // through \u{...}; printable Unicode stays raw (cvc5 reads UTF-8). + if (!k.length && this.sort === "String" && typeof this.c === "string") s = smtStr(this.c); + else if (!k.length) s = this.op; // var/literal pre-rendered as op + else if (this.op === "ite") s = `(ite ${k[0]!} ${k[1]!} ${k[2]!})`; + else s = `(${k.map(String).join(" ")})`; + this.p = s; + return s; + } + + /** DAG-safe structural hash (memoized, linear in DAG size). Use this — + * NEVER toString() — for equality checks and dedupe keys on terms that + * may be large: tree-printing a shared DAG is exponential. */ + private h?: number; + hash(): number { + if (this.h !== undefined) return this.h; + let acc = 2166136261; + const mix = (x: number): void => { acc ^= x; acc = Math.imul(acc, 16777619); }; + for (let i = 0; i < this.op.length; i++) mix(this.op.charCodeAt(i)); + mix(this.sort.length); + if (this.c !== undefined) mix(typeof this.c === "number" ? this.c : typeof this.c === "boolean" ? (this.c ? 1 : 0) : this.c.length); + for (const k of this.kids) mix(k.hash()); + this.h = acc >>> 0; + return this.h; + } +} + +/** SMT-LIB 2.6 string literal: quotes double; control chars as \u{...}. */ +function smtStr(v: string): string { + let out = '"'; + for (const ch of v) { + const cp = ch.codePointAt(0)!; + if (ch === '"') out += '""'; + else if (cp < 0x20 || cp === 0x7f) out += `\\u{${cp.toString(16).padStart(4, "0")}}`; + else out += ch; + } + return out + '"'; +} + +function fmtReal(n: number): string { + return Number.isInteger(n) ? `${n}.0` : String(n); +} + +const lit = (ctx: Context, s: string, sort: Sort, c?: number | string | boolean): Term => + ctx.intern(s, sort, [], c); + +export class Context { + readonly decls: string[] = []; + readonly declSort = new Map(); + readonly pendingFns = new Set(); + readonly pendingFnsR1 = new Set(); + readonly pendingFnsS1 = new Set(); + /** (String, String) → String, deterministic in both args. */ + readonly pendingFnsS2 = new Set(); + private readonly varCache = new Map(); + private nextVar = 0; + + fresh(prefix: string): string { + return `__${prefix}_${this.nextVar++}`; + } + + intern(op: string, sort: Sort, kids: readonly Term[], c?: number | string | boolean): Term { + // NOTE: deliberately NOT hash-consed into a global table — a retaining + // intern map blew the heap exponentially on loop-heavy merges (every + // discarded intermediate wrapper stayed reachable). Structural sharing + // happens at smt2Of emission time via canonical keys instead. + return new Term(op, sort, kids, c, undefined, this); + } + + app1(f: string, a: Term, sort: Sort): Term { + return this.intern("app1", sort, [lit(this, f, "String"), a]); + } + app2(f: string, a: Term, b: Term, sort: Sort): Term { + return this.intern("app2", sort, [lit(this, f, "String"), a, b]); + } + app3(f: string, a: Term, b: Term, c: Term, sort: Sort): Term { + return this.intern("app3", sort, [lit(this, f, "String"), a, b, c]); + } + + Real = { + val: (n: number): Term => lit(this, fmtReal(n), "Real", n), + const: (name: string): Term => this.declare(name, "Real"), + }; + Int = { + val: (n: number): Term => lit(this, String(n), "Int", n), + const: (name: string): Term => this.declare(name, "Int"), + }; + Bool = { + val: (b: boolean): Term => lit(this, b ? "true" : "false", "Bool", b), + const: (name: string): Term => this.declare(name, "Bool"), + }; + String = { + val: (s: string): Term => lit(this, JSON.stringify(s), "String", s), + const: (name: string): Term => this.declare(name, "String"), + }; + + private declare(name: string, sort: Sort): Term { + if (!this.decls.includes(name)) { + this.decls.push(name); + this.declSort.set(name, sort); + } + // leaf vars get canonical objects so identity-based folding works for + // repeated const(name) calls (bounded by declared names — no blowup risk) + const key = `${sort}:${name}`; + let t = this.varCache.get(key); + if (!t) { t = this.intern(name, sort, [], undefined); this.varCache.set(key, t); } + return t; + } + + If(c: Term, a: Term, b: Term): Term { + if (c.c === true) return a; + if (c.c === false) return b; + if (a === b) return a; + return this.intern("ite", a.sort, [c, a, b]); + } + ToReal(a: Term): Term { + if (a.sort === "Real") return a; + return this.app1("to_real", a, "Real"); + } + ToInt(a: Term): Term { + if (a.sort === "Int") return a; + if (a.c !== undefined && typeof a.c === "number") return lit(this, String(a.c), "Int", a.c); + return this.app1("to_int", a, "Int"); + } + IsInt(a: Term): Term { + if (a.sort === "Int") return lit(this, "true", "Bool", true); + if (a.c !== undefined && typeof a.c === "number") + return lit(this, String(Number.isInteger(a.c)), "Bool", Number.isInteger(a.c)); + return this.app1("is_int", a, "Bool"); + } +} + +// --------------------------------------------------------------------------- +// arithmetic / boolean / string ops — bound per-Context onto Term.prototype +// --------------------------------------------------------------------------- + +const numSortOf = (a: Term, b: Term): Sort => (a.sort === "Int" && b.sort === "Int" ? "Int" : "Real"); +const asReal = (t: Term): Term => (t.sort === "Int" ? t.ctx!.ToReal(t) : t); + +function foldArith(ctx: Context, op: string, a: Term, b: Term, sign: number, mul: boolean): Term { + const sort = numSortOf(a, b); + const av = typeof a.c === "number" ? a.c : undefined; + const bv = typeof b.c === "number" ? b.c : undefined; + if (av !== undefined && bv !== undefined) { + const v = mul ? av * bv : sign > 0 ? av + bv : av - bv; + return sort === "Real" ? lit(ctx, fmtReal(v), "Real", v) : lit(ctx, String(v), "Int", v); + } + if (mul) { + if (av === 0 || bv === 0) return sort === "Real" ? lit(ctx, "0.0", "Real", 0) : lit(ctx, "0", "Int", 0); + if (av === 1) return b; + if (bv === 1) return a; + } else { + if (sign > 0 && av === 0) return b; + if (sign > 0 && bv === 0) return a; + if (sign < 0 && bv === 0) return a; + } + return ctx.app2(op, a, b, sort); +} + +function foldCmp(ctx: Context, op: string, a: Term, b: Term): Term { + const av = typeof a.c === "number" ? a.c : undefined; + const bv = typeof b.c === "number" ? b.c : undefined; + if (av !== undefined && bv !== undefined) { + const v = op === "<" ? av < bv : op === "<=" ? av <= bv : op === ">" ? av > bv : av >= bv; + return lit(ctx, String(v), "Bool", v); + } + if (a === b && (op === "<=" || op === ">=")) return lit(ctx, "true", "Bool", true); + return ctx.app2(op, a, b, "Bool"); +} + +function foldBool(ctx: Context, op: string, a: Term, b: Term): Term { + const av = typeof a.c === "boolean" ? a.c : undefined; + const bv = typeof b.c === "boolean" ? b.c : undefined; + if (op === "and") { + if (av === false || bv === false) return lit(ctx, "false", "Bool", false); + if (av === true) return b; + if (bv === true) return a; + if (a === b) return a; + return ctx.app2("and", a, b, "Bool"); + } + if (op === "or") { + if (av === true || bv === true) return lit(ctx, "true", "Bool", true); + if (av === false) return b; + if (bv === false) return a; + if (a === b) return a; + return ctx.app2("or", a, b, "Bool"); + } + if (op === "xor") { + if (av !== undefined && bv !== undefined) return lit(ctx, String(av !== bv), "Bool", av !== bv); + if (a === b) return lit(ctx, "false", "Bool", false); + return ctx.app2("xor", a, b, "Bool"); + } + // implies + if (av === false || bv === true) return lit(ctx, "true", "Bool", true); + if (av === true) return b; + return ctx.app2("=>", a, b, "Bool"); +} + +function foldStrEq(ctx: Context, eq: boolean, a: Term, b: Term): Term { + const av = typeof a.c === "string" ? a.c : undefined; + const bv = typeof b.c === "string" ? b.c : undefined; + if (av !== undefined && bv !== undefined) { + const v = eq ? av === bv : av !== bv; + return lit(ctx, String(v), "Bool", v); + } + if (a === b) return lit(ctx, String(eq), "Bool", eq); + return ctx.app2(eq ? "=" : "distinct", a, b, "Bool"); +} + +function bindMethods(ctx: Context): void { + const proto: Record Term> = { + eq(o) { + const c = this.ctx ?? ctx; + if (this.sort === "Bool" && o.sort === "Bool") { + if (this.c !== undefined && o.c !== undefined) return lit(c, String(this.c === o.c), "Bool", this.c === o.c); + if (this === o) return lit(c, "true", "Bool", true); + return foldBool(c, "xor", this.not.call(this), o); // (= a b) ≡ ¬a⊕b folded via not+xor + } + if (this.sort === "String" || o.sort === "String") return foldStrEq(c, true, this, o); + if (this.sort !== o.sort) return foldCmp(c, "=", asReal(this), asReal(o)); + return foldCmp(c, "=", this, o); + }, + neq(o) { + const c = this.ctx ?? ctx; + if (this.sort === "String" || o.sort === "String") return foldStrEq(c, false, this, o); + const e = proto.eq!.call(this, o); + if (e.c !== undefined) return lit(c, String(!e.c), "Bool", !e.c); + return e.not.call(e); + }, + ge(o) { return foldCmp(this.ctx ?? ctx, ">=", asReal(this), asReal(o)); }, + le(o) { return foldCmp(this.ctx ?? ctx, "<=", asReal(this), asReal(o)); }, + gt(o) { return foldCmp(this.ctx ?? ctx, ">", asReal(this), asReal(o)); }, + lt(o) { return foldCmp(this.ctx ?? ctx, "<", asReal(this), asReal(o)); }, + add(o) { return foldArith(this.ctx ?? ctx, "+", asReal(this), asReal(o), 1, false); }, + sub(o) { return foldArith(this.ctx ?? ctx, "-", asReal(this), asReal(o), -1, false); }, + mul(o) { return foldArith(this.ctx ?? ctx, "*", asReal(this), asReal(o), 1, true); }, + div(o) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "number" ? this.c : undefined; + const bv = typeof o.c === "number" ? o.c : undefined; + if (av !== undefined && bv !== undefined && Number.isInteger(av) && Number.isInteger(bv) && bv !== 0 && av % bv === 0) + return lit(c, `${av / bv}.0`, "Real", av / bv); + return c.app2("/", asReal(this), asReal(o), "Real"); + }, + mod(o) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "number" ? this.c : undefined; + const bv = typeof o.c === "number" ? o.c : undefined; + if (av !== undefined && bv !== undefined && Number.isInteger(av) && Number.isInteger(bv) && bv !== 0) + return lit(c, String(av % bv), "Int", av % bv); + return c.app2("mod", this, o, "Int"); + }, + neg() { + const av = typeof this.c === "number" ? this.c : undefined; + if (av !== undefined) return this.sort === "Real" ? lit(this.ctx ?? ctx, fmtReal(-av), "Real", -av) : lit(this.ctx ?? ctx, String(-av), "Int", -av); + return (this.ctx ?? ctx).app1("-", this, this.sort); + }, + and(o) { return foldBool(this.ctx ?? ctx, "and", this, o); }, + or(o) { return foldBool(this.ctx ?? ctx, "or", this, o); }, + not() { + const av = typeof this.c === "boolean" ? this.c : undefined; + if (av !== undefined) return lit(this.ctx ?? ctx, String(!av), "Bool", !av); + return (this.ctx ?? ctx).app1("not", this, "Bool"); + }, + xor(o) { return foldBool(this.ctx ?? ctx, "xor", this, o); }, + implies(o) { return foldBool(this.ctx ?? ctx, "=>", this, o); }, + power(o) { + const c = this.ctx ?? ctx; + const sv = typeof this.c === "string" ? this.c : undefined; + const nv = typeof o.c === "number" ? o.c : undefined; + if (sv !== undefined && nv !== undefined && sv.length * Math.max(0, nv) <= 4096) { + const s = sv.repeat(Math.max(0, nv)); + return lit(c, JSON.stringify(s), "String", s); + } + c.pendingFns.add("__str_repeat"); + return c.app2("__str_repeat", this, o, "String"); + }, + concat(o) { + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof o.c === "string" ? o.c : undefined; + if (av !== undefined && bv !== undefined) return lit(this.ctx ?? ctx, JSON.stringify(av + bv), "String", av + bv); + if (av === "") return o; + if (bv === "") return this; + return (this.ctx ?? ctx).app2("str.++", this, o, "String"); + }, + length() { + const av = typeof this.c === "string" ? this.c : undefined; + if (av !== undefined) return lit(this.ctx ?? ctx, String(av.length), "Int", av.length); + return (this.ctx ?? ctx).app1("str.len", this, "Int"); + }, + at(i) { return (this.ctx ?? ctx).app2("str.at", this, i, "String"); }, + extract(off, len) { return (this.ctx ?? ctx).app3("str.extract", this, off, len, "String"); }, + indexOf(s, from) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof s.c === "string" ? s.c : undefined; + if (av !== undefined && bv !== undefined && from === undefined) + return lit(c, String(av.indexOf(bv)), "Int", av.indexOf(bv)); + return from === undefined + ? c.app2("str.indexof", this, s, "Int") + : c.app3("str.indexof", this, s, from, "Int"); + }, + contains(s) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof s.c === "string" ? s.c : undefined; + if (av !== undefined && bv !== undefined) return lit(c, String(av.includes(bv)), "Bool", av.includes(bv)); + if (this === s) return lit(c, "true", "Bool", true); + return c.app2("str.contains", this, s, "Bool"); + }, + prefixOf(s) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof s.c === "string" ? s.c : undefined; + if (av !== undefined && bv !== undefined) return lit(c, String(bv.startsWith(av)), "Bool", bv.startsWith(av)); + return c.app2("str.prefixof", this, s, "Bool"); + }, + suffixOf(s) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof s.c === "string" ? s.c : undefined; + if (av !== undefined && bv !== undefined) return lit(c, String(bv.endsWith(av)), "Bool", bv.endsWith(av)); + return c.app2("str.suffixof", this, s, "Bool"); + }, + replace(a, b) { + const c = this.ctx ?? ctx; + const sv = typeof this.c === "string" ? this.c : undefined; + const av = typeof a.c === "string" ? a.c : undefined; + const bv = typeof b.c === "string" ? b.c : undefined; + if (sv !== undefined && av !== undefined && bv !== undefined) + return lit(c, JSON.stringify(sv.replace(av, bv)), "String", sv.replace(av, bv)); + return c.app3("str.replace", this, a, b, "String"); + }, + toCode() { + const av = typeof this.c === "string" ? this.c : undefined; + if (av !== undefined) return lit(this.ctx ?? ctx, String(av.codePointAt(0) ?? 0), "Int", av.codePointAt(0) ?? 0); + return (this.ctx ?? ctx).app1("str.to_code", this, "Int"); + }, + toInt() { + const av = typeof this.c === "string" ? this.c : undefined; + if (av !== undefined) { + const n = Number(av); + return lit(this.ctx ?? ctx, String(n), "Int", n); + } + return (this.ctx ?? ctx).app1("str.to_int", this, "Int"); + }, + }; + for (const [name, fn] of Object.entries(proto)) { + (Term.prototype as any)[name] = fn; + } +} + +/** convenience: build a string-constant Term bound to ctx */ +export function strc(ctx: Context, sv: string): Term { + return ctx.String.val(sv); +} + +export function newCtx(): Context { + const ctx = new Context(); + bindMethods(ctx); + return ctx; +} + +// --------------------------------------------------------------------------- +// SMT2 emission with let-sharing +// --------------------------------------------------------------------------- + +interface Smt2Out { + smt2: string; + lets: number; +} + +export function smt2Of( + ctx: Context, + asserts: readonly Term[], + getValue: readonly string[] = [], +): Smt2Out { + // canonical structural key per node (bottom-up, memoized by object) + const canon = new Map(); + let seq = 0; + // DAG-safe hashing: string keys blow up quadratically on deep chains + // (Invalid string length). Numbers via Term.hash() are memoized+linear. + const seenObj = new Set(); + const counts = new Map(); + const walk = (t: Term): void => { + if (seenObj.has(t)) return; + seenObj.add(t); + for (const k of t.kids) { + counts.set(k.hash(), (counts.get(k.hash()) ?? 0) + 1); + walk(k); + } + }; + for (const a of asserts) { + counts.set(a.hash(), (counts.get(a.hash()) ?? 0) + 1); + walk(a); + } + const sizeCache = new Map(); + const sizeOf = (t: Term): number => { + const h = t.hash(); + const hit = sizeCache.get(h); + if (hit !== undefined) return hit; + let n = 1; + for (const kk of t.kids) n += sizeOf(kk); + sizeCache.set(h, n); + return n; + }; + // assign let names by hash, post-order + const names = new Map(); + let counter = 0; + const seen2 = new Set(); + const assign = (t: Term): void => { + if (seen2.has(t)) return; + seen2.add(t); + for (const k of t.kids) assign(k); + const key = t.hash(); + if (t.kids.length && (counts.get(key) ?? 0) > 1 && sizeOf(t) > 24 && !names.has(key)) { + names.set(key, `!t${counter++}`); + } + }; + for (const a of asserts) assign(a); + const render = (t: Term, allowName = true): string => { + const key = t.hash(); + const n = allowName ? names.get(key) : undefined; + if (n !== undefined) return n; + if (!t.kids.length) return t.toString(); + if (t.op === "ite") return `(ite ${render(t.kids[0]!)} ${render(t.kids[1]!)} ${render(t.kids[2]!)})`; + if (t.op === "app1" || t.op === "app2" || t.op === "app3") + return `(${t.kids.map((k) => render(k)).join(" ")})`; // kids keep allowName; kid0 is the fn name + return `(${t.op} ${t.kids.map((k) => render(k)).join(" ")})`; // kids keep allowName + }; + + const lines: string[] = ["(set-logic ALL)"]; + for (const fn of ctx.pendingFns) lines.push(`(declare-fun ${fn} (String Int) String)`); + for (const fn of ctx.pendingFnsR1) lines.push(`(declare-fun ${fn} (Real) String)`); + for (const fn of ctx.pendingFnsS1) lines.push(`(declare-fun ${fn} (String) String)`); + for (const fn of (ctx as unknown as { pendingFnsS2: Set }).pendingFnsS2) lines.push(`(declare-fun ${fn} (String String) String)`); + for (const d of ctx.decls) lines.push(`(declare-fun ${d} () ${ctx.declSort.get(d) ?? "Real"})`); + if (names.size) { + // Shared subterms become top-level define-funs — sequential commands, + // so later definitions may reference earlier ones. (SMT-LIB `let` + // bindings are PARALLEL: a binding cannot reference its siblings — + // cvc5 rejects chained lets with "Symbol not declared".) + const defFor = new Map(); + const seen3 = new Set(); + const collect = (t: Term): void => { + if (seen3.has(t)) return; + seen3.add(t); + const k = t.hash(); + if (names.has(k) && !defFor.has(k)) defFor.set(k, t); + t.kids.forEach(collect); + }; + asserts.forEach(collect); + for (const [k, n] of names) { + const d = defFor.get(k)!; + lines.push(`(define-fun ${n} () ${d.sort} ${render(d, false)})`); + } + } + for (const a of asserts) lines.push(`(assert ${render(a)})`); + lines.push("(check-sat)"); + if (getValue.length) lines.push(`(get-value (${getValue.join(" ")}))`); + lines.push("(exit)"); + return { smt2: lines.join("\n") + "\n", lets: names.size }; +} From 54bb967c68a82a853bb1fc26b3a51449679e292d Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:55:34 +0800 Subject: [PATCH 28/29] kernel: every applied truncation clip comes out strictly shorter (#3935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clip trigger only checked original.length > keepPrefix + keepSuffix; prefix + marker + suffix briefly exceeded the source just above the window (4001 → 4064), rewriting banked prefix-cache bytes and INFLATING the context while reporting saved tokens. The keep window is now capped to the room the original actually leaves below the marker (floor 64 chars, final strictly-shorter guard), large outputs keep the full 2000+2000 window byte-identical, and candidates with no room stay verbatim. --- kernel/src/truncate-tools.ts | 22 ++- .../tests/issue3935-truncation-shrink.test.ts | 127 ++++++++++++++++++ 2 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 kernel/tests/issue3935-truncation-shrink.test.ts diff --git a/kernel/src/truncate-tools.ts b/kernel/src/truncate-tools.ts index 25cebb433b..8da1cf7c6a 100644 --- a/kernel/src/truncate-tools.ts +++ b/kernel/src/truncate-tools.ts @@ -100,13 +100,29 @@ export function truncateLargeToolOutputs( if (original.length <= opts.keepPrefixChars + opts.keepSuffixChars) { continue; } - const prefix = clampPrefix(original, opts.keepPrefixChars); + // #3935: the clip must come out STRICTLY shorter than the original — + // prefix + suffix + marker briefly exceeded the source text just above + // the keep window (4001 → 4064), rewriting banked prefix-cache bytes and + // INFLATING the context while "saving" tokens. Cap the total keep + // window to whatever room the original actually leaves below the + // marker; candidates too small to leave room are skipped verbatim. + const marker = `\n\n...${TRUNCATION_MARKER} — original ~${tokens} tokens]...\n\n`; + const maxKeep = original.length - marker.length - 4; + if (maxKeep < 64) continue; + const keep = Math.min( + opts.keepPrefixChars + opts.keepSuffixChars, + maxKeep, + ); + const prefixKeep = Math.floor(keep / 2); + const suffixKeep = keep - prefixKeep; + const prefix = clampPrefix(original, prefixKeep); const suffix = clampWindow( original, - original.length - opts.keepSuffixChars, + original.length - suffixKeep, original.length, ); - const replacement = `${prefix}\n\n...${TRUNCATION_MARKER} — original ~${tokens} tokens]...\n\n${suffix}`; + const replacement = `${prefix}${marker}${suffix}`; + if (replacement.length >= original.length) continue; replacements.set(candidate.id, replacement); truncatedIds.push(candidate.id); truncatedCount++; diff --git a/kernel/tests/issue3935-truncation-shrink.test.ts b/kernel/tests/issue3935-truncation-shrink.test.ts new file mode 100644 index 0000000000..59d137fc19 --- /dev/null +++ b/kernel/tests/issue3935-truncation-shrink.test.ts @@ -0,0 +1,127 @@ +// #3935: an "emergency truncation" that produces text LONGER than the +// original is a cache-seam bug two ways — it rewrites mid-history bytes that +// the prefix cache had already banked, and it inflates the context instead of +// relieving it (solver witness: 4001-char tool output clipped to 4064 — +// prefix 2000 + marker ~63 + suffix 2000; savedTokens NEGATIVE). The trigger +// only checked original.length > keepPrefix + keepSuffix; it never required +// the replacement to be strictly shorter. These pins freeze the fixed +// contract: every applied clip is strictly shorter than its original, big +// outputs keep the full keep-window (byte-stable with the #3941-era pins), +// and the whole thing stays deterministic and idempotent. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { defaultConfig } from "../src/index.js"; +import { truncateLargeToolOutputs } from "../src/truncate-tools.js"; +import type { CoreMessage } from "../src/types.js"; + +const WINDOW = 100_000; +const config = () => ({ ...defaultConfig(WINDOW) }); +const countTokens = (text: string) => Math.ceil(text.length / 4); + +function toolResult(id: string, text: string): CoreMessage { + return { + id, + role: "tool", + contentType: "tool-result", + toolCallId: `call-${id}`, + toolName: "read", + text, + }; +} + +function history(text: string): CoreMessage[] { + return [ + toolResult("t1", text), + { id: "p1", role: "user", contentType: "text", text: "q1" }, + { id: "p2", role: "user", contentType: "text", text: "q2" }, + { id: "p3", role: "user", contentType: "text", text: "q3" }, + ]; +} + +test("pin: a just-over-window output clips to something strictly shorter (#3935)", () => { + const original = "A".repeat(4001); + const out = truncateLargeToolOutputs( + history(original), + 96_000, + config(), + countTokens, + ); + + assert.equal(out.truncatedCount, 1); + const text = out.messages[0]!.text!; + assert.ok(text.includes("[truncated for context space]")); + assert.ok( + text.length < original.length, + `replacement (${text.length}) must be strictly shorter than original (${original.length})`, + ); + assert.ok(out.savedTokens > 0); +}); + +test("pin: the strictly-shorter guarantee holds across the whole bloat window", () => { + for (const size of [4001, 4010, 4040, 4060, 4067, 4068, 4200, 6000]) { + const original = "A".repeat(size); + const out = truncateLargeToolOutputs( + history(original), + 96_000, + config(), + countTokens, + ); + assert.equal(out.truncatedCount, 1, `size ${size}`); + const text = out.messages[0]!.text!; + assert.ok( + text.length < size, + `size ${size}: replacement ${text.length} must be < original`, + ); + } +}); + +test("pin: large outputs keep the full keep-window (byte-stable with the #3941 pins)", () => { + const out = truncateLargeToolOutputs( + history("A".repeat(400_000)), + 96_000, + config(), + countTokens, + ); + assert.equal(out.truncatedCount, 1); + assert.equal(out.messages[0]!.text!.length, 4066); +}); + +test("pin: clipping is deterministic — same input, same bytes", () => { + const original = "B".repeat(4037); + const a = truncateLargeToolOutputs(history(original), 96_000, config(), countTokens); + const b = truncateLargeToolOutputs(history(original), 96_000, config(), countTokens); + assert.equal(a.messages[0]!.text, b.messages[0]!.text); +}); + +test("pin: already-clipped text is never re-clipped (idempotency)", () => { + const first = truncateLargeToolOutputs( + history("C".repeat(5000)), + 96_000, + config(), + countTokens, + ); + const clipped = first.messages[0]!.text!; + const second = truncateLargeToolOutputs( + history(clipped), + 96_000, + config(), + countTokens, + ); + assert.equal(second.truncatedCount, 0); + assert.equal(second.messages[0]!.text, clipped); +}); + +test("pin: a candidate too small to shrink meaningfully stays verbatim", () => { + // keep windows of 0/0 with a 60-char text: the marker alone (~64 chars) is + // longer than the original — clipping would bloat, so the candidate is + // skipped entirely. + const out = truncateLargeToolOutputs( + history("short payload"), + 96_000, + config(), + countTokens, + { keepPrefixChars: 0, keepSuffixChars: 0, minOutputTokens: 1 }, + ); + assert.equal(out.truncatedCount, 0); + assert.equal(out.messages[0]!.text, "short payload"); +}); From ad90feaa832a625fc5fa41a068a2d4f09eb9ab00 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:55:34 +0800 Subject: [PATCH 29/29] kernel: BLOCKED binding migration requires no same-id sibling protected (#3945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A BLOCKED ref binding is keyed by id but shared by every message with that id, while protection is decided per message — refs.ts migrated the binding as soon as ONE sibling lost protection, re-tagging a still-protected sibling's already-sent bytes. assignRefs now precomputes same-id protection and skips the migration while any sibling stays protected; verif harnesses re-baselined (generic Q2 UNSAT = collision theft unreachable; induction gains a protected-append UNSAT and documents the remaining collision SAT as the legitimate #2607 protection-lift migration). --- kernel/package-lock.json | 4 +- kernel/package.json | 2 +- kernel/src/refs.ts | 17 ++- kernel/tests/issue3945-id-collision.test.ts | 110 ++++++++++++++++++ package-lock.json | 2 +- tests/verif-generic-append-stability.test.ts | 34 +++--- .../verif-induction-append-stability.test.ts | 31 +++-- 7 files changed, 174 insertions(+), 26 deletions(-) create mode 100644 kernel/tests/issue3945-id-collision.test.ts diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 1d88f406cb..514bb51cc7 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.143", + "version": "0.0.144", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.143", + "version": "0.0.144", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 0457970312..476f41bf03 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.143", + "version": "0.0.144", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/refs.ts b/kernel/src/refs.ts index 21606cda23..63c309d2f2 100644 --- a/kernel/src/refs.ts +++ b/kernel/src/refs.ts @@ -90,6 +90,17 @@ export function assignRefs( let newlyAssigned = 0; const migratedIds: string[] = []; + // #3945: a BLOCKED binding is keyed by id but SHARED by every message + // carrying that id, while protection is decided per message. Migrating the + // binding the moment ONE sibling loses protection would re-tag a + // still-protected sibling's already-sent bytes. Migration requires that NO + // message with that id is protected this pass. + const siblingProtected = new Set(); + for (const message of messages) { + if (!message.id || options.shouldSkip?.(message)) continue; + if (options.isProtected?.(message)) siblingProtected.add(message.id); + } + for (const message of messages) { if (!message.id || options.shouldSkip?.(message)) continue; @@ -100,7 +111,11 @@ export function assignRefs( // is lifted (#2607: #1188 pinned all media; archivable images now fold // and must be citable via decompress({ imageRef })). Fresh number only — // the id-never-reused contract holds. - if (prev === BLOCKED_REF && !protectedNow) { + if ( + prev === BLOCKED_REF && + !protectedNow && + !siblingProtected.has(message.id) + ) { delete map.byRaw[message.id]; migratedIds.push(message.id); } else continue; diff --git a/kernel/tests/issue3945-id-collision.test.ts b/kernel/tests/issue3945-id-collision.test.ts new file mode 100644 index 0000000000..d9a60ff819 --- /dev/null +++ b/kernel/tests/issue3945-id-collision.test.ts @@ -0,0 +1,110 @@ +// #3945: duplicate-id messages flip bytes of an ALREADY-SENT message. The +// BLOCKED binding in messageRefs.byRaw is keyed by message id but SHARED by +// every message carrying that id, while protection is decided per message. +// refs.ts migrated a stale BLOCKED entry to a fresh numeric ref as soon as +// ANY one message with that id lost protection — deleting the binding a +// still-protected sibling depended on, so the sibling's next render gained a +// tag it never had (mid-history rewrite of bytes the prefix cache banked). +// Found by the verif solver (tests/verif-generic-append-stability.test.ts Q2); +// these pins freeze the fixed contract: migration requires that NO message +// with that id is protected, and the unprotected-sibling path renders verbatim. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { assignRefs, BLOCKED_REF } from "../src/refs.js"; +import { renderVisibleRefs } from "../src/render-refs.js"; +import { createInitialState } from "../src/state.js"; +import type { CoreMessage } from "../src/types.js"; + +const countTokens = (text: string) => Math.ceil(text.length / 4); + +const user = (id: string, text: string): CoreMessage => ({ + id, + role: "user", + contentType: "text", + text, +}); + +test("pin: an unprotected sibling never steals a BLOCKED binding from a protected one (#3945)", () => { + const m1 = user("C", "shared id, protected"); + const m2 = user("D", "distinct id"); + const m3 = user("C", "shared id, NOT protected"); + const isProtected = (m: CoreMessage) => m === m1; + + const pass = assignRefs([m1, m2, m3], { + existing: { byRaw: {}, byRef: {} }, + nextIndex: 1, + isProtected, + }); + + // The binding survives: no migration was recorded for "C"… + assert.equal(pass.map.byRaw["C"], BLOCKED_REF); + assert.deepEqual(pass.migratedIds, []); + // …and the sibling pair renders verbatim (BLOCKED ⇒ as-is, no tag). + const state = createInitialState(); + state.messageRefs = pass.map; + const rendered = renderVisibleRefs([m1, m2, m3], state, countTokens); + assert.equal(rendered[0]!.text, "shared id, protected"); + assert.equal(rendered[2]!.text, "shared id, NOT protected"); +}); + +test("pin: round-over-round bytes stay stable under the collision shape", () => { + const m1 = user("C", "shared id, protected"); + const m2 = user("D", "distinct id"); + const m3 = user("C", "shared id, NOT protected"); + const isProtected = (m: CoreMessage) => m === m1; + + const state = createInitialState(); + state.messageRefs = { byRaw: {}, byRef: {} }; + let next = 1; + let prevRender: string[] | null = null; + for (let round = 0; round < 3; round++) { + const pass = assignRefs([m1, m2, m3], { + existing: state.messageRefs, + nextIndex: next, + isProtected, + }); + state.messageRefs = pass.map; + next = pass.nextIndex; + const rendered = renderVisibleRefs([m1, m2, m3], state, countTokens); + const texts = rendered.map((m) => m.text ?? ""); + if (prevRender) assert.deepEqual(texts, prevRender); + prevRender = texts; + } + // The distinct-id message keeps its stable tag; the collided pair never + // gains one. + assert.ok((prevRender![1] ?? "").startsWith(" { + const m1 = user("C", "shared id, was protected"); + const m3 = user("C", "shared id, never protected"); + const pass1 = assignRefs([m1], { + existing: { byRaw: {}, byRef: {} }, + nextIndex: 1, + isProtected: () => true, + }); + assert.equal(pass1.map.byRaw["C"], BLOCKED_REF); + + const pass2 = assignRefs([m1, m3], { + existing: pass1.map, + nextIndex: pass1.nextIndex, + isProtected: () => false, + }); + assert.equal(pass2.map.byRaw["C"], "m00001"); + assert.deepEqual(pass2.migratedIds, ["C"]); +}); + +test("pin: all-unprotected duplicate ids mint exactly one ref", () => { + const m1 = user("C", "first"); + const m3 = user("C", "second"); + const pass = assignRefs([m1, m3], { + existing: { byRaw: {}, byRef: {} }, + nextIndex: 1, + isProtected: () => false, + }); + assert.equal(pass.map.byRaw["C"], "m00001"); + assert.equal(pass.map.byRef["m00001"], "C"); + assert.equal(pass.newlyAssigned, 1); +}); diff --git a/package-lock.json b/package-lock.json index 7848be48d0..542ae2f6c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.143", + "version": "0.0.144", "dev": true, "license": "MIT", "devDependencies": { diff --git a/tests/verif-generic-append-stability.test.ts b/tests/verif-generic-append-stability.test.ts index 22c36a01b2..748b3b47bd 100644 --- a/tests/verif-generic-append-stability.test.ts +++ b/tests/verif-generic-append-stability.test.ts @@ -25,9 +25,11 @@ import type { CoreMessage } from "../kernel/src/types.js"; * Q1: violation reachable? (expect SAT — at minimum the known #3805 * durable-ref migration re-emerges from the general model) * Q2: violation reachable WITHOUT any protection dropping? - * (p_i → q_i for all i). Expect UNSAT: no NEW violation shapes within - * bounds (≤3 messages, text length ≤ 200, unroll bounds). SAT here = - * a candidate NEW bug → must replay on real code. + * (p_i → q_i for all i). Expect UNSAT since #3945 was fixed: the + * id-collision re-tag (m3 reusing m1's raw id while unprotected stole + * the shared BLOCKED binding and re-tagged m1's sent bytes) is no + * longer reachable — refs.ts keeps the binding while ANY same-id + * message is protected. */ const REPO = new URL("..", import.meta.url).pathname; @@ -141,20 +143,21 @@ test("generic append-only stability: arbitrary inputs over assignRefs + renderVi const w: Record = {}; (q1res.values ?? []).forEach((v, i) => { w[names[i]!] = v; }); - // Q2: violation without ANY protection drop (p_i → q_i)? SAT here found a - // NEW shape the scenario harnesses never covered: m3 reusing m1's raw id - // while being unprotected — the per-id BLOCKED binding is shared but - // protection is per-message, so the colliding message steals the binding - // (refs.ts:99 delete + fresh alloc re-tags m1's already-sent bytes). + // Q2: violation without ANY protection drop (p_i → q_i)? UNSAT since the + // #3945 fix — the per-id BLOCKED binding is shared but protection is + // per-message; refs.ts now migrates only when NO same-id message is + // protected, so the colliding message can no longer steal the binding + // (refs.ts delete + fresh alloc used to re-tag m1's already-sent bytes). const noDrop = p1.implies(q1).and(p2.implies(q2)); const q2res = solveSmt2(smt2Of(ctx, [...baseAsserts, noDrop, violation] as never[], [] as never), []); - assert.equal(q2res.status, "sat", "Q2 expected SAT — the id-collision re-tag must be reachable without protection drops"); + assert.equal(q2res.status, "unsat", "Q2 expected UNSAT — the id-collision re-tag is fixed (#3945): no violation without protection drops"); // Q3: with pairwise-distinct ids and no protection drop, NO violation may - // remain within bounds (bounded absence — the known shapes are gone). + // remain within bounds (kept as a belt beyond Q2 — Q2 already proves the + // collision shape is gone). const idt = (n: string) => ctx.String.const(n); const distinct = idt("m1_id").neq(idt("m2_id")).and(idt("m1_id").neq(idt("m3_id"))).and(idt("m2_id").neq(idt("m3_id"))); const q3res = solveSmt2(smt2Of(ctx, [...baseAsserts, noDrop, distinct, violation] as never[], [] as never), []); - assert.equal(q3res.status, "unsat", "Q3 expected UNSAT — nothing beyond protection-drop and id-collision shapes within bounds"); + assert.equal(q3res.status, "unsat", "Q3 expected UNSAT — nothing beyond protection-drop shapes within bounds"); // ---- replay on REAL code with a witness-shaped input ---- // Interpret witness booleans through the real durable classifier semantics @@ -187,6 +190,8 @@ test("generic append-only stability: arbitrary inputs over assignRefs + renderVi assert.ok(realFlip === true || lifted === false, `witness p=${p1w},${p2w} q=${q1w},${q2w},${q3w} replayFlip=${realFlip} lifted=${lifted}`); // Q2 shape replayed for real: duplicate raw id, protection NOT dropped. + // Fixed contract (#3945): the shared BLOCKED binding survives while any + // same-id message is protected — no flip, no re-tag, both passes verbatim. const mkDup = (id: string, text: string, prot: boolean): CoreMessage => ({ id, role: "user", contentType: "text", text: prot ? `${marker} catalog ${id}` : text } as CoreMessage); const d1 = mkDup("C", "marker body catalog", true); @@ -197,8 +202,9 @@ test("generic append-only stability: arbitrary inputs over assignRefs + renderVi const dupRA = renderVisibleRefs([d1, d2], { messageRefs: dupA.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; const dupB = assignRefs([d1, d2, d3], { existing: dupA.map, nextIndex: dupA.nextIndex, isProtected: dupProt as never }); const dupRB = renderVisibleRefs([d1, d2, d3], { messageRefs: dupB.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; - assert.ok(dupRA[0]!.text !== dupRB[0]!.text, - "id-collision witness must flip real code: protected m1 verbatim in A, re-tagged in B"); + assert.ok(dupRA[0]!.text === dupRB[0]!.text, + "id-collision fixed (#3945): protected m1 stays verbatim across the append — no re-tag"); assert.equal(dupA.map.byRaw.C, "BLOCKED"); - assert.notEqual(dupB.map.byRaw.C, "BLOCKED"); + assert.equal(dupB.map.byRaw.C, "BLOCKED"); + assert.equal(dupB.migratedIds.length, 0, "no migration while a same-id sibling is protected"); }); diff --git a/tests/verif-induction-append-stability.test.ts b/tests/verif-induction-append-stability.test.ts index 8ffebecc42..11fc1266b1 100644 --- a/tests/verif-induction-append-stability.test.ts +++ b/tests/verif-induction-append-stability.test.ts @@ -24,11 +24,15 @@ * bytes. The binding is the only channel through which append can act — * and the step closes it. * - * Side condition (documented, not hidden): id collisions. The step formula - * WITHOUT the distinctness constraint must be SAT — that witness is exactly - * issue #3945, replayed here on real code as the induction's honest failure - * case. A real-code randomized spot-check pins the locality argument on the - * implementation itself. + * Side condition (documented, not hidden): id collisions. Since the #3945 + * fix the only remaining collision change is the LEGITIMATE #2607 migration + * (binding BLOCKED → numeric once no same-id message is protected — in this + * single-message abstraction there IS no protected sibling in the call, so + * the solver reaching it is correct behavior, not a bug). The sibling-theft + * shape — a protected same-id sibling present in the call — is proven + * unreachable by the generic harness (Q2 UNSAT) and pinned on real code by + * kernel/tests/issue3945-id-collision.test.ts. A real-code randomized + * spot-check pins the locality argument on the implementation itself. */ import { test } from "node:test"; import assert from "node:assert/strict"; @@ -112,10 +116,23 @@ test("induction step: appending a distinct-id message cannot touch any observed const step = solveSmt2(smt2Of(ctx, [...base, mId.neq(idP), changedReal] as never[], [] as never), []); assert.equal(step.status, "unsat", "induction step must hold: appending a distinct-id message cannot alter the observed binding, for ANY prefix state"); - // Q-collision: WITHOUT distinctness, the change IS reachable — that is #3945. + // Q-collision: WITHOUT distinctness, the change IS reachable when the + // binding is BLOCKED and the appended message is unprotected. Since the + // #3945 fix this is the legitimate #2607 protection-lift migration (no + // protected same-id sibling exists in this call — the abstraction's + // single message is the whole batch); the sibling-theft shape needs the + // sibling present, which generic Q2 proves unreachable. const col = solveSmt2(smt2Of(ctx, [...base, mId.eq(idP), changedReal] as never[], [] as never), []); assert.equal(col.status, "sat", - "id-collision shape must break the step — this SAT is issue #3945 talking"); + "collision SAT is the legitimate #2607 BLOCKED→numeric migration (protection lifted, no protected sibling in the call)"); + // Q-collision-protected: a PROTECTED same-id append may never touch the + // observed binding — that is the #3945 theft shape, closed. (r_p is + // constrained non-empty: an ""-keyed entry is not a binding, and the + // protected-assign path writing BLOCKED over it is not a theft.) + const realBinding = (rP as unknown as { neq: (x: unknown) => unknown }).neq(ctx.String.val("")); + const colProt = solveSmt2(smt2Of(ctx, [...base, mId.eq(idP), prot, realBinding, changedReal] as never[], [] as never), []); + assert.equal(colProt.status, "unsat", + "a protected same-id append must not alter the observed binding (#3945 fixed)"); }); test("induction anchor: real code keeps append local (randomized spot-check, 60 runs)", () => {